golang: Go modules support; Prepare to deprecate GOPATH

Add support for using Go modules to golang/gowork infrastructure:

- Users can now request to install a module via gowork:install as. e.g.
  in the following example:

        [gowork]
        install =
            lab.nexedi.com/kirr/neo/go/...@v0.0.0-20210103165133-f3effa6c535f
            golang.org/x/tools/gopls@v0.4.3
            ${helloweb:location}/go:./...

  The first two request to install programs from an external module at particular
  revision/version. The latter requests to install programs from locally
  cloned/checked-out module source.

  The documentation now talks only about programs, because "package
  installation" became unnecessary long time ago as Go toolchain uses
  right packages and recompiles things as needed automatically since
  introduction of the Go build cache in go 1.10.

- The change comes accompanied by corresponding helloweb change that
  reworks it to a) become a module itself, and b) to use other modules -
  that are not explicitly cloned by buildout - so that we can be sure
  that module way of fetching/building things actually works.

  kirr/helloweb@a7c788ae

- Non-module way - e.g. build via GOPATH - is still supported (because
  e.g. software/gitlab still uses it), but not explicitly documented and 
  scheduled to be deprecated and removed. The reason for this is that
  upstream Go is going to remove support for GOPATH and leave only
  module-based approach in Go1.17

  https://github.com/golang/go/issues/37755#issuecomment-771879911

/cc @jerome, @luke, @tomo, @alain.takoudjou
/reviewed-on nexedi/slapos!924

component/golang/buildout.cfg

@@ -80,30 +80,39 @@ environment-extra =
 
 # ---- infrastructure to build Go workspaces / projects ----
 
-# gowork is a top-level section representing workspace
+# gowork is the top-level section that defines Go workspace.
 #
-# users should add `install` field to [gowork] to describe packages they want to
-# be installed (+ automatically their dependencies are installed too). e.g.
+# It specifies global settings that are used to build Go programs:
+#
+# - Go toolchain to use, e.g.
 #
 #   [gowork]
-#   install =
-#       lab.nexedi.com/kirr/neo/go/...
-#       github.com/pkg/profile
-#       golang.org/x/perf/cmd/benchstat
+#   golang = ${golang1.15:location}
 #
-# For Cgo support pkg-config is made pre-available by gowork, and users
-# should list paths where to search for pkg-config files, e.g. this way:
+# - For Cgo support pkg-config is made pre-available by gowork, and users
+#   should list paths where to search for pkg-config files, e.g. this way:
 #
 #   [gowork]
 #   cpkgpath =
 #       ${sqlite3:location}/lib/pkgconfig
 #       ${zlib:location}/lib/pkgconfig
 #
-# It is also possible to specify Go build flags used for compilation e.g. this way:
+# - It is also possible to specify Go build flags used for compilation e.g. this way:
 #
 #   [gowork]
 #   buildflags = -race
 #
+#
+# Users can also add `install` field to [gowork] to request Go programs to be
+# automatically installed, for example:
+#
+#   [gowork]
+#   install =
+#       lab.nexedi.com/kirr/neo/go/...@v0.0.0-20210103165133-f3effa6c535f
+#       golang.org/x/tools/gopls@v0.4.3
+#       ${helloweb:location}/go:./...
+#
+#
 # ${go:exe} is standalone executable that runs go in activated gowork environment.
 [go]
 recipe  = slapos.recipe.template:jinja2
@@ -116,9 +125,9 @@ template= inline:
   exec go "$@"
 
 [gowork]
-directory = ${buildout:directory}/go.work
-src	= ${:directory}/src
-bin	= ${:directory}/bin
+directory = ${gowork.dir:directory}
+src	= ${gowork.dir:src}
+bin	= ${gowork.dir:bin}
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
@@ -130,6 +139,9 @@ buildflags =
 # empty pkg-config path by default
 cpkgpath =
 
+# by default don't `go install` anything
+install =
+
 # everything is done by dependent parts
 recipe  = plone.recipe.command
 command = :
@@ -140,24 +152,52 @@ env.sh  = ${gowork-env.sh:output}
 [gowork-env.sh]
 recipe	= slapos.recipe.template
 url     = ${:_profile_base_location_}/goenv.sh.in
-output	= ${gowork:directory}/env.sh
-depends = ${gowork.mkdir:recipe}
+output	= ${gowork.dir:directory}/env.sh
+depends = ${gowork.dir:recipe}
 md5sum	= 6efdf5d63381c240c7d206d7939a63f7
 
-[gowork.mkdir]
+[gowork.dir]
+directory = ${buildout:directory}/go.work
+src	= ${:directory}/src
+bin	= ${:directory}/bin
 # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
 # uses slapos.cookbook:* in recipe - slapos.cookbook will get compiled against system
 # libxml/libxslt or fail to bootstrap at all if those are not present.
 recipe  = plone.recipe.command
-command = mkdir -p ${gowork:directory}
+command = mkdir -p ${:directory}
 update-command = ${:command}
 stop-on-error = true
 
-# install go packages
-# clients should put package list to install to gowork:install ("..." requests installing everything)
+# install go programs
+# clients can put program list to install to gowork:install
 [gowork.goinstall]
 recipe  = plone.recipe.command
-command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
+command = bash -c ". ${gowork:env.sh} &&
+	for x in $(echo -n '${gowork:install}' |tr '\n' ' '); do
+		case "\$x" in
+
+		# external module, e.g. golang.org/x/tools/gopls@v0.4.3
+		*@*)
+			echo GOMOD \$x
+			GO111MODULE=on go install ${gowork:buildflags} -v \$x
+			;;
+
+		# locally-cloned module source, e.g. <module-src-location>:./...
+		*:*)
+			echo GOMODSRC \$x
+			dir=\"\$${x%%:*}\"
+			arg=\"\$${x#*:}\"
+			(cd \$dir && GO111MODULE=on go install ${gowork:buildflags} -v \$arg)
+			;;
+
+		# non-module
+		*)
+			echo GOPKG \$x
+			GO111MODULE=off go install ${gowork:buildflags} -v \$x
+			;;
+		esac
+	done
+	"
 update-command = ${:command}
 stop-on-error = true
 
@@ -175,4 +215,4 @@ git-executable = ${git:location}/bin/git
 # generated with the help of gowork-snapshot tool.
 [go-git-package]
 <= git-repository
-location = ${gowork:src}/${:go.importpath}
+location = ${gowork.dir:src}/${:go.importpath}

component/helloweb/buildout.cfg

@@ -4,7 +4,6 @@ extends =
   ../git/buildout.cfg
   ../ruby/buildout.cfg
   ../golang/buildout.cfg
-  gowork.cfg
 
 parts =
   helloweb-python
@@ -21,9 +20,15 @@ parts =
 # cloning+building manually. However to be able to use third-party Go packages
 # we need gowork support, and other languages can use helloweb repository from
 # under gowork as well.
+[helloweb]
+<= go-git-package
+go.importpath = lab.nexedi.com/nexedi/helloweb
+repository    = https://lab.nexedi.com/nexedi/helloweb.git
+revision      = a7c788ae71
+
 [gowork]
 install =
-    lab.nexedi.com/nexedi/helloweb/go/...
+    ${helloweb:location}/go:./...
 
 golang  = ${golang1.16:location}
 
@@ -44,7 +49,7 @@ input   = inline:
 [helloweb-egg]
 recipe  = zc.recipe.egg:develop
 egg     = helloweb
-setup   = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/python/
+setup   = ${helloweb:location}/python/
 
 [helloweb-python]
 recipe  = zc.recipe.egg:scripts
@@ -86,7 +91,7 @@ environment =
 
 [helloweb-ruby-bundle]
 recipe  = slapos.recipe.cmmi
-path    = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/ruby/
+path    = ${helloweb:location}/ruby/
 
 configure-command = :
 make-binary =

component/helloweb/gowork.cfg

-# Code generated by gowork-snapshot; DO NOT EDIT.
-
-# list of go git repositories to fetch
-[gowork.goinstall]
-depends_gitfetch  =
-    ${go_lab.nexedi.com_nexedi_helloweb:recipe}
-
-
-[go_lab.nexedi.com_nexedi_helloweb]
-<= go-git-package
-go.importpath = lab.nexedi.com/nexedi/helloweb
-repository    = https://lab.nexedi.com/nexedi/helloweb.git
-revision      = 8bfedac656