neotest: Draft software-release to run NEO/go & friends tests/benchmarks under webrunner

We provide neotest SR which builds current NEO/go together with the
tools which are needed to run the benchmarks.

The SR does not yet provide automated service to run the tests
automatically periodically and to upload the results to ERP5 (all marked
as TODO).

However with present state it provides all the infrastructure for people
to try to run NEO tests in their webrunner: a `neotest` program is
installed in buildout top-level bin/ . One should use it to run the
tests.

An example run output could be seen here:

https://lab.nexedi.com/snippets/258

With benchstat (https://godoc.org/golang.org/x/perf/cmd/benchstat) it can be summarized or compared to another run. Here is e.g. summarization:

```
name                                                                 pystone/s
vifibcloud-onlinenet-hosting-004/pystone                              156k ± 6%

name                                                                 µs/op
vifibcloud-onlinenet-hosting-004/sha1/py/1024B                        1.71 ± 6%
vifibcloud-onlinenet-hosting-004/sha1/go/1024B                        2.26 ±73%
vifibcloud-onlinenet-hosting-004/sha1/py/4096B                        6.39 ± 4%
vifibcloud-onlinenet-hosting-004/sha1/go/4096B                        5.66 ± 2%

name                                                                 us/op
vifibcloud-onlinenet-hosting-004/disk/randread/direct/4K-min          75.7 ± 4%
vifibcloud-onlinenet-hosting-004/disk/randread/direct/4K-avg          95.2 ± 1%
vifibcloud-onlinenet-hosting-004/disk/randread/pagecache/4K-avg       1.84 ± 0%

name                                                                 time/op
vifibcloud-onlinenet-hosting-004/disk/randread/pagecache/4K-min      288ns ± 5%
vifibcloud-onlinenet-hosting-004/disk/randread/pagecache/4K-avg      876ns ± 9%

name                                                                 µs/object
dataset:wczblk1-8
vifibcloud-onlinenet-hosting-004/fs1/zhash.py                         22.2 ±11%
vifibcloud-onlinenet-hosting-004/fs1/zhash.py-P16                     40.6 ±19%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go                         4.00 ±40%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go+prefetch128             5.58 ±24%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go-P16                     3.68 ±88%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py                          626 ±40%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py-P16                    2.17k ± 3%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py                823 ±17%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py-P16          2.04k ± 3%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go                666 ± 9%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go+prefetch128    138 ±10%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go-P16          1.96k ± 0%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py                 1.02k ±71%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py-P16             3.09k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go                   602 ±29%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go+prefetch128       213 ±14%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go-P16             4.66k ± 1%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py                       326 ±32%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py-P16                   554 ± 7%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go                      39.6 ± 7%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go+prefetch128          24.3 ±18%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go-P16                   213 ± 6%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go               42.8 ±13%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go+prefetch128   24.5 ±19%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go-P16           70.2 ±62%
dataset:prod1-1024
vifibcloud-onlinenet-hosting-004/fs1/zhash.py                         18.8 ±22%
vifibcloud-onlinenet-hosting-004/fs1/zhash.py-P16                     33.6 ±21%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go                         3.46 ±25%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go+prefetch128             5.36 ±25%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go-P16                     2.56 ±45%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py                          617 ±23%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py-P16                    2.19k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py                717 ±46%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py-P16          2.02k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go                297 ±14%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go+prefetch128    130 ±10%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go-P16          1.86k ± 7%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py                   680 ±61%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py-P16             4.76k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go                   269 ±13%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go+prefetch128       178 ± 8%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go-P16             3.03k ± 1%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py                       227 ±10%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py-P16                   654 ± 5%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go                      32.4 ± 8%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go+prefetch128          15.5 ±12%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go-P16                   261 ±12%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go               24.2 ± 1%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go+prefetch128   13.9 ±25%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go-P16            270 ±10%
```

The infrastructure to build Go projects under SlapOS is also introduced along the way
(was applied separetely as nexedi/slapos@1b540151).

Please see details in the individual commit messages. 

/cc @nexedi 

P.S. the results are noisy becuase under regular webrunner I do not have root and cannot run e.g. `cpupower frequency-set -g performance`.

/reviewed-on nexedi/slapos!242

component/ethtool/buildout.cfg

+# ethtool - query or control network driver and hardware settings
+# https://www.kernel.org/pub/software/network/ethtool
+[buildout]
+parts   = ethtool
+
+[ethtool]
+recipe  = slapos.recipe.cmmi
+url     = https://www.kernel.org/pub/software/network/ethtool/ethtool-4.11.tar.xz
+md5sum  = 16d38f4ebe23e44f96f7d8b38ed3652c

component/ioping/buildout.cfg

+# ioping - simple disk I/O latency measuring tool
+# https://github.com/koct9i/ioping
+[buildout]
+extends =
+    ../git/buildout.cfg
+
+parts   =
+    ioping
+
+[ioping-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+# NOTE we use a bit patched ioping version which shows not only avg latency but also its distribution
+# repository = https://github.com/koct9i/ioping.git
+repository = https://lab.nexedi.com/kirr/ioping.git
+revision = v1.0-9-g34c97f7636
+
+[ioping]
+recipe  = slapos.recipe.cmmi
+path    = ${ioping-repository:location}
+configure-command = :
+make-binary =
+# XXX without vvv PREFIX=${:location} does not work
+location= ${buildout:parts-directory}/${:_buildout_section_name_}
+make-targets= make install PREFIX=${:location}

component/lmbench/buildout.cfg

+# LMbench - Tools for Performance Analysis
+# http://www.bitmover.com/lmbench/
+[buildout]
+extends =
+    ../git/buildout.cfg
+    ../golang/buildout.cfg
+
+parts   =
+    lmbench
+
+
+[lmbench-repository]
+<= git-repository
+# NOTE we use a bit patched lmbench version with fixes to lat_tcp for errors not to go unnoticed and other addons
+# repository = https://svn.code.sf.net/p/lmbench/code
+repository = https://lab.nexedi.com/kirr/lmbench.git
+revision = 9b108b6ff3
+
+[lmbench]
+recipe  = slapos.recipe.cmmi
+path    = ${lmbench-repository:location}
+configure-command = :
+make-binary =
+# XXX without vvv BASE=${:location} does not work
+location= ${buildout:parts-directory}/${:_buildout_section_name_}
+make-targets= cd ${lmbench-repository:location}/lmbench3/src
+              && make install BASE=${:location}
+              && bash -c ". ${gowork:env.sh} && go build -o ${:location}/bin/lat_tcp_go lat_tcp.go"

software/neoppod/software-common.cfg

@@ -121,7 +121,7 @@ slapos.toolbox = 0.73
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0
-zodbtools = 0.0.0.dev3
+zodbtools = 0.0.0.dev4
 
 # Required by:
 # slapos.toolbox==0.71
@@ -153,3 +153,20 @@ paramiko = 2.1.2
 # Required by:
 # slapos.toolbox==0.71
 passlib = 1.7.1
+
+# Required by:
+# zodbtools==0.0.0dev4
+zodburi = 2.3.0
+
+# Required by:
+# zodburi==2.0
+# ZEO 5 requires transaction >= 2
+ZEO = 4.3.1
+
+# Required by:
+# zodburi==2.0
+mock = 2.0.0
+
+# Required by:
+# ZEO==4.3.1
+zdaemon = 4.2.0

software/neotest/gowork.cfg

+# Code generated by gowork-snapshot; DO NOT EDIT.
+
+# list of go git repositories to fetch
+[gowork.goinstall]
+depends_gitfetch  =
+    ${go_github.com_cznic_strutil:recipe}
+    ${go_github.com_golang_glog:recipe}
+    ${go_github.com_kisielk_og-rek:recipe}
+    ${go_github.com_kylelemons_godebug:recipe}
+    ${go_github.com_pkg_errors:recipe}
+    ${go_github.com_pkg_profile:recipe}
+    ${go_github.com_someonegg_gocontainer:recipe}
+    ${go_golang.org_x_net:recipe}
+    ${go_golang.org_x_perf:recipe}
+    ${go_golang.org_x_sync:recipe}
+    ${go_golang.org_x_tools:recipe}
+    ${go_lab.nexedi.com_kirr_go123:recipe}
+    ${go_lab.nexedi.com_kirr_neo:recipe}
+
+
+[go_github.com_cznic_strutil]
+<= go-git-package
+go.importpath = github.com/cznic/strutil
+repository    = https://github.com/cznic/strutil
+revision  = 529a34b1c1
+
+[go_github.com_golang_glog]
+<= go-git-package
+go.importpath = github.com/golang/glog
+repository    = https://github.com/golang/glog
+revision  = 23def4e6c1
+
+[go_github.com_kisielk_og-rek]
+<= go-git-package
+go.importpath = github.com/kisielk/og-rek
+repository    = https://github.com/kisielk/og-rek
+revision  = dd41cde712
+
+[go_github.com_kylelemons_godebug]
+<= go-git-package
+go.importpath = github.com/kylelemons/godebug
+repository    = https://github.com/kylelemons/godebug
+revision  = d65d576e93
+
+[go_github.com_pkg_errors]
+<= go-git-package
+go.importpath = github.com/pkg/errors
+repository    = https://github.com/pkg/errors
+revision  = v0.8.0-6-g2b3a18b5f0
+
+[go_github.com_pkg_profile]
+<= go-git-package
+go.importpath = github.com/pkg/profile
+repository    = https://github.com/pkg/profile
+revision  = v1.2.1-0-g5b67d42886
+
+[go_github.com_someonegg_gocontainer]
+<= go-git-package
+go.importpath = github.com/someonegg/gocontainer
+repository    = https://github.com/someonegg/gocontainer
+revision  = fc2c7e84b5
+
+[go_golang.org_x_net]
+<= go-git-package
+go.importpath = golang.org/x/net
+repository    = https://go.googlesource.com/net
+revision  = 1087133bc4
+
+[go_golang.org_x_perf]
+<= go-git-package
+go.importpath = golang.org/x/perf
+repository    = https://go.googlesource.com/perf
+revision  = 4469e6ce8c
+
+[go_golang.org_x_sync]
+<= go-git-package
+go.importpath = golang.org/x/sync
+repository    = https://go.googlesource.com/sync
+revision  = 8e0aa688b6
+
+[go_golang.org_x_tools]
+<= go-git-package
+go.importpath = golang.org/x/tools
+repository    = https://go.googlesource.com/tools
+revision  = 9bd2f44268
+
+[go_lab.nexedi.com_kirr_go123]
+<= go-git-package
+go.importpath = lab.nexedi.com/kirr/go123
+repository    = https://lab.nexedi.com/kirr/go123.git
+revision  = 2578d58311
+
+[go_lab.nexedi.com_kirr_neo]
+<= go-git-package
+go.importpath = lab.nexedi.com/kirr/neo
+repository    = https://lab.nexedi.com/kirr/neo.git
+revision  = v1.8-1326-g4d0cd89484

software/neotest/instance.cfg.in

+# TODO instance which runs this test periodically automatically and ingests results to ERP5

software/neotest/neotest-env.sh.in

+# env.sh for putting everything needed to run neotest on path
+
+. ${gowork:env.sh}
+
+PATH="${ethtool:location}/sbin:$PATH"
+PATH="${ioping:location}/bin:$PATH"
+PATH="${lmbench:location}/bin:$PATH"
+PATH="${mariadb:location}/bin:$PATH"
+PATH="${mariadb:location}/scripts:$PATH"
+PATH="${python2.7:location}/bin:$PATH"
+
+# add all eggs and develop-eggs to py path
+#
+# XXX better leverage something like zc.recipe.egg to generate eggs path buildout way
+# XXX see e.g. https://lab.nexedi.com/nexedi/slapos/merge_requests/242#note_49644 for why
+# TODO(kirr) try doing so
+PYTHONPATH=${buildout:develop-eggs-directory}:$PYTHONPATH       # for .egg-link to be found by setuptools
+for egglink in ${buildout:develop-eggs-directory}/*.egg-link; do
+    PYTHONPATH=`cat $egglink |head -1`:$PYTHONPATH
+done
+
+for egg in ${buildout:develop-eggs-directory}/*.egg ${buildout:eggs-directory}/*.egg; do
+        export PYTHONPATH=$egg:$PYTHONPATH
+done
+
+
+export PATH
+export PYTHONPATH
+
+export PS1="(neotest-`basename ${buildout:directory}`) $PS1"

software/neotest/neotest.in

+#!/bin/bash -e
+# neotest wraper so it could be run without any environment preset
+
+. ${buildout:directory}/neotest-env.sh
+exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/neotest "$@"

software/neotest/software-dev.cfg

+# neotest SR with main components taken from latest git versions
+[buildout]
+extends = software.cfg
+
+
+[go_lab.nexedi.com_kirr_go123]
+branch	= master
+revision=
+
+[go_lab.nexedi.com_kirr_neo]
+branch	= t
+revision=
+
+[lmbench-repository]
+branch  = x/kirr
+revision=
+
+[ioping-repository]
+branch  = x/hist
+revision=

software/neotest/software.cfg

+# NEO test software-release
+#
+# This software-release prepares environment so that neotest NEO
+# testing/benchmarking can be run in a SlapOS WebRunner.
+[buildout]
+extends =
+    ../../stack/slapos.cfg
+    ../neoppod/software-common.cfg
+    ../../component/golang/buildout.cfg
+    gowork.cfg
+    ../../component/wendelin.core/buildout.cfg
+    ../../component/ethtool/buildout.cfg
+    ../../component/ioping/buildout.cfg
+    ../../component/lmbench/buildout.cfg
+
+parts =
+    gowork
+    lmbench
+    ioping
+    ethtool
+
+    neoppod-develop
+    neoppod
+    wendelin.core-dev
+    eggs
+
+    neotest-env.sh
+    neotest
+
+#   for instance
+    slapos-deps-eggs
+    slapos-cookbook
+#   instance.cfg
+
+
+# go packages to install (+ automatically their dependencies)
+[gowork]
+install =
+    lab.nexedi.com/kirr/neo/go/...  \
+    github.com/pkg/profile          \
+    golang.org/x/perf/cmd/benchstat
+
+# dev-install neo from go checkout
+[neoppod-develop]
+setup   = ${go_lab.nexedi.com_kirr_neo:location}
+
+
+# neotest-env.sh + neotest wrapper
+[buildout-template]
+recipe  = slapos.recipe.template
+url     = ${:_profile_base_location_}/${:_buildout_section_name_}.in
+output  = ${buildout:directory}/${:_buildout_section_name_}
+
+[neotest-env.sh]
+<= buildout-template
+md5sum  = 6e2203c7a86e8a3e8e56b8086115f5d4
+
+[neotest]
+<= buildout-template
+output  = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+md5sum  = fb3b4109128c1db1739ef5bb6abd1d94
+
+
+# instance (TODO)
+[instance.cfg]
+<= buildout-template
+md5sum  = 410e1b2d72829824b28cc0299adb472e
+
+
+# eggs:
+[eggs]
+recipe  = zc.recipe.egg
+eggs    =
+# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
+    ZODB3
+# also for wc
+    numpy
+# to install not only wendelin.core modules but also scripts
+    wendelin.core
+# for ZEO scripts (runzeo)
+    ZEO
+
+
+# wendelin.core: latest not yet released
+[wendelin.core-repository]
+revision= v0.11-4-g38fbc83ceb
+
+# ping eggs versions
+[versions]
+pyasn1 = 0.3.7
+ZODB3 = 3.11.0
+numpy = 1.13.3
+zope.testing = 4.6.2
+
+# Required by:
+# ZEO==4.3.1
+# ZODB==4.4.5
+# zodburi==2.3.0
+ZConfig = 3.2.0
+
+# Required by:
+# ZEO==4.3.1
+# ZODB==4.4.5
+zc.lockfile = 1.2.1
+
+# Required by:
+# slapos.toolbox==0.73
+erp5.util = 0.4.49

stack/erp5/buildout.cfg

@@ -688,8 +688,6 @@ PyXML = 0.8.5
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
-# ZEO 5 requires transaction >= 2
-ZEO = 4.3.1
 ZODB3 = 3.11.0
 # astroid 1.4.1 breaks testDynamicClassGeneration
 astroid = 1.3.8