Commit fd43ba62 authored by Kirill Smelkov's avatar Kirill Smelkov

neotest: Draft software-release to run NEO/go & friends tests/benchmarks under webrunner

We provide neotest SR which builds current NEO/go together with the
tools which are needed to run the benchmarks.

The SR does not yet provide automated service to run the tests
automatically periodically and to upload the results to ERP5 (all marked
as TODO).

However with present state it provides all the infrastructure for people
to try to run NEO tests in their webrunner: a `neotest` program is
installed in buildout top-level bin/ . One should use it to run the
tests.

An example run output could be seen here:

https://lab.nexedi.com/snippets/258

With benchstat (https://godoc.org/golang.org/x/perf/cmd/benchstat) it can be summarized or compared to another run. Here is e.g. summarization:

```
name                                                                 pystone/s
vifibcloud-onlinenet-hosting-004/pystone                              156k ± 6%

name                                                                 µs/op
vifibcloud-onlinenet-hosting-004/sha1/py/1024B                        1.71 ± 6%
vifibcloud-onlinenet-hosting-004/sha1/go/1024B                        2.26 ±73%
vifibcloud-onlinenet-hosting-004/sha1/py/4096B                        6.39 ± 4%
vifibcloud-onlinenet-hosting-004/sha1/go/4096B                        5.66 ± 2%

name                                                                 us/op
vifibcloud-onlinenet-hosting-004/disk/randread/direct/4K-min          75.7 ± 4%
vifibcloud-onlinenet-hosting-004/disk/randread/direct/4K-avg          95.2 ± 1%
vifibcloud-onlinenet-hosting-004/disk/randread/pagecache/4K-avg       1.84 ± 0%

name                                                                 time/op
vifibcloud-onlinenet-hosting-004/disk/randread/pagecache/4K-min      288ns ± 5%
vifibcloud-onlinenet-hosting-004/disk/randread/pagecache/4K-avg      876ns ± 9%

name                                                                 µs/object
dataset:wczblk1-8
vifibcloud-onlinenet-hosting-004/fs1/zhash.py                         22.2 ±11%
vifibcloud-onlinenet-hosting-004/fs1/zhash.py-P16                     40.6 ±19%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go                         4.00 ±40%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go+prefetch128             5.58 ±24%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go-P16                     3.68 ±88%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py                          626 ±40%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py-P16                    2.17k ± 3%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py                823 ±17%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py-P16          2.04k ± 3%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go                666 ± 9%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go+prefetch128    138 ±10%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go-P16          1.96k ± 0%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py                 1.02k ±71%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py-P16             3.09k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go                   602 ±29%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go+prefetch128       213 ±14%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go-P16             4.66k ± 1%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py                       326 ±32%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py-P16                   554 ± 7%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go                      39.6 ± 7%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go+prefetch128          24.3 ±18%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go-P16                   213 ± 6%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go               42.8 ±13%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go+prefetch128   24.5 ±19%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go-P16           70.2 ±62%
dataset:prod1-1024
vifibcloud-onlinenet-hosting-004/fs1/zhash.py                         18.8 ±22%
vifibcloud-onlinenet-hosting-004/fs1/zhash.py-P16                     33.6 ±21%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go                         3.46 ±25%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go+prefetch128             5.36 ±25%
vifibcloud-onlinenet-hosting-004/fs1/zhash.go-P16                     2.56 ±45%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py                          617 ±23%
vifibcloud-onlinenet-hosting-004/zeo/zhash.py-P16                    2.19k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py                717 ±46%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.py-P16          2.02k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go                297 ±14%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go+prefetch128    130 ±10%
vifibcloud-onlinenet-hosting-004/neo/py/sqlite/zhash.go-P16          1.86k ± 7%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py                   680 ±61%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.py-P16             4.76k ± 1%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go                   269 ±13%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go+prefetch128       178 ± 8%
vifibcloud-onlinenet-hosting-004/neo/py/sql/zhash.go-P16             3.03k ± 1%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py                       227 ±10%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.py-P16                   654 ± 5%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go                      32.4 ± 8%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go+prefetch128          15.5 ±12%
vifibcloud-onlinenet-hosting-004/neo/go/zhash.go-P16                   261 ±12%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go               24.2 ± 1%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go+prefetch128   13.9 ±25%
vifibcloud-onlinenet-hosting-004/neo/go(!sha1)/zhash.go-P16            270 ±10%
```

The infrastructure to build Go projects under SlapOS is also introduced along the way
(was applied separetely as nexedi/slapos@1b540151).

Please see details in the individual commit messages. 

/cc @nexedi 

P.S. the results are noisy becuase under regular webrunner I do not have root and cannot run e.g. `cpupower frequency-set -g performance`.

/reviewed-on nexedi/slapos!242
parents 13fb6b54 31fe231f
# ethtool - query or control network driver and hardware settings
# https://www.kernel.org/pub/software/network/ethtool
[buildout]
parts = ethtool
[ethtool]
recipe = slapos.recipe.cmmi
url = https://www.kernel.org/pub/software/network/ethtool/ethtool-4.11.tar.xz
md5sum = 16d38f4ebe23e44f96f7d8b38ed3652c
# ioping - simple disk I/O latency measuring tool
# https://github.com/koct9i/ioping
[buildout]
extends =
../git/buildout.cfg
parts =
ioping
[ioping-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
# NOTE we use a bit patched ioping version which shows not only avg latency but also its distribution
# repository = https://github.com/koct9i/ioping.git
repository = https://lab.nexedi.com/kirr/ioping.git
revision = v1.0-9-g34c97f7636
[ioping]
recipe = slapos.recipe.cmmi
path = ${ioping-repository:location}
configure-command = :
make-binary =
# XXX without vvv PREFIX=${:location} does not work
location= ${buildout:parts-directory}/${:_buildout_section_name_}
make-targets= make install PREFIX=${:location}
# LMbench - Tools for Performance Analysis
# http://www.bitmover.com/lmbench/
[buildout]
extends =
../git/buildout.cfg
../golang/buildout.cfg
parts =
lmbench
[lmbench-repository]
<= git-repository
# NOTE we use a bit patched lmbench version with fixes to lat_tcp for errors not to go unnoticed and other addons
# repository = https://svn.code.sf.net/p/lmbench/code
repository = https://lab.nexedi.com/kirr/lmbench.git
revision = 9b108b6ff3
[lmbench]
recipe = slapos.recipe.cmmi
path = ${lmbench-repository:location}
configure-command = :
make-binary =
# XXX without vvv BASE=${:location} does not work
location= ${buildout:parts-directory}/${:_buildout_section_name_}
make-targets= cd ${lmbench-repository:location}/lmbench3/src
&& make install BASE=${:location}
&& bash -c ". ${gowork:env.sh} && go build -o ${:location}/bin/lat_tcp_go lat_tcp.go"
......@@ -121,7 +121,7 @@ slapos.toolbox = 0.73
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
zodbtools = 0.0.0.dev3
zodbtools = 0.0.0.dev4
# Required by:
# slapos.toolbox==0.71
......@@ -153,3 +153,20 @@ paramiko = 2.1.2
# Required by:
# slapos.toolbox==0.71
passlib = 1.7.1
# Required by:
# zodbtools==0.0.0dev4
zodburi = 2.3.0
# Required by:
# zodburi==2.0
# ZEO 5 requires transaction >= 2
ZEO = 4.3.1
# Required by:
# zodburi==2.0
mock = 2.0.0
# Required by:
# ZEO==4.3.1
zdaemon = 4.2.0
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_cznic_strutil:recipe}
${go_github.com_golang_glog:recipe}
${go_github.com_kisielk_og-rek:recipe}
${go_github.com_kylelemons_godebug:recipe}
${go_github.com_pkg_errors:recipe}
${go_github.com_pkg_profile:recipe}
${go_github.com_someonegg_gocontainer:recipe}
${go_golang.org_x_net:recipe}
${go_golang.org_x_perf:recipe}
${go_golang.org_x_sync:recipe}
${go_golang.org_x_tools:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_lab.nexedi.com_kirr_neo:recipe}
[go_github.com_cznic_strutil]
<= go-git-package
go.importpath = github.com/cznic/strutil
repository = https://github.com/cznic/strutil
revision = 529a34b1c1
[go_github.com_golang_glog]
<= go-git-package
go.importpath = github.com/golang/glog
repository = https://github.com/golang/glog
revision = 23def4e6c1
[go_github.com_kisielk_og-rek]
<= go-git-package
go.importpath = github.com/kisielk/og-rek
repository = https://github.com/kisielk/og-rek
revision = dd41cde712
[go_github.com_kylelemons_godebug]
<= go-git-package
go.importpath = github.com/kylelemons/godebug
repository = https://github.com/kylelemons/godebug
revision = d65d576e93
[go_github.com_pkg_errors]
<= go-git-package
go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors
revision = v0.8.0-6-g2b3a18b5f0
[go_github.com_pkg_profile]
<= go-git-package
go.importpath = github.com/pkg/profile
repository = https://github.com/pkg/profile
revision = v1.2.1-0-g5b67d42886
[go_github.com_someonegg_gocontainer]
<= go-git-package
go.importpath = github.com/someonegg/gocontainer
repository = https://github.com/someonegg/gocontainer
revision = fc2c7e84b5
[go_golang.org_x_net]
<= go-git-package
go.importpath = golang.org/x/net
repository = https://go.googlesource.com/net
revision = 1087133bc4
[go_golang.org_x_perf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://go.googlesource.com/perf
revision = 4469e6ce8c
[go_golang.org_x_sync]
<= go-git-package
go.importpath = golang.org/x/sync
repository = https://go.googlesource.com/sync
revision = 8e0aa688b6
[go_golang.org_x_tools]
<= go-git-package
go.importpath = golang.org/x/tools
repository = https://go.googlesource.com/tools
revision = 9bd2f44268
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = 2578d58311
[go_lab.nexedi.com_kirr_neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.8-1326-g4d0cd89484
# TODO instance which runs this test periodically automatically and ingests results to ERP5
# env.sh for putting everything needed to run neotest on path
. ${gowork:env.sh}
PATH="${ethtool:location}/sbin:$PATH"
PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH"
PATH="${mariadb:location}/bin:$PATH"
PATH="${mariadb:location}/scripts:$PATH"
PATH="${python2.7:location}/bin:$PATH"
# add all eggs and develop-eggs to py path
#
# XXX better leverage something like zc.recipe.egg to generate eggs path buildout way
# XXX see e.g. https://lab.nexedi.com/nexedi/slapos/merge_requests/242#note_49644 for why
# TODO(kirr) try doing so
PYTHONPATH=${buildout:develop-eggs-directory}:$PYTHONPATH # for .egg-link to be found by setuptools
for egglink in ${buildout:develop-eggs-directory}/*.egg-link; do
PYTHONPATH=`cat $egglink |head -1`:$PYTHONPATH
done
for egg in ${buildout:develop-eggs-directory}/*.egg ${buildout:eggs-directory}/*.egg; do
export PYTHONPATH=$egg:$PYTHONPATH
done
export PATH
export PYTHONPATH
export PS1="(neotest-`basename ${buildout:directory}`) $PS1"
#!/bin/bash -e
# neotest wraper so it could be run without any environment preset
. ${buildout:directory}/neotest-env.sh
exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/neotest "$@"
# neotest SR with main components taken from latest git versions
[buildout]
extends = software.cfg
[go_lab.nexedi.com_kirr_go123]
branch = master
revision=
[go_lab.nexedi.com_kirr_neo]
branch = t
revision=
[lmbench-repository]
branch = x/kirr
revision=
[ioping-repository]
branch = x/hist
revision=
# NEO test software-release
#
# This software-release prepares environment so that neotest NEO
# testing/benchmarking can be run in a SlapOS WebRunner.
[buildout]
extends =
../../stack/slapos.cfg
../neoppod/software-common.cfg
../../component/golang/buildout.cfg
gowork.cfg
../../component/wendelin.core/buildout.cfg
../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg
../../component/lmbench/buildout.cfg
parts =
gowork
lmbench
ioping
ethtool
neoppod-develop
neoppod
wendelin.core-dev
eggs
neotest-env.sh
neotest
# for instance
slapos-deps-eggs
slapos-cookbook
# instance.cfg
# go packages to install (+ automatically their dependencies)
[gowork]
install =
lab.nexedi.com/kirr/neo/go/... \
github.com/pkg/profile \
golang.org/x/perf/cmd/benchstat
# dev-install neo from go checkout
[neoppod-develop]
setup = ${go_lab.nexedi.com_kirr_neo:location}
# neotest-env.sh + neotest wrapper
[buildout-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:_buildout_section_name_}.in
output = ${buildout:directory}/${:_buildout_section_name_}
[neotest-env.sh]
<= buildout-template
md5sum = 6e2203c7a86e8a3e8e56b8086115f5d4
[neotest]
<= buildout-template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = fb3b4109128c1db1739ef5bb6abd1d94
# instance (TODO)
[instance.cfg]
<= buildout-template
md5sum = 410e1b2d72829824b28cc0299adb472e
# eggs:
[eggs]
recipe = zc.recipe.egg
eggs =
# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
ZODB3
# also for wc
numpy
# to install not only wendelin.core modules but also scripts
wendelin.core
# for ZEO scripts (runzeo)
ZEO
# wendelin.core: latest not yet released
[wendelin.core-repository]
revision= v0.11-4-g38fbc83ceb
# ping eggs versions
[versions]
pyasn1 = 0.3.7
ZODB3 = 3.11.0
numpy = 1.13.3
zope.testing = 4.6.2
# Required by:
# ZEO==4.3.1
# ZODB==4.4.5
# zodburi==2.3.0
ZConfig = 3.2.0
# Required by:
# ZEO==4.3.1
# ZODB==4.4.5
zc.lockfile = 1.2.1
# Required by:
# slapos.toolbox==0.73
erp5.util = 0.4.49
......@@ -688,8 +688,6 @@ PyXML = 0.8.5
Pympler = 0.4.3
StructuredText = 2.11.1
WSGIUtils = 0.7
# ZEO 5 requires transaction >= 2
ZEO = 4.3.1
ZODB3 = 3.11.0
# astroid 1.4.1 breaks testDynamicClassGeneration
astroid = 1.3.8
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment