Update Release Candidate

d7115cef · Alain Takoudjou · de5c1132 · ab944d3a · de5c1132 · d7115cef
Commit d7115cef authored Mar 05, 2020 by Alain Takoudjou
39 changed files
--- a/CHANGES.rst
+++ b/CHANGES.rst
-Changes
-=======
-
-1.0.138 (2020-03-03)
--------------------
-
- Update postgresql recipe for postgres version 10 and later
-
-1.0.124 (2020-01-30)
--------------------
-
- cookbook:erp5testnode: new shared_part_list option
-
-
-1.0.123 (2019-10-03)
--------------------
-
- wrapper: accept hash-files already existing inside the partition directory
-
-
-1.0.122 (2019-09-24)
--------------------
-
- wrapper: add hash-existing-files option
-
-
-1.0.121 (2019-09-12)
--------------------
-
- generic.mysql.wrap_update_mysql: prepare for MariaDB 10.4
- publish-early: process -init entries in specified order
- Partial support of Python 3
- Remove unused generic.mysql recipe
-
-
-1.0.119 (2019-08-14)
--------------------
-
- * publish_early: rework API
-
-
-1.0.118 (2019-08-13)
--------------------
-
-* NEO: new recipe to fix/optimize propagation of the 'masters' parameter
-* publish_early: new '-update' option, keep published values out of buildout installed file
-* publish: new -publish option to list explicitly options to publish
-* re6stnet: Fix typo
-* librecipe: Try to reuse existing file to avoid excessive IO on update and other minor optimisations
-* certificate_authority: unique_subject = no
-* wrapper: handle "=" in environment variables' content
- 
-
-1.0.92 (2019-02-21)
-------------------
-
-* plugin recipe: improve recipe to correctly generate promise with parameters which contain control characters
-
-1.0.85 (2018-12-28)
-----------------------
-
-* Drop ``slapos.recipe:xvfb``, use simple ``slapos.recipe:wrapper`` instead.
-* Drop ``slapos.recipe:seleniumrunner`` and ``slapos.recipe:firefox``, they
-  were not used.
-* Encode unicode to UTF-8 on ``slapos.recipe:request`` and 
-  ``slapos.recipe:slapconfiguration`` 
-
-1.0.75 (2018-09-04)
-------------------
-
-* erp5_test: stop using erp5_test recipe
-* random: fix password generation with newlines
-* erp5testnode: enable password authentication for scalability test system
-* pbs: Ignore numerical IDs (UID/GID) when push
-* request: add requestoptional.serialised
-
-1.0.65 (2018-06-22)
-------------------
-
-* Automatic restart of services when configuration changes
-* erp5_test: define cloudooo-retry-count value in test
-* userinfo: expose values as string
-
-1.0.62 (2018-04-10)
-------------------
-
-* promise.plugin: new recipe for python promises plugin script generation
-
-1.0.59 (2018-03-15)
-------------------
-* librecipe.execute: fix convert process arguments to string formatting.
-
-1.0.58 (2018-03-14)
-------------------
-
-* generic.mysql: unregister UDFs before (re)adding UDFs
-* Remove obsolete/unused recipes.
-* neoppod: add support for new --dedup storage option.
-* Use inotify-simple instead of inotifyx.
-* erp5.test: remove duplicated code.
-* librecipe: bugfixes found by pylint, performance improvements, and major
-  refactoring of executable wrappers.
-* GenericBaseRecipe.createWrapper: remove 'comments' parameter.
-* Drop the 'parameters-extra' option and always forward extra parameters.
-* wrapper: new 'private-dev-shm' option (useful for wendelin.core).
-* generic.cloudooo: OnlyOffice converter support odf.
-* erp5testnode: don't tell git to ignore SSL errors.
-
-1.0.53 (2017-09-13)
-------------------
-
-* check_port_listening: workaround for shebang limitation, reduce to a single file
-* erp5.test: pass new --conversion_server_url option to runUnitTest
-
-1.0.52 (2017-07-04)
-------------------
-
-* wrapper: Add option to reserve CPU core
-* slapconfiguration: Recipe reads partitions resource file
-* neoppod: add support for new --disable-drop-partitions storage option
-* random: Fix the monkeypatch in random.py to incorporate the recent changes in buildout 'get' function
-* random: Add Integer recipe.
-* librecipe.execute: Notify on file moved
-* zero_knowledge: allow to set destination folder of configuration file
-
-
-1.0.50 (2017-04-18)
-------------------
-
-* pbs: Do not parallelize calculus when the heaviest task is IO
-* re6st-registry: Refactor integration with re6st registry
-* erp5testnode: make shellinabox reusing password file of pwgen
-
-1.0.48 (2017-01-31)
-------------------
-
-* random-recipe: add option create-once to prevent storage file deletion by buildout
-
-1.0.45 (2017-01-09)
-------------------
-
-* recipe: set default timeout of check url promise to 20 seconds
-
-1.0.44 (2016-12-30)
-------------------
-
-* pbs: handles the fact that some parameters are not present when slaves are down
-* recipe: allow usage of pidfile in wrapper recipe
-* sshd: fix generation of authorized_keys
-
-1.0.43 (2016-11-24)
-------------------
-
-* pbs: fixes trap command for dash intepreter
-* pbs: remove infinite loops from pbs scripts.
-* random.py: new file containing recipes generating random values.
-* testnode: disallow frontend access to all folders, avoiding publishing private repositories
-
-1.0.41 (2016-10-26)
-------------------
-
-* dcron: new parameter to get a random time, with a frequency of once a day
-* softwaretype: fix parse error on '+ =' when using buildout 2
-* pbs: General Improvement and fixes.
-
-1.0.35 (2016-09-19)
-------------------
-
-* pbs: fix/accelerates deployment of resilient instances
-* recipe: new recipe to get a free network port
-* Remove url-list parameter to download fonts from fontconfig instance
-
-1.0.31 (2016-05-30)
-------------------
-
-* Implement cross recipe cache for registerComputerPartition
-* Fix workaround for long shebang (place script on bin)
-
-1.0.30 (2016-05-23)
-------------------
-
-* Implement a workarround for long shebang
-* Implement Validation for user inputs ssl certificates
-
-1.0.25 (2016-04-15)
-------------------
-
-* fixup slap configuration: provide instance and root instance title
-
-1.0.22 (2016-04-01)
-------------------
-
-* slap configuration: provide instance and root instance title
-
-1.0.16 (2015-10.27)
-------------------
-
-* kvm recipe: fix bugs dowload image and disk creation
-
-1.0.14 (2015-10.26)
-------------------
-
-* kvm recipe: Allow to set keyboard layout language used by qemu and VNC
-* simplehttpserver-recipe: fix encoding error
-
-For older entries, see https://lab.nexedi.com/nexedi/slapos/blob/a662db75cc840df9d4664a9d048ef28ebfff4d50/CHANGES.rst
--- a/component/git/buildout.cfg
+++ b/component/git/buildout.cfg
@@ -18,8 +18,8 @@ parts =
 [git]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.23.0.tar.xz
-md5sum = 93ee0f867f81a39e0ef29eabfb1d2c5b
+url = https://www.kernel.org/pub/software/scm/git/git-2.25.1.tar.xz 
+md5sum = 92bf65673b4fc08b64108d807f36f4d9
 configure-options =
  --with-curl=${curl:location}
  --with-openssl=${openssl:location}

--- a/component/nodejs/buildout.cfg
+++ b/component/nodejs/buildout.cfg
@@ -31,6 +31,10 @@ md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
 version = v8.6.0
 md5sum = 0c95e08220667d8a18b97ecec8218ac6

+[nodejs-8.12.0]
+<= nodejs-base
+version = v8.12.0
+md5sum = 5690333b77964edf81945fc724f6ea85

 [nodejs-base]
 # Server-side Javascript.

--- a/component/ruby/buildout.cfg
+++ b/component/ruby/buildout.cfg
@@ -36,6 +36,10 @@ md5sum = f18ed96bd1d5890f97a17d0d17aaefdd
 url = http://ftp.ruby-lang.org/pub/ruby/2.2/ruby-2.2.2.tar.xz
 md5sum = dbce9b9d79d90f213ba8d448b0b6ed86

+[ruby2.3]
+<= ruby-common
+url = http://ftp.ruby-lang.org/pub/ruby/2.3/ruby-2.3.8.tar.xz
+md5sum = 927e1857f3dd5a1bdec26892dbae2a05

 [ruby]
 <= ruby2.2
--- a/setup.py
+++ b/setup.py
@@ -28,10 +28,9 @@ from setuptools import setup, find_packages
 import glob
 import os

-version = '1.0.138'
+version = '1.0.139'
 name = 'slapos.cookbook'
-long_description = open("README.rst").read() + "\n" + \
-    open("CHANGES.rst").read() + "\n"
+long_description = open("README.rst").read()

 for f in sorted(glob.glob(os.path.join('slapos', 'recipe', 'README.*.rst'))):
  long_description += '\n' + open(f).read() + '\n'

--- a/slapos/recipe/librecipe/execute.py
+++ b/slapos/recipe/librecipe/execute.py
 from __future__ import print_function

+import errno
 import sys
 import os
 import signal
@@ -50,7 +51,7 @@ def _libc():
  return mount, unshare

 def generic_exec(args, extra_environ=None, wait_list=None,
-                 pidfile=None, reserve_cpu=False, private_dev_shm=None,
+                 pidfile=None, reserve_cpu=False, private_tmpfs=(),
                 #shebang_workaround=False, # XXX: still needed ?
                 ):
  args = list(args)
@@ -83,7 +84,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
  if wait_list:
    _wait_files_creation(wait_list)

-  if private_dev_shm:
+  if private_tmpfs:
    mount, unshare = _libc()
    CLONE_NEWNS   = 0x00020000
    CLONE_NEWUSER = 0x10000000
@@ -93,7 +94,13 @@ def generic_exec(args, extra_environ=None, wait_list=None,
    with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
    with open('/proc/self/uid_map',   'wb') as f: f.write('%s %s 1' % (uid, uid))
    with open('/proc/self/gid_map',   'wb') as f: f.write('%s %s 1' % (gid, gid))
-    mount('tmpfs', '/dev/shm', 'tmpfs', 0, 'size=' + private_dev_shm)
+    for size, path in private_tmpfs:
+      try:
+        os.mkdir(path)
+      except OSError as e:
+        if e.errno != errno.EEXIST:
+          raise
+      mount('tmpfs', path, 'tmpfs', 0, 'size=' + size)

  if extra_environ:
    env = os.environ.copy()

--- a/slapos/recipe/librecipe/generic.py
+++ b/slapos/recipe/librecipe/generic.py
@@ -43,7 +43,8 @@ from six.moves.urllib.parse import urlunparse


 import pkg_resources
-import zc.buildout
+from zc.buildout import easy_install, UserError
+from zc.recipe.egg import Egg

 from slapos.recipe.librecipe import shlex

@@ -85,8 +86,7 @@ class GenericBaseRecipe(object):

  def getWorkingSet(self):
    """If you want do override the default working set"""
-    egg = zc.recipe.egg.Egg(self.buildout, 'slapos.cookbook',
-                                  self.options.copy())
+    egg = Egg(self.buildout, 'slapos.cookbook', self.options.copy())
    requirements, ws = egg.working_set()
    return ws

@@ -156,10 +156,20 @@ class GenericBaseRecipe(object):
    args = itertools.chain(map(repr, args),
                           map('%s=%r'.__mod__, six.iteritems(kw)))

-    return zc.buildout.easy_install.scripts(
+    return easy_install.scripts(
      [(filename, module, function)], self._ws, sys.executable,
      path, arguments=', '.join(args))[0]

+  def parsePrivateTmpfs(self):
+    private_tmpfs = []
+    for line in (self.options.get('private-tmpfs') or '').splitlines():
+      if line:
+        x = line.split(None, 1)
+        if len(x) != 2:
+          raise UserError("failed to split %r into size and path" % line)
+        private_tmpfs.append(tuple(x))
+    return private_tmpfs
+
  def createWrapper(self, path, args, env=None, **kw):
    """Create a wrapper script for process replacement"""
    assert args

--- a/slapos/recipe/neoppod.py
+++ b/slapos/recipe/neoppod.py
@@ -88,7 +88,9 @@ class NeoBaseRecipe(GenericBaseRecipe):
        )
    args += self._getOptionList()
    args += shlex.split(options.get('extra-options', ''))
-    return self.createWrapper(options['wrapper'], args)
+    private_tmpfs = self.parsePrivateTmpfs()
+    kw = {'private_tmpfs': private_tmpfs} if private_tmpfs else {}
+    return self.createWrapper(options['wrapper'], args, **kw)

  def _getBindingAddress(self):
    options = self.options

--- a/slapos/recipe/postgres/__init__.py
+++ b/slapos/recipe/postgres/__init__.py
@@ -91,16 +91,13 @@ class Recipe(GenericBaseRecipe):
                # run we won't update it.
                shutil.rmtree(pgdata)
                raise
-
-
-
-        # install() methods usually return the pathnames of managed files.
-        # If they are missing, they will be rebuilt.
-        # In this case, we already check for the existence of pgdata,
-        # so we don't need to return anything here.
+        else:
+            self.createConfig()
+            self.createRunScript()

        return []

+    update = install

    def check_exists(self, path):
        if not os.path.isfile(path):

--- a/slapos/recipe/wrapper.py
+++ b/slapos/recipe/wrapper.py
@@ -38,7 +38,7 @@ class Recipe(GenericBaseRecipe):
    :param lines hash-files: list of buildout-generated files to be checked by hash
    :param lines hash-existing-files: list of existing files to be checked by hash
    :param str pidfile: path to pidfile ensure exclusivity for the process
-    :param str private-dev-shm: size of private /dev/shm, using user namespaces
+    :param lines private-tmpfs: list of "<size> <path>" private tmpfs, using user namespaces
    :param bool reserve-cpu: command will ask for an exclusive CPU core
    """

@@ -72,13 +72,14 @@ class Recipe(GenericBaseRecipe):
          raise UserError(
            "hash-files must only list files that are generated by buildout:"
            "\n  " + "\n  ".join(self._existing))
-        args = shlex.split(self.options['command-line'])
-        wait_files = self.options.get('wait-for-files')
-        pidfile = self.options.get('pidfile')
-        private_dev_shm = self.options.get('private-dev-shm')
+        options = self.options
+        args = shlex.split(options['command-line'])
+        wait_files = options.get('wait-for-files')
+        pidfile = options.get('pidfile')
+        private_tmpfs = self.parsePrivateTmpfs()

        environment = {}
-        for line in (self.options.get('environment') or '').splitlines():
+        for line in (options.get('environment') or '').splitlines():
          line = line.strip()
          if line:
            k, v = line.split('=', 1)
@@ -89,9 +90,9 @@ class Recipe(GenericBaseRecipe):
          kw['wait_list'] = wait_files.split()
        if pidfile:
          kw['pidfile'] = pidfile
-        if private_dev_shm:
-          kw['private_dev_shm'] = private_dev_shm
-        if self.isTrueValue(self.options.get('reserve-cpu')):
+        if private_tmpfs:
+          kw['private_tmpfs'] = private_tmpfs
+        if self.isTrueValue(options.get('reserve-cpu')):
          kw['reserve_cpu'] = True
        return self.createWrapper(self.getWrapperPath(),
                                  args, environment, **kw)

--- a/software/gitlab/buildout.hash.cfg
+++ b/software/gitlab/buildout.hash.cfg
@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 36252abb4d857da08d62bf3eb26faae1
+md5sum = dc3f318e8a3aa7a59f9394118543e9e3

 [watcher]
 _update_hash_filename_ = watcher.in
@@ -34,27 +34,31 @@ md5sum = 7782f5c5d75663c2586e28d029c51e49

 [gitlab-parameters.cfg]
 _update_hash_filename_ = gitlab-parameters.cfg
-md5sum = 8f4537cb8a0c9a8e0058c30cb687681c
+md5sum = c2e23c0f7baa1633df0436ca4e728424

 [gitlab-shell-config.yml.in]
 _update_hash_filename_ = template/gitlab-shell-config.yml.in
-md5sum = 58c09b1e609f903e483a76fe9e57366c
+md5sum = 52d18b521b8cd16352fc88b1e1d79d53

 [gitlab-unicorn-startup.in]
 _update_hash_filename_ = gitlab-unicorn-startup.in
-md5sum = a9cb347f60aad3465932fd36cd4fe25d
+md5sum = aff91edaf9786c213db8ea703ab3571e

 [gitlab.yml.in]
 _update_hash_filename_ = template/gitlab.yml.in
-md5sum = 0ddf4093dcf4427e5a160707e6017950
+md5sum = f4cc0bc898b8d59010d61473e2adc53b
+
+[gitaly-config.toml.in]
+_update_hash_filename_ = template/gitaly-config.toml.in
+md5sum = 056d7ed09e1bf20d022d3ef6b9363e00

 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = d794631233626d03b04894ca6b6d8496
+md5sum = f5e7f9717eaa999fbf11ce4b6c1abb1c

 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
-md5sum = 319d7dbe3ad9b260c1e292cfc0d13b11
+md5sum = 2af7dcf63f74e5edc53a3ff11fa4989b

 [instance-gitlab-test.cfg.in]
 _update_hash_filename_ = instance-gitlab-test.cfg.in
@@ -66,11 +70,11 @@ md5sum = a56a44e96f65f5ed20211bb6a54279f4

 [nginx-gitlab-http.conf.in]
 _update_hash_filename_ = template/nginx-gitlab-http.conf.in
-md5sum  = e74695aa1be60f0ffac64ddbe1c8eaf1
+md5sum = 79d2b4e8a32abf7a74a3d4528844c593

 [nginx.conf.in]
 _update_hash_filename_ = template/nginx.conf.in
-md5sum = 1374f38ab6f295b850d45ea0019ec05d
+md5sum = 8c904510eb39dc212204f68f2b81b068

 [rack_attack.rb.in]
 _update_hash_filename_ = template/rack_attack.rb.in
@@ -82,7 +86,7 @@ md5sum = 7c89a730889e3224548d9abe51a2d719

 [smtp_settings.rb.in]
 _update_hash_filename_ = template/smtp_settings.rb.in
-md5sum = 4e1ced687a86e4cfff2dde91237e3942
+md5sum = e2144b03f7247636143c65dc81550d75

 [template-gitlab-resiliency-restore.sh.in]
 _update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
@@ -90,4 +94,4 @@ md5sum = 590fcadf26085fdd17487175bc0a469d

 [unicorn.rb.in]
 _update_hash_filename_ = template/unicorn.rb.in
-md5sum = 83921db1835d9e81cbbe808631cc40a9
+md5sum = 67728235a2c4c9425c80f0c856749885
--- a/software/gitlab/gitlab-parameters.cfg
+++ b/software/gitlab/gitlab-parameters.cfg
@@ -45,7 +45,7 @@ configuration.default_projects_features.issues          = true
 configuration.default_projects_features.merge_requests  = true
 configuration.default_projects_features.wiki            = true
 configuration.default_projects_features.snippets        = true
-#configuration.default_projects_features.builds          = false
+configuration.default_projects_features.builds          = true

 configuration.webhook_timeout           = 10

@@ -102,6 +102,10 @@ configuration.nginx_gzip_proxied        = any
 configuration.nginx_gzip_types          = text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json
 configuration.nginx_keepalive_timeout   = 65
 configuration.nginx_header_allow_origin = $http_origin
+configuration.nginx_hsts_max_age        = 31536000
+configuration.nginx_hsts_include_subdomains = false
+configuration.nginx_gzip_enabled        = true
+

 # configuring trusted proxies
 # GitLab is behind a reverse proxy, so we don't want the IP address of the proxy

--- a/software/gitlab/gitlab-unicorn-startup.in
+++ b/software/gitlab/gitlab-unicorn-startup.in
@@ -27,7 +27,7 @@ psql() {
 # ( first quering PG several times waiting a bit till postgresql is started and ready )
 tpgwait=5
 while true; do
-    pgtables="$(psql -c '\d')" && break
+    pgtables="$(psql -c '\d' 2>&1)" && break
    tpgwait=$(( $tpgwait - 1 ))
    test $tpgwait = 0 && die "pg query problem"
    echo "I: PostgreSQL is not ready (yet ?); will retry $tpgwait times..." 1>&2
@@ -38,10 +38,11 @@ echo "I: PostgreSQL ready." 1>&2
 # make sure pg_trgm extension is enabled for gitlab db
 psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'   || die "pg_trgm setup failed"

-if echo "$pgtables" | grep -q '^No relations found' ; then
+if echo "$pgtables" | grep -q '^Did not find any relations' ; then
    $RAKE db:schema:load db:seed_fu  || die "initial db setup failed"
 fi

+
 # re-build ssh keys
 # (we do not use them - just for cleannes)
 force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"

--- a/software/gitlab/gowork.cfg
+++ b/software/gitlab/gowork.cfg
@@ -6,7 +6,6 @@ depends_gitfetch  =
    ${go_github.com_pkg_errors:recipe}
    ${go_lab.nexedi.com_kirr_git-backup:recipe}
    ${go_lab.nexedi.com_kirr_go123:recipe}
-    ${go_gitlab.com_gitlab-org_gitlab-workhorse:recipe}


 [go_github.com_libgit2_git2go]
@@ -14,7 +13,7 @@ depends_gitfetch  =
 go.importpath = github.com/libgit2/git2go
 repository    = https://github.com/libgit2/git2go.git
 # branch 'next' is required by git-backup
-revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184
+revision = next-g5d0a4c752a74258a5f42e40fccd2908ac4e336b8

 [go_github.com_pkg_errors]
 <= go-git-package
@@ -26,16 +25,10 @@ revision      = v0.8.0-12-g816c908556
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/git-backup
 repository    = https://lab.nexedi.com/kirr/git-backup.git
-revision = cc6ac54f451dfa6e343d6340dcfa25aa6eac9565
+revision      = 3f6c4deec8834bdcd2c28c7c5eeacd8211e759b5

 [go_lab.nexedi.com_kirr_go123]
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/go123
 repository    = https://lab.nexedi.com/kirr/go123.git
-revision      = d9250d6332
-
-[go_gitlab.com_gitlab-org_gitlab-workhorse]
-<= go-git-package
-go.importpath = gitlab.com/gitlab-org/gitlab-workhorse
-repository    = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
-revision      = v1.3.0-8-g5f44f59cbb
\ No newline at end of file
+revision      = 56bf8f815a
\ No newline at end of file
--- a/software/gitlab/instance-gitlab-export.cfg.in
+++ b/software/gitlab/instance-gitlab-export.cfg.in
@@ -44,6 +44,7 @@ command	= ${exporter:wrapper-path}
 recipe = collective.recipe.template
 input = inline: gitlab-shell-work*
  gitlab-work*
+  var/log/**
  var/backup/**
  var/repositories*
  var/repositories/**

--- a/software/gitlab/instance-gitlab.cfg.in
+++ b/software/gitlab/instance-gitlab.cfg.in
@@ -30,8 +30,12 @@ parts =
    service-nginx
    service-postgresql
    service-redis
+    promise-redis

-    service-cron
+    service-gitaly
+    cron-service
+    cron-entry-logrotate
+    logrotate-entry-cron

    on-reinstantiate

@@ -68,7 +72,6 @@ configuration.nginx_worker_processes    = {{ multiprocessing.cpu_count() }}
 configuration.icp_license  =


-
 # for convenience
 [external-url]
 recipe  = slapos.cookbook:urlparse
@@ -107,7 +110,7 @@ srv     = ${:home}/srv
 # slapos startup/service/promise scripts live here:
 startup = ${:etc}/run
 service = ${:etc}/service
-promise.slow = ${:promise}.slow
+promise.slow = ${:etc}/promise.slow

 # gitlab: etc/ log/ ...
 [gitlab-dir]
@@ -124,6 +127,8 @@ artifacts = ${:shared}/artifacts
 lfs-objects = ${:shared}/lfs-objects
 builds  = ${:var}/builds
 backup  = ${directory:var}/backup
+public  = ${:var}/public
+pages   = ${:shared}/pages

 [gitlab-repo-dir]
 recipe  = slapos.cookbook:mkdirectory
@@ -150,6 +155,8 @@ lfs-objects = ${gitlab-dir:lfs-objects}
 builds  = ${gitlab-dir:builds}
 backup  = ${gitlab-dir:backup}
 repositories = ${gitlab-repo-xdir:repositories}
+public  = ${gitlab-dir:public}
+pages   = ${gitlab-dir:shared}/pages


 # gitlab-shell: etc/ log/ gitlab_shell_secret ...
@@ -162,6 +169,7 @@ log     = ${directory:log}/gitlab-shell
 etc     = ${gitlab-shell-dir:etc}
 log     = ${gitlab-shell-dir:log}
 secret  = ${secrets:secrets}/gitlab_shell_secret
+hook    =


 # place to keep all secrets
@@ -170,8 +178,19 @@ recipe  = slapos.cookbook:mkdirectory
 secrets = ${directory:var}/secrets
 mode    = 0700

-
-
+[gitaly-dir]
+recipe  = slapos.cookbook:mkdirectory
+gitaly  = ${directory:var}/gitaly
+sockets = ${:gitaly}/sockets
+internal = ${:sockets}/internal
+log     = ${directory:log}/gitaly
+
+[gitaly]
+socket  = ${gitaly-dir:sockets}/gitaly.socket
+log     = ${gitaly-dir:log}
+location = {{ gitaly_location }}
+pid     = ${directory:run}/gitaly.pid
+internal_socket = ${gitaly-dir:internal}

 # 2. configuration files
 [etc-template]
@@ -218,6 +237,7 @@ context-extra =
    import  urllib                  urllib
    section gitlab                  gitlab
    section gitlab_shell            gitlab-shell
+    section gitlab_shell_work       gitlab-shell-work
    section unicorn                 unicorn
    section service_redis           service-redis
    raw     redis_binprefix         {{ redis_binprefix }}
@@ -230,12 +250,14 @@ context-extra =
    section gitlab                  gitlab
    section gitlab_shell            gitlab-shell
    section gitlab_shell_work       gitlab-shell-work
+    section gitaly                  gitaly

 [nginx.conf]
 <= nginx-etc-template
 template= {{ nginx_conf_in }}
 context-extra =
    section directory               directory
+    section gitlab_workhorse        gitlab-workhorse
    raw     nginx_mime_types        {{ nginx_mime_types }}
    raw     nginx_gitlab_http_conf  ${nginx-gitlab-http.conf:rendered}

@@ -247,6 +269,16 @@ context-extra =
    section gitlab_work             gitlab-work
    section gitlab_workhorse        gitlab-workhorse

+[gitaly-config.toml]
+<= etc-template
+template= {{ gitaly_config_toml_in }}
+rendered= ${directory:etc}/${:_buildout_section_name_}
+context-extra =
+    import  urllib                  urllib
+    section gitlab                  gitlab
+    section gitlab_shell_work       gitlab-shell-work
+    section gitaly                  gitaly
+
 [rack_attack.rb]
 <= gitlab-etc-template
 template = {{ rack_attack_rb_in }}
@@ -280,7 +312,7 @@ recipe  = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:_buildout_section_name_}
 # NOTE $HOME needed to pick gitconfig
 environment  =
-    PATH = {{ node_bin_location }}:{{ gopath_bin }}:$PATH
+    PATH = {{ node_bin_location }}:{{ gopath_bin }}:{{ yarn_location }}/bin:/usr/local/bin:/usr/bin:/bin
    BUNDLE_GEMFILE = {{ gitlab_repository_location }}/Gemfile
    HOME = ${directory:home}
    RAILS_ENV = production
@@ -361,6 +393,10 @@ update-command =
 <= work-base
 software = {{ gitlab_repository_location }}
 tune-command =
+# Initialise secrets
+    if [ ! -s "${secrets:secrets}/gitlab_secrets.yml" ]; then
+      cp config/secrets.yml.example ${secrets:secrets}/gitlab_secrets.yml;
+    fi
 # secret* tmp/ log/ shared/ builds/ node_modules/
    rm -f .secret  &&
    rm -rf log tmp shared builds node_modules  &&
@@ -370,6 +406,7 @@ tune-command =
    ln -sf ${gitlab:shared} shared  &&
    ln -sf ${gitlab:builds} builds  &&
    ln -sf {{ gitlab_repository_location }}/node_modules node_modules &&
+    ln -sf ${gitlab-workhorse:secret} .gitlab_workhorse_secret
 # config/
    cd config  &&
    ln -sf ${unicorn.rb:rendered} unicorn.rb  &&
@@ -405,8 +442,8 @@ tune-command =
 # [promise-<something>] to check <something> by url
 [promise-byurl]
 <= monitor-promise-base
-module = check_url_available
-name   = !py! '${:_buildout_section_name_}'[8:] + '.py'
+module = check_command_execute
+name   = ${:_buildout_section_name_}.py
 config-http_code   = 200


@@ -447,14 +484,14 @@ depend  =
    ${promise-postgresql:recipe}

 [promise-postgresql]
-<= monitor-base-promise
+<= monitor-promise-base
 module = check_command_execute
 name = promise-postgresql.py
 config-command =
-    {{ postgresql_location }}/bin/psql
-        -h ${service-postgresql:pgdata-directory}
-        -U ${service-postgresql:superuser}
-        -d ${service-postgresql:dbname}
+    {{ postgresql_location }}/bin/psql \
+        -h ${service-postgresql:pgdata-directory} \
+        -U ${service-postgresql:superuser} \
+        -d ${service-postgresql:dbname} \
        -c '\q'

 # postgresql logs to stdout/stderr - logs are handled by slapos not us
@@ -473,7 +510,7 @@ log     = ${directory:log}/redis
 [service-redis]
 recipe  = slapos.cookbook:redis.server
 wrapper = ${directory:service}/redis
-promise_wrapper = ${directory:promise}/redis
+promise_wrapper = ${directory:bin}/redis-promise

 server_dir  = ${redis:srv}
 config_file = ${directory:etc}/redis.conf
@@ -489,12 +526,18 @@ server_bin  = {{ redis_binprefix }}/redis-server
 depend  =
    ${logrotate-entry-redis:recipe}

+[promise-redis]
+<= monitor-promise-base
+module = check_command_execute
+name = promise-redis.py
+config-command = ${service-redis:promise_wrapper}

 # NOTE slapos.cookbook:redis.server setups promise automatically

 [logrotate-entry-redis]
-<= logrotate-entry
+<= logrotate-entry-base
 log     = ${redis:log}/*.log
+name = redis


 ########################
@@ -503,10 +546,13 @@ log     = ${redis:log}/*.log
 [gitlab-workhorse-dir]
 recipe  = slapos.cookbook:mkdirectory
 srv     = ${directory:srv}/gitlab-workhorse
+log     = ${directory:log}/workhorse

 [gitlab-workhorse]
 srv     = ${gitlab-workhorse-dir:srv}
 socket  = ${gitlab-workhorse:srv}/gitlab-workhorse.socket
+log     = ${gitlab-workhorse-dir:log}/gitlab-workhorse.log
+secret  = ${secrets:secrets}/gitlab_workhorse_secret

 [service-gitlab-workhorse]
 recipe  = slapos.cookbook:wrapper
@@ -516,7 +562,8 @@ command-line    = {{ gitlab_workhorse }}
    -listenAddr ${gitlab-workhorse:socket}
    -authSocket ${unicorn:socket}
    -documentRoot ${gitlab-work:location}/public
-    -secretPath ${gitlab-work:location}/.gitlab_workhorse_secret
+    -secretPath ${gitlab-workhorse:secret}
+    -logFile ${gitlab-workhorse:log}
 # NOTE for profiling
 #   -pprofListenAddr ...

@@ -530,13 +577,14 @@ environment =

 depend  =
    ${promise-gitlab-workhorse:recipe}
+    ${logrotate-entry-gitlab-workhorse:recipe}


 [promise-gitlab-workhorse]
 <= promise-byurl
 # http://localhost/users/statics.css will not redirect to /users/sign_in anymore because of this commit:
 #  https://lab.nexedi.com/nexedi/gitlab-workhorse/commit/c81f109a62fecf2a847fb17ceed012b380dab49f#c1215002e6d745f05eaaf9ee1dad7752e85d866f_318_331
-config-url     = --unix-socket ${gitlab-workhorse:socket}  http://localhost/users/sign_in
+config-command     = {{ curl_bin }} --unix-socket ${gitlab-workhorse:socket}  http://localhost/users/sign_in


 # gitlab-workhorse logs to stdout/stderr - logs are handled by slapos not us
@@ -578,7 +626,7 @@ depend  =

 [promise-unicorn]
 <= promise-byurl
-config-url     = --unix-socket ${unicorn:socket}  http://localhost/
+config-command = {{ curl_bin }} --unix-socket ${unicorn:socket}  http://localhost/

 [promise-rakebase]
 recipe  = slapos.cookbook:wrapper
@@ -601,17 +649,24 @@ command-line    = ${:rake} gitlab:gitlab_shell:check


 [logrotate-entry-unicorn]
-<= logrotate-entry
+<= logrotate-entry-base
 log     = ${unicorn:log}/*.log
+name    = unicorn 

 [logrotate-entry-gitlab]
-<= logrotate-entry
+<= logrotate-entry-base
 log     = ${gitlab:log}/*.log
+name    = gitlab

 [logrotate-entry-gitlab-shell]
-<= logrotate-entry
+<= logrotate-entry-base
 log     = ${gitlab-shell:log}/*.log
+name    = gitlab-shell

+[logrotate-entry-gitlab-workhorse]
+<= logrotate-entry-base
+log     = ${gitlab-workhorse-dir:log}//*.log
+name    = gitlab-shell

 #######################################
 #   sidekiq background jobs manager   #
@@ -658,8 +713,9 @@ depend  =
 command-line    = ${:rake} gitlab:sidekiq:check

 [logrotate-entry-sidekiq]
-<= logrotate-entry
+<= logrotate-entry-base
 log     = ${sidekiq:log}/*.log
+name    = sidekiq


 ######################
@@ -726,42 +782,12 @@ depend  =
 # XXX this depends on gitlab-workhorse being up
 #     (nginx is configured to proxy all requests to gitlab-workhorse)
 config-url     = ${backend-info:url}/users/sign_in
+module = check_url_available

 [logrotate-entry-nginx]
-<= logrotate-entry
+<= logrotate-entry-base
 log     = ${nginx:log}/*.log
-
-
-
-#############
-#   cron    #
-#############
-[cron-dir]
-recipe  = slapos.cookbook:mkdirectory
-cron.d  = ${directory:etc}/cron.d
-crontabs= ${directory:srv}/cron/crontabs
-cronstamps = ${directory:var}/cron/cronstamps
-log     = ${directory:log}/cron
-
-[service-cron]
-recipe  = slapos.cookbook:cron
-binary  = ${directory:service}/crond
-cron-entries    = ${cron-dir:cron.d}
-crontabs        = ${cron-dir:crontabs}
-cronstamps      = ${cron-dir:cronstamps}
-catcher         = ${cron-simplelogger:wrapper}
-
-dcrond-binary   = {{ dcron_bin }}
-
-depends =
-    ${logrotate-entry-cron:recipe}
-
-# "mailer" that cron uses to emit messages to logfile
-[cron-simplelogger]
-recipe  = slapos.cookbook:simplelogger
-wrapper = ${directory:bin}/${:_buildout_section_name_}
-log     = ${cron-dir:log}/cron.log
-
+name    = nginx

 # base entry for clients who registers to cron
 [cron-entry]
@@ -771,63 +797,22 @@ recipe  = slapos.cookbook:cron.d
 name    = !py!'${:_buildout_section_name_}' [11:]
 # NOTE _not_ ${service-cron:cron-entries}  - though the value is the same we do
 # not want service-cron to be instantiated just if a cron-entry is registered.
-cron-entries = ${cron-dir:cron.d}
+cron-entries = ${cron:cron-entries}

-# cron logs are also rotated
-[logrotate-entry-cron]
-<= logrotate-entry
-log     = ${cron-dir:log}/*.log

+######################
+#   gitaly worker    #
+######################

-#######################################
-#   logrotate base for all services   #
-#######################################
-[logrotate-dir]
-recipe  = slapos.cookbook:mkdirectory
-srv     = ${directory:srv}/logrotate
-entries = ${directory:etc}/logrotate.d
-
-[logrotate]
-recipe  = slapos.cookbook:logrotate
-wrapper = ${directory:bin}/${:_buildout_section_name_}
-conf    = ${directory:etc}/logrotate.conf
-logrotate-entries   = ${logrotate-dir:entries}
-state-file  = ${logrotate-dir:srv}/logrotate.status
-
-logrotate-binary    = {{ logrotate_bin }}
-gzip-binary     = {{ gzip_bin }}
-gunzip-binary   = {{ gunzip_bin }}
-
-depend  = ${cron-entry-logrotate:recipe}
-
-
-# base entry for clients who registers to logrotate
-[logrotate-entry]
-recipe  = slapos.cookbook:logrotate.d
-logrotate-entries   = ${logrotate:logrotate-entries}
-# name  = <section-name>.strip_prefix('logrotate-entry-')
-# XXX len is not available in !py! - 16 hardcoded
-name    = !py!'${:_buildout_section_name_}'[16:]
-# NOTE frequency is hardcoded to `daily` in slapos.cookbook:logrotate.d
-# NOTE backup is also used to add custom logrotate options (hack)
-backup  = ...
-# TODO settle whether we need/want olddir or not
-    noolddir
-# override create emitted by slapos.cookbook:logrotate.d
-    nocreate
-# do not move log file and this way we do not need to signal its program to
-# reopen the log. There are a lot of bugs when on such reopen / restart /
-# graceful-restart something bad happens. Even if copytruncate is a bit racy
-# and can loose some data, it is better to keep the system the stable way.
-    copytruncate
-
-
-# hook logrotate into cron
-[cron-entry-logrotate]
-<= cron-entry
-time    = daily
-command = ${logrotate:wrapper}
+# https://docs.gitlab.com/ee/install/installation.html
+[service-gitaly]
+recipe  = slapos.cookbook:wrapper
+wrapper-path    = ${directory:service}/gitaly
+#command-line    = ${gitlab-work:location}/bin/daemon_with_pidfile ${gitaly:pid}
+command-line    = {{ gitaly_location }}/gitaly ${gitaly-config.toml:rendered}

+environment =
+    PATH={{ bundler_1_17_3_dir }}:{{ ruby_location }}/bin:/bin:/usr/bin


 # 6. on-reinstantiate actions
@@ -842,7 +827,17 @@ rake    = ${gitlab-rake:wrapper-path}
 # run command on every reinstantiation
 update-command = ${:command}

+# https://gitlab.com/gitlab-org/gitlab-foss/issues/38457
+# we need to manually install ajv@^4.0.0 with yarn to fix the bug 'yarn check failed!'
 command =
    ${:rake} gitlab:assets:clean  &&
-    ${:rake} gitlab:assets:compile  &&
+    ${:rake} gettext:compile RAILS_ENV=production &&
+    cd ${gitlab-work:location} &&
+    PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn add ajv@^4.11.2 &&
+    PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn install --production --pure-lockfile &&
+    ${:rake} gitlab:assets:compile NODE_ENV=production NODE_OPTIONS="--max_old_space_size=4096" &&
    true
+
+
+# Promise, gitlab can connect to gitaly:
+# sudo gitlab-rake gitlab:tcp_check[GITALY_SERVER_IP,GITALY_LISTEN_PORT]
\ No newline at end of file
--- a/software/gitlab/instance.cfg.in
+++ b/software/gitlab/instance.cfg.in
@@ -27,6 +27,7 @@ context =
    import pwd pwd
    import multiprocessing multiprocessing

+    key bin_directory           buildout:bin-directory
    key eggs_directory          buildout:eggs-directory
    key develop_eggs_directory  buildout:develop-eggs-directory
    raw gitlab_repository_location          ${gitlab-repository:location}
@@ -36,11 +37,13 @@ context =
    raw bash_bin                    ${bash:location}/bin/bash
    raw bzip2_location              ${bzip2:location}
    raw bundler_4gitlab             ${bundler-4gitlab:bundle}
+    raw bundler_1_17_3_dir          ${bundler-4gitlab:bundle1.17.3}
    raw coreutils_location          ${coreutils:location}
    raw curl_bin                    ${curl:location}/bin/curl
    raw dcron_bin                   ${dcron-output:crond}
    raw git                         ${git:location}/bin/git
    raw git_location                ${git:location}
+    raw gitaly_location             ${gitaly-repository:location}
    raw gitlab_export               ${gitlab-export:rendered}
    raw gitlab_workhorse            ${gowork:bin}/gitlab-workhorse
    raw gopath_bin                  ${gowork:bin}
@@ -51,14 +54,15 @@ context =
    raw logrotate_bin               ${logrotate:location}/usr/sbin/logrotate
    raw nginx_bin                   ${nginx-output:nginx}
    raw nginx_mime_types            ${nginx-output:mime}
-    raw node_bin_location           ${nodejs-8.6.0:location}/bin/
+    raw node_bin_location           ${nodejs-8.12.0:location}/bin/
    raw openssl_bin                 ${openssl-output:openssl}
-    raw postgresql_location         ${postgresql92:location}
+    raw postgresql_location         ${postgresql10:location}
    raw redis_binprefix             ${redis28:location}/bin
    raw ruby_location               ${bundler-4gitlab:ruby-location}
    raw tar_location                ${tar:location}
    raw watcher                     ${watcher:rendered}
    raw xnice_repository_location   ${xnice-repository:location}
+    raw yarn_location               ${yarn:location}

 # config files
    raw database_yml_in             ${database.yml.in:target}
@@ -68,6 +72,7 @@ context =
    raw gitlab_shell_config_yml_in  ${gitlab-shell-config.yml.in:target}
    raw gitlab_unicorn_startup_in   ${gitlab-unicorn-startup.in:target}
    raw gitlab_yml_in               ${gitlab.yml.in:target}
+    raw gitaly_config_toml_in       ${gitaly-config.toml.in:target}
    raw macrolib_cfg_in             ${macrolib.cfg.in:target}
    raw nginx_conf_in               ${nginx.conf.in:target}
    raw nginx_gitlab_http_conf_in   ${nginx-gitlab-http.conf.in:target}

--- a/software/gitlab/software.cfg
+++ b/software/gitlab/software.cfg
@@ -15,6 +15,7 @@ extends =
    ../../component/openssl/buildout.cfg
    ../../component/nginx/buildout.cfg
    ../../component/zlib/buildout.cfg
+    ../../component/icu/buildout.cfg
    gowork.cfg

 #   for instance
@@ -29,10 +30,10 @@ extends =
    ../../component/logrotate/buildout.cfg

 parts =
-    ruby2.1
-    golang19
+    ruby2.3
+    golang1.12
    git
-    postgresql92
+    postgresql10
    redis28
    cmake
    icu
@@ -40,6 +41,8 @@ parts =
    nginx-output

    gowork
+    gitlab-workhorse
+    gitaly-build
    python-4gitlab
    gitlab-shell/vendor
    gitlab/vendor/bundle
@@ -64,6 +67,13 @@ parts =
 [slapos.cookbook-repository]
 revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261

+
+[yarn]
+# need this version of Yarn
+recipe = slapos.recipe.build:download-unpacked
+url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
+md5sum = db82fa09c996e9318f2f1d2ab99228f9
+
 ############################
 #   Software compilation   #
 ############################
@@ -78,20 +88,22 @@ eggs    =
 # rubygemsrecipe with fixed url and this way pinned rubygems version
 [rubygemsrecipe]
 recipe  = rubygemsrecipe
-url     = https://rubygems.org/rubygems/rubygems-2.5.2.zip
-
+url     = https://rubygems.org/rubygems/rubygems-3.1.2.zip

 # bundler, that we'll use to
 # - install gems for gitlab
 # - run gitlab services / jobs  (via `bundle exec ...`)
 [bundler-4gitlab]
 <= rubygemsrecipe
-ruby-location = ${ruby2.1:location}
+ruby-location = ${ruby2.3:location}
 ruby-executable = ${:ruby-location}/bin/ruby
-gems    = bundler==1.11.2
+gems    =
+  bundler==1.17.3

 # bin installed here
 bundle  = ${buildout:bin-directory}/bundle
+# Gitaly need bundler 1.17.3 which is not the default version at the end
+bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/gems/1.8/gems/bundler-1.17.3/exe/

 # install together with dependencies of gitlab, which we cannot specify using
 #   --with-... gem option
@@ -109,7 +121,8 @@ bundle  = ${buildout:bin-directory}/bundle
 # gitlab (via github-markup) wants to convert rst -> html via running: python2 (with docutils egg)
 # (python-4gitlab puts interpreter into ${buildout:bin-directory})
 environment =
-  PATH    = ${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs-8.6.0:location}/bin:${postgresql92:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
+
+  PATH    = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs-8.12.0:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s


 # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
@@ -120,21 +133,31 @@ git-executable = ${git:location}/bin/git

 [gitlab-repository]
 <= git-repository
-#repository = https://gitlab.com/gitlab-org/gitlab-ce.git
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
-# 8.17.X + NXD patches:
-revision = v8.17.8-12-g611cf13b90
+# 9.5.10 + NXD patches:
+revision = v9.5.10-8-gc290e22a08cb
 location = ${buildout:parts-directory}/gitlab

 [gitlab-shell-repository]
 <= git-repository
-#repository = https://gitlab.com/gitlab-org/gitlab-shell.git
-repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
-# gitlab 8.17 wants gitlab-shell 4.1.1
-# 4.1.1 + NXD patches
-revision = v4.1.1-1-g64603b4da2
+#repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
+repository = https://gitlab.com/gitlab-org/gitlab-shell.git
+# gitlab 9.5.10 wants gitlab-shell 5.6.1
+revision = v5.6.1-10-g1e587d3b7f
 location = ${buildout:parts-directory}/gitlab-shell

+[gitaly-repository]
+<= git-repository
+repository = https://gitlab.com/gitlab-org/gitaly.git
+# for version v0.35.0 (gitlab 9.5.10)
+revision = v0.35.0-0-gf99a57b19a
+location = ${buildout:parts-directory}/gitaly
+
+[gitlab-workhorse-repository]
+<= git-repository
+repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
+revision = v3.0.0-8-g74793ad3cc
+
 # Patch github markup to not call "python2 -S /path/to/rest2html" but only "python2 /path/to/rest2html"
 # NOTE github-markup invokes it as `python2`, that's why we are naming it this way
 # https://github.com/github/markup/blob/5393ae93/lib/github/markups.rb#L36
@@ -158,11 +181,23 @@ bundle  = ${bundler-4gitlab:bundle}

 configure-command = cd ${:path} &&
    ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}  &&
-    ${:bundle} config --local build.pg  --with-pg-config=${postgresql92:location}/bin/pg_config
+    ${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config &&
+    ${:bundle} config --local build.re2 --with-re2-dir=${re2:location}

 make-binary =
 make-targets= cd ${:path} &&
-    ${:bundle} install --deployment  --without development test  mysql kerberos
+    ${:bundle} install --deployment  --without development test mysql aws kerberos
+environment =
+  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+
+################## Google re2
+[re2]
+recipe = slapos.recipe.cmmi
+url = https://github.com/google/re2/archive/2019-12-01.tar.gz
+md5sum = 527eab0c75d6a1a0044c6eefd816b2fb
+configure-command = :
+

 [gitlab_npm]
 recipe  = slapos.recipe.cmmi
@@ -173,7 +208,7 @@ make-binary =
 make-targets= cd ${:path} && npm install

 environment =
-  PATH=${nodejs-8.6.0:location}/bin/:%(PATH)s
+  PATH=${nodejs-8.12.0:location}/bin/:%(PATH)s

 #our go infrastructure not currently supporting submodules, IIRC
 # https://lab.nexedi.com/nexedi/slapos/merge_requests/337
@@ -184,25 +219,39 @@ configure-command = :
 make-binary =
 make-targets= cd ${go_github.com_libgit2_git2go:location}
              && git submodule update --init
+              && sed -i 's/.*--build.*/cmake --build . --target install/' script/build-libgit2-static.sh
              && make install
 environment =
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
-  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang19:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.12:location}/bin:${buildout:bin-directory}:%(PATH)s
  GOPATH=${gowork:directory}

+[gowork.goinstall]
+git2go = ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install
+command = bash -c ". ${gowork:env.sh} && CGO_CFLAGS=-I${:git2go}/include CGO_LDFLAGS='-L${:git2go}/lib -lgit2' go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
+
 [gowork]
-golang  = ${golang19:location}
-install = 
+golang  = ${golang1.12:location}
+gcc-bin-directory = ${golang1.12:gcc-bin-directory}
+# gitlab.com/gitlab-org/gitlab-workhorse
+# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
+# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
+install =
  lab.nexedi.com/kirr/git-backup
-  gitlab.com/gitlab-org/gitlab-workhorse
-  gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
-  gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
-
 cpkgpath =
-    ${openssl:location}/lib/pkgconfig
+    ${openssl-1.0:location}/lib/pkgconfig
    ${zlib:location}/lib/pkgconfig
-before-install =
-  ${go_github.com_libgit2_git2go_prepare:recipe}
+    ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install/lib/pkgconfig
+buildflags = --tags "static"
+
+[gitlab-workhorse]
+recipe = slapos.recipe.cmmi
+path = ${gitlab-workhorse-repository:location}
+md5sum = 2988c944d58c4a08880498c4981cc7b7
+configure-command = :
+make-binary =
+make-targets = 
+  . ${gowork:env.sh} && make install PREFIX=${gowork:directory}

 [gitlab-backup]
 recipe = plone.recipe.command
@@ -210,6 +259,21 @@ command =
  cp -a ${go_lab.nexedi.com_kirr_git-backup:location}/contrib/gitlab-backup ${gowork:bin}
 update-command = ${:command}

+[gitaly-build]
+recipe = slapos.recipe.cmmi
+path = ${gitaly-repository:location}
+bundle  = ${bundler-4gitlab:bundle}
+
+configure-command = cd ${:path}/ruby &&
+    ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}
+make-binary =
+make-targets =
+  . ${gowork:env.sh} && make
+environment =
+  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:${ruby2.3:location}/bin:%(PATH)s
+
+
 [xnice-repository]
 # to get kirr's misc repo containing xnice script for executing processes
 # with lower priority (used for backup script inside the cron)
@@ -231,6 +295,7 @@ make-binary =
 make-targets= cd ${:path} &&
    ${:bundle} install --deployment  --without development test

+
 ###############################
 #   Trampoline for instance   #
 ###############################
@@ -293,6 +358,9 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
 [gitlab.yml.in]
 <= download-file

+[gitaly-config.toml.in]
+<= download-file
+
 [instance-gitlab.cfg.in]
 <= download-file

@@ -336,6 +404,6 @@ strip-top-level-dir = true
 cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
-rubygemsrecipe  = 0.2.2+slapos001
-slapos.recipe.template = 4.4
+rubygemsrecipe = 0.2.2+slapos002
+slapos.recipe.template = 4.3
 z3c.recipe.scripts = 1.0.1
--- a/software/gitlab/template/gitaly-config.toml.in
+++ b/software/gitlab/template/gitaly-config.toml.in
+# Example Gitaly configuration file
+# Documentation lives at https://docs.gitlab.com/ee/administration/gitaly/ and
+# https://docs.gitlab.com/ee//administration/gitaly/reference
+
+socket_path = "{{ gitaly.socket }}"
+
+# The directory where Gitaly's executables are stored
+bin_dir = "{{ gitaly.location }}"
+
+# # Optional: listen on a TCP socket. This is insecure (no authentication)
+# listen_addr = "localhost:9999"
+# tls_listen_addr = "localhost:8888
+
+# # Optional: export metrics via Prometheus
+# prometheus_listen_addr = "localhost:9236"
+
+
+# # Git settings
+[git]
+bin_path = "{{ git }}"
+
+[[storage]]
+name = "default"
+path = "{{ gitlab.repositories }}"
+
+# # You can optionally configure more storages for this Gitaly instance to serve up
+#
+# [[storage]]
+# name = "other_storage"
+# path = "/mnt/other_storage/repositories"
+#
+
+# # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
+# [logging]
+# format = "json"
+# # Additionally exceptions can be reported to Sentry
+# sentry_dsn = "https://<key>:<secret>@sentry.io/<project>
+
+
+# # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
+# [prometheus]
+# grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0]
+
+[gitaly-ruby]
+# The directory where gitaly-ruby is installed
+dir = "{{ gitaly.location }}/ruby"
+
+
+[gitlab-shell]
+# The directory where gitlab-shell is installed
+dir = "{{ gitlab_shell_work.location }}"
+
--- a/software/gitlab/template/gitlab-shell-config.yml.in
+++ b/software/gitlab/template/gitlab-shell-config.yml.in
@@ -24,7 +24,7 @@ http_settings:
 # Give the canonicalized absolute pathname,
 # REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!!
 # Check twice that none of the components is a symlink, including "/home".
-repos_path: "{{ gitlab.repositories }}"
+# repos_path: "{{ gitlab.repositories }}"

 # File used as authorized_keys for gitlab user
 # NOTE not used in slapos version (all access via https only)
@@ -34,6 +34,9 @@ auth_file: "{{ gitlab.var }}/sshkeys-notused"
 # Default is .gitlab_shell_secret in the root directory.
 secret_file: "{{ gitlab_shell.secret }}"

+# Parent directory for global custom hook directories (pre-receive.d, update.d, post-receive.d)
+# Default is hooks in the gitlab-shell directory.
+custom_hooks_dir: "{{ gitlab_shell_work.location }}/hooks/"

 # Redis settings used for pushing commit notices to gitlab
 redis:
@@ -41,11 +44,6 @@ redis:
  host: {# <%= @redis_host %> #}
  port: {# <%= @redis_port %> #}
  socket: {{ service_redis.unixsocket }}
-{# we don't use password for redis
-<% if @redis_password %>
-  pass: <%= @redis_password %>
-<% end %>
-#}
  database: {# <%= @redis_database %> #}
  namespace: resque:gitlab


--- a/software/gitlab/template/gitlab.yml.in
+++ b/software/gitlab/template/gitlab.yml.in
@@ -32,6 +32,29 @@ production: &base
    relative_url_root: <%= @gitlab_relative_url %>
    #}

+    # Content Security Policy
+    # See https://guides.rubyonrails.org/security.html#content-security-policy
+    content_security_policy:
+      enabled: true
+      report_only: false
+      directives:
+        base_uri:
+        child_src:
+        connect_src: "'self' http://localhost:* ws://localhost:* wss://localhost:*"
+        default_src: "'self'"
+        font_src:
+        form_action:
+        frame_ancestors: "'self'"
+        frame_src: "'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com"
+        img_src: "* data: blob:"
+        manifest_src:
+        media_src:
+        object_src: "'none'"
+        script_src: "'self' 'unsafe-eval' http://localhost:* https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://www.gstatic.com/recaptcha/ https://apis.google.com"
+        style_src: "'self' 'unsafe-inline'"
+        worker_src: "'self' blob:"
+        report_uri:
+
    # Trusted Proxies
    # Customize if you have GitLab behind a reverse proxy which is running on a different machine.
    # Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
@@ -84,7 +107,7 @@ production: &base
      merge_requests:   {{ cfg('default_projects_features.merge_requests') }}
      wiki:             {{ cfg('default_projects_features.wiki') }}
      snippets:         {{ cfg('default_projects_features.snippets') }}
-      builds: false {# builds not supported yet <%= @gitlab_default_projects_features_builds %> #}
+      builds:           {{ cfg('default_projects_features.builds') }}
      {# container_registry: <%= @gitlab_default_projects_features_container_registry %> #}

    ## Webhook settings
@@ -148,6 +171,7 @@ production: &base
    storage_path: <%= @lfs_storage_path %>
  #}

+
  {# we do not support container registry
  ## Container Registry
  registry:
@@ -191,6 +215,9 @@ production: &base
    ssl_url:   <%= single_quote(@gravatar_ssl_url) %>    # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon
    #}

+  ## Sidekiq
+  sidekiq:
+    log_format: json # (default is the original format)

  {# XXX cron jobs are disabled for now - we do not support CI and EE features or we are ok with defaults
  ## Auxiliary jobs
@@ -375,19 +402,18 @@ production: &base
    path: <%= @shared_path %>
  #}

+  # Gitaly settings
+  gitaly:
+    # Default Gitaly authentication token. Can be overriden per storage. Can
+    # be left blank when Gitaly is running locally on a Unix socket, which
+    # is the normal way to deploy Gitaly.
+    token:
+
+
  #
  # 4. Advanced settings
  # ==========================

-  # GitLab Satellites
-  # Important: keep the satellites.path setting until GitLab 9.0 at
-  # least. This setting is fed to 'rm -rf' in
-  # db/migrate/20151023144219_remove_satellites.rb
-  satellites:
-    # Relative paths are relative to Rails.root (default: tmp/repo_satellites/)
-    path: /dev/null
-    timeout: 0
-
  ## Repositories settings
  repositories:
    # Paths where repositories can be stored. Give the canonicalized absolute pathname.
@@ -395,7 +421,11 @@ production: &base
    # gitlab-shell invokes Dir.pwd inside the repository path and that results
    # real path not the symlink.
    storages: # You must have at least a `default` storage path.
-      default: {{ gitlab.repositories }}
+      default:
+        path: {{ gitlab.repositories }}
+        gitaly_address: unix:{{ gitaly.socket }} # TCP connections are supported too (e.g. tcp://host:port). TLS connections are also supported using the system certificate pool (eg: tls://host:port).
+        # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage.
+

  ## Backup settings
  backup:
@@ -420,8 +450,8 @@ production: &base
  ## GitLab Shell settings
  gitlab_shell:
    path: {{ gitlab_shell_work.location }}
+    authorized_keys_file: {{ gitlab.var }}/sshkeys-notused

-    # REPOS_PATH MUST NOT BE A SYMLINK!!!
    repos_path: {{ gitlab.repositories }}
    hooks_path: {{ gitlab_shell_work.location }}/hooks/
    secret_file: {{ gitlab_shell.secret }}
@@ -430,6 +460,9 @@ production: &base
    upload_pack: true
    receive_pack: true

+    # Git import/fetch timeout, in seconds. Defaults to 3 hours.
+    # git_timeout: 10800
+
    {# Git over SSH is disabled elsewhere (so we don't care about ssh_port)
    # If you use non-standard ssh port you need to specify it
    ssh_port: <%= @gitlab_shell_ssh_port %>
@@ -452,7 +485,6 @@ production: &base
    # Git timeout to read a commit, in seconds
    timeout: {{ cfg('git_timeout') }}

-
  #
  # 5. Extra customization
  # ==========================

--- a/software/gitlab/template/nginx-gitlab-http.conf.in
+++ b/software/gitlab/template/nginx-gitlab-http.conf.in
@@ -111,16 +111,71 @@ server {
  set_real_ip_from {{ trusted_address }};
  {% endfor %}

+  ## HSTS Config
+  ## https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/
+  {% if cfg("nginx_hsts_max_age") > 0 -%}
+  {%   if '{{ cfg("nginx_hsts_include_subdomains") }}' == 'true' -%}
+  add_header Strict-Transport-Security "max-age={{ cfg('nginx_hsts_max_age') }}; includeSubDomains"
+  {%   else -%}
+  add_header Strict-Transport-Security "max-age={{ cfg('nginx_hsts_max_age') }}";
+  {%   endif -%}
+  {% endif -%}
+
  ## Individual nginx logs for this GitLab vhost
  access_log  {{ nginx.log }}/gitlab_access.log gitlab_access;
  error_log   {{ nginx.log }}/gitlab_error.log;
+  
+  # Set CORS header
+  add_header 'Access-Control-Allow-Origin' {{ cfg('nginx_header_allow_origin') }};
+  add_header 'Access-Control-Allow-Credentials' true;
+  #{{ 'gzip off;' if cfg_https else ''}}
+  {% if '{{ cfg("nginx_gzip_enabled") }}' == 'true' -%}
+  gzip on;
+  gzip_static on;
+  gzip_comp_level 2;
+  gzip_http_version 1.1;
+  gzip_vary on;
+  gzip_disable "msie6";
+  gzip_min_length 10240;
+  gzip_proxied no-cache no-store private expired auth;
+  gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/json application/xml application/rss+xml;
+  {% endif -%}
+
+  ## https://github.com/gitlabhq/gitlabhq/issues/694
+  ## Some requests take more than 30 seconds.
+  proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
+  proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
+  proxy_redirect          off;
+
+  proxy_http_version 1.1;

  {# we do not support relative URL - path is always "/" #}
  {% set path = "/" %}
+  
+  #if ($http_host = "") {
+  #  set $http_host_with_default "<%= default_host %>";
+  #}
+  #if ($http_host != "") {
+  #  set $http_host_with_default $http_host;
+  #}
+  location ~ (\.git/gitlab-lfs/objects|\.git/info/lfs/objects/batch$) {
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    {% if cfg_https %}
+    proxy_set_header    X-Forwarded-Ssl     on;
+    {% endif %}
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
+
+    proxy_pass http://gitlab-workhorse;
+  }
+
  location {{ path }} {
-    # Set CORS header
-    add_header 'Access-Control-Allow-Origin' {{ cfg('nginx_header_allow_origin') }};
-    add_header 'Access-Control-Allow-Credentials' true;
+    # NOTE(slapos) proxy headers are defined upstream in omnibus-gitlab in:
+    #   - files/gitlab-config-template/gitlab.rb.template       nginx['proxy_set_headers']
+    #   - files/gitlab-cookbooks/gitlab/attributes/default.rb   default['gitlab']['nginx']['proxy_set_headers']
+    #   - files/gitlab-cookbooks/gitlab/libraries/gitlab.rb     parse_nginx_proxy_headers()
+    # (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
    if ($request_method = OPTIONS ) {
        add_header Allow "GET, OPTIONS";
        add_header Content-Type text/plain;
@@ -128,23 +183,7 @@ server {
        add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Authorization, Content-Type, Accept";
        return 200;
    }
-    ## If you use HTTPS make sure you disable gzip compression
-    ## to be safe against BREACH attack.
-    {{ 'gzip off;' if cfg_https else ''}}
-
-    ## https://github.com/gitlabhq/gitlabhq/issues/694
-    ## Some requests take more than 30 seconds.
-    proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
-    proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
-    proxy_redirect          off;
-
-    proxy_http_version 1.1;
-
-    # NOTE(slapos) proxy headers are defined upstream in omnibus-gitlab in:
-    #   - files/gitlab-config-template/gitlab.rb.template       nginx['proxy_set_headers']
-    #   - files/gitlab-cookbooks/gitlab/attributes/default.rb   default['gitlab']['nginx']['proxy_set_headers']
-    #   - files/gitlab-cookbooks/gitlab/libraries/gitlab.rb     parse_nginx_proxy_headers()
-    # (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+    proxy_cache off;
    proxy_set_header    Host                $http_host;
    proxy_set_header    X-Real-IP           $remote_addr;
    {% if cfg_https %}
@@ -153,7 +192,12 @@ server {
    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};

-    proxy_pass http://gitlab-workhorse;
+    proxy_pass  http://gitlab-workhorse;
+  }
+
+  location ~ ^/(assets)/ {
+    proxy_cache off;
+    proxy_pass  http://gitlab-workhorse;
  }

  error_page 404 /404.html;
@@ -169,3 +213,4 @@ server {
  <%= @custom_gitlab_server_config %>
  #}
 }
+
--- a/software/gitlab/template/nginx.conf.in
+++ b/software/gitlab/template/nginx.conf.in
@@ -50,6 +50,42 @@ http {

  include {{ nginx_gitlab_http_conf }};

+  map $http_upgrade $connection_upgrade {
+      default upgrade;
+      ''      close;
+  }
+
+  # Remove private_token from the request URI
+  # In:  /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&...
+  # Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
+  map $request_uri $temp_request_uri_1 {
+    default $request_uri;
+    ~(?i)^(?<start>.*)(?<temp>[\?&]private[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+  }
+
+  # Remove authenticity_token from the request URI
+  # In:  /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
+  # Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
+  map $temp_request_uri_1 $temp_request_uri_2 {
+    default $temp_request_uri_1;
+    ~(?i)^(?<start>.*)(?<temp>[\?&]authenticity[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+  }
+
+  # Remove rss_token from the request URI
+  # In:  /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
+  # Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&...
+  map $temp_request_uri_2 $filtered_request_uri {
+    default $temp_request_uri_2;
+    ~(?i)^(?<start>.*)(?<temp>[\?&]rss[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+  }
+
+  # A version of the referer without the query string
+  map $http_referer $filtered_http_referer {
+    default $http_referer;
+    ~^(?<temp>.*)\? $temp;
+  }
+
+
  {# we don't need: ci, pages, mattermost, registry
  include <%= @gitlab_ci_http_config %>
  include <%= @gitlab_pages_http_config %>;

--- a/software/gitlab/template/smtp_settings.rb.in
+++ b/software/gitlab/template/smtp_settings.rb.in
@@ -29,3 +29,4 @@ end
 # SMTP disabled in instance configuration (see `smtp_enable` parameter).
 # Mail sending, if enabled (see `email_enabled`), will be done via sendmail.
 {% endif %}
+
--- a/software/gitlab/template/unicorn.rb.in
+++ b/software/gitlab/template/unicorn.rb.in
@@ -17,8 +17,20 @@ working_directory '{{ gitlab_work.location }}'
 # What the timeout for killing busy workers is, in seconds
 timeout {{ cfg('unicorn_worker_timeout') }}

-# Whether the app should be pre-loaded
+# combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
+# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
 preload_app true
+GC.respond_to?(:copy_on_write_friendly=) and
+  GC.copy_on_write_friendly = true
+
+
+# Enable this flag to have unicorn test client connections by writing the
+# beginning of the HTTP headers before calling the application.  This
+# prevents calling the application for connections that have disconnected
+# while queued.  This is only guaranteed to detect clients on the same
+# host unicorn runs on, and unlikely to detect disconnects even on a
+# fast LAN.
+check_client_connection false

 # How many worker processes
 worker_processes {{ cfg('unicorn_worker_processes') }}
@@ -35,6 +47,10 @@ before_fork do |server, worker|
  # defined?(ActiveRecord::Base) and
  #   ActiveRecord::Base.connection.disconnect!

+  # The following is only recommended for memory/DB-constrained
+  # installations.  It is not needed if your system can house
+  # twice as many worker_processes as you have configured.
+  #
  # This allows a new master process to incrementally
  # phase out the old master process with SIGTTOU to avoid a
  # thundering herd (especially in the "preload_app false" case)
@@ -48,8 +64,15 @@ before_fork do |server, worker|
    rescue Errno::ENOENT, Errno::ESRCH
    end
  end
+  #
+  # Throttle the master from forking too quickly by sleeping.  Due
+  # to the implementation of standard Unix signal handlers, this
+  # helps (but does not completely) prevent identical, repeated signals
+  # from being lost when the receiving process is busy.
+  # sleep 1
 end

+
 # What to do after we fork a worker
 after_fork do |server, worker|
  # per-process listener ports for debugging/admin/migrations
@@ -60,6 +83,17 @@ after_fork do |server, worker|
  # # the following is *required* for Rails + "preload_app true",
  # defined?(ActiveRecord::Base) and
  #   ActiveRecord::Base.establish_connection
+  
+  # reset prometheus client, this will cause any opened metrics files to be closed
+  #defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
+  #  Prometheus::Client.reinitialize_on_pid_change
+
+  # if preload_app is true, then you may also want to check and
+  # restart any other shared sockets/descriptors such as Memcached,
+  # and Redis.  TokyoCabinet file handles are safe to reuse
+  # between any number of forked children (assuming your kernel
+  # correctly implements pread()/pwrite() system calls)
+
 end



--- a/software/gitlab/test/README.md
+++ b/software/gitlab/test/README.md
+Tests for Gitlab software release
--- a/software/gitlab/test/setup.py
+++ b/software/gitlab/test/setup.py
+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.gitlab'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Gitlab",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)
--- a/software/gitlab/test/test.py
+++ b/software/gitlab/test/test.py
+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import logging
+from six.moves.urllib.parse import urlparse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestGitlab(SlapOSInstanceTestCase):
+  __partition_reference__ = 'G'  # solve path too long for postgresql and unicorn
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'gitlab-test'
+
+  def setUp(self):
+    self.backend_url = self.computer_partition.getConnectionParameterDict(
+    )['backend_url']
+
+  def test_http_get(self):
+    resp = requests.get(self.backend_url, verify=False)
+    self.assertTrue(
+      resp.status_code in [requests.codes.ok, requests.codes.found])
--- a/software/neoppod/buildout.hash.cfg
+++ b/software/neoppod/buildout.hash.cfg
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9

 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = 574acb0cae9af8ec2af52825fb2436d8
+md5sum = 512383220488335ac186013c2ffdc7c1

 [template-neo-my-cnf]
 filename = my.cnf.in
@@ -46,4 +46,4 @@ md5sum = 5afd326de385563b5aeac81039f23341

 [runTestSuite.in]
 _update_hash_filename_ = runTestSuite.in
-md5sum = b656e805c5dbc7f9c73716398b3e032e
+md5sum = 7a0d5d259eb7f90fc0421d1264fbe7b5
--- a/software/neoppod/instance-neo-input-schema.json
+++ b/software/neoppod/instance-neo-input-schema.json
@@ -114,6 +114,10 @@
              ],
              "type": "string"
            },
+            "private-tmpfs": {
+              "description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
+              "type": "string"
+            },
            "mysql": {
              "description": "Dictionary containing parameters for MySQL.",
              "default": {},

--- a/software/neoppod/instance-neo.cfg.in
+++ b/software/neoppod/instance-neo.cfg.in
@@ -2,22 +2,36 @@
 {% set part_list = [] -%}
 {% set init_list = [] -%}

+{% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
 {% set storage_type = slapparameter_dict.get('storage-type') or (
     'MySQL' if mariadb_location is defined else 'SQLite') -%}
 {% set mysql = storage_type == 'MySQL' -%}
 {% if mysql -%}

 [{{ section('mysqld') }}]
+{% if private_tmpfs -%}
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:etc_run}/mariadb
+private-tmpfs = {{ private_tmpfs }} ${my-cnf-parameters:tmp-directory}
+command-line = ${mariadb-ns:rendered}
+[mariadb-ns]
+rendered = ${directory:bin}/mariadb-ns
+{% else -%}
+rendered = ${directory:etc_run}/mariadb
+{% endif -%}
 recipe = slapos.recipe.template:jinja2
 template = {{ template_mysqld_wrapper }}
-rendered = ${directory:etc_run}/mariadb
 context =
  key defaults_file my-cnf:rendered
  key datadir my-cnf-parameters:data-directory

 [my-cnf-parameters]
 socket = ${directory:var_run}/mariadb.sock
+{% if private_tmpfs -%}
+data-directory = ${:tmp-directory}/mariadb
+{% else -%}
 data-directory = ${directory:srv}/mariadb
+{% endif -%}
 tmp-directory = ${directory:tmp}
 pid-file = ${directory:var_run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
@@ -105,6 +119,9 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
 {%- if mysql %}
 {%-   do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~  i ~ ';') %}
 database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
+{%- elif private_tmpfs %}
+private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
+database-parameters = ${directory:tmp}/db.sqlite
 {%- else %}
 database-parameters = ${directory:db-{{i}}}/db.sqlite

@@ -148,6 +165,14 @@ post = ${binary-wrap-mysql:command-line} -e "FLUSH LOGS"
 {% if runTestSuite_in is defined -%}
 # bin/runTestSuite to run NEO tests
 [{{ section('runTestSuite') }}]
+{%- if private_tmpfs %}
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/${:_buildout_section_name_}
+private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
+command-line = ${runTestSuite-ns:rendered}
+
+[runTestSuite-ns]
+{%- endif %}
 recipe = slapos.recipe.template:jinja2
 rendered = ${directory:bin}/${:_buildout_section_name_}
 template = {{ runTestSuite_in }}
@@ -157,6 +182,13 @@ context =
    section my_cnf_parameters my-cnf-parameters
    raw     bin_directory     {{ bin_directory }}
    raw     prepend_path      {{ mariadb_location }}/bin
+{%- if private_tmpfs %}
+    key     datadir           my-cnf-parameters:data-directory
+    key     results_directory directory:results
+
+[directory]
+results = ${directory:srv}/tests
+{%- endif %}
 {%- endif %}

 {%- endif %}

--- a/software/neoppod/runTestSuite.in
+++ b/software/neoppod/runTestSuite.in
@@ -2,7 +2,7 @@
 """
  Script to run NEO test suite using Nexedi's test node framework.
 """
-import argparse, os, re, shutil, subprocess, sys, traceback
+import argparse, errno, json, os, re, shutil, subprocess, sys, traceback
 from erp5.util import taskdistribution
 from time import gmtime, sleep, strftime, time

@@ -13,12 +13,12 @@ SUMMARY_RE = re.compile(
  r' (.*) (?P<duration>\d+(\.\d*)?|\.\d+)s', re.MULTILINE)

 PATH = os.environ['PATH']
-PATH = '{{ prepend_path }}' + (PATH and ':' + PATH)
+PATH = {{ repr(prepend_path) }} + (PATH and ':' + PATH)

 # NEO specific environment
-TEMP_DIRECTORY  = '{{directory.tmp}}'
-NEO_DB_SOCKET = '{{my_cnf_parameters.socket}}'
-RUN_NEO_TESTS_COMMAND = '{{ bin_directory }}/neotestrunner'
+TEMP_DIRECTORY  = {{ repr(directory.tmp) }}
+NEO_DB_SOCKET = {{ repr(my_cnf_parameters.socket) }}
+RUN_NEO_TESTS_COMMAND = {{ repr(bin_directory + '/neotestrunner') }}

 def parseTestStdOut(data):
  """
@@ -52,6 +52,28 @@ def parseTestStdOut(data):

  return test_count, unexpected_count, expected_count, skip_count, duration

+class DummyTestResult:
+
+  class DummyTestResultLine:
+
+    def stop(self, **kw):
+      with open(self.name + '.json', 'w') as f:
+        json.dump(kw, f)
+
+  done = 0
+
+  def __init__(self, test_name_list):
+    self.test_name_list = test_name_list
+
+  def start(self):
+    test_result_line = self.DummyTestResultLine()
+    try:
+      test_result_line.name = self.test_name_list[self.done]
+    except IndexError:
+      return
+    self.done += 1
+    return test_result_line
+
 def main():
  parser = argparse.ArgumentParser(description='Run a test suite.')
  parser.add_argument('--test_suite', help='The test suite name')
@@ -67,18 +89,20 @@ def main():
  args = parser.parse_args()

  test_suite_title = args.test_suite_title or args.test_suite
-  revision = args.revision

  test_name_list = 'SQLite', 'MySQL'

-  tool = taskdistribution.TaskDistributor(portal_url = args.master_url)
-  test_result = tool.createTestResult(revision = revision,
-                                      test_name_list = test_name_list,
-                                      node_title = args.test_node_title,
-                                      test_title = test_suite_title,
-                                      project_title = args.project_title)
-  if test_result is None:
-    return
+  if args.master_url:
+    tool = taskdistribution.TaskDistributor(portal_url = args.master_url)
+    test_result = tool.createTestResult(args.revision,
+                                        test_name_list,
+                                        args.test_node_title,
+                                        test_title=test_suite_title,
+                                        project_title=args.project_title)
+    if test_result is None:
+      return
+  else:
+    test_result = DummyTestResult(test_name_list)
  # run NEO tests
  while 1:
    test_result_line = test_result.start()
@@ -106,6 +130,14 @@ def main():
          if timeout < time():
            raise RuntimeError("MySQL server not started")
          sleep(1)
+{%- if datadir is defined %}
+        # fake path for neostorage (getTopologyPath)
+        try:
+          os.mkdir({{ repr(datadir) }})
+        except OSError as e:
+          if e.errno != errno.EEXIST:
+            raise
+{%- endif %}
      with open(os.devnull) as stdin:
        p = subprocess.Popen(args, stdin=stdin, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, env=env)
@@ -137,6 +169,13 @@ def main():
      date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
      stderr=stderr,
      **status_dict)
+{%- if results_directory is defined %}
+
+    results = {{ repr(results_directory + '/') }} + adapter
+    if os.path.exists(results):
+      shutil.rmtree(results)
+    shutil.move(temp, results)
+{%- endif %}

 if __name__ == "__main__":
    main()
--- a/software/neoppod/software-common.cfg
+++ b/software/neoppod/software-common.cfg
@@ -131,14 +131,15 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
 mode = 644
 template =
  inline:{% raw %}#!/bin/sh -e
+  basedir='${mariadb:location}'
  datadir='{{datadir}}'
  [ -e "$datadir" ] || {
    rm -vrf "$datadir.new"
-    '${mariadb:location}/scripts/mysql_install_db' \
+    "$basedir/scripts/mysql_install_db" \
      --defaults-file='{{defaults_file}}' \
      --skip-name-resolve \
      --auth-root-authentication-method=normal \
-      --basedir='${mariadb:location}' \
+      --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
      --datadir="$datadir.new"
    mv -v "$datadir.new" "$datadir"
  }
@@ -147,9 +148,7 @@ template =
  {{ variable }} \
  {%-   endfor %}
  {%- endif %}
-  exec '${mariadb:location}/bin/mysqld' \
-    --defaults-file='{{defaults_file}}' \
-    "$@"
+  exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
  {% endraw %}

 [versions]

--- a/software/slapos-sr-testing/buildout.hash.cfg
+++ b/software/slapos-sr-testing/buildout.hash.cfg
@@ -15,4 +15,4 @@

 [template]
 filename = instance.cfg
-md5sum = 1cbab58e896ff63575f6a67db530d183
+md5sum = 3ad1b06673000d9f424a1e7187c6a1fa
--- a/software/slapos-sr-testing/instance.cfg
+++ b/software/slapos-sr-testing/instance.cfg
@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
 working-dir = $${buildout:directory}/tmp

 [test-list]
-path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup}
+path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup},${slapos.test.gitlab-setup:setup}

 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper

--- a/software/slapos-sr-testing/software.cfg
+++ b/software/slapos-sr-testing/software.cfg
@@ -112,6 +112,11 @@ setup = ${slapos-repository:location}/software/theia/test/
 egg = slapos.test.grafana
 setup = ${slapos-repository:location}/software/grafana/test/

+[slapos.test.gitlab-setup]
+<= setup-develop-egg
+egg = slapos.test.gitlab
+setup = ${slapos-repository:location}/software/gitlab/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git

--- a/stack/erp5/buildout.hash.cfg
+++ b/stack/erp5/buildout.hash.cfg
@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9

 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
+md5sum = e9032f39c6e5db684342491fdeb4624c

 [template-balancer]
 filename = instance-balancer.cfg.in

--- a/stack/erp5/instance-zope.cfg.in
+++ b/stack/erp5/instance-zope.cfg.in
@@ -189,7 +189,10 @@ wrapped-command-line = '{{ bin_directory }}/runwsgi' {% if webdav %}-w{% endif %
 {% else -%}
 wrapped-command-line = '{{ bin_directory }}/runzope' -C '${:configuration-file}'
 {%- endif %}
-private-dev-shm = {{ slapparameter_dict['private-dev-shm'] }}
+{%- set private_dev_shm = slapparameter_dict['private-dev-shm'] %}
+{%- if private_dev_shm %}
+private-tmpfs = {{ private_dev_shm }} /dev/shm
+{%- endif %}

 [{{ section('zcml') }}]
 recipe = slapos.cookbook:copyfilelist

--- a/stack/slapos.cfg
+++ b/stack/slapos.cfg
@@ -137,7 +137,7 @@ pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
 six = 1.12.0
-slapos.cookbook = 1.0.138
+slapos.cookbook = 1.0.139
 slapos.core = 1.5.9
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0