gitlab: start automation of gitlab-backup

  - git-clone and build git-backup sources
  - add a cron job doing `gitlab-backup-pull` every 4 hours by default
  - add instance parameter to change backup frequency
  - add xnice for using less resources while backuping
  - add lock to avoid concurrent calls of backup script
  - add a specific software type `gitlab-export` doing all this instead of in
    `default`; this can be changed in Services > parameters (kirr's changes)

`coreutils`, `grep` and `tar` components are used by `gitlab-backup-pull`
script, as it is called from a cron and the PATH has to include these
binaries paths

software/gitlab/gitlab-export.in

+#!{{ bash.location }}/bin/bash -e
+# export gitlab site via `gitlab-backup pull` to a "for-export" git-backup repository
+# gitlab-export <exportto-repo>
+#
+# NOTE gitlab-backup, gitlab-rails, ... all have to be on $PATH.
+#      which site to export is determined by which gitlab-rails is on $PATH.
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: gitlab-export <exportto-repo>" 1>&2
+    exit 1
+fi
+exportto_repo="$1"
+
+# create / setup export repository if it does not exist yet
+mkdir -p "$exportto_repo"
+
+cd "$exportto_repo"
+
+# verify we are in a git repository
+if ! git rev-parse --is-inside-git-dir ; then
+  git init --bare
+fi
+
+exec gitlab-backup pull

software/gitlab/instance-gitlab-export.cfg.in

+# GitLab instance + site export support
+[buildout]
+extends = {{ instance_gitlab_cfg }}
+# TODO + ${pbsready-export:output}
+
+parts +=
+    cron-entry-gitlab-backup
+
+# -export specific instance parameters
+[instance-parameter]
+# cron frequency for gitlab backup (default: every 4h)
+configuration.backup_frequency  = 0 */4 * * *
+
+[gitlab-backup-directory]
+recipe            = slapos.cookbook:mkdirectory
+srv               = ${buildout:directory}/srv
+backup            = ${:srv}/backup
+backup-gitlab.git = ${:backup}/backup-gitlab.git
+var               = ${buildout:directory}/var
+pid               = ${:var}/pid
+
+# instance exporter script
+[exporter]
+recipe        = slapos.cookbook:wrapper
+wrapper-path  = ${buildout:directory}/bin/gitlab-exporter
+command-line  = {{ xnice_repository_location }}/bin/xnice {{ gitlab_export }} ${gitlab-backup-directory:backup-gitlab.git}
+pidfile       = ${gitlab-backup-directory:pid}/gitlab-exporter.pid
+environment   =
+# XXX: `/usr/bin` has to be in the PATH environment variable to be able to use
+# `which` command in gitlab-backup, `chrt` in xnice, ...
+  PATH=/usr/bin:${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin
+
+[cron-entry-gitlab-backup]
+<= cron-entry
+# run backup script on a regular basis (given as instance parameter)
+frequency = ${instance-parameter:configuration.backup_frequency}
+command	= ${exporter:wrapper-path}

software/gitlab/instance.cfg.in

@@ -10,15 +10,17 @@ offline = true
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = $${instance-gitlab.cfg:rendered}
-# TODO -export, -import, -pull-backup
+gitlab  = $${instance-gitlab.cfg:rendered}
+gitlab-export   = $${instance-gitlab-export.cfg:rendered}
+default = $${:gitlab}
+# TODO -import, -pull-backup
 
 
-[instance-gitlab.cfg]
+# macro: render instance-*.cfg from instance-*.cfg.in
+[instance-cfg]
 recipe  = slapos.recipe.template:jinja2
 mode    = 0644
-template= ${instance-gitlab.cfg.in:target}
-rendered= $${buildout:directory}/instance-gitlab.cfg
+rendered= $${buildout:directory}/$${:_buildout_section_name_}
 context =
     import os os
     import pwd pwd
@@ -33,12 +35,16 @@ context =
     raw bash_bin                    ${bash:location}/bin/bash
     raw bzip2_location              ${bzip2:location}
     raw bundler_4gitlab             ${bundler-4gitlab:bundle}
+    raw coreutils_location          ${coreutils:location}
     raw curl_bin                    ${curl:location}/bin/curl
     raw dcron_bin                   ${dcron-output:crond}
     raw git                         ${git:location}/bin/git
     raw git_location                ${git:location}
+    raw gitlab_export               ${gitlab-export:rendered}
     raw gitlab_workhorse            ${gitlab-workhorse:location}/gitlab-workhorse
+    raw gopath_bin                  ${gopath:bin}
     raw gunzip_bin                  ${gzip:location}/bin/gunzip
+    raw grep_location               ${grep:location}
     raw gzip_bin                    ${gzip:location}/bin/gzip
     raw gzip_location               ${gzip:location}
     raw logrotate_bin               ${logrotate:location}/usr/sbin/logrotate
@@ -48,7 +54,9 @@ context =
     raw postgresql_location         ${postgresql92:location}
     raw redis_binprefix             ${redis28:location}/bin
     raw ruby_location               ${bundler-4gitlab:ruby-location}
+    raw tar_location                ${tar:location}
     raw watcher_sigkill             ${watcher-sigkill:rendered}
+    raw xnice_repository_location   ${xnice-repository:location}
 
 # config files
     raw database_yml_in             ${database.yml.in:target}
@@ -64,3 +72,16 @@ context =
     raw resque_yml_in               ${resque.yml.in:target}
     raw smtp_settings_rb_in         ${smtp_settings.rb.in:target}
     raw unicorn_rb_in               ${unicorn.rb.in:target}
+
+    $${:context-extra}
+context-extra =
+
+[instance-gitlab.cfg]
+<= instance-cfg
+template= ${instance-gitlab.cfg.in:target}
+
+[instance-gitlab-export.cfg]
+<= instance-cfg
+template= ${instance-gitlab-export.cfg.in:target}
+context-extra =
+    raw instance_gitlab_cfg         $${instance-gitlab.cfg:rendered}

software/gitlab/software.cfg

@@ -14,9 +14,12 @@ extends =
     ../../component/nginx/buildout.cfg
 
 #   for instance
+    ../../component/coreutils/buildout.cfg
     ../../component/bash/buildout.cfg
+    ../../component/grep/buildout.cfg
     ../../component/bzip2/buildout.cfg
     ../../component/curl/buildout.cfg
+    ../../component/tar/buildout.cfg
     ../../component/gzip/buildout.cfg
     ../../component/dcron/buildout.cfg
     ../../component/logrotate/buildout.cfg
@@ -36,6 +39,7 @@ parts =
     gitlab-shell/vendor
     gitlab/vendor/bundle
     gitlab-workhorse
+    git-backup
 
 #   for instance
     instance.cfg
@@ -48,6 +52,7 @@ parts =
     bash
     curl
     watcher-sigkill
+    gitlab-export
     gzip
     dcron-output
     logrotate
@@ -157,6 +162,52 @@ make-binary =
 make-targets= cd ${:path} &&
     ${:bundle} install --deployment  --without development test  mysql kerberos
 
+# directories and repositories required by gitlab-backup
+[gopath]
+directory = ${buildout:directory}/go.work
+src	= ${:directory}/src
+bin	= ${:directory}/bin
+
+[go-git-repository]
+<= git-repository
+repository = https://${:go.importpath}.git
+location = ${gopath:src}/${:go.importpath}
+
+[git2go-repository]
+<= go-git-repository
+go.importpath = github.com/libgit2/git2go
+# branch 'next' is required by git-backup
+revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184
+
+[git-backup-repository]
+<= go-git-repository
+go.importpath = lab.nexedi.com/kirr/git-backup
+revision = 3ba6cf73ba224c40f67f1fb87c855b915eb91f58
+
+
+[git-backup]
+# install git2go, git-backup, gitlab-backup in <gopath>/bin
+recipe  = slapos.recipe.cmmi
+path    = ${git-backup-repository:location}
+configure-command = :
+make-binary =
+make-targets= cd ${git2go-repository:location}
+              && git submodule update --init
+              && make install
+              && install -d ${gopath:bin}
+              && go install ${git-backup-repository:go.importpath}
+              && cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin}
+environment =
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s
+  GOPATH=${gopath:directory}
+
+[xnice-repository]
+# to get kirr's misc repo containing xnice script for executing processes
+# with lower priority (used for backup script inside the cron)
+<= git-repository
+repository = https://lab.nexedi.com/kirr/misc.git
+revision =  4073572ea700bf1b115f3a135aebebe5b3b824e4
+location = ${buildout:parts-directory}/misc
 
 # build needed-by-gitlab-shell gems via bundler
 # ( there is not vendor/ dir in gitlab-shell, so to avoid having buildout error
@@ -200,17 +251,26 @@ eggs    =
 recipe  = slapos.recipe.template
 url     = ${:_profile_base_location_}/instance.cfg.in
 output  = ${buildout:directory}/instance.cfg
-md5sum  = ef85f02c4f6070c586d773b859a2f4e2
+md5sum  = b99a99b161c0b292845002fc3fee50cd
 
-[watcher-sigkill]
+# macro: download a shell script and put it rendered into <software>/bin/
+[binsh]
 recipe  = slapos.recipe.template:jinja2
 template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
 rendered= ${buildout:bin-directory}/${:_buildout_section_name_}
 mode    = 0755
-md5sum  = 2986dcb006dc9e8508ff81f646656131
 context =
     section bash    bash
 
+[watcher-sigkill]
+<= binsh
+md5sum  = 2986dcb006dc9e8508ff81f646656131
+
+[gitlab-export]
+<= binsh
+md5sum  = a7b32680e80f34276f0a32a5e22dad50
+
+
 
 # macro: download a file named as section name
 #
@@ -256,6 +316,10 @@ md5sum  = 176939a6428a7aca4767a36421b0af2b
 <= download-file
 md5sum  = 89914e4a225f6cdebfa196d46359f6f2
 
+[instance-gitlab-export.cfg.in]
+<= download-file
+md5sum  = a295e524046a9e2af569ad4b55f210cd
+
 [macrolib.cfg.in]
 <= download-file
 md5sum  = a56a44e96f65f5ed20211bb6a54279f4