gitlab: Add empty instance

Add stub instance configuration which just establishes a way to have
several software types(*), pass all needed info from software to
instance, organizes base directory and establishes log rotation base for
upcoming services.

Log rotation is done with the help of cron periodicallly calling
logrotate. The rotation is done in "copytruncate" mode - i.e. log file
is not moved away and signal sent for service to reopen it, but instead
log content is just copied to outside and there is no need for a service
to reopen it's log file.

The reason it is done this way, is that there is a chance of not
handling such "reopen-log-file" callbacks correctly on a service side,
and so the net is full of crashing reports, e.g. like this:

    http://serverfault.com/questions/627521/why-is-logrotate-causing-apache-to-seg-fault-each-time

That's why we take a safer approach instead, even if "copytruncate" mode
is risking to loose several log entries(**) on rotation.

NOTE services will organize log rotation with just

    [logrotate-entry-<service>]
    <= logrotate-entry
    log     = path/to/log/files/*.log

For this to work some "!py!" magic (our way to serialize object into
executable python and process it in buildout recipes) is used to process
section names.

The approach trick is also used for cron, e.g. logrotate registers to
cron this way:

    [cron-entry-logrotate]
    <= cron-entry
    time    = daily
    command = ${logrotate:wrapper}

NOTE2 instance md5 are not fixed yet - we'll fix them after applying all
    patches in gitlab series.

(*) for now there is only 1 - "gitlab", but we'll need to have "-export"
    and "-import" for resiliency in the future.

(**) ideally such things should be done with logfs - a filesystem
    specializeing in logging - for client services it will look like as
    they just continue to write to log file, and on log service side, the
    rotation can happen, all transparent to client service.

/cc @kazuhiko, @jerome

software/gitlab/instance-gitlab.cfg.in

+# GitLab instance
+[buildout]
+parts =
+    directory
+
+    service-cron
+
+# std stuff for slapos instance
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+
+##################################
+#   GitLab instance parameters   #
+##################################
+
+[instance-parameter]
+# std stuff to fetch slapos instance parameters
+recipe  = slapos.cookbook:slapconfiguration
+computer= ${slap-connection:computer-id}
+partition=${slap-connection:partition-id}
+url     = ${slap-connection:server-url}
+key     = ${slap-connection:key-file}
+cert    = ${slap-connection:cert-file}
+
+
+
+#############################
+#   GitLab instance setup   #
+#############################
+
+# 1. directories
+[directory]
+recipe  = slapos.cookbook:mkdirectory
+home    = ${buildout:directory}
+bin     = ${:home}/bin
+etc     = ${:home}/etc
+var     = ${:home}/var
+log     = ${:var}/log
+run     = ${:var}/run
+srv     = ${:home}/srv
+# slapos startup/service/promise scripts live here:
+startup = ${:etc}/run
+service = ${:etc}/service
+promise = ${:etc}/promise
+
+
+
+
+# 2. services
+
+#############
+#   cron    #
+#############
+[cron-dir]
+recipe  = slapos.cookbook:mkdirectory
+cron.d  = ${directory:etc}/cron.d
+crontabs= ${directory:srv}/cron/crontabs
+cronstamps = ${directory:var}/cron/cronstamps
+log     = ${directory:log}/cron
+
+[service-cron]
+recipe  = slapos.cookbook:cron
+binary  = ${directory:service}/crond
+cron-entries    = ${cron-dir:cron.d}
+crontabs        = ${cron-dir:crontabs}
+cronstamps      = ${cron-dir:cronstamps}
+catcher         = ${cron-simplelogger:wrapper}
+
+dcrond-binary   = {{ dcron_bin }}
+
+depends =
+    ${logrotate-entry-cron:recipe}
+
+# "mailer" that cron uses to emit messages to logfile
+[cron-simplelogger]
+recipe  = slapos.cookbook:simplelogger
+wrapper = ${directory:bin}/${:_buildout_section_name_}
+log     = ${cron-dir:log}/cron.log
+
+
+# base entry for clients who registers to cron
+[cron-entry]
+recipe  = slapos.cookbook:cron.d
+# name  = <section-name>.strip_prefix('cron-entry-')
+# XXX len() is not available in !py! - 11 hardcoded
+name    = !py!'${:_buildout_section_name_}' [11:]
+# NOTE _not_ ${service-cron:cron-entries}  - though the value is the same we do
+# not want service-cron to be instantiated just if a cron-entry is registered.
+cron-entries = ${cron-dir:cron.d}
+
+# cron logs are also rotated
+[logrotate-entry-cron]
+<= logrotate-entry
+log     = ${cron-dir:log}/*.log
+
+
+#######################################
+#   logrotate base for all services   #
+#######################################
+[logrotate-dir]
+recipe  = slapos.cookbook:mkdirectory
+srv     = ${directory:srv}/logrotate
+entries = ${directory:etc}/logrotate.d
+
+[logrotate]
+recipe  = slapos.cookbook:logrotate
+wrapper = ${directory:bin}/${:_buildout_section_name_}
+conf    = ${directory:etc}/logrotate.conf
+logrotate-entries   = ${logrotate-dir:entries}
+state-file  = ${logrotate-dir:srv}/logrotate.status
+
+logrotate-binary    = {{ logrotate_bin }}
+gzip-binary     = {{ gzip_bin }}
+gunzip-binary   = {{ gunzip_bin }}
+
+depend  = ${cron-entry-logrotate:recipe}
+
+
+# base entry for clients who registers to logrotate
+[logrotate-entry]
+recipe  = slapos.cookbook:logrotate.d
+logrotate-entries   = ${logrotate:logrotate-entries}
+# name  = <section-name>.strip_prefix('logrotate-entry-')
+# XXX len is not available in !py! - 16 hardcoded
+name    = !py!'${:_buildout_section_name_}'[16:]
+# NOTE frequency is hardcoded to `daily` in slapos.cookbook:logrotate.d
+# NOTE backup is also used to add custom logrotate options (hack)
+backup  = ...
+# TODO settle whether we need/want olddir or not
+    noolddir
+# override create emitted by slapos.cookbook:logrotate.d
+    nocreate
+# do not move log file and this way we do not need to signal its program to
+# reopen the log. There are a lot of bugs when on such reopen / restart /
+# graceful-restart something bad happens. Even if copytruncate is a bit racy
+# and can loose some data, it is better to keep the system the stable way.
+    copytruncate
+
+
+# hook logrotate into cron
+[cron-entry-logrotate]
+<= cron-entry
+time    = daily
+command = ${logrotate:wrapper}

software/gitlab/instance.cfg.in

+# GitLab "switch-softwaretype" instance
+[buildout]
+parts = switch-softwaretype
+
+# std stuff for slapos instance
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${instance-gitlab.cfg:rendered}
+# TODO -export, -import, -pull-backup
+
+
+[instance-gitlab.cfg]
+recipe  = slapos.recipe.template:jinja2
+mode    = 0644
+template= ${instance-gitlab.cfg.in:target}
+rendered= $${buildout:directory}/instance-gitlab.cfg
+context =
+    key eggs_directory          buildout:eggs-directory
+    key develop_eggs_directory  buildout:develop-eggs-directory
+
+# program binaries
+    raw dcron_bin                   ${dcron-output:crond}
+    raw gunzip_bin                  ${gzip:location}/bin/gunzip
+    raw gzip_bin                    ${gzip:location}/bin/gzip
+    raw logrotate_bin               ${logrotate:location}/usr/sbin/logrotate

software/gitlab/software.cfg

@@ -13,6 +13,11 @@ extends =
     ../../component/openssl/buildout.cfg
     ../../component/nginx/buildout.cfg
 
+#   for instance
+    ../../component/gzip/buildout.cfg
+    ../../component/dcron/buildout.cfg
+    ../../component/logrotate/buildout.cfg
+
 parts =
     ruby2.1
     golang15
@@ -28,6 +33,16 @@ parts =
     gitlab/vendor/bundle
     gitlab-workhorse
 
+#   for instance
+    instance.cfg
+
+    slapos-cookbook
+    eggs
+
+    gzip
+    dcron-output
+    logrotate
+
 
 ############################
 #   Software compilation   #
@@ -135,7 +150,41 @@ environment =
   PATH=${golang15:location}/bin:%(PATH)s
 
 
+###############################
+#   Trampoline for instance   #
+###############################
+
+# eggs for instance.cfg
+[eggs]
+recipe  = zc.recipe.egg
+eggs    =
+    plone.recipe.command
+
+
+[instance.cfg]
+recipe  = slapos.recipe.template
+url     = ${:_profile_base_location_}/instance.cfg.in
+output  = ${buildout:directory}/instance.cfg
+# md5sum = TODO
+
+
+# macro: download a file named as section name
+#
+#   [filename]
+#   <= download-file
+#   md5sum = ...
+[download-file]
+recipe  = slapos.recipe.build:download
+url     = ${:_profile_base_location_}/${:_buildout_section_name_}
+destination = ${buildout:directory}/${:_buildout_section_name_}
+
+
+[instance-gitlab.cfg.in]
+<= download-file
+# md5sum = TODO
 
 
 [versions]
+plone.recipe.command = 1.1
 rubygemsrecipe  = 0.2.2
+slapos.recipe.template = 2.9