gitlab: Compile assets on instantiation and make sure DB is properly...

gitlab: Compile assets on instantiation and make sure DB is properly setup/migrated before unicorn runs

There are several actions that needs to be done on gitlab instance
upgrade:

    - we have to (re-)compile assets
    - we have to migrate DB

and also before the first run

    - we have to initialize DB

We can compile assets as part of instantiation process, but regarding
DB migration / setup - it is not currently possible to do that as part
of instantiation - for that operations we need PG & Redis to be already
running, but the first time slapos instantiates an SR it first prepares
all services, and only after instantiation is done, starts them all. There is
currently no way to hook into starting process, and run some scripts
after one service is started but before another service startup...

So the solution is: to perform such actions in delayed mode as part of
application - unicorn service - startup: it makes sure PG is running and
initializes it and does other actions which needs to be done to migrate
the DB. Only if/after they succeed the main application is started.

NOTE the comment about unicorn/gitlab startup slowness from the previous
    patch still holds true - so in order to get "all ok" after
    instantiation, it is required to perform the instantiation several
    times, because unicorn promise initially fails.

/cc @kazuhiko, @jerome

software/gitlab/gitlab-unicorn-startup.in

+#!{{ bash_bin }}
+# start up gitlab's unicorn with first making sure db is properly setup and all
+# migrations are up as pre-condition.
+
+RAKE={{ gitlab_rake }}
+
+die() {
+    echo "$*" 1>&2
+    exit 1
+}
+
+# 1. what to do when instance is initially setup
+# see
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/database_migrations.rb
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb
+
+# initial db setup
+pgtables="$({{ psql_bin }}   \
+    -h {{ pgsql['pgdata-directory'] }}  \
+    -U {{ pgsql.superuser }}            \
+    -d {{ pgsql.dbname }}               \
+    -c '\d')" || die "pg query problem"
+
+if echo "$pgtables" | grep -q '^No relations found' ; then
+    $RAKE db:schema:load db:seed_fu  || die "initial db setup failed"
+fi
+
+# re-build ssh keys
+# (we do not use them - just for cleannes)
+force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"
+
+
+# 2. what to do when instance is upgraded
+# see
+#   https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/deploy/deploy.sh
+#   https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/upgrader.rb
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb
+#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-ctl-commands/upgrade.rb
+#
+# Assets compilation is handled at instance deployment time. We do everything else here.
+
+# make sure all migrations are up
+migrate_log="{{ log_dir }}/db-migrate-`date +%s`.log"
+$RAKE db:migrate >$migrate_log 2>&1  || die "db:migrate failed"
+# if it was a no-op "migration" - we don't need info about that - only keep
+# logs of actual migration run.
+test -s $migrate_log || rm $migrate_log
+
+
+# clear cache
+$RAKE cache:clear   || die "cache:clear failed"
+
+
+
+# 3. finally exec to unicorn
+exec {{ gitlab_unicorn }}   \
+    -E production   \
+    -c {{ unicorn_rb.rendered }}    \
+    {{ gitlab_work.location }}/config.ru

software/gitlab/instance-gitlab.cfg.in

@@ -10,7 +10,7 @@ parts =
 
 #   gitlab-<prog>
 # ? mailroom
-{% set gitlab_progv = 'rails rake unicorn sidekiq' .split() %}
+{% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %}
 {% for prog in gitlab_progv %}
     gitlab-{{ prog }}
 {% endfor %}
@@ -25,6 +25,8 @@ parts =
 
     service-cron
 
+    on-reinstantiate
+
 # std stuff for slapos instance
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
@@ -236,6 +238,22 @@ prog    = {{ prog }}
 {% endfor %}
 
 
+[gitlab-unicorn-startup]
+recipe  = slapos.recipe.template:jinja2
+mode    = 0755
+template= {{ gitlab_unicorn_startup_in }}
+rendered= ${directory:bin}/${:_buildout_section_name_}
+context =
+    raw     bash_bin                {{ bash_bin }}
+    raw     gitlab_rake             ${gitlab-rake:wrapper-path}
+    raw     gitlab_unicorn          ${gitlab-unicorn:wrapper-path}
+    raw     psql_bin                {{ postgresql_location }}/bin/psql
+    section pgsql                   service-postgresql
+    raw     log_dir                 ${gitlab:log}
+    section unicorn_rb              unicorn.rb
+    section gitlab_work             gitlab-work
+
+
 # 4. gitlab- & gitlab-shell- work directories
 #
 # Gitlab/Rails operation is tightened that config/ lives inside code, which goes
@@ -439,11 +457,10 @@ socket  = ${:srv}/unicorn.socket
 [service-unicorn]
 recipe  = slapos.cookbook:wrapper
 wrapper-path    = ${directory:service}/unicorn
-command-line    = ${gitlab-unicorn:wrapper-path}
-    -E production
-    -c ${unicorn.rb:rendered}
-    ${gitlab-work:location}/config.ru
-
+# NOTE we perform db setup / migrations as part of unicorn startup.
+# Those operations require PG and Redis to be up and running already, that's
+# why we do it here. See gitlab-unicorn-startup for details.
+command-line    = ${gitlab-unicorn-startup:rendered}
 
 depend  =
     ${promise-unicorn:recipe}
@@ -589,3 +606,22 @@ backup  = ...
 <= cron-entry
 time    = daily
 command = ${logrotate:wrapper}
+
+
+
+# 6. on-reinstantiate actions
+
+# NOTE here we only recompile assets. Other on-reinstantiate actions, which
+# require pg and redis running, are performed as part of unicorn service -
+# right before its startup (see gitlab-unicorn-startup).
+[on-reinstantiate]
+recipe  = plone.recipe.command
+stop-on-error   = true
+rake    = ${gitlab-rake:wrapper-path}
+# run command on every reinstantiation
+update-command = ${:command}
+
+command =
+    ${:rake} assets:clean  &&
+    ${:rake} assets:precompile  &&
+    true

software/gitlab/instance.cfg.in

@@ -46,6 +46,7 @@ context =
     raw database_yml_in             ${database.yml.in:target}
     raw gitlab_parameters_cfg       ${gitlab-parameters.cfg:target}
     raw gitlab_shell_config_yml_in  ${gitlab-shell-config.yml.in:target}
+    raw gitlab_unicorn_startup_in   ${gitlab-unicorn-startup.in:target}
     raw gitlab_yml_in               ${gitlab.yml.in:target}
     raw macrolib_cfg_in             ${macrolib.cfg.in:target}
     raw rack_attack_rb_in           ${rack_attack.rb.in:target}

software/gitlab/software.cfg

@@ -204,6 +204,10 @@ url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
 <= download-template
 # md5sum = TODO
 
+[gitlab-unicorn-startup.in]
+<= download-file
+# md5sum = TODO
+
 [gitlab.yml.in]
 <= download-template
 # md5sum = TODO