gitlab: Upgrade to 8.6

- GitLab Software + patches ported to GitLab 8.6.X;
- Configs synced with upstream;
- Base software upgraded where appropriate;
- misc adjustments.

Demo instance: https://softinst64196.host.vifib.net/

@jerome @kazuhiko @iv Please have a look. I've verified it works but there is always a chance one can miss some detail. If all ok I'd like to deploy this tomorrow (3 Aug) evening to lab.nexedi.com

Thanks beforehand for feedback,  
Kirill


/reviewed-on nexedi/slapos!92

component/postgresql/buildout.cfg

@@ -27,11 +27,13 @@ configure-options =
   --without-ossp-uuid
   --without-libxml
   --without-libxslt
+# build core PostgreSQL + pg_trgm contrib extension for GitLab
+make-targets = install && make -C contrib/pg_trgm/ install
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${perl:location}/libs-c -Wl,-rpath=${perl:location}/libs-c
 
 [postgresql92]
 <= postgresql-common
-url = http://ftp.postgresql.org/pub/source/v9.2.16/postgresql-9.2.16.tar.bz2
-md5sum = a6ced0ccd4ce3279acfef9bac16a95ee
+url = http://ftp.postgresql.org/pub/source/v9.2.17/postgresql-9.2.17.tar.bz2
+md5sum = a75d4a82eae1edda04eda2e60656e74c

software/gitlab/gitlab-parameters.cfg

@@ -6,7 +6,7 @@
 #
 # TODO better autogenerate from ^^^ (?)
 #
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 [gitlab-parameters]
 configuration.external_url              = http://lab.example.com

software/gitlab/gitlab-unicorn-startup.in

@@ -9,6 +9,15 @@ die() {
     exit 1
 }
 
+# run psql on gitlab db
+psql() {
+    {{ psql_bin }}  \
+        -h {{ pgsql['pgdata-directory'] }}  \
+        -U {{ pgsql.superuser }}            \
+        -d {{ pgsql.dbname }}               \
+        "$@"
+}
+
 # 1. what to do when instance is initially setup
 # see
 #   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/database_migrations.rb
@@ -18,12 +27,7 @@ die() {
 # ( first quering PG several times waiting a bit till postgresql is started and ready )
 tpgwait=5
 while true; do
-    pgtables="$({{ psql_bin }}   \
-        -h {{ pgsql['pgdata-directory'] }}  \
-        -U {{ pgsql.superuser }}            \
-        -d {{ pgsql.dbname }}               \
-        -c '\d')" && break
-
+    pgtables="$(psql -c '\d')" && break
     tpgwait=$(( $tpgwait - 1 ))
     test $tpgwait = 0 && die "pg query problem"
     echo "I: PostgreSQL is not ready (yet ?); will retry $tpgwait times..." 1>&2
@@ -31,6 +35,9 @@ while true; do
 done
 echo "I: PostgreSQL ready." 1>&2
 
+# make sure pg_trgm extension is enabled for gitlab db
+psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'   || die "pg_trgm setup failed"
+
 if echo "$pgtables" | grep -q '^No relations found' ; then
     $RAKE db:schema:load db:seed_fu  || die "initial db setup failed"
 fi

software/gitlab/instance-gitlab.cfg.in

@@ -633,7 +633,7 @@ log     = ${sidekiq-dir:log}
 # NOTE see queue list here:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 [service-sidekiq]
 recipe  = slapos.cookbook:wrapper
 wrapper-path    = ${directory:service}/sidekiq

software/gitlab/software.cfg

@@ -23,7 +23,7 @@ extends =
 
 parts =
     ruby2.1
-    golang15
+    golang16
     git
     postgresql92
     redis28
@@ -113,26 +113,25 @@ git-executable = ${git:location}/bin/git
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-ce.git
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
-# 8.5.X + NXD patches:
-revision = v8.5.3-15-g827d39147af496bcd4696c845e9094a07b5a09d3
+# 8.6.X + NXD patches:
+revision = v8.6.9-9-g1aafce0d0634afa9b3cfaf1e3a5b090c8c980076
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-shell.git
 repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
-# gitlab 8.5 wants gitlab-shell 2.6.10
-# 2.6.10 + no-hooks.old patch
-revision = v2.6.10-13-gcb8f331c955aa780efe10263d0d45b332f030f42
+# gitlab 8.6 wants gitlab-shell 2.6.12
+# 2.6.12 + NXD patches
+revision = v2.6.12-1-g03df9d7f9cfaa9e058f4c7053a497069b9b52657
 location = ${buildout:parts-directory}/gitlab-shell
 
 [gitlab-workhorse-repository]
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-workhorse.git
 repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
-# 0.6.4 + NXD patches for blob download speedup
-# (https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/17)
-revision = v0.6.4-3-g32ec77c4db3556dc66e8c863326325f8ec970849
+# 0.7.1 + NXD patches
+revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48
 location = ${buildout:parts-directory}/gitlab-workhorse
 
 
@@ -175,7 +174,7 @@ configure-command = :
 make-targets= ${:_buildout_section_name_}
 
 environment =
-  PATH=${golang15:location}/bin:%(PATH)s
+  PATH=${golang16:location}/bin:%(PATH)s
 
 
 ###############################
@@ -224,35 +223,35 @@ url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
 
 [config.ru.in]
 <= download-template
-md5sum  = 91737a2067be80f3ae48157e5a0738b4
+md5sum  = 4f6191c6e6bbaf2cd39c6f155fe192e1
 
 [database.yml.in]
 <= download-template
-md5sum  = 76d8f20532d63282ecd3617a3937fcf1
+md5sum  = 60504181bf40fefd023b2b8facff291d
 
 [gitconfig.in]
 <= download-template
-md5sum  = d8b3611386c4982605edd9a31832ee28
+md5sum  = 56b135596e013c02bda5555dda6b376b
 
 [gitlab-parameters.cfg]
 <= download-file
-md5sum  = b04b72949f0b9b5da96870ffbf8c1ff4
+md5sum  = 8cbcdb8d9e942df67d2fbb4db5fbfaad
 
 [gitlab-shell-config.yml.in]
 <= download-template
-md5sum  = 9c62aa1bf7396f207a528e39973aa135
+md5sum  = cdcb2036c33da547a2cf5d0515cf48ff
 
 [gitlab-unicorn-startup.in]
 <= download-file
-md5sum  = 14c5632182d830c03f7788c85d6f4da1
+md5sum  = a9cb347f60aad3465932fd36cd4fe25d
 
 [gitlab.yml.in]
 <= download-template
-md5sum  = 8f1c52ed223ee2c2d07c6c79bcd2f6c0
+md5sum  = 043804b76affe91a0bd3686f93f80ac8
 
 [instance-gitlab.cfg.in]
 <= download-file
-md5sum  = d3158d58e7fea9a656a8ef6e238a919b
+md5sum  = 0e7c13b5a5a3ad8748a79eea177b0ce3
 
 [macrolib.cfg.in]
 <= download-file
@@ -260,27 +259,27 @@ md5sum  = a56a44e96f65f5ed20211bb6a54279f4
 
 [nginx-gitlab-http.conf.in]
 <= download-template
-md5sum  = be7ddd26b0b0196fde8d699e50e84b2f
+md5sum  = 27feb252c2eba8e665164ea3b496fcd7
 
 [nginx.conf.in]
 <= download-template
-md5sum  = 71b4221cd91e0e1e20a72b97991f2665
+md5sum  = 5799afef0a60d07c9e6b52d0efaccec7
 
 [rack_attack.rb.in]
 <= download-template
-md5sum  = 2cf56b210a906cee80f86cde7112f468
+md5sum  = 6218a2f01fea29da1f61d69584635b52
 
 [resque.yml.in]
 <= download-template
-md5sum  = 1fa21655a845602f68a901f23a1da89a
+md5sum  = acd686b43ae37c4282e67b668786986e
 
 [smtp_settings.rb.in]
 <= download-template
-md5sum  = 07b640122b8c4840e853360f2479d0ac
+md5sum  = a4dda9c94f2ee72851343f505d5f495e
 
 [unicorn.rb.in]
 <= download-template
-md5sum  = 4f7c14147fd60b4cc9aefe7f64524abe
+md5sum  = 293143fecab603299bbc0a3dfd907ac3
 
 
 [versions]

software/gitlab/template/config.ru.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config.ru
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-rails-config.ru.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 # This file is used by Rack-based servers to start the application.
 

software/gitlab/template/database.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/database.yml.erb
-# (last updated for 8.5.1+ce.0-1-ge732b39)
+# (last updated for 8.6.5+ce.0-0-g342f8be)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/gitconfig.in

@@ -3,7 +3,7 @@
 # see:
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitconfig.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 #
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/gitlab-shell-config.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-shell-config.yml.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 # GitLab user. git by default
 user: {{ backend_info.user }}

software/gitlab/template/gitlab.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_https, external_url  with context %}
 
@@ -124,7 +124,7 @@ production: &base
   artifacts:
     enabled: <%= @artifacts_enabled %>
     # The location where Build Artifacts are stored (default: shared/artifacts).
-    storage_path: <%= @artifacts_path %>
+    path: <%= @artifacts_path %>
   #}
 
   {# we do not support LFS
@@ -223,6 +223,7 @@ production: &base
     enabled: false
     {# just disabled
     enabled: <%= @ldap_enabled %>
+    sync_time: <%= @ldap_sync_time %>
   <% if @ldap_servers.any? %>
     servers:
     <% @ldap_servers.each do |provider_id, settings| %>

software/gitlab/template/nginx-gitlab-http.conf.in

 {{ autogenerated }}
 # see:
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn  with context %}
 
@@ -66,7 +66,6 @@ server {
 
   server_name {{ fqdn }};
   server_tokens off; ## Don't show the nginx version number, a security best practice
-  root {{ gitlab_work.location }}/public;
 
   ## Increase this if you want to upload large attachments
   ## Or if you want to accept large git objects over http
@@ -135,6 +134,15 @@ server {
     proxy_pass http://gitlab-workhorse;
   }
 
+  error_page 404 /404.html;
+  error_page 422 /422.html;
+  error_page 500 /500.html;
+  error_page 502 /502.html;
+  location ~ ^/(404|422|500|502)\.html$ {
+    root {{ gitlab_work.location }}/public;
+    internal;
+  }
+
   {# we don't support custom nginx configs
   <%= @custom_gitlab_server_config %>
   #}

software/gitlab/template/nginx.conf.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 
@@ -41,6 +41,13 @@ http {
 
   include {{ nginx_mime_types }};
 
+  {# we do not do nginx caching:
+     - gitlab-workhorse serves raw blobs fast
+     - we have caches on frontend nodes
+  proxy_cache_path <%= @proxy_cache_path %>;
+  proxy_cache <%= @proxy_cache %>;
+  #}
+
   include {{ nginx_gitlab_http_conf }};
 
   {# we don't need: ci, pages, mattermost

software/gitlab/template/rack_attack.rb.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/resque.yml.in

@@ -2,6 +2,6 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb
-# (last udpdated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last udpdated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 production: unix://{{ redis.unixsocket }}

software/gitlab/template/smtp_settings.rb.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_bool  with context %}
 

software/gitlab/template/unicorn.rb.in

@@ -3,7 +3,7 @@
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.6.5+ce.0-0-g342f8be)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}