WIP: start gitlab upgrade

software/gitlab/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = d1ca30a1b910b6b775f4f95bd91123a6
+md5sum = ef827795ea729358c54292714da1c554
 
 [watcher]
 _update_hash_filename_ = watcher.in

software/gitlab/instance-gitlab-export.cfg.in

@@ -55,6 +55,7 @@ input = inline: gitlab-shell-work*
   etc/service/postgres-start
   srv/redis/**
   srv/unicorn/unicorn.socket
+  .cache
 output = ${directory:srv}/exporter.exclude
 
 [gitlab-resiliency-restore-script]

software/gitlab/instance-gitlab.cfg.in

@@ -288,6 +288,7 @@ context-extra =
     import  urllib                  urllib
     section gitlab                  gitlab
     section gitlab_shell_work       gitlab-shell-work
+    section gitlab_workhorse        gitlab_workhorse
     section gitaly                  gitaly
 
 [rack_attack.rb]

software/gitlab/instance.cfg.in

@@ -75,8 +75,8 @@ context =
     raw nginx_mime_types            ${nginx-output:mime}
     raw node_bin_location           ${nodejs:location}/bin/
     raw openssl_bin                 ${openssl-output:openssl}
-    raw postgresql_location         ${postgresql10:location}
-    raw redis_binprefix             ${redis28:location}/bin
+    raw postgresql_location         ${postgresql:location}
+    raw redis_binprefix             ${redis:location}/bin
     raw ruby_location               ${bundler-4gitlab:ruby-location}
     raw tar_location                ${tar:location}
     raw watcher                     ${watcher:output}

software/gitlab/software.cfg

@@ -30,17 +30,15 @@ extends =
     ../../component/logrotate/buildout.cfg
 
 parts =
-    golang1.13
+    golang1.15
     git
-    postgresql10
-    redis28
+    postgresql
     cmake
     icu
     pkgconfig
     nginx-output
 
     gowork
-    gitlab-workhorse
     gitaly-build
     gitlab-shell/vendor
     gitlab/vendor/bundle
@@ -65,7 +63,7 @@ parts =
 revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
 
 [nodejs]
-<= nodejs-12.18.3
+<= nodejs-14.16.0
 [yarn]
 <= yarn-1.16.0
 
@@ -148,7 +146,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/
 # gitlab (via github-markup) wants to convert rst -> html via running: python (with docutils egg)
 environment =
 
-  PATH    = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH    = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql:location}/bin:${redis:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
 
 
 # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
@@ -160,25 +158,26 @@ git-executable = ${git:location}/bin/git
 [gitlab-repository]
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
-revision = v12.10.14-12-g7ce27b49193
+#revision = v12.10.14-12-g7ce27b49193
+branch = v13.12.15-nxd
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]
 <= git-repository
 repository = https://gitlab.com/gitlab-org/gitlab-shell.git
-revision = v12.2.0
+revision = v13.18.1
 location = ${buildout:parts-directory}/gitlab-shell
 
 [gitaly-repository]
 <= git-repository
 repository = https://gitlab.com/gitlab-org/gitaly.git
-revision = v12.10.14
+revision = v13.12.15
 location = ${buildout:parts-directory}/gitaly
 
-[gitlab-workhorse-repository]
-<= git-repository
-repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
-revision = v8.30.3-19-g919c9b532c
+#[gitlab-workhorse-repository]
+#<= git-repository
+#repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
+#revision = v8.30.3-19-g919c9b532c
 
 # build needed-by-gitlab gems via bundler
 [gitlab/vendor/bundle]
@@ -188,7 +187,7 @@ bundle  = ${bundler-4gitlab:bundle}
 
 configure-command = cd ${:path} &&
     ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}  &&
-    ${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config &&
+    ${:bundle} config --local build.pg --with-pg-config=${postgresql:location}/bin/pg_config &&
     ${:bundle} config --local build.re2 --with-re2-dir=${re2:location} &&
     ${:bundle} config --local build.nokogiri --with-zlib-dir=${zlib:location} --with-cflags=-I${xz-utils:location}/include --with-ldflags="-L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib"
     ${:bundle} config set without 'development test mysql aws kerberos'
@@ -233,7 +232,7 @@ make-targets= cd ${go_github.com_libgit2_git2go:location}
               && make install
 environment =
   PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
-  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.13:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.15:location}/bin:${buildout:bin-directory}:%(PATH)s
   GOPATH=${gowork:directory}
 
 [gowork.goinstall]
@@ -241,10 +240,7 @@ git2go = ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install
 command = bash -c ". ${gowork:env.sh} && CGO_CFLAGS=-I${:git2go}/include CGO_LDFLAGS='-L${:git2go}/lib -lgit2' go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ') && go test -v lab.nexedi.com/kirr/git-backup"
 
 [gowork]
-golang  = ${golang1.13:location}
-# gitlab.com/gitlab-org/gitlab-workhorse
-# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
-# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
+golang  = ${golang1.15:location}
 install =
   lab.nexedi.com/kirr/git-backup
 cpkgpath =
@@ -255,7 +251,7 @@ buildflags = --tags "static"
 
 [gitlab-workhorse]
 recipe = slapos.recipe.cmmi
-path = ${gitlab-workhorse-repository:location}
+path = ${gitlab-repository:location}/workhorse
 configure-command = :
 make-binary =
 make-targets =
@@ -285,7 +281,8 @@ post-install =
   chmod 755 ${:path}/ruby/git-hooks/gitlab-shell-hook
 environment =
   PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
-  PATH=${pkgconfig:location}/bin:${ruby2.6:location}/bin:%(PATH)s
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${ruby2.6:location}/bin:%(PATH)s
+  OPENSSL_ROOT_DIR=${openssl-1.0:location}
 
 [xnice-repository]
 # to get kirr's misc repo containing xnice script for executing processes

software/gitlab/template/gitaly-config.toml.in

@@ -7,7 +7,12 @@ socket_path = "{{ gitaly.socket }}"
 # The directory where Gitaly's executables are stored
 bin_dir = "{{ gitaly.location }}"
 
-# # Optional: listen on a TCP socket. This is insecure (no authentication)
+# # Optional. The directory where Gitaly can create all files required to
+# # properly operate at runtime. If not set, Gitaly will create a directory in
+# # the global temporary directory. This directory must exist.
+# runtime_dir = "/home/git/gitaly/run"
+
+# # Optional if socket_path is set. TCP address for Gitaly to listen on. This is insecure (unencrypted connection).
 # listen_addr = "localhost:9999"
 # tls_listen_addr = "localhost:8888
 
@@ -31,8 +36,16 @@ internal_socket_dir = "{{ gitaly.internal_socket }}"
 # # Git settings
 [git]
 bin_path = "{{ git }}"
+# # Maximum number of cached 'cat-file' processes, which constitute a pair of 'git cat-file --batch' and
+# # 'git cat-file --batch-check' processes. Defaults to '100'.
 # catfile_cache_size = 100
 
+# [[git.config]]
+# key = fetch.fsckObjects
+# value = true
+# # Storages are the directories where Gitaly stores its data such as the repositories and runtime state.
+# # Each storage must have a unique name.
+
 [[storage]]
 name = "default"
 path = "{{ gitlab.repositories }}"
@@ -57,8 +70,8 @@ level = "warn"
 #
 # # Additionally exceptions from the Go server can be reported to Sentry
 # sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
-# # Exceptions from gitaly-ruby can also be reported to Sentry
-# ruby_sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
+# # Sentry Environment for exception monitoring.
+sentry_environment = ""
 
 
 # # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
@@ -89,7 +102,34 @@ dir = "{{ gitaly.location }}/ruby"
 # The directory where gitlab-shell is installed
 dir = "{{ gitlab_shell_work.location }}"
 
+[hooks]
+custom_hooks_dir = "{{ gitlab_shell_work.location }}/hooks/"
+
+[gitlab]
+secret_file = "{{ gitlab_shell.secret }}"
+url = "http+unix://{{ urllib.parse.unquote_plus(gitlab_workhorse.socket) }}"
+# Only needed if a UNIX socket is used in `url` and GitLab is configured to
+# use a relative path (e.g. /gitlab).
+# relative_url_root = '/'
+
+[gitlab.http-settings]
+# read_timeout = 300
+# user = someone
+# password = somepass
+# ca_file = /etc/ssl/cert.pem
+# ca_path = /etc/pki/tls/certs
+# self_signed_cert = false
+
 # # You can adjust the concurrency of each RPC endpoint
 # [[concurrency]]
 # rpc = "/gitaly.RepositoryService/GarbageCollect"
 # max_per_repo = 1
+
+# Daily maintenance designates time slots to run daily to optimize and maintain
+# enabled storages.
+# [daily_maintenance]
+# start_hour = 23
+# start_minute = 30
+# duration = "45m"
+# storages = ["default"]
+# disabled = false

software/gitlab/template/gitlab-shell-config.yml.in

@@ -14,6 +14,7 @@ http_settings:
 {# we don't need any
   <%= @http_settings.to_json if @http_settings %>
 #}
+#  read_timeout: 300
 #  user: someone
 #  password: somepass
 #  ca_file: /etc/ssl/cert.pem
@@ -34,35 +35,17 @@ auth_file: "{{ gitlab.var }}/sshkeys-notused"
 # Default is .gitlab_shell_secret in the root directory.
 secret_file: "{{ gitlab_shell.secret }}"
 
-# Parent directory for global custom hook directories (pre-receive.d, update.d, post-receive.d)
-# Default is hooks in the gitlab-shell directory.
-custom_hooks_dir: "{{ gitlab_shell_work.location }}/hooks/"
-
-# Redis settings used for pushing commit notices to gitlab
-redis:
-  bin: {{ redis_binprefix }}/redis-cli
-  host: {# <%= @redis_host %> #}
-  port: {# <%= @redis_port %> #}
-  socket: {{ service_redis.unixsocket }}
-  database: {# <%= @redis_database %> #}
-  namespace: resque:gitlab
-
 # Log file.
 # Default is gitlab-shell.log in the root directory.
 log_file: "{{ gitlab_shell.log }}/gitlab-shell.log"
 
 # Log level. INFO by default
-log_level:
+log_level: INFO
+
+# Log format. 'text' by default
+log_format: text
 
 # Audit usernames.
 # Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
 # incurs an extra API call on every gitlab-shell command.
-audit_usernames:
-
-# Enable git-annex support
-# git-annex allows managing files with git, without checking the file contents into git
-# See https://git-annex.branchable.com/ for documentation
-# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup
-# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex
-# For CentOS: sudo yum install epel-release && sudo yum install git-annex
-git_annex_enabled:
+audit_usernames: false

software/gitlab/template/gitlab.yml.in

@@ -18,6 +18,9 @@ production: &base
     host: {{ external_url.hostname }}
     port: {{ external_url.port or default_port[external_url.scheme] }}
     https: {{ cfg_https }}
+    # The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout.
+    # Default is 95% of the worker timeout
+    max_request_duration_seconds: 57
 
     {# ssh is disabled completely in slapos version
     # Uncommment this line below if your ssh host is different from HTTP/HTTPS one
@@ -55,6 +58,8 @@ production: &base
         worker_src: "'self' blob:"
         report_uri:
 
+    allowed_hosts: []
+
     # Trusted Proxies
     # Customize if you have GitLab behind a reverse proxy which is running on a different machine.
     # Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
@@ -122,6 +127,15 @@ production: &base
     repository_downloads_path: <%= @gitlab_repository_downloads_path %>
     #}
 
+    ## Impersonation settings
+    impersonation_enabled: true
+
+    ## Disable jQuery and CSS animations
+    # disable_animations: true
+
+    ## Application settings cache expiry in seconds (default: 60)
+    # application_settings_cache_seconds: 60
+
   {# we do not support reply by email
   ## Reply by email
   # Allow users to comment on issues and merge requests by replying to notification emails.
@@ -414,7 +428,9 @@ production: &base
 
   # Gitaly settings
   gitaly:
-    # Default Gitaly authentication token. Can be overriden per storage. Can
+    # Path to the directory containing Gitaly client executables.
+    client_path: {{ gitaly.location }}
+    # Default Gitaly authentication token. Can be overridden per storage. Can
     # be left blank when Gitaly is running locally on a Unix socket, which
     # is the normal way to deploy Gitaly.
     token:
@@ -462,8 +478,6 @@ production: &base
     path: {{ gitlab_shell_work.location }}
     authorized_keys_file: {{ gitlab.var }}/sshkeys-notused
 
-    repos_path: {{ gitlab.repositories }}
-    hooks_path: {{ gitlab_shell_work.location }}/hooks/
     secret_file: {{ gitlab_shell.secret }}
 
     # Git over HTTP
@@ -488,12 +502,6 @@ production: &base
   # Use the default values unless you really know what you are doing
   git:
     bin_path: {{ git }}
-    # The next value is the maximum memory size grit can use
-    # Given in number of bytes per git object (e.g. a commit)
-    # This value can be increased if you have very large commits
-    max_size: {{ cfg('git_max_size') }}
-    # Git timeout to read a commit, in seconds
-    timeout: {{ cfg('git_timeout') }}
 
   #
   # 5. Extra customization

software/gitlab/template/unicorn.rb.in

@@ -12,11 +12,23 @@
 listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }}
 #listen "127.0.0.1:8888", :tcp_nopush => true
 
+# Where to drop a pidfile
+pid '{{ directory.run }}/unicorn.pid'
+
+# Where stderr gets logged
+stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
+
+# Where stdout gets logged
+stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
+
 working_directory '{{ gitlab_work.location }}'
 
 # What the timeout for killing busy workers is, in seconds
 timeout {{ cfg('unicorn_worker_timeout') }}
 
+# How many worker processes
+worker_processes {{ cfg('unicorn_worker_processes') }}
+
 # combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
 # http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
 preload_app true
@@ -37,8 +49,7 @@ before_exec do |server|
   Gitlab::Cluster::LifecycleEvents.do_before_master_restart
 end
 
-# How many worker processes
-worker_processes {{ cfg('unicorn_worker_processes') }}
+run_once = true
 
 # about before_fork / after_fork - see:
 #   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb
@@ -46,8 +57,18 @@ worker_processes {{ cfg('unicorn_worker_processes') }}
 
 # What to do before we fork a worker
 before_fork do |server, worker|
+  if run_once
+    # There is a difference between Puma and Unicorn:
+    # - Puma calls before_fork once when booting up master process
+    # - Unicorn runs before_fork whenever new work is spawned
+    # To unify this behavior we call before_fork only once (we use
+    # this callback for deleting Prometheus files so for our purposes
+    # it makes sense to align behavior with Puma)
+    run_once = false
+
     # Signal application hooks that we're about to fork
     Gitlab::Cluster::LifecycleEvents.do_before_fork
+  end
 
   # The following is only recommended for memory/DB-constrained
   # installations.  It is not needed if your system can house
@@ -74,8 +95,6 @@ before_fork do |server, worker|
   # sleep 1
 end
 
-
-# What to do after we fork a worker
 after_fork do |server, worker|
   # Signal application hooks of worker start
   Gitlab::Cluster::LifecycleEvents.do_worker_start
@@ -83,18 +102,11 @@ after_fork do |server, worker|
   # per-process listener ports for debugging/admin/migrations
   # addr = "127.0.0.1:#{9293 + worker.nr}"
   # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
-
 end
 
-
-# Where to drop a pidfile
-pid '{{ directory.run }}/unicorn.pid'
-
-# Where stderr gets logged
-stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
-
-# Where stdout gets logged
-stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
+# Configure the default logger to use a custom formatter that formats the
+# timestamps to be in UTC and in ISO8601.3 format
+Configurator::DEFAULTS[:logger].formatter = Gitlab::LogTimestampFormatter.new
 
 {# we do not support Relative url
 <%- if @relative_url %>