Commit 5691fd6f authored by Alain Takoudjou's avatar Alain Takoudjou

Upgrade Gitlab to v13.12

See merge request nexedi/slapos!1558
parents 8376de82 af88b7c8
...@@ -33,6 +33,9 @@ environment = ...@@ -33,6 +33,9 @@ environment =
PATH=${curl:location}/bin:${gettext:location}/bin:${tar:location}/bin:${xz-utils:location}/bin:%(PATH)s PATH=${curl:location}/bin:${gettext:location}/bin:${tar:location}/bin:${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${curl:location}/lib -Wl,-rpath=${libexpat:location}/lib LDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${curl:location}/lib -Wl,-rpath=${libexpat:location}/lib
patch-options = -p1
patches =
${:_profile_base_location_}/prevent-git-fetch-pack-segfault.patch#f18e84a5ff8951aad094eddf32fbf8a1
[gitweb] [gitweb]
<= git <= git
......
From 93be30997053105b5e3bad9203e5d07cbea0f0e6 Mon Sep 17 00:00:00 2001
From: Alain Takoudjou <alain.takoudjou@nexedi.com>
Date: Fri, 7 Jun 2024 12:13:50 +0200
Subject: [PATCH] prevent-git-fetch-pack-segfault
---
fetch-pack.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fetch-pack.c b/fetch-pack.c
index 65c1ff4bb4..a9b1558ce2 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -1037,7 +1037,7 @@ static int get_pack(struct fetch_pack_args *args,
cmd.git_cmd = 1;
if (start_command(&cmd))
die(_("fetch-pack: unable to fork off %s"), cmd_name);
- if (do_keep && (pack_lockfiles || fsck_objects)) {
+ if (do_keep && (pack_lockfiles /*|| fsck_objects*/)) {
int is_well_formed;
char *pack_lockfile = index_pack_lockfile(cmd.out, &is_well_formed);
--
2.34.1
...@@ -96,6 +96,15 @@ patches += ...@@ -96,6 +96,15 @@ patches +=
https://lab.nexedi.com/kirr/go/commit/6dfc2256e2cd1ad46f3a5e9da85af6a70be7fba3.patch#fa9de83838b198490abec0a6a924ad8d https://lab.nexedi.com/kirr/go/commit/6dfc2256e2cd1ad46f3a5e9da85af6a70be7fba3.patch#fa9de83838b198490abec0a6a924ad8d
https://lab.nexedi.com/kirr/go/commit/28fbdd01d6c89db1e81e12bd05910fdec4c3b9b3.patch#c75e2eb7aaaeffdd3c507ec1beacd3ca https://lab.nexedi.com/kirr/go/commit/28fbdd01d6c89db1e81e12bd05910fdec4c3b9b3.patch#c75e2eb7aaaeffdd3c507ec1beacd3ca
[golang1.15]
<= golang-common-pre-1.19
url = https://go.dev/dl/go1.15.15.src.tar.gz
md5sum = 05fedd8289291eb2d91cd0c092b41aaa
# go1.1 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
[golang1.16] [golang1.16]
<= golang-common-pre-1.19 <= golang-common-pre-1.19
url = https://golang.org/dl/go1.16.15.src.tar.gz url = https://golang.org/dl/go1.16.15.src.tar.gz
......
From 386b9a90dec9be74a271258833acb8c7ae993eed Mon Sep 17 00:00:00 2001
From: Alain Takoudjou <alain.takoudjou@nexedi.com>
Date: Tue, 23 Apr 2024 20:51:04 +0200
Subject: [PATCH] prefer use python 3 for tests
---
tests/CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6f8a18ec0..04ccf99e7 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,3 +1,4 @@
+set(Python_ADDITIONAL_VERSIONS 3 2.7)
FIND_PACKAGE(PythonInterp)
IF(NOT PYTHONINTERP_FOUND)
--
2.42.0
[buildout]
extends =
../defaults.cfg
../cmake/buildout.cfg
../openssl/buildout.cfg
../pkgconfig/buildout.cfg
../pcre/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
parts = libgit2
[libgit2]
recipe = slapos.recipe.cmmi
url = https://github.com/libgit2/libgit2/archive/refs/tags/v${:version}.tar.gz
md5sum = 9f4ca15249e703ab88cbc929187750cd
version = 1.1.0
shared = true
configure-command =
cmake
configure-options =
-Bbuild
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
-DCMAKE_C_FLAGS="-I${openssl:location}/include -I${zlib:location}/include -I${pcre:location}/include"
-DCMAKE_INSTALL_RPATH=${zlib:location}/lib:${openssl:location}/lib:${pcre:location}/lib
-G"Unix Makefiles"
patch-options = -p1
patches =
${:_profile_base_location_}/0001-prefer-use-python-3-for-tests.patch#6f2a6e83db45b33fc7da86279f06595b
make-options = -C build
environment =
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
PATH=${python3:location}/bin:${pkgconfig:location}/bin:${cmake:location}/bin:%(PATH)s
LDFLAGS=-L${openssl-1.0:location}/lib -Wl,-rpath=${openssl:location}/lib -L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib
[buildout]
extends =
../gnutls/buildout.cfg
../gnupg/buildout.cfg
parts =
libgpgme
[libgpgme]
recipe = slapos.recipe.cmmi
url = https://www.gnupg.org/ftp/gcrypt/gpgme/gpgme-1.23.2.tar.bz2
md5sum = 01a8c05b409847e87daf0543e91f8c37
configure-options =
--disable-gpg-test
--with-libgpg-error-prefix=${libgpg-error:location}
--with-libassuan-prefix=${libassuan:location}
make-options =
PYTHONS=
environment =
LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib -Wl,-rpath=${libassuan:location}/lib
...@@ -33,7 +33,7 @@ configure-options = ...@@ -33,7 +33,7 @@ configure-options =
# build core PostgreSQL + pg_trgm contrib extension for GitLab # build core PostgreSQL + pg_trgm contrib extension for GitLab
# unaccent contrib extension is for peertube # unaccent contrib extension is for peertube
# citext contrib extension is for metabase # citext contrib extension is for metabase
make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install && make -C contrib/citext/ install make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install && make -C contrib/citext/ install && make -C contrib/btree_gist/ install
environment = environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/include CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/include
......
...@@ -8,20 +8,18 @@ extends = ...@@ -8,20 +8,18 @@ extends =
../tcl/buildout.cfg ../tcl/buildout.cfg
[redis] [redis]
<= redis28
[redis28]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://download.redis.io/releases/redis-2.8.24.tar.gz url = https://download.redis.io/releases/redis-6.2.9.tar.gz
md5sum = 7b6eb6e4ccc050c351df8ae83c55a035 md5sum = 5251b9cfe7d799e6b63b7bd07410f97c
configure-command = true configure-command = true
prefix = prefix =
make-options = make-options =
PREFIX=${buildout:parts-directory}/${:_buildout_section_name_} PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
V=1 V=1
make-targets = make-targets =
install install
test # There are many tests failing, so disable make test for now
# test
environment = environment =
PATH=${patch:location}/bin:${tcl:location}/bin:%(PATH)s PATH=${patch:location}/bin:${tcl:location}/bin:%(PATH)s
patch-options = -p1 patch-options = -p1
......
...@@ -25,10 +25,10 @@ environment = ...@@ -25,10 +25,10 @@ environment =
PKG_CONFIG_PATH=${libyaml:location}/lib/ PKG_CONFIG_PATH=${libyaml:location}/lib/
[ruby2.6] [ruby2.7]
<= ruby-common <= ruby-common
url = https://ftp.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.xz url = https://ftp.ruby-lang.org/pub/ruby/2.7/ruby-2.7.8.tar.xz
md5sum = b8a4e2bdbb76485c3d6690e57be67750 md5sum = 27af2c340d0524ab272d564ddfd733d9
[ruby] [ruby]
<= ruby2.6 <= ruby2.7
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[instance.cfg] [instance.cfg]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = d1ca30a1b910b6b775f4f95bd91123a6 md5sum = 956ae53af22b551fbb087415e835868b
[watcher] [watcher]
_update_hash_filename_ = watcher.in _update_hash_filename_ = watcher.in
...@@ -30,35 +30,35 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef ...@@ -30,35 +30,35 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef
[gitconfig.in] [gitconfig.in]
_update_hash_filename_ = template/gitconfig.in _update_hash_filename_ = template/gitconfig.in
md5sum = eb1230fee50067924ba89f4dc6e82fa9 md5sum = c559a24ab6281268b608ed3bccb8e4ce
[gitlab-parameters.cfg] [gitlab-parameters.cfg]
_update_hash_filename_ = gitlab-parameters.cfg _update_hash_filename_ = gitlab-parameters.cfg
md5sum = cfda6d959bb90bf0b9c947383f45ce0a md5sum = 95b18789111ed239146d243e39ffefbe
[gitlab-shell-config.yml.in] [gitlab-shell-config.yml.in]
_update_hash_filename_ = template/gitlab-shell-config.yml.in _update_hash_filename_ = template/gitlab-shell-config.yml.in
md5sum = 69e8ed76b06233d11932a5c0ef16f03b md5sum = 70d394305f4e1482a5c1a673b0762c6a
[gitlab-unicorn-startup.in] [gitlab-puma-startup.in]
_update_hash_filename_ = gitlab-unicorn-startup.in _update_hash_filename_ = gitlab-puma-startup.in
md5sum = 705825e6d8c6b37699f1321805d09de3 md5sum = 838209b9246fa86d6a21fef910f17e25
[gitlab.yml.in] [gitlab.yml.in]
_update_hash_filename_ = template/gitlab.yml.in _update_hash_filename_ = template/gitlab.yml.in
md5sum = 673c393e6728a8d82e6b9a44886785a8 md5sum = 6f8df1467a6168bde7d2f5f42abb1012
[gitaly-config.toml.in] [gitaly-config.toml.in]
_update_hash_filename_ = template/gitaly-config.toml.in _update_hash_filename_ = template/gitaly-config.toml.in
md5sum = 58e3d5bbda32583d00cd8f44ec0525b0 md5sum = d769ea27820e932c596c35bbbf3f2902
[instance-gitlab.cfg.in] [instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in _update_hash_filename_ = instance-gitlab.cfg.in
md5sum = b913c4a1f199a87ad71da6d102adffa4 md5sum = 6d8d20ded84622339d49c60b0e61380c
[instance-gitlab-export.cfg.in] [instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in _update_hash_filename_ = instance-gitlab-export.cfg.in
md5sum = b8dea5ca4c6f9fc1ca54eb0265e1fdee md5sum = c8231583d04bf0d3fe2d26230b94d78d
[macrolib.cfg.in] [macrolib.cfg.in]
_update_hash_filename_ = macrolib.cfg.in _update_hash_filename_ = macrolib.cfg.in
...@@ -72,22 +72,18 @@ md5sum = 4980c1571a4dd7753aaa60d065270849 ...@@ -72,22 +72,18 @@ md5sum = 4980c1571a4dd7753aaa60d065270849
_update_hash_filename_ = template/nginx.conf.in _update_hash_filename_ = template/nginx.conf.in
md5sum = 8c904510eb39dc212204f68f2b81b068 md5sum = 8c904510eb39dc212204f68f2b81b068
[rack_attack.rb.in]
_update_hash_filename_ = template/rack_attack.rb.in
md5sum = 7d0e6dc6b826f6df6b20d8574a29e2f8
[resque.yml.in] [resque.yml.in]
_update_hash_filename_ = template/resque.yml.in _update_hash_filename_ = template/resque.yml.in
md5sum = 7c89a730889e3224548d9abe51a2d719 md5sum = 7c89a730889e3224548d9abe51a2d719
[smtp_settings.rb.in] [smtp_settings.rb.in]
_update_hash_filename_ = template/smtp_settings.rb.in _update_hash_filename_ = template/smtp_settings.rb.in
md5sum = 4e1ced687a86e4cfff2dde91237e3942 md5sum = b1becd9ec4c2eeefe573af4bb53c9751
[template-gitlab-resiliency-restore.sh.in] [template-gitlab-resiliency-restore.sh.in]
_update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in _update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
md5sum = 87f16b4f4a2370acada46b2751ef3366 md5sum = 8ce31a27e814e750dfd38c92a278fb9e
[unicorn.rb.in] [puma.rb.in]
_update_hash_filename_ = template/unicorn.rb.in _update_hash_filename_ = template/puma.rb.in
md5sum = b4758129a8d0c47b2c3adb10fefb8275 md5sum = 707c0c713af41518d21724c1be8efe22
...@@ -15,8 +15,11 @@ configuration.external_url = https://lab.example.com ...@@ -15,8 +15,11 @@ configuration.external_url = https://lab.example.com
configuration.db_pool = 10 configuration.db_pool = 10
# rack-attack # rack-attack
configuration.rate_limit_requests_per_period = 10 configuration.rack_attack_enable = true
configuration.rate_limit_period = 60 configuration.rack_attack_max_retry = 10
configuration.rack_attack_find_time = 60
configuration.rack_attack_ban_time = 3600
configuration.rack_attack_ip_whitelist = 127.0.0.1
configuration.time_zone = UTC configuration.time_zone = UTC
...@@ -64,8 +67,10 @@ configuration.sidekiq_memory_killer_max_rss = 1000000 ...@@ -64,8 +67,10 @@ configuration.sidekiq_memory_killer_max_rss = 1000000
# unicorn # unicorn
configuration.unicorn_worker_timeout = 60 configuration.puma_worker_timeout = 60
configuration.unicorn_worker_processes = 2 configuration.puma_worker_processes = 2
configuration.puma_min_threads = 1
configuration.puma_max_threads = 16
# unicorn advanced # unicorn advanced
configuration.unicorn_backlog_socket = 1024 configuration.unicorn_backlog_socket = 1024
......
...@@ -38,6 +38,8 @@ echo "I: PostgreSQL ready." 1>&2 ...@@ -38,6 +38,8 @@ echo "I: PostgreSQL ready." 1>&2
# make sure pg_trgm extension is enabled for gitlab db # make sure pg_trgm extension is enabled for gitlab db
psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' || die "pg_trgm setup failed" psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' || die "pg_trgm setup failed"
psql -c 'CREATE EXTENSION IF NOT EXISTS btree_gist;' || die "btree_gist setup failed"
if echo "$pgtables" | grep -q '^Did not find any relations' ; then if echo "$pgtables" | grep -q '^Did not find any relations' ; then
$RAKE gitlab:setup RAILS_ENV=production force=yes || die "initial db setup failed" $RAKE gitlab:setup RAILS_ENV=production force=yes || die "initial db setup failed"
fi fi
...@@ -70,8 +72,7 @@ $RAKE cache:clear || die "cache:clear failed" ...@@ -70,8 +72,7 @@ $RAKE cache:clear || die "cache:clear failed"
force=yes $RAKE gitlab:shell:setup || die "gitlab:shell:setup failed" force=yes $RAKE gitlab:shell:setup || die "gitlab:shell:setup failed"
# 3. finally exec to unicorn # 3. finally exec to puma
exec {{ gitlab_unicorn }} \ exec {{ gitlab_puma }} \
-E production \ -e production \
-c {{ unicorn_rb.output }} \ -C {{ puma_rb.output }}
{{ gitlab_work.location }}/config.ru
...@@ -6,6 +6,7 @@ depends_gitfetch = ...@@ -6,6 +6,7 @@ depends_gitfetch =
${go_github.com_pkg_errors:recipe} ${go_github.com_pkg_errors:recipe}
${go_lab.nexedi.com_kirr_git-backup:recipe} ${go_lab.nexedi.com_kirr_git-backup:recipe}
${go_lab.nexedi.com_kirr_go123:recipe} ${go_lab.nexedi.com_kirr_go123:recipe}
${go_golang.org_x_crypto:recipe}
[go_github.com_libgit2_git2go] [go_github.com_libgit2_git2go]
...@@ -13,7 +14,7 @@ depends_gitfetch = ...@@ -13,7 +14,7 @@ depends_gitfetch =
go.importpath = github.com/libgit2/git2go go.importpath = github.com/libgit2/git2go
repository = https://github.com/libgit2/git2go.git repository = https://github.com/libgit2/git2go.git
# branch 'next' is required by git-backup # branch 'next' is required by git-backup
revision = next-g5d0a4c752a74258a5f42e40fccd2908ac4e336b8 revision = cbca5b82b8c22c08c183a1f44cad4b8b51ba6f25
[go_github.com_pkg_errors] [go_github.com_pkg_errors]
<= go-git-package <= go-git-package
...@@ -25,10 +26,17 @@ revision = v0.8.0-12-g816c908556 ...@@ -25,10 +26,17 @@ revision = v0.8.0-12-g816c908556
<= go-git-package <= go-git-package
go.importpath = lab.nexedi.com/kirr/git-backup go.importpath = lab.nexedi.com/kirr/git-backup
repository = https://lab.nexedi.com/kirr/git-backup.git repository = https://lab.nexedi.com/kirr/git-backup.git
revision = da754af24da351291c99caa421a103db09e7a4c4 revision = 3327aedfbe903b210366105b8a4f9f85a313a0a3
[go_lab.nexedi.com_kirr_go123] [go_lab.nexedi.com_kirr_go123]
<= go-git-package <= go-git-package
go.importpath = lab.nexedi.com/kirr/go123 go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git repository = https://lab.nexedi.com/kirr/go123.git
revision = 95433de34f revision = 8299741f
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto.git
revision = 75b288015ac94e66e3d6715fb68a9b41bf046ec2
...@@ -54,7 +54,8 @@ input = inline: gitlab-shell-work* ...@@ -54,7 +54,8 @@ input = inline: gitlab-shell-work*
srv/backup/logrotate/** srv/backup/logrotate/**
etc/service/postgres-start etc/service/postgres-start
srv/redis/** srv/redis/**
srv/unicorn/unicorn.socket srv/puma/puma.socket
.cache
output = ${directory:srv}/exporter.exclude output = ${directory:srv}/exporter.exclude
[gitlab-resiliency-restore-script] [gitlab-resiliency-restore-script]
...@@ -70,13 +71,14 @@ context = ...@@ -70,13 +71,14 @@ context =
raw git_location {{ git_location }} raw git_location {{ git_location }}
raw bin_directory ${directory:bin} raw bin_directory ${directory:bin}
raw etc_directory ${directory:etc} raw etc_directory ${directory:etc}
raw run_directory ${directory:run} raw var_directory ${directory:var}
raw postgress_script ${service-postgresql:services}/postgres-start raw postgress_script ${service-postgresql:services}/postgres-start
raw redis_script ${service-redis:wrapper} raw redis_script ${service-redis:wrapper}
raw unicorn_script ${service-unicorn:wrapper-path} raw puma_script ${service-puma:wrapper-path}
raw sidekiq_script ${service-sidekiq:wrapper-path} raw sidekiq_script ${service-sidekiq:wrapper-path}
raw gitlab_backup_dir ${gitlab-backup-directory:backup-gitlab.git} raw gitlab_backup_dir ${gitlab-backup-directory:backup-gitlab.git}
raw redis_pid_file ${service-redis:pid-file} raw redis_pid_file ${service-redis:pid-file}
raw postgres_pid_file ${service-postgresql:pgdata-directory}/postmaster.pid raw postgres_pid_file ${service-postgresql:pgdata-directory}/postmaster.pid
raw puma_pid_file ${puma:pid}/puma.pid
raw gitlab_work_location ${gitlab-work:location} raw gitlab_work_location ${gitlab-work:location}
raw promise_lab_location ${directory:promise.slow} raw promise_lab_location ${directory:promise.slow}
...@@ -12,7 +12,7 @@ parts = ...@@ -12,7 +12,7 @@ parts =
# gitlab-<prog> # gitlab-<prog>
# ? mailroom # ? mailroom
{% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %} {% set gitlab_progv = 'rails rake puma sidekiq puma-startup' .split() %}
{% for prog in gitlab_progv %} {% for prog in gitlab_progv %}
gitlab-{{ prog }} gitlab-{{ prog }}
{% endfor %} {% endfor %}
...@@ -23,7 +23,7 @@ parts = ...@@ -23,7 +23,7 @@ parts =
gitlab-shell-work gitlab-shell-work
service-gitlab-workhorse service-gitlab-workhorse
service-unicorn service-puma
service-sidekiq service-sidekiq
service-nginx service-nginx
...@@ -51,29 +51,29 @@ offline = true ...@@ -51,29 +51,29 @@ offline = true
[worker-processes] [worker-processes]
recipe = slapos.recipe.build recipe = slapos.recipe.build
unicorn-worker-processes = {{ instance_parameter_dict['configuration.unicorn_worker_processes'] }} puma-worker-processes = {{ instance_parameter_dict['configuration.puma_worker_processes'] }}
init = init =
import multiprocessing import multiprocessing
worker_count = int(options['unicorn-worker-processes']) worker_count = int(options['puma-worker-processes'])
if worker_count == 0: if worker_count == 0:
# automatically load all available CPUs # automatically load all available CPUs
worker_count = multiprocessing.cpu_count() + 1 worker_count = multiprocessing.cpu_count() + 1
worker_count = 2 if worker_count < 2 else worker_count worker_count = 2 if worker_count < 2 else worker_count
options['unicorn-worker-processes'] = worker_count options['puma-worker-processes'] = worker_count
options['nginx-worker-processes'] = worker_count -1 options['nginx-worker-processes'] = worker_count -1
[instance-parameter] [instance-parameter]
{#- There are dangerous keys like recipe, etc #} {#- There are dangerous keys like recipe, etc #}
{#- XXX: Some other approach would be useful #} {#- XXX: Some other approach would be useful #}
{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert', {%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert',
'configuration.unicorn_worker_processes', 'configuration.nginx_worker_processes'] %} 'configuration.puma_worker_processes', 'configuration.nginx_worker_processes'] %}
{%- for key, value in instance_parameter_dict.items() -%} {%- for key, value in instance_parameter_dict.items() -%}
{%- if key not in DROP_KEY_LIST %} {%- if key not in DROP_KEY_LIST %}
{{ key }} = {{ value }} {{ key }} = {{ value }}
{%- endif -%} {%- endif -%}
{%- endfor %} {%- endfor %}
# settings for worker processes: # settings for worker processes:
configuration.unicorn_worker_processes = ${worker-processes:unicorn-worker-processes} configuration.puma_worker_processes = ${worker-processes:puma-worker-processes}
configuration.nginx_worker_processes = ${worker-processes:nginx-worker-processes} configuration.nginx_worker_processes = ${worker-processes:nginx-worker-processes}
...@@ -186,16 +186,18 @@ mode = 0700 ...@@ -186,16 +186,18 @@ mode = 0700
[gitaly-dir] [gitaly-dir]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
gitaly = ${directory:var}/gitaly gitaly = ${directory:var}/gitaly
sockets = ${:gitaly}/sockets sockets = ${:gitaly}/s
internal = ${directory:var}/int internal = ${:sockets}/int
log = ${directory:log}/gitaly log = ${directory:log}/gitaly
[gitaly] [gitaly]
socket = ${directory:var}/gitaly.socket socket = ${gitaly-dir:sockets}/gitaly.socket
log = ${gitaly-dir:log} logdir = ${gitaly-dir:log}
location = {{ gitaly_location }} location = {{ gitaly_location }}
pid = ${directory:run}/gitaly.pid pid = ${directory:run}/gitaly.pid
internal_socket = ${gitaly-dir:internal} internal_socket = ${gitaly-dir:internal}
basedir = ${gitaly-dir:gitaly}
num_workers = 2
[gitaly-socket-listening-promise] [gitaly-socket-listening-promise]
<= monitor-promise-base <= monitor-promise-base
...@@ -249,7 +251,7 @@ context-extra = ...@@ -249,7 +251,7 @@ context-extra =
section gitlab gitlab section gitlab gitlab
section gitlab_shell gitlab-shell section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work section gitlab_shell_work gitlab-shell-work
section unicorn unicorn section puma puma
section service_redis service-redis section service_redis service-redis
raw redis_binprefix {{ redis_binprefix }} raw redis_binprefix {{ redis_binprefix }}
...@@ -261,6 +263,7 @@ context-extra = ...@@ -261,6 +263,7 @@ context-extra =
section gitlab gitlab section gitlab gitlab
section gitlab_shell gitlab-shell section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work section gitlab_shell_work gitlab-shell-work
section gitlab_workhorse gitlab-workhorse
section gitaly gitaly section gitaly gitaly
[nginx.conf] [nginx.conf]
...@@ -288,12 +291,10 @@ context-extra = ...@@ -288,12 +291,10 @@ context-extra =
import urllib urllib import urllib urllib
section gitlab gitlab section gitlab gitlab
section gitlab_shell_work gitlab-shell-work section gitlab_shell_work gitlab-shell-work
section gitlab_shell gitlab-shell
section gitlab_workhorse gitlab-workhorse
section gitaly gitaly section gitaly gitaly
[rack_attack.rb]
<= gitlab-etc-template
url = {{ rack_attack_rb_in }}
[resque.yml] [resque.yml]
<= gitlab-etc-template <= gitlab-etc-template
url = {{ resque_yml_in }} url = {{ resque_yml_in }}
...@@ -306,11 +307,11 @@ url = {{ smtp_settings_rb_in }} ...@@ -306,11 +307,11 @@ url = {{ smtp_settings_rb_in }}
# contains smtp password # contains smtp password
mode = 0600 mode = 0600
[unicorn.rb] [puma.rb]
<= gitlab-etc-template <= gitlab-etc-template
url = {{ unicorn_rb_in }} url = {{ puma_rb_in }}
context-extra = context-extra =
section unicorn unicorn section puma puma
section directory directory section directory directory
section gitlab_work gitlab-work section gitlab_work gitlab-work
...@@ -340,20 +341,20 @@ prog = {{ prog }} ...@@ -340,20 +341,20 @@ prog = {{ prog }}
{% endfor %} {% endfor %}
[gitlab-unicorn-startup] [gitlab-puma-startup]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
mode = 0755 mode = 0755
url = {{ gitlab_unicorn_startup_in }} url = {{ gitlab_puma_startup_in }}
output= ${directory:bin}/${:_buildout_section_name_} output= ${directory:bin}/${:_buildout_section_name_}
context = context =
raw bash_bin {{ bash_bin }} raw bash_bin {{ bash_bin }}
raw gitlab_rake ${gitlab-rake:wrapper-path} raw gitlab_rake ${gitlab-rake:wrapper-path}
raw gitlab_unicorn ${gitlab-unicorn:wrapper-path} raw gitlab_puma ${gitlab-puma:wrapper-path}
raw psql_bin {{ postgresql_location }}/bin/psql raw psql_bin {{ postgresql_location }}/bin/psql
section pgsql service-postgresql section pgsql service-postgresql
raw log_dir ${gitlab:log} raw log_dir ${gitlab:log}
raw var_dir ${directory:var} raw var_dir ${directory:var}
section unicorn_rb unicorn.rb section puma_rb puma.rb
section gitlab_work gitlab-work section gitlab_work gitlab-work
...@@ -421,14 +422,13 @@ tune-command = ...@@ -421,14 +422,13 @@ tune-command =
ln -sf ${gitlab-workhorse:secret} .gitlab_workhorse_secret ln -sf ${gitlab-workhorse:secret} .gitlab_workhorse_secret
# config/ # config/
cd config && cd config &&
ln -sf ${unicorn.rb:output} unicorn.rb && ln -sf ${puma.rb:output} puma.rb &&
ln -sf ${gitlab.yml:output} gitlab.yml && ln -sf ${gitlab.yml:output} gitlab.yml &&
ln -sf ${database.yml:output} database.yml && ln -sf ${database.yml:output} database.yml &&
ln -sf ${resque.yml:output} resque.yml && ln -sf ${resque.yml:output} resque.yml &&
ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml && ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml &&
# config/initializers/ # config/initializers/
cd initializers && cd initializers &&
ln -sf ${rack_attack.rb:output} rack_attack.rb &&
ln -sf ${smtp_settings.rb:output} smtp_settings.rb && ln -sf ${smtp_settings.rb:output} smtp_settings.rb &&
# public/ # public/
cd ../../public && cd ../../public &&
...@@ -573,11 +573,12 @@ wrapper-path = ${directory:service}/gitlab-workhorse ...@@ -573,11 +573,12 @@ wrapper-path = ${directory:service}/gitlab-workhorse
command-line = {{ gitlab_workhorse }} command-line = {{ gitlab_workhorse }}
-listenNetwork unix -listenNetwork unix
-listenAddr ${gitlab-workhorse:socket} -listenAddr ${gitlab-workhorse:socket}
-authSocket ${unicorn:socket} -authSocket ${puma:socket}
-documentRoot ${gitlab-work:location}/public -documentRoot ${gitlab-work:location}/public
-secretPath ${gitlab-workhorse:secret} -secretPath ${gitlab-workhorse:secret}
-logFile ${gitlab-workhorse:log} -logFile ${gitlab-workhorse:log}
-repoPath ${gitlab-repo-dir:repositories} # repoPath is for patched gitlab-workhorse
# -repoPath ${gitlab-repo-dir:repositories}
# NOTE for profiling # NOTE for profiling
# -pprofListenAddr ... # -pprofListenAddr ...
...@@ -606,41 +607,47 @@ config-command = {{ curl_bin }} --unix-socket ${gitlab-workhorse:socket} ht ...@@ -606,41 +607,47 @@ config-command = {{ curl_bin }} --unix-socket ${gitlab-workhorse:socket} ht
###################### ######################
# unicorn worker # # puma worker #
###################### ######################
[unicorn-dir] [puma-dir]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/unicorn srv = ${directory:srv}/puma
log = ${directory:log}/unicorn log = ${directory:log}/puma
pid = ${directory:srv}/pids
[unicorn] [puma]
srv = ${unicorn-dir:srv} srv = ${puma-dir:srv}
log = ${unicorn-dir:log} log = ${puma-dir:log}
socket = ${directory:srv}/unicorn.socket socket = ${puma-dir:srv}/puma.socket
pid = ${puma-dir:pid}
[service-unicorn] [service-puma]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/unicorn wrapper-path = ${directory:service}/puma
# NOTE we perform db setup / migrations as part of unicorn startup. # NOTE we perform db setup / migrations as part of puma startup.
# Those operations require PG and Redis to be up and running already, that's # Those operations require PG and Redis to be up and running already, that's
# why we do it here. See gitlab-unicorn-startup for details. # why we do it here. See gitlab-puma-startup for details.
command-line = ${gitlab-unicorn-startup:output} command-line = ${gitlab-puma-startup:output}
{% if instance_parameter_dict.get('configuration.root-password') -%}
environment =
GITLAB_ROOT_PASSWORD={{ instance_parameter_dict['configuration.root-password'] }}
{% endif %}
depend = depend =
${promise-unicorn:recipe} ${promise-puma:recipe}
${promise-gitlab-app:recipe} ${promise-gitlab-app:recipe}
${promise-gitlab-shell:recipe} ${promise-gitlab-shell:recipe}
${logrotate-entry-unicorn:recipe} ${logrotate-entry-puma:recipe}
# gitlab is a service "run" under unicorn # gitlab is a service "run" under puma
# gitlab-shell is called by gitlab # gitlab-shell is called by gitlab
# -> associate their logs rotation to here # -> associate their logs rotation to here
${logrotate-entry-gitlab:recipe} ${logrotate-entry-gitlab:recipe}
[promise-unicorn] [promise-puma]
<= promise-byurl <= promise-byurl
config-command = {{ curl_bin }} --unix-socket ${unicorn:socket} http://localhost/ config-command = {{ curl_bin }} --unix-socket ${puma:socket} http://localhost/
[promise-rakebase] [promise-rakebase]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -662,10 +669,10 @@ command-line = ${:rake} gitlab:gitlab_shell:check ...@@ -662,10 +669,10 @@ command-line = ${:rake} gitlab:gitlab_shell:check
# rake gitlab:repo:check (fsck all repos) # rake gitlab:repo:check (fsck all repos)
[logrotate-entry-unicorn] [logrotate-entry-puma]
<= logrotate-entry-base <= logrotate-entry-base
log = ${unicorn:log}/*.log log = ${puma:log}/*.log
name = unicorn name = puma
copytruncate = true copytruncate = true
[logrotate-entry-gitlab] [logrotate-entry-gitlab]
...@@ -682,8 +689,8 @@ copytruncate = true ...@@ -682,8 +689,8 @@ copytruncate = true
[logrotate-entry-gitlab-workhorse] [logrotate-entry-gitlab-workhorse]
<= logrotate-entry-base <= logrotate-entry-base
log = ${gitlab-workhorse-dir:log}//*.log log = ${gitlab-workhorse-dir:log}/*.log
name = gitlab-shell name = gitlab-workhorse
copytruncate = true copytruncate = true
####################################### #######################################
...@@ -828,18 +835,17 @@ cron-entries = ${cron:cron-entries} ...@@ -828,18 +835,17 @@ cron-entries = ${cron:cron-entries}
[service-gitaly] [service-gitaly]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/gitaly wrapper-path = ${directory:service}/gitaly
#command-line = ${gitlab-work:location}/bin/daemon_with_pidfile ${gitaly:pid}
command-line = {{ gitaly_location }}/gitaly ${gitaly-config.toml:output} command-line = {{ gitaly_location }}/gitaly ${gitaly-config.toml:output}
environment = environment =
PATH={{ bundler_1_17_3_dir }}:{{ ruby_location }}/bin:/bin:/usr/bin PATH={{ buildout_bin_directory }}:{{ ruby_location }}/bin:/bin:/usr/bin
# 6. on-reinstantiate actions # 6. on-reinstantiate actions
# NOTE here we only recompile assets. Other on-reinstantiate actions, which # NOTE here we only recompile assets. Other on-reinstantiate actions, which
# require pg and redis running, are performed as part of unicorn service - # require pg and redis running, are performed as part of puma service -
# right before its startup (see gitlab-unicorn-startup). # right before its startup (see gitlab-puma-startup).
[on-reinstantiate] [on-reinstantiate]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
......
...@@ -53,6 +53,7 @@ context = ...@@ -53,6 +53,7 @@ context =
section instance_parameter_dict slap-configuration section instance_parameter_dict slap-configuration
# program binaries # program binaries
raw buildout_bin_directory ${buildout:bin-directory}
raw bash_bin ${bash:location}/bin/bash raw bash_bin ${bash:location}/bin/bash
raw bzip2_location ${bzip2:location} raw bzip2_location ${bzip2:location}
raw bundler_4gitlab ${bundler-4gitlab:bundle} raw bundler_4gitlab ${bundler-4gitlab:bundle}
...@@ -64,7 +65,7 @@ context = ...@@ -64,7 +65,7 @@ context =
raw git_location ${git:location} raw git_location ${git:location}
raw gitaly_location ${gitaly-repository:location} raw gitaly_location ${gitaly-repository:location}
raw gitlab_export ${gitlab-export:output} raw gitlab_export ${gitlab-export:output}
raw gitlab_workhorse ${gowork:bin}/gitlab-workhorse raw gitlab_workhorse ${gitlab-workhorse:binary}
raw gopath_bin ${gowork:bin} raw gopath_bin ${gowork:bin}
raw gunzip_bin ${gzip:location}/bin/gunzip raw gunzip_bin ${gzip:location}/bin/gunzip
raw grep_location ${grep:location} raw grep_location ${grep:location}
...@@ -75,8 +76,8 @@ context = ...@@ -75,8 +76,8 @@ context =
raw nginx_mime_types ${nginx-output:mime} raw nginx_mime_types ${nginx-output:mime}
raw node_bin_location ${nodejs:location}/bin/ raw node_bin_location ${nodejs:location}/bin/
raw openssl_bin ${openssl-output:openssl} raw openssl_bin ${openssl-output:openssl}
raw postgresql_location ${postgresql10:location} raw postgresql_location ${postgresql:location}
raw redis_binprefix ${redis28:location}/bin raw redis_binprefix ${redis:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location} raw ruby_location ${bundler-4gitlab:ruby-location}
raw tar_location ${tar:location} raw tar_location ${tar:location}
raw watcher ${watcher:output} raw watcher ${watcher:output}
...@@ -88,17 +89,16 @@ context = ...@@ -88,17 +89,16 @@ context =
raw gitconfig_in ${gitconfig.in:target} raw gitconfig_in ${gitconfig.in:target}
raw monitor_template ${monitor2-template:output} raw monitor_template ${monitor2-template:output}
raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target} raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target}
raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target} raw gitlab_puma_startup_in ${gitlab-puma-startup.in:target}
raw gitlab_yml_in ${gitlab.yml.in:target} raw gitlab_yml_in ${gitlab.yml.in:target}
raw gitaly_config_toml_in ${gitaly-config.toml.in:target} raw gitaly_config_toml_in ${gitaly-config.toml.in:target}
raw macrolib_cfg_in ${macrolib.cfg.in:target} raw macrolib_cfg_in ${macrolib.cfg.in:target}
raw nginx_conf_in ${nginx.conf.in:target} raw nginx_conf_in ${nginx.conf.in:target}
raw nginx_gitlab_http_conf_in ${nginx-gitlab-http.conf.in:target} raw nginx_gitlab_http_conf_in ${nginx-gitlab-http.conf.in:target}
raw rack_attack_rb_in ${rack_attack.rb.in:target}
raw resque_yml_in ${resque.yml.in:target} raw resque_yml_in ${resque.yml.in:target}
raw smtp_settings_rb_in ${smtp_settings.rb.in:target} raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw gitlab_restore_sh_in ${template-gitlab-resiliency-restore.sh.in:target} raw gitlab_restore_sh_in ${template-gitlab-resiliency-restore.sh.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target} raw puma_rb_in ${puma.rb.in:target}
$${:context-extra} $${:context-extra}
context-extra = context-extra =
......
...@@ -5,8 +5,10 @@ extends = ...@@ -5,8 +5,10 @@ extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/nodejs.cfg ../../stack/nodejs.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../component/libgit2/buildout.cfg
../../component/ruby/buildout.cfg ../../component/ruby/buildout.cfg
../../component/golang/buildout.cfg ../../component/golang/buildout.cfg
../../component/perl-Image-ExifTool/buildout.cfg
../../component/postgresql/buildout.cfg ../../component/postgresql/buildout.cfg
../../component/redis/buildout.cfg ../../component/redis/buildout.cfg
../../component/cmake/buildout.cfg ../../component/cmake/buildout.cfg
...@@ -28,19 +30,19 @@ extends = ...@@ -28,19 +30,19 @@ extends =
../../component/gzip/buildout.cfg ../../component/gzip/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg ../../component/logrotate/buildout.cfg
../../component/libgpgme/buildout.cfg
../../component/patchelf/buildout.cfg
parts = parts =
golang1.13 golang1.15
git git
postgresql10 postgresql
redis28
cmake cmake
icu icu
pkgconfig pkgconfig
nginx-output nginx-output
gowork gowork
gitlab-workhorse
gitaly-build gitaly-build
gitlab-shell/vendor gitlab-shell/vendor
gitlab/vendor/bundle gitlab/vendor/bundle
...@@ -65,16 +67,16 @@ parts = ...@@ -65,16 +67,16 @@ parts =
revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261 revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
[nodejs] [nodejs]
<= nodejs-12.18.3 <= nodejs-14.16.0
[yarn] [yarn]
<= yarn-1.16.0 <= yarn-1.16.0
[openssl]
<= openssl-1.1
# Gitlab backup (git-backup) is failing (segfault) with recent git version > 2.30.9 [libgit2]
# We will use git 2.30.9 version for production upgrade # This version is for rugged 1.1.0 needed by gitlab and gitaly 13.12.15,
# TODO: fix the issue with git and use latest version # see: https://github.com/libgit2/rugged/tree/v1.1.0/vendor
[git] version = 1.1.0
url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.30.9.tar.xz
md5sum = c1d42936036cc44a448738329c821569
############################ ############################
# Software compilation # # Software compilation #
...@@ -122,7 +124,7 @@ url = https://rubygems.org/rubygems/rubygems-3.1.2.zip ...@@ -122,7 +124,7 @@ url = https://rubygems.org/rubygems/rubygems-3.1.2.zip
# - run gitlab services / jobs (via `bundle exec ...`) # - run gitlab services / jobs (via `bundle exec ...`)
[bundler-4gitlab] [bundler-4gitlab]
<= rubygemsrecipe <= rubygemsrecipe
ruby-location = ${ruby2.6:location} ruby-location = ${ruby:location}
ruby-executable = ${:ruby-location}/bin/ruby ruby-executable = ${:ruby-location}/bin/ruby
gems = gems =
bundler==1.17.3 bundler==1.17.3
...@@ -148,7 +150,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/ ...@@ -148,7 +150,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/
# gitlab (via github-markup) wants to convert rst -> html via running: python (with docutils egg) # gitlab (via github-markup) wants to convert rst -> html via running: python (with docutils egg)
environment = environment =
PATH = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s PATH = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql:location}/bin:${redis:location}/bin:${git:location}/bin:${libgpgme:location}/bin:${buildout:bin-directory}:%(PATH)s
# gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
...@@ -160,26 +162,21 @@ git-executable = ${git:location}/bin/git ...@@ -160,26 +162,21 @@ git-executable = ${git:location}/bin/git
[gitlab-repository] [gitlab-repository]
<= git-repository <= git-repository
repository = https://lab.nexedi.com/nexedi/gitlab-ce.git repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
revision = v12.10.14-12-g7ce27b49193 revision = v13.12.15-10-gf4b76cb11f
location = ${buildout:parts-directory}/gitlab location = ${buildout:parts-directory}/gitlab
[gitlab-shell-repository] [gitlab-shell-repository]
<= git-repository <= git-repository
repository = https://gitlab.com/gitlab-org/gitlab-shell.git repository = https://gitlab.com/gitlab-org/gitlab-shell.git
revision = v12.2.0 revision = v13.18.1
location = ${buildout:parts-directory}/gitlab-shell location = ${buildout:parts-directory}/gitlab-shell
[gitaly-repository] [gitaly-repository]
<= git-repository <= git-repository
repository = https://gitlab.com/gitlab-org/gitaly.git repository = https://gitlab.com/gitlab-org/gitaly.git
revision = v12.10.14 revision = v13.12.15
location = ${buildout:parts-directory}/gitaly location = ${buildout:parts-directory}/gitaly
[gitlab-workhorse-repository]
<= git-repository
repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
revision = v8.30.3-19-g919c9b532c
# build needed-by-gitlab gems via bundler # build needed-by-gitlab gems via bundler
[gitlab/vendor/bundle] [gitlab/vendor/bundle]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
...@@ -188,19 +185,30 @@ bundle = ${bundler-4gitlab:bundle} ...@@ -188,19 +185,30 @@ bundle = ${bundler-4gitlab:bundle}
configure-command = cd ${:path} && configure-command = cd ${:path} &&
${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location} && ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location} &&
${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config && ${:bundle} config --local build.pg --with-pg-config=${postgresql:location}/bin/pg_config &&
${:bundle} config --local build.re2 --with-re2-dir=${re2:location} && ${:bundle} config --local build.re2 --with-re2-dir=${re2:location} &&
${:bundle} config --local build.nokogiri --with-zlib-dir=${zlib:location} --with-cflags=-I${xz-utils:location}/include --with-ldflags="-L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib" ${:bundle} config --local build.nokogiri --with-zlib-dir=${zlib:location} --with-cflags=-I${xz-utils:location}/include --with-ldflags="-L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib"
${:bundle} config --local build.rugged --use-system-libraries --with-git2-dir=${libgit2:location}
${:bundle} config --local build.openssl --with-openssl-dir=${openssl:location}
${:bundle} config --local build.puma --with-openssl-dir=${openssl:location}
${:bundle} config --local build.gpgme --use-system-libraries
${:bundle} config set without 'development test mysql aws kerberos' ${:bundle} config set without 'development test mysql aws kerberos'
${:bundle} config set deployment 'true' ${:bundle} config set deployment 'true'
make-binary = make-binary =
make-targets= cd ${:path} && ${:bundle} install make-targets= cd ${:path} && ${:bundle} install
environment = environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig:${libgit2:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:%(PATH)s
CFLAGS=-I${xz-utils:location}/include CFLAGS=-I${xz-utils:location}/include
gpme-rpath = ${ruby:location}/lib:${gdbm:location}/lib:${libffi:location}/lib:${libyaml:location}/lib:${ncurses:location}/lib:${readline:location}/lib:${zlib:location}/lib:${libgpgme:location}/lib:${libgpg-error:location}/lib:${libassuan:location}/lib
post-install =
# bundle is not setting libgpgme lib and dependencies in gpgme_n.so rpath
# it result to `libgpgme.so.11 => not found`
find ${bundler-4gitlab:location}/lib/ruby/gems/ -type f -name gpgme_n.so -exec \
${patchelf:location}/bin/patchelf --set-rpath ${:gpme-rpath} {} \;
################## Google re2 ################## Google re2
[re2] [re2]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
...@@ -208,7 +216,6 @@ url = https://github.com/google/re2/archive/2019-12-01.tar.gz ...@@ -208,7 +216,6 @@ url = https://github.com/google/re2/archive/2019-12-01.tar.gz
md5sum = 527eab0c75d6a1a0044c6eefd816b2fb md5sum = 527eab0c75d6a1a0044c6eefd816b2fb
configure-command = : configure-command = :
[gitlab_npm] [gitlab_npm]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
path = ${gitlab-repository:location} path = ${gitlab-repository:location}
...@@ -229,37 +236,39 @@ configure-command = : ...@@ -229,37 +236,39 @@ configure-command = :
make-binary = make-binary =
make-targets= cd ${go_github.com_libgit2_git2go:location} make-targets= cd ${go_github.com_libgit2_git2go:location}
&& git submodule update --init && git submodule update --init
&& sed -i 's/.*--build.*/cmake --build . --target install/' script/build-libgit2-static.sh && make install-static
&& make install
environment = environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.13:location}/bin:${buildout:bin-directory}:%(PATH)s PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.15:location}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gowork:directory} GOPATH=${gowork:directory}
[gowork.goinstall] [gowork.goinstall]
git2go = ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install git2go = ${go_github.com_libgit2_git2go_prepare:path}/static-build/install
command = bash -c ". ${gowork:env.sh} && CGO_CFLAGS=-I${:git2go}/include CGO_LDFLAGS='-L${:git2go}/lib -lgit2' go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ') && go test -v lab.nexedi.com/kirr/git-backup" command = bash -c ". ${gowork:env.sh} && CGO_CFLAGS=-I${:git2go}/include CGO_LDFLAGS='-L${:git2go}/lib -lgit2' go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ') && go test -v lab.nexedi.com/kirr/git-backup"
[gowork] [gowork]
golang = ${golang1.13:location} golang = ${golang1.15:location}
# gitlab.com/gitlab-org/gitlab-workhorse
# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
install = install =
lab.nexedi.com/kirr/git-backup lab.nexedi.com/kirr/git-backup
cpkgpath = cpkgpath =
${openssl-1.0:location}/lib/pkgconfig ${openssl:location}/lib/pkgconfig
${zlib:location}/lib/pkgconfig ${zlib:location}/lib/pkgconfig
${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install/lib/pkgconfig ${go_github.com_libgit2_git2go_prepare:path}/static-build/install/lib/pkgconfig
buildflags = --tags "static" buildflags = --tags "static"
[gitlab-workhorse] [gitlab-workhorse]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
path = ${gitlab-workhorse-repository:location} path = ${gitlab-repository:location}/workhorse
configure-command = : configure-command = :
make-binary = make-binary =
make-targets = make-targets =
. ${gowork:env.sh} && make test && make install PREFIX=${gowork:directory} . ${gowork:env.sh} && make test && make install PREFIX=${gowork:directory}
binary = ${gowork:bin}/${:_buildout_section_name_}
# adding ExifTool in env so make test can work.
# ExifTool is not used for now in gitlab-workhorse service for security reason.
# see: https://lab.nexedi.com/nexedi/slapos/-/merge_requests/1558?diff_id=54289&start_sha=725054eeef9dcd8018abfbf6147cbbbfccfbba5c#note_208105
environment =
PATH=${perl-Image-ExifTool:location}/bin/:%(PATH)s
[gitlab-backup] [gitlab-backup]
recipe = plone.recipe.command recipe = plone.recipe.command
...@@ -275,6 +284,7 @@ bundle = ${bundler-4gitlab:bundle} ...@@ -275,6 +284,7 @@ bundle = ${bundler-4gitlab:bundle}
configure-command = cd ${:path}/ruby && configure-command = cd ${:path}/ruby &&
${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location} ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}
${:bundle} config --local build.rugged --use-system-libraries --with-git2-dir=${libgit2:location}
make-binary = make-binary =
make-targets = make-targets =
. ${gowork:env.sh} && . ${gowork:env.sh} &&
...@@ -284,8 +294,9 @@ post-install = ...@@ -284,8 +294,9 @@ post-install =
# solve the problem error="not executable: ruby/git-hooks/pre-receive" # solve the problem error="not executable: ruby/git-hooks/pre-receive"
chmod 755 ${:path}/ruby/git-hooks/gitlab-shell-hook chmod 755 ${:path}/ruby/git-hooks/gitlab-shell-hook
environment = environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig:${libgit2:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${ruby2.6:location}/bin:%(PATH)s PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${ruby:location}/bin:%(PATH)s
OPENSSL_ROOT_DIR=${openssl:location}
[xnice-repository] [xnice-repository]
# to get kirr's misc repo containing xnice script for executing processes # to get kirr's misc repo containing xnice script for executing processes
...@@ -310,7 +321,7 @@ make-targets= cd ${:path} && ...@@ -310,7 +321,7 @@ make-targets= cd ${:path} &&
. ${gowork:env.sh} && make build && . ${gowork:env.sh} && make build &&
${:bundle} install --deployment --without development test ${:bundle} install --deployment --without development test
environment = environment =
PATH=${ruby2.6:location}/bin:%(PATH)s PATH=${ruby:location}/bin:%(PATH)s
############################### ###############################
# Trampoline for instance # # Trampoline for instance #
...@@ -368,7 +379,7 @@ destination = ${buildout:directory}/${:_buildout_section_name_} ...@@ -368,7 +379,7 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
[gitlab-shell-config.yml.in] [gitlab-shell-config.yml.in]
<= download-file <= download-file
[gitlab-unicorn-startup.in] [gitlab-puma-startup.in]
<= download-file <= download-file
[gitlab.yml.in] [gitlab.yml.in]
...@@ -392,9 +403,6 @@ destination = ${buildout:directory}/${:_buildout_section_name_} ...@@ -392,9 +403,6 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
[nginx.conf.in] [nginx.conf.in]
<= download-file <= download-file
[rack_attack.rb.in]
<= download-file
[resque.yml.in] [resque.yml.in]
<= download-file <= download-file
...@@ -404,16 +412,12 @@ destination = ${buildout:directory}/${:_buildout_section_name_} ...@@ -404,16 +412,12 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
[template-gitlab-resiliency-restore.sh.in] [template-gitlab-resiliency-restore.sh.in]
<= download-file <= download-file
[unicorn.rb.in] [puma.rb.in]
<= download-file <= download-file
[gitlab-demo-backup.git]
recipe = slapos.recipe.build:download-unpacked
url = https://lab.nexedi.com/alain.takoudjou/labdemo.backup/repository/archive.tar.gz?ref=master
md5sum = d40e5e211dc9a4e5ada9c0250377c639
[versions] [versions]
docutils = 0.16 docutils = 0.16
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1 plone.recipe.command = 1.1
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
beautifulsoup4 = 4.12.3
...@@ -7,7 +7,12 @@ socket_path = "{{ gitaly.socket }}" ...@@ -7,7 +7,12 @@ socket_path = "{{ gitaly.socket }}"
# The directory where Gitaly's executables are stored # The directory where Gitaly's executables are stored
bin_dir = "{{ gitaly.location }}" bin_dir = "{{ gitaly.location }}"
# # Optional: listen on a TCP socket. This is insecure (no authentication) # # Optional. The directory where Gitaly can create all files required to
# # properly operate at runtime. If not set, Gitaly will create a directory in
# # the global temporary directory. This directory must exist.
runtime_dir = "{{ gitaly.basedir }}"
# # Optional if socket_path is set. TCP address for Gitaly to listen on. This is insecure (unencrypted connection).
# listen_addr = "localhost:9999" # listen_addr = "localhost:9999"
# tls_listen_addr = "localhost:8888 # tls_listen_addr = "localhost:8888
...@@ -33,6 +38,10 @@ internal_socket_dir = "{{ gitaly.internal_socket }}" ...@@ -33,6 +38,10 @@ internal_socket_dir = "{{ gitaly.internal_socket }}"
bin_path = "{{ git }}" bin_path = "{{ git }}"
# catfile_cache_size = 100 # catfile_cache_size = 100
# [[git.config]]
# key = fetch.fsckObjects
# value = true
[[storage]] [[storage]]
name = "default" name = "default"
path = "{{ gitlab.repositories }}" path = "{{ gitlab.repositories }}"
...@@ -47,9 +56,9 @@ path = "{{ gitlab.repositories }}" ...@@ -47,9 +56,9 @@ path = "{{ gitlab.repositories }}"
# You can optionally configure Gitaly to output JSON-formatted log messages to stdout # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
[logging] [logging]
# The directory where Gitaly stores extra log files # The directory where Gitaly stores extra log files
dir = "{{ gitaly.log }}" dir = "{{ gitaly.logdir }}"
# format = "json" # format = "text"
# format = "json" format = "json"
# # Optional: Set log level to only log entries with that severity or above # # Optional: Set log level to only log entries with that severity or above
# # One of, in order: debug, info, warn, errror, fatal, panic # # One of, in order: debug, info, warn, errror, fatal, panic
# # Defaults to "info" # # Defaults to "info"
...@@ -79,7 +88,7 @@ dir = "{{ gitaly.location }}/ruby" ...@@ -79,7 +88,7 @@ dir = "{{ gitaly.location }}/ruby"
# restart_delay = "5m" # restart_delay = "5m"
# #
# # Number of gitaly-ruby worker processes # # Number of gitaly-ruby worker processes
# num_workers = 2 num_workers = {{ gitaly.num_workers }}
# #
# # Search path for system gitconfig file (e.g. /etc, /opt/gitlab/embedded/etc) # # Search path for system gitconfig file (e.g. /etc, /opt/gitlab/embedded/etc)
# # NOTE: This only affects RPCs that use Rugged. # # NOTE: This only affects RPCs that use Rugged.
...@@ -89,7 +98,47 @@ dir = "{{ gitaly.location }}/ruby" ...@@ -89,7 +98,47 @@ dir = "{{ gitaly.location }}/ruby"
# The directory where gitlab-shell is installed # The directory where gitlab-shell is installed
dir = "{{ gitlab_shell_work.location }}" dir = "{{ gitlab_shell_work.location }}"
[hooks]
custom_hooks_dir = "{{ gitlab_shell_work.location }}/hooks/"
[gitlab]
secret_file = "{{ gitlab_shell.secret }}"
url = "http+unix://{{ urllib.parse.unquote_plus(gitlab_workhorse.socket) }}"
# Only needed if a UNIX socket is used in `url` and GitLab is configured to
# use a relative path (e.g. /gitlab).
# relative_url_root = '/'
[gitlab.http-settings]
# read_timeout = 300
# user = someone
# password = somepass
# ca_file = /etc/ssl/cert.pem
# ca_path = /etc/pki/tls/certs
# self_signed_cert = false
# # You can adjust the concurrency of each RPC endpoint # # You can adjust the concurrency of each RPC endpoint
# [[concurrency]] # [[concurrency]]
# rpc = "/gitaly.RepositoryService/GarbageCollect" # rpc = "/gitaly.RepositoryService/GarbageCollect"
# max_per_repo = 1 # max_per_repo = 1
# Daily maintenance designates time slots to run daily to optimize and maintain
# enabled storages.
# [daily_maintenance]
# start_hour = 23
# start_minute = 30
# duration = "45m"
# storages = ["default"]
# disabled = false
# [cgroups]
# count = 10
# mountpoint = "/sys/fs/cgroup"
# hierarchy_root = "gitaly"
# [cgroups.memory]
# enabled = true
# limit = 1048576
# [cgroups.cpu]
# enabled = true
# shares = 512
...@@ -12,15 +12,24 @@ ...@@ -12,15 +12,24 @@
[pack] [pack]
threads = 1 threads = 1
# Enable packfile bitmaps
[repack]
writeBitmaps = true
# don't allow corrupt/broken objects to go in # don't allow corrupt/broken objects to go in
# Enable push (advertisePushOptions) options
[receive] [receive]
fsckObjects = true fsckObjects = true
advertisePushOptions = true
[user] [user]
name = {{ cfg('email_display_name') }} name = {{ cfg('email_display_name') }}
email = {{ cfg('email_from') }} email = {{ cfg('email_from') }}
# Enable fsyncObjectFiles to reduce risk of repository corruption if the server crashes
[core] [core]
autocrlf = input autocrlf = input
fsyncObjectFiles = true
[gc] [gc]
auto = 0 auto = 0
...@@ -7,13 +7,15 @@ ...@@ -7,13 +7,15 @@
# GitLab user. git by default # GitLab user. git by default
user: {{ backend_info.user }} user: {{ backend_info.user }}
# Url to gitlab instance. Used for api calls. Should end with a slash. # URL to GitLab instance, used for API calls. Default: http://localhost:8080.
gitlab_url: "http+unix://{{ urllib.parse.quote_plus(unicorn.socket) }}/" # For relative URL support read http://doc.gitlab.com/ce/install/relative_url.html
gitlab_url: "http+unix://{{ urllib.parse.quote_plus(puma.socket) }}/"
http_settings: http_settings:
{# we don't need any {# we don't need any
<%= @http_settings.to_json if @http_settings %> <%= @http_settings.to_json if @http_settings %>
#} #}
# read_timeout: 300
# user: someone # user: someone
# password: somepass # password: somepass
# ca_file: /etc/ssl/cert.pem # ca_file: /etc/ssl/cert.pem
...@@ -34,35 +36,17 @@ auth_file: "{{ gitlab.var }}/sshkeys-notused" ...@@ -34,35 +36,17 @@ auth_file: "{{ gitlab.var }}/sshkeys-notused"
# Default is .gitlab_shell_secret in the root directory. # Default is .gitlab_shell_secret in the root directory.
secret_file: "{{ gitlab_shell.secret }}" secret_file: "{{ gitlab_shell.secret }}"
# Parent directory for global custom hook directories (pre-receive.d, update.d, post-receive.d)
# Default is hooks in the gitlab-shell directory.
custom_hooks_dir: "{{ gitlab_shell_work.location }}/hooks/"
# Redis settings used for pushing commit notices to gitlab
redis:
bin: {{ redis_binprefix }}/redis-cli
host: {# <%= @redis_host %> #}
port: {# <%= @redis_port %> #}
socket: {{ service_redis.unixsocket }}
database: {# <%= @redis_database %> #}
namespace: resque:gitlab
# Log file. # Log file.
# Default is gitlab-shell.log in the root directory. # Default is gitlab-shell.log in the root directory.
log_file: "{{ gitlab_shell.log }}/gitlab-shell.log" log_file: "{{ gitlab_shell.log }}/gitlab-shell.log"
# Log level. INFO by default # Log level. INFO by default
log_level: log_level: INFO
# Log format. 'text' by default
log_format: text
# Audit usernames. # Audit usernames.
# Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but # Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
# incurs an extra API call on every gitlab-shell command. # incurs an extra API call on every gitlab-shell command.
audit_usernames: audit_usernames: false
# Enable git-annex support
# git-annex allows managing files with git, without checking the file contents into git
# See https://git-annex.branchable.com/ for documentation
# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup
# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex
# For CentOS: sudo yum install epel-release && sudo yum install git-annex
git_annex_enabled:
{{ autogenerated }} {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %} {% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %}
# # # # # # # # # # # # # # # # # #
# GitLab application config file #
# # # # # # # # # # # # # # # # # #
#
########################### NOTE #####################################
# This file should not receive new settings. All configuration options #
# * are being moved to ApplicationSetting model! #
# If a setting requires an application restart say so in that screen. #
# If you change this file in a merge request, please also create #
# a MR on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests. #
# For more details see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/gitlab.yml.md #
########################################################################
#
#
# How to use:
# 1. Copy file as gitlab.yml
# 2. Update gitlab -> host with your fully qualified domain name
# 3. Update gitlab -> email_from
# 4. If you installed Git from source, change git -> bin_path to /usr/local/bin/git
# IMPORTANT: If Git was installed in a different location use that instead.
# You can check with `which git`. If a wrong path of Git is specified, it will
# result in various issues such as failures of GitLab CI builds.
# 5. Review this configuration file for other settings you may want to adjust
production: &base production: &base
# #
# 1. GitLab app settings # 1. GitLab app settings
...@@ -18,6 +40,9 @@ production: &base ...@@ -18,6 +40,9 @@ production: &base
host: {{ external_url.hostname }} host: {{ external_url.hostname }}
port: {{ external_url.port or default_port[external_url.scheme] }} port: {{ external_url.port or default_port[external_url.scheme] }}
https: {{ cfg_https }} https: {{ cfg_https }}
# The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout.
# Default is 95% of the worker timeout
max_request_duration_seconds: 57
{# ssh is disabled completely in slapos version {# ssh is disabled completely in slapos version
# Uncommment this line below if your ssh host is different from HTTP/HTTPS one # Uncommment this line below if your ssh host is different from HTTP/HTTPS one
...@@ -55,6 +80,8 @@ production: &base ...@@ -55,6 +80,8 @@ production: &base
worker_src: "'self' blob:" worker_src: "'self' blob:"
report_uri: report_uri:
allowed_hosts: []
# Trusted Proxies # Trusted Proxies
# Customize if you have GitLab behind a reverse proxy which is running on a different machine. # Customize if you have GitLab behind a reverse proxy which is running on a different machine.
# Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address. # Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
...@@ -122,6 +149,15 @@ production: &base ...@@ -122,6 +149,15 @@ production: &base
repository_downloads_path: <%= @gitlab_repository_downloads_path %> repository_downloads_path: <%= @gitlab_repository_downloads_path %>
#} #}
## Impersonation settings
impersonation_enabled: true
## Disable jQuery and CSS animations
# disable_animations: true
## Application settings cache expiry in seconds (default: 60)
# application_settings_cache_seconds: 60
{# we do not support reply by email {# we do not support reply by email
## Reply by email ## Reply by email
# Allow users to comment on issues and merge requests by replying to notification emails. # Allow users to comment on issues and merge requests by replying to notification emails.
...@@ -414,7 +450,9 @@ production: &base ...@@ -414,7 +450,9 @@ production: &base
# Gitaly settings # Gitaly settings
gitaly: gitaly:
# Default Gitaly authentication token. Can be overriden per storage. Can # Path to the directory containing Gitaly client executables.
client_path: {{ gitaly.location }}
# Default Gitaly authentication token. Can be overridden per storage. Can
# be left blank when Gitaly is running locally on a Unix socket, which # be left blank when Gitaly is running locally on a Unix socket, which
# is the normal way to deploy Gitaly. # is the normal way to deploy Gitaly.
token: token:
...@@ -463,7 +501,6 @@ production: &base ...@@ -463,7 +501,6 @@ production: &base
authorized_keys_file: {{ gitlab.var }}/sshkeys-notused authorized_keys_file: {{ gitlab.var }}/sshkeys-notused
repos_path: {{ gitlab.repositories }} repos_path: {{ gitlab.repositories }}
hooks_path: {{ gitlab_shell_work.location }}/hooks/
secret_file: {{ gitlab_shell.secret }} secret_file: {{ gitlab_shell.secret }}
# Git over HTTP # Git over HTTP
...@@ -483,17 +520,16 @@ production: &base ...@@ -483,17 +520,16 @@ production: &base
# gitlab-shell needs to be set to true # gitlab-shell needs to be set to true
git_annex_enabled: <%= @git_annex_enabled %> git_annex_enabled: <%= @git_annex_enabled %>
workhorse:
# File that contains the secret key for verifying access for gitlab-workhorse.
secret_file: {{ gitlab_workhorse.secret }}
## Git settings ## Git settings
# CAUTION! # CAUTION!
# Use the default values unless you really know what you are doing # Use the default values unless you really know what you are doing
git: git:
bin_path: {{ git }} bin_path: {{ git }}
# The next value is the maximum memory size grit can use
# Given in number of bytes per git object (e.g. a commit)
# This value can be increased if you have very large commits
max_size: {{ cfg('git_max_size') }}
# Git timeout to read a commit, in seconds
timeout: {{ cfg('git_timeout') }}
# #
# 5. Extra customization # 5. Extra customization
...@@ -515,11 +551,6 @@ production: &base ...@@ -515,11 +551,6 @@ production: &base
<% end %> <% end %>
#} #}
{# we are ok (for now) with default rack-attack git settings
rack_attack:
git_basic_auth: <%= @rack_attack_git_basic_auth.to_json if @rack_attack_git_basic_auth %>
#}
## Site ICP License ## Site ICP License
# XXX unquote needed only for slapos.core earlier than # XXX unquote needed only for slapos.core earlier than
...@@ -530,6 +561,22 @@ production: &base ...@@ -530,6 +561,22 @@ production: &base
{# ICP: '{{ cfg("icp_license") }}' #} {# ICP: '{{ cfg("icp_license") }}' #}
{% endif %} {% endif %}
rack_attack:
git_basic_auth:
# Rack Attack IP banning enabled
enabled: {{ cfg("rack_attack_enable") }}
#
# Whitelist requests from 127.0.0.1 for web proxies (NGINX/Apache) with incorrect headers
ip_whitelist: [{{ cfg("rack_attack_ip_whitelist")}}]
#
# Limit the number of Git HTTP authentication attempts per IP
maxretry: {{ cfg("rack_attack_max_retry") }}
#
# Reset the auth attempt counter per IP after 60 seconds
findtime: {{ cfg("rack_attack_find_time") }}
#
# Ban an IP for one hour (3600s) after too many auth attempts
bantime: {{ cfg("rack_attack_ban_time") }}
development: development:
<<: *base <<: *base
......
# see: https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-cookbooks/gitlab/templates/default/puma.rb.erb
{% from 'macrolib.cfg.in' import cfg with context %}
# frozen_string_literal: true
# Load "path" as a rackup file.
#
# The default is "config.ru".
#
rackup 'config.ru'
pidfile '{{ puma.pid }}/puma.pid'
state_path '{{ puma.pid }}/puma.state'
stdout_redirect '{{ puma.log }}/puma.stdout.log',
'{{ puma.log }}/puma.stderr.log',
true
# Configure "min" to be the minimum number of threads to use to answer
# requests and "max" the maximum.
#
# The default is "0, 16".
#
threads {{ cfg("puma_min_threads") }}, {{ cfg("puma_max_threads") }}
# By default, workers accept all requests and queue them to pass to handlers.
# When false, workers accept the number of simultaneous requests configured.
#
# Queueing requests generally improves performance, but can cause deadlocks if
# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
#
# When set to false this may require a reverse proxy to handle slow clients and
# queue requests before they reach puma. This is due to disabling HTTP keepalive
queue_requests false
# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
# accepted protocols.
bind 'unix://{{ puma.socket }}'
directory '{{ gitlab_work.location }}'
workers {{ cfg("puma_worker_processes") }}
require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events"
require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/puma_worker_killer_initializer"
on_restart do
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
end
options = { workers: {{ cfg("puma_worker_processes") }} }
before_fork do
# Signal to the puma killer
Gitlab::Cluster::PumaWorkerKillerInitializer.start options unless ENV['DISABLE_PUMA_WORKER_KILLER']
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
end
Gitlab::Cluster::LifecycleEvents.set_puma_options options
on_worker_boot do
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
end
# Preload the application before starting the workers; this conflicts with
# phased restart feature. (off by default)
preload_app!
tag 'gitlab-puma-worker'
# Verifies that all workers have checked in to the master process within
# the given timeout. If not the worker process will be restarted. Default
# value is 60 seconds.
#
worker_timeout {{ cfg("puma_worker_timeout") }}
# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#lower-latency-better-throughput
wait_for_less_busy_worker ENV.fetch('PUMA_WAIT_FOR_LESS_BUSY_WORKER', 0.001).to_f
# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#nakayoshi_fork
nakayoshi_fork unless ENV['DISABLE_PUMA_NAKAYOSHI_FORK'] == 'true'
# Use json formatter
require_relative "{{ gitlab_work.location }}/lib/gitlab/puma_logging/json_formatter"
json_formatter = Gitlab::PumaLogging::JSONFormatter.new
log_formatter do |str|
json_formatter.call(str)
end
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg with context %}
# 1. Rename this file to rack_attack.rb
# 2. Review the paths_to_be_protected and add any other path you need protecting
#
paths_to_be_protected = [
"#{Rails.application.config.relative_url_root}/users/password",
"#{Rails.application.config.relative_url_root}/users/sign_in",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session",
"#{Rails.application.config.relative_url_root}/users",
"#{Rails.application.config.relative_url_root}/users/confirmation",
"#{Rails.application.config.relative_url_root}/unsubscribes/"
]
# Create one big regular expression that matches strings starting with any of
# the paths_to_be_protected.
paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ })
rack_attack_enabled = Gitlab.config.rack_attack.git_basic_auth['enabled']
unless Rails.env.test? || !rack_attack_enabled
Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req|
if req.post? && req.path =~ paths_regex
req.ip
end
end
end
...@@ -21,8 +21,6 @@ if Rails.env.production? ...@@ -21,8 +21,6 @@ if Rails.env.production?
enable_starttls_auto: {{ cfg('smtp_enable_starttls_auto') }}, enable_starttls_auto: {{ cfg('smtp_enable_starttls_auto') }},
# ssl: # ssl:
openssl_verify_mode: '{{ cfg("smtp_openssl_verify_mode") }}' openssl_verify_mode: '{{ cfg("smtp_openssl_verify_mode") }}'
# ca_path:
# ca_file:
} }
end end
{% else %} {% else %}
......
...@@ -21,15 +21,15 @@ redis_pid_file="{{ redis_pid_file }}" ...@@ -21,15 +21,15 @@ redis_pid_file="{{ redis_pid_file }}"
postgres_pid_file="{{ postgres_pid_file }}" postgres_pid_file="{{ postgres_pid_file }}"
bin_location="{{ bin_directory }}" bin_location="{{ bin_directory }}"
run_location="{{ run_directory }}"
git_location="{{ git_location }}" git_location="{{ git_location }}"
go_work_bin="{{ go_work_bin }}" go_work_bin="{{ go_work_bin }}"
etc_location="{{ etc_directory }}" etc_location="{{ etc_directory }}"
gitlab_work="{{ gitlab_work_location }}" gitlab_work="{{ gitlab_work_location }}"
promise_check="{{ promise_lab_location }}" promise_check="{{ promise_lab_location }}"
unicorn_script="{{ unicorn_script }}" puma_script="{{ puma_script }}"
puma_pid_file="{{ puma_pid_file }}"
sidekiq_script="{{ sidekiq_script }}" sidekiq_script="{{ sidekiq_script }}"
var_location="{{ run_directory }}/.." var_location="{{ var_directory }}"
# export GIT_EXEC_PATH=$git_location/libexec/git-core/ # export GIT_EXEC_PATH=$git_location/libexec/git-core/
...@@ -56,7 +56,7 @@ kill_process () { ...@@ -56,7 +56,7 @@ kill_process () {
check_process $postgres_pid_file "Postgres" check_process $postgres_pid_file "Postgres"
check_process $redis_pid_file "Redis" check_process $redis_pid_file "Redis"
check_process $run_location/unicorn.pid "Unicorn" check_process $puma_pid_file "Puma"
if [ -f "$postgres_pid_file" ]; then if [ -f "$postgres_pid_file" ]; then
rm $postgres_pid_file rm $postgres_pid_file
...@@ -90,14 +90,14 @@ echo "Checking gitlab promises..." ...@@ -90,14 +90,14 @@ echo "Checking gitlab promises..."
echo "[info] Not all promises are checked!" echo "[info] Not all promises are checked!"
$promise_check/gitlab-app $promise_check/gitlab-app
echo "Starting Unicorn to check gitlab-shell promise..." echo "Starting Puma to check gitlab-shell promise..."
$unicorn_script & $puma_script &
unicorn_pid=$! puma_pid=$!
trap "kill $postgres_pid $redis_pid $unicorn_pid" EXIT TERM INT trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
sleep 60 sleep 60
if [ -s "$run_location/unicorn.pid" ]; then if [ -s "$puma_pid_file" ]; then
unicorn_ppid=$(head -n 1 $run_location/unicorn.pid) > /dev/null 2>&1 puma_pid=$(head -n 1 $puma_pid_file) > /dev/null 2>&1
trap "kill $postgres_pid $redis_pid $unicorn_ppid" EXIT TERM INT trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
fi fi
$promise_check/gitlab-shell $promise_check/gitlab-shell
...@@ -109,7 +109,7 @@ $promise_check/gitlab-shell ...@@ -109,7 +109,7 @@ $promise_check/gitlab-shell
kill_process $postgres_pid kill_process $postgres_pid
kill_process $redis_pid kill_process $redis_pid
kill_process $unicorn_pid kill_process $puma_pid
RESTORE_EXIT_CODE=$? RESTORE_EXIT_CODE=$?
......
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
{% from 'macrolib.cfg.in' import cfg with context %}
# What ports/sockets to listen on, and what options for them.
# we listen only on unix socket
listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }}
#listen "127.0.0.1:8888", :tcp_nopush => true
working_directory '{{ gitlab_work.location }}'
# What the timeout for killing busy workers is, in seconds
timeout {{ cfg('unicorn_worker_timeout') }}
# combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
preload_app true
# Enable this flag to have unicorn test client connections by writing the
# beginning of the HTTP headers before calling the application. This
# prevents calling the application for connections that have disconnected
# while queued. This is only guaranteed to detect clients on the same
# host unicorn runs on, and unlikely to detect disconnects even on a
# fast LAN.
check_client_connection false
require_relative '{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events'
before_exec do |server|
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
end
# How many worker processes
worker_processes {{ cfg('unicorn_worker_processes') }}
# about before_fork / after_fork - see:
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb
# http://bogomips.org/unicorn.git/tree/examples/unicorn.conf.rb?id=3312aca8#n75
# What to do before we fork a worker
before_fork do |server, worker|
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
# The following is only recommended for memory/DB-constrained
# installations. It is not needed if your system can house
# twice as many worker_processes as you have configured.
#
# This allows a new master process to incrementally
# phase out the old master process with SIGTTOU to avoid a
# thundering herd (especially in the "preload_app false" case)
# when doing a transparent upgrade. The last worker spawned
# will then kill off the old master process with a SIGQUIT.
old_pid = "#{server.config[:pid]}.oldbin"
if old_pid != server.pid
begin
sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
Process.kill(sig, File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
end
end
#
# Throttle the master from forking too quickly by sleeping. Due
# to the implementation of standard Unix signal handlers, this
# helps (but does not completely) prevent identical, repeated signals
# from being lost when the receiving process is busy.
# sleep 1
end
# What to do after we fork a worker
after_fork do |server, worker|
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
# per-process listener ports for debugging/admin/migrations
# addr = "127.0.0.1:#{9293 + worker.nr}"
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
end
# Where to drop a pidfile
pid '{{ directory.run }}/unicorn.pid'
# Where stderr gets logged
stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
# Where stdout gets logged
stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
{# we do not support Relative url
<%- if @relative_url %>
# Relative url from where GitLab is served
ENV['RAILS_RELATIVE_URL_ROOT'] = "<%= @relative_url %>"
<%- end %>
#}
# Min memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MIN'] = ({{ cfg('unicorn_worker_memory_limit_min') }}).to_s
# Max memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MAX'] = ({{ cfg('unicorn_worker_memory_limit_max') }}).to_s
...@@ -46,6 +46,7 @@ setup( ...@@ -46,6 +46,7 @@ setup(
'erp5.util', 'erp5.util',
'supervisor', 'supervisor',
'requests', 'requests',
'beautifulsoup4'
], ],
zip_safe=True, zip_safe=True,
test_suite='test', test_suite='test',
......
...@@ -26,10 +26,10 @@ ...@@ -26,10 +26,10 @@
############################################################################## ##############################################################################
import os import os
import logging
from urllib.parse import urlparse
import requests import requests
import functools
import bs4
from urllib.parse import urljoin
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
...@@ -46,6 +46,10 @@ class TestGitlab(SlapOSInstanceTestCase): ...@@ -46,6 +46,10 @@ class TestGitlab(SlapOSInstanceTestCase):
def getInstanceSoftwareType(cls): def getInstanceSoftwareType(cls):
return 'gitlab' return 'gitlab'
@classmethod
def getInstanceParameterDict(cls):
return {'root-password': 'admin1234'}
def setUp(self): def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict( self.backend_url = self.computer_partition.getConnectionParameterDict(
)['backend_url'] )['backend_url']
...@@ -54,3 +58,33 @@ class TestGitlab(SlapOSInstanceTestCase): ...@@ -54,3 +58,33 @@ class TestGitlab(SlapOSInstanceTestCase):
resp = requests.get(self.backend_url, verify=False) resp = requests.get(self.backend_url, verify=False)
self.assertTrue( self.assertTrue(
resp.status_code in [requests.codes.ok, requests.codes.found]) resp.status_code in [requests.codes.ok, requests.codes.found])
def test_rack_attack_sign_in_rate_limiting(self):
session = requests.session()
# Load the login page to get a CSRF token.
response = session.get(urljoin(self.backend_url, 'users/sign_in'), verify=False)
self.assertEqual(response.status_code, 200)
# Extract the CSRF token and param.
bsoup = bs4.BeautifulSoup(response.text, 'html.parser')
csrf_param = bsoup.find('meta', dict(name='csrf-param'))['content']
csrf_token = bsoup.find('meta', dict(name='csrf-token'))['content']
request_data = {
'user[login]': 'test',
'user[password]': 'random',
csrf_param: csrf_token}
sign_in = functools.partial(
session.post,
response.url,
data=request_data,
verify=False)
for _ in range(10):
sign_in(headers={'X-Forwarded-For': '1.2.3.4'})
# after 10 authentication failures, this client is rate limited
self.assertEqual(sign_in(headers={'X-Forwarded-For': '1.2.3.4'}).status_code, 429)
# but other clients are not
self.assertNotEqual(sign_in(headers={'X-Forwarded-For': '5.6.7.8'}).status_code, 429)
...@@ -332,6 +332,7 @@ eggs += ...@@ -332,6 +332,7 @@ eggs +=
${recurls-setup:egg} ${recurls-setup:egg}
${pillow-python:egg} ${pillow-python:egg}
${pycurl:egg} ${pycurl:egg}
beautifulsoup4
caucase caucase
erp5.util erp5.util
${python-pynacl:egg} ${python-pynacl:egg}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment