diff --git a/software/gitlab/gitlab-parameters.cfg b/software/gitlab/gitlab-parameters.cfg new file mode 100644 index 0000000000000000000000000000000000000000..91b3582338d39fdd9bc9bd2285fbf53cab1c918b --- /dev/null +++ b/software/gitlab/gitlab-parameters.cfg @@ -0,0 +1,105 @@ +# Upstream parameters for a GitLab instance +# +# Selected parameters - main ones - names and advanced defaults taken from omnibus-gitlab +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb +# +# TODO better autogenerate from ^^^ (?) +# +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +[gitlab-parameters] +configuration.external_url = http://lab.example.com + +# db advanced +configuration.db_pool = 10 + +# rack-attack +configuration.rate_limit_requests_per_period = 10 +configuration.rate_limit_period = 60 + +configuration.time_zone = UTC + +configuration.email_enabled = true +configuration.email_from = lab@example.com +configuration.email_display_name = GitLab +configuration.email_reply_to = noreply@example.com + +configuration.smtp_enable = true +configuration.smtp_address = smtp.server +configuration.smtp_port = 465 +configuration.smtp_user_name = smtp user +configuration.smtp_password = smtp password +configuration.smtp_domain = lab.example.com +configuration.smtp_authentication = login +configuration.smtp_enable_starttls_auto = true + +# none | peer | client_once | fail_if_no_peer_cert -> see gitlab-omnibus links at top +configuration.smtp_openssl_verify_mode = peer + +configuration.default_can_create_group = true +configuration.username_changing_enabled = true +configuration.default_theme = 2 + +configuration.default_projects_features.issues = true +configuration.default_projects_features.merge_requests = true +configuration.default_projects_features.wiki = true +configuration.default_projects_features.snippets = true +# NOTE can be public|private|internal +configuration.default_projects_features.visibility_level= public +#configuration.default_projects_features.builds = false + +configuration.webhook_timeout = 10 + +# 0 means forever (seconds) +configuration.backup_keep_time = 0 + +# NOTE empty = default gitlab limits +configuration.git_max_size = +configuration.git_timeout = + + +# sidekiq +configuration.sidekiq_shutdown_timeout = 4 +configuration.sidekiq_concurrency = 25 +configuration.sidekiq_memory_killer_max_rss = 1000000 + + +# unicorn +configuration.unicorn_worker_timeout = 60 +configuration.unicorn_worker_processes = 2 + +# unicorn advanced +configuration.unicorn_backlog_socket = 1024 + +configuration.unicorn_worker_memory_limit_min = 200*(1024**2) +configuration.unicorn_worker_memory_limit_max = 250*(1024**2) + + +# nginx +configuration.nginx_client_max_body_size = 250m + +# NOTE: we don't really need old ciphers - usually we talk directly to frontend only +configuration.nginx_ssl_ciphers = ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4 +configuration.nginx_ssl_prefer_server_ciphers = on +configuration.nginx_ssl_protocols = TLSv1 TLSv1.1 TLSv1.2 +# the following is gitlab-omnibus default but not nginx's default +configuration.nginx_ssl_session_cache = builtin:1000 shared:SSL:10m +configuration.nginx_ssl_session_timeout = 5m + +configuration.nginx_proxy_read_timeout = 300 +configuration.nginx_proxy_connect_timeout = 300 + +# nginx advanced +configuration.nginx_worker_processes = 4 +configuration.nginx_worker_connections = 10240 +configuration.nginx_log_format = $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" +configuration.nginx_sendfile = on +configuration.nginx_tcp_nopush = on +configuration.nginx_tcp_nodelay = on +configuration.nginx_gzip = on +configuration.nginx_gzip_http_version = 1.0 +configuration.nginx_gzip_comp_level = 2 +configuration.nginx_gzip_proxied = any +configuration.nginx_gzip_types = text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json +configuration.nginx_keepalive_timeout = 65 diff --git a/software/gitlab/gitlab-unicorn-startup.in b/software/gitlab/gitlab-unicorn-startup.in new file mode 100644 index 0000000000000000000000000000000000000000..590304e701c153e81ae2ed4735716015f594b649 --- /dev/null +++ b/software/gitlab/gitlab-unicorn-startup.in @@ -0,0 +1,59 @@ +#!{{ bash_bin }} +# start up gitlab's unicorn with first making sure db is properly setup and all +# migrations are up as pre-condition. + +RAKE={{ gitlab_rake }} + +die() { + echo "$*" 1>&2 + exit 1 +} + +# 1. what to do when instance is initially setup +# see +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/database_migrations.rb +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb + +# initial db setup +pgtables="$({{ psql_bin }} \ + -h {{ pgsql['pgdata-directory'] }} \ + -U {{ pgsql.superuser }} \ + -d {{ pgsql.dbname }} \ + -c '\d')" || die "pg query problem" + +if echo "$pgtables" | grep -q '^No relations found' ; then + $RAKE db:schema:load db:seed_fu || die "initial db setup failed" +fi + +# re-build ssh keys +# (we do not use them - just for cleannes) +force=yes $RAKE gitlab:shell:setup || die "gitlab:shell:setup failed" + + +# 2. what to do when instance is upgraded +# see +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/deploy/deploy.sh +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/upgrader.rb +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-ctl-commands/upgrade.rb +# +# Assets compilation is handled at instance deployment time. We do everything else here. + +# make sure all migrations are up +migrate_log="{{ log_dir }}/db-migrate-`date +%s`.log" +$RAKE db:migrate >$migrate_log 2>&1 || die "db:migrate failed" +# if it was a no-op "migration" - we don't need info about that - only keep +# logs of actual migration run. +test -s $migrate_log || rm $migrate_log + + +# clear cache +$RAKE cache:clear || die "cache:clear failed" + + + +# 3. finally exec to unicorn +exec {{ gitlab_unicorn }} \ + -E production \ + -c {{ unicorn_rb.rendered }} \ + {{ gitlab_work.location }}/config.ru diff --git a/software/gitlab/instance-gitlab.cfg.in b/software/gitlab/instance-gitlab.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..3d2548bb0b24a271bd434aec22ec894623a4bf55 --- /dev/null +++ b/software/gitlab/instance-gitlab.cfg.in @@ -0,0 +1,852 @@ +# GitLab instance +# NOTE instance/software layout is inspired by gitlab omnibus +# NOTE all services are interconnected via unix sockets - because of easier +# security and performance reasons (unix has 2x less latency and more +# throughput compared to tcp over loopback). +[buildout] +extends = {{ gitlab_parameters_cfg }} +parts = + directory + publish-instance-info + +# gitlab-<prog> +# ? mailroom +{% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %} +{% for prog in gitlab_progv %} + gitlab-{{ prog }} +{% endfor %} + + gitconfig + + gitlab-work + gitlab-shell-work + + service-gitlab-workhorse + service-unicorn + service-sidekiq + + service-nginx + service-postgresql + service-redis + + service-cron + + on-reinstantiate + +# std stuff for slapos instance +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true + + +################################## +# GitLab instance parameters # +################################## + +[instance-parameter] +# std stuff to fetch slapos instance parameters +recipe = slapos.cookbook:slapconfiguration +computer= ${slap-connection:computer-id} +partition=${slap-connection:partition-id} +url = ${slap-connection:server-url} +key = ${slap-connection:key-file} +cert = ${slap-connection:cert-file} + +# autogenerated gitlab instance parameters +<= gitlab-parameters + +# adjust/override some default settings: + +# automatically load all available CPUs +configuration.unicorn_worker_processes = {{ multiprocessing.cpu_count() + 1 }} +configuration.nginx_worker_processes = {{ multiprocessing.cpu_count() }} + + +# gitlab non-native parameters +configuration.icp_license = + + + +# for convenience +[external-url] +recipe = slapos.cookbook:urlparse +url = ${instance-parameter:configuration.external_url} + +[backend-info] +host = ${instance-parameter:ipv6-random} +port = 7777 +# whether to use http or https - determined by external url +url = ${external-url:scheme}://[${:host}]:${:port} + +# current slapuserX +user = {{ pwd.getpwuid(os.getuid())[0] }} + + +[publish-instance-info] +recipe = slapos.cookbook:publish +backend_url = ${backend-info:url} + + + +############################# +# GitLab instance setup # +############################# + +# 1. directories +[directory] +recipe = slapos.cookbook:mkdirectory +home = ${buildout:directory} +bin = ${:home}/bin +etc = ${:home}/etc +var = ${:home}/var +log = ${:var}/log +run = ${:var}/run +srv = ${:home}/srv +# slapos startup/service/promise scripts live here: +startup = ${:etc}/run +service = ${:etc}/service +promise = ${:etc}/promise +promise.slow = ${:promise}.slow + +# gitlab: etc/ log/ ... +[gitlab-dir] +recipe = slapos.cookbook:mkdirectory +etc = ${directory:etc}/gitlab +log = ${directory:log}/gitlab + +var = ${directory:var}/gitlab +tmp = ${:var}/tmp +uploads = ${:var}/uploads +assets = ${:var}/assets +backup = ${directory:var}/backup + +[gitlab-repo-dir] +recipe = slapos.cookbook:mkdirectory +repositories = ${directory:var}/repositories +# gitlab wants it to be drwxrws--- +# FIXME setting such mode with :mkdirectory is not possible, because mkdir(2) +# does & 0777 and also there is umask. So we workaround: +[gitlab-repo-xdir] +recipe = plone.recipe.command +stop-on-error = yes +repositories = ${gitlab-repo-dir:repositories} +command = chmod 02770 ${:repositories} + +[gitlab] +etc = ${gitlab-dir:etc} +log = ${gitlab-dir:log} +var = ${gitlab-dir:var} +tmp = ${gitlab-dir:tmp} +uploads = ${gitlab-dir:uploads} +assets = ${gitlab-dir:assets} +backup = ${gitlab-dir:backup} +repositories = ${gitlab-repo-xdir:repositories} + + +# gitlab-shell: etc/ log/ gitlab_shell_secret ... +[gitlab-shell-dir] +recipe = slapos.cookbook:mkdirectory +etc = ${directory:etc}/gitlab-shell +log = ${directory:log}/gitlab-shell + +[gitlab-shell] +etc = ${gitlab-shell-dir:etc} +log = ${gitlab-shell-dir:log} +secret = ${secrets:secrets}/gitlab_shell_secret + + +# place to keep all secrets +[secrets] +recipe = slapos.cookbook:mkdirectory +secrets = ${directory:var}/secrets +mode = 0700 + + + + +# 2. configuration files +[etc-template] +recipe = slapos.recipe.template:jinja2 +extensions = jinja2.ext.do +mode = 0640 +import-list = + rawfile macrolib.cfg.in {{ macrolib_cfg_in }} +context = + raw autogenerated # This file was autogenerated. (DO NOT EDIT - changes will be lost) + section instance_parameter instance-parameter + section backend_info backend-info + import urlparse urlparse + raw git {{ git }} + ${:context-extra} +context-extra = + +[gitlab-etc-template] +<= etc-template +rendered= ${gitlab:etc}/${:_buildout_section_name_} + +[nginx-etc-template] +<= etc-template +rendered= ${nginx:etc}/${:_buildout_section_name_} + + +[config.ru] +<= gitlab-etc-template +template = {{ config_ru_in }} + +[database.yml] +<= gitlab-etc-template +template= {{ database_yml_in }} +context-extra = + section pgsql service-postgresql + +[gitconfig] +<= etc-template +template= {{ gitconfig_in }} +# NOTE put directly into $HOME/ - this way git will pick it up +rendered= ${directory:home}/.${:_buildout_section_name_} + +[gitlab-shell-config.yml] +<= etc-template +template= {{ gitlab_shell_config_yml_in }} +rendered= ${gitlab-shell:etc}/config.yml +context-extra = + import urllib urllib + section gitlab gitlab + section gitlab_shell gitlab-shell + section unicorn unicorn + section service_redis service-redis + raw redis_binprefix {{ redis_binprefix }} + +[gitlab.yml] +<= gitlab-etc-template +template= {{ gitlab_yml_in }} +context-extra = + import urllib urllib + section gitlab gitlab + section gitlab_shell gitlab-shell + section gitlab_shell_work gitlab-shell-work + +[nginx.conf] +<= nginx-etc-template +template= {{ nginx_conf_in }} +context-extra = + section directory directory + raw nginx_mime_types {{ nginx_mime_types }} + raw nginx_gitlab_http_conf ${nginx-gitlab-http.conf:rendered} + +[nginx-gitlab-http.conf] +<= nginx-etc-template +template= {{ nginx_gitlab_http_conf_in }} +context-extra = + section nginx nginx + section gitlab_work gitlab-work + section gitlab_workhorse gitlab-workhorse + section unicorn unicorn + +[rack_attack.rb] +<= gitlab-etc-template +template = {{ rack_attack_rb_in }} + +[resque.yml] +<= gitlab-etc-template +template= {{ resque_yml_in }} +context-extra = + section redis service-redis + +[smtp_settings.rb] +<= gitlab-etc-template +template= {{ smtp_settings_rb_in }} +# contains smtp password +mode = 0600 + +[unicorn.rb] +<= gitlab-etc-template +template = {{ unicorn_rb_in }} +context-extra = + section unicorn unicorn + section directory directory + section gitlab_work gitlab-work + + + +# 3. bin/ +# gitlab-<prog> +[gitlab-bin] +recipe = slapos.cookbook:wrapper +wrapper-path = ${directory:bin}/${:_buildout_section_name_} +# NOTE $HOME needed to pick gitconfig +environment = + BUNDLE_GEMFILE = {{ gitlab_repository_location }}/Gemfile + HOME = ${directory:home} + RAILS_ENV = production + SIDEKIQ_MEMORY_KILLER_MAX_RSS = ${instance-parameter:configuration.sidekiq_memory_killer_max_rss} + +# NOTE sys.argv[1:] implicitly appended +# (by slapos.recipe.librecipe.execute.generic_exec() at runtime) +command-line = + {{ bundler_4gitlab }} exec sh -c + 'cd ${gitlab-work:location} && ${:prog} "$@"' ${:prog} + +{% for prog in gitlab_progv %} +[gitlab-{{ prog }}] +<= gitlab-bin +prog = {{ prog }} +{% endfor %} + + +[gitlab-unicorn-startup] +recipe = slapos.recipe.template:jinja2 +mode = 0755 +template= {{ gitlab_unicorn_startup_in }} +rendered= ${directory:bin}/${:_buildout_section_name_} +context = + raw bash_bin {{ bash_bin }} + raw gitlab_rake ${gitlab-rake:wrapper-path} + raw gitlab_unicorn ${gitlab-unicorn:wrapper-path} + raw psql_bin {{ postgresql_location }}/bin/psql + section pgsql service-postgresql + raw log_dir ${gitlab:log} + section unicorn_rb unicorn.rb + section gitlab_work gitlab-work + + +# 4. gitlab- & gitlab-shell- work directories +# +# Gitlab/Rails operation is tightened that config/ lives inside code, which goes +# against having ability to create several instances configured differently +# from 1 SR. +# +# One possibility to overcome this could be to make another Gitlab root +# symbolically linked to original SR _and_ several configuration files +# symbolically linked to instance place. Unfortunately this does not work - +# Ruby determines realpath on module import and Gitlab and Rails lookup config +# files relative to imported modules. +# +# we clone cloned gitlab and add proper links to vendor/bundle and instance +# config files. +# XXX there is no need for full clone - we only need worktree checkout (a-la `git +# worktree add`, but without creating files in original clone) +# +# This way Gitlab/Rails still think they work in 1 code / 1 instance way, +# and we can reuse SR. +# XXX better do such tricks with bind mounting, but that requires user namespaces + +[work-base] +recipe = plone.recipe.command +stop-on-error = yes +location = ${directory:home}/${:_buildout_section_name_} +command = +# make sure we start from well-defined empty state +# (needed e.g. if previous install failed in the middle) + rm -rf ${:location} && +# init work repository and add `software` remote pointing to main repo in SR software/... + {{ git }} init ${:location} && + cd ${:location} && + {{ git }} remote add software ${:software} && + ${:update-command} + +update-command = + cd ${:location} && + {{ git }} fetch software && + {{ git }} reset --hard `cd ${:software} && {{ git }} rev-parse HEAD` && + ${:tune-command} + + +# NOTE there is no need to link/create .gitlab_shell_secret - we set path to it +# in gitlab & gitlab-shell configs, and gitlab creates it on its first start +[gitlab-work] +<= work-base +software = {{ gitlab_repository_location }} +tune-command = +# secret* config.ru tmp/ log/ + rm -f .secret && + rm -f config.ru && + rm -rf log tmp && + ln -sf ${secrets:secrets}/gitlab_rails_secret .secret && + ln -sf ${config.ru:rendered} config.ru && + ln -sf ${gitlab:log} log && + ln -sf ${gitlab:tmp} tmp && +# config/ + cd config && + ln -sf ${unicorn.rb:rendered} unicorn.rb && + ln -sf ${gitlab.yml:rendered} gitlab.yml && + ln -sf ${database.yml:rendered} database.yml && + ln -sf ${resque.yml:rendered} resque.yml && + ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml && +# config/initializers/ + cd initializers && + ln -sf ${rack_attack.rb:rendered} rack_attack.rb && + ln -sf ${smtp_settings.rb:rendered} smtp_settings.rb && +# public/ + cd ../../public && + rm -rf uploads assets && + ln -sf ${gitlab:uploads} uploads && + ln -sf ${gitlab:assets} assets && + true + + +# ----//---- for gitlab-shell +[gitlab-shell-work] +<= work-base +software = {{ gitlab_shell_repository_location }} + +tune-command = + ln -sf ${gitlab-shell-config.yml:rendered} config.yml && + true + + + +# 5. services + +# [promise-<something>] to generate promise wrapper <something> +[promise-wrapper] +recipe = slapos.cookbook:wrapper +wrapper-path = !py! '${directory:promise}/' + '${:_buildout_section_name_}'[8:] + +# [promise-<something>] to check <something> by url +[promise-byurl] +recipe = slapos.cookbook:check_url_available +path = !py! '${directory:promise}/' + '${:_buildout_section_name_}'[8:] +dash_path = {{ bash_bin }} +curl_path = {{ curl_bin }} +http_code = 200 + + + +##################### +# Postgresql db # +##################### + +# XXX gitlab-omnibus also tunes: +# - shared_buffers +# - work_mem +# - checkpoint_* +# - effective_check_size +# - lc_* en_US.UTF-8 -> C (?) +[service-postgresql] +recipe = slapos.cookbook:postgres +bin = {{ postgresql_location }}/bin +services= ${directory:service} + +dbname = gitlabhq_production +# NOTE db name must match to what was used in KVM on lab.nexedi.com (restore script grants access to this user) +superuser = gitlab-psql +# no password - pgsql will listen only on unix sockets (see below) thus access +# is protected with filesystem-level permissions. +# ( besides, if we use slapos.cookbook:generate.password and do `password = ...` +# the password is stored in plain text in .installed and thus becomes insecure ) +password= + +pgdata-directory = ${directory:srv}/postgresql + +# empty addresses - listen only on unix socket +ipv4 = !py!set([]) +ipv6 = !py!set([]) +ipv6-random = +port = + +depend = + ${promise-postgresql:recipe} + +[promise-postgresql] +<= promise-wrapper +command-line = + {{ postgresql_location }}/bin/psql + -h ${service-postgresql:pgdata-directory} + -U ${service-postgresql:superuser} + -d ${service-postgresql:dbname} + -c '\q' + +# postgresql logs to stdout/stderr - logs are handled by slapos not us +# [logrotate-entry-postgresql] + + +############# +# Redis # +############# +[redis] +recipe = slapos.cookbook:mkdirectory +srv = ${directory:srv}/redis +log = ${directory:log}/redis + + +[service-redis] +recipe = slapos.cookbook:redis.server +wrapper = ${directory:service}/redis +promise_wrapper = ${directory:promise}/redis + +server_dir = ${redis:srv} +config_file = ${directory:etc}/redis.conf +log_file = ${redis:log}/redis.log +pid_file = ${directory:run}/redis.pid +use_passwd = false +unixsocket = ${:server_dir}/redis.socket +# port = 0 means "don't listen on TCP at all" - listen only on unix socket +ipv6 = ::1 +port = 0 + +server_bin = {{ redis_binprefix }}/redis-server +depend = + ${logrotate-entry-redis:recipe} + + +# NOTE slapos.cookbook:redis.server setups promise automatically + +[logrotate-entry-redis] +<= logrotate-entry +log = ${redis:log}/*.log + + +######################## +# gitlab-workhorse # +######################## +[gitlab-workhorse-dir] +recipe = slapos.cookbook:mkdirectory +srv = ${directory:srv}/gitlab-workhorse + +[gitlab-workhorse] +srv = ${gitlab-workhorse-dir:srv} +socket = ${gitlab-workhorse:srv}/gitlab-workhorse.socket + +[service-gitlab-workhorse] +recipe = slapos.cookbook:wrapper +wrapper-path = ${directory:service}/gitlab-workhorse +command-line = {{ gitlab_workhorse }} + -listenNetwork unix + -listenAddr ${gitlab-workhorse:socket} + -authSocket ${unicorn:socket} +# NOTE for profiling +# -pprofListenAddr ... + +# NOTE environment for: +# - git to be available on path +# - ruby to be available on path (gitlab-workhorse -> gitlab-shell -> hooks on push) +# - gitconfig be found from ~/.gitconfig +environment = + PATH={{ git_location }}/bin:{{ ruby_location }}/bin:%(PATH)s + HOME=${directory:home} + +depend = + ${promise-gitlab-workhorse:recipe} + + +[promise-gitlab-workhorse] +<= promise-byurl +# gitlab-workhorse works on repositories. Here we only check it accepts an +# serves requests, so request is non-existent URL and expected code is 403 +url = --unix-socket ${gitlab-workhorse:socket} http:/non-existent +http_code = 403 + + +# gitlab-workhorse logs to stdout/stderr - logs are handled by slapos not us +# [logrotate-entry-gitlab-workhorse] + + +###################### +# unicorn worker # +###################### +[unicorn-dir] +recipe = slapos.cookbook:mkdirectory +srv = ${directory:srv}/unicorn +log = ${directory:log}/unicorn + +[unicorn] +srv = ${unicorn-dir:srv} +log = ${unicorn-dir:log} +socket = ${:srv}/unicorn.socket + +[service-unicorn] +recipe = slapos.cookbook:wrapper +wrapper-path = ${directory:service}/unicorn +# NOTE we perform db setup / migrations as part of unicorn startup. +# Those operations require PG and Redis to be up and running already, that's +# why we do it here. See gitlab-unicorn-startup for details. +command-line = ${gitlab-unicorn-startup:rendered} + +depend = + ${promise-unicorn:recipe} + ${promise-gitlab-app:recipe} + ${promise-gitlab-shell:recipe} + + ${logrotate-entry-unicorn:recipe} +# gitlab is a service "run" under unicorn +# gitlab-shell is called by gitlab +# -> associate their logs rotation to here + ${logrotate-entry-gitlab:recipe} + + +[promise-unicorn] +<= promise-byurl +url = --unix-socket ${unicorn:socket} http:/ + +[promise-rakebase] +recipe = slapos.cookbook:wrapper +# FIXME gitlab-rake is too slow to load and promise timeouts +# that's why we instantiate to <promise>.slow/ (and this way promises are not run) +wrapper-path = !py!'${directory:promise.slow}/' + '${:_buildout_section_name_}'[8:] +rake = ${gitlab-rake:wrapper-path} + + +[promise-gitlab-app] +<= promise-rakebase +command-line = ${:rake} gitlab:app:check + +[promise-gitlab-shell] +<= promise-rakebase +command-line = ${:rake} gitlab:gitlab_shell:check + +# very slow +# rake gitlab:repo:check (fsck all repos) + + +[logrotate-entry-unicorn] +<= logrotate-entry +log = ${unicorn:log}/*.log + +[logrotate-entry-gitlab] +<= logrotate-entry +log = ${gitlab:log}/*.log + +[logrotate-entry-gitlab-shell] +<= logrotate-entry +log = ${gitlab-shell:log}/*.log + + +####################################### +# sidekiq background jobs manager # +####################################### +[sidekiq-dir] +recipe = slapos.cookbook:mkdirectory +log = ${directory:log}/sidekiq + +[sidekiq] +log = ${sidekiq-dir:log} + +# NOTE see queue list here: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb +# (last updated for ominbus-gitlab 8.2.3+ce.0-0-g8eda093) +[service-sidekiq] +recipe = slapos.cookbook:wrapper +wrapper-path = ${directory:service}/sidekiq +command-line = +# NOTE Sidekiq memory killer just makes sidekiq processes to be SIGKILL +# terminated and relies on managing service to restart it. In slapos we don't +# have mechanism to set autorestart=true, nor bang/watchdog currently work with +# slapproxy, so we do the monitoring ourselves. + {{ watcher_sigkill }} + + ${gitlab-sidekiq:wrapper-path} +# XXX -q runner ? (present in gitlab-ce/Procfile but not in omnibus) +# XXX -P ? (pidfile) + -e production + -r ${gitlab-work:location} + -t ${instance-parameter:configuration.sidekiq_shutdown_timeout} + -c ${instance-parameter:configuration.sidekiq_concurrency} + -L ${sidekiq:log}/sidekiq.log + + -q post_receive + -q mailer + -q archive_repo + -q system_hook + -q project_web_hook + -q gitlab_shell + -q incoming_email + -q common + -q default + +depend = + ${promise-sidekiq:recipe} + ${logrotate-entry-sidekiq:recipe} + +[promise-sidekiq] +<= promise-rakebase +command-line = ${:rake} gitlab:sidekiq:check + +[logrotate-entry-sidekiq] +<= logrotate-entry +log = ${sidekiq:log}/*.log + + +###################### +# Nginx frontend # +###################### + +# srv/nginx/ prefix + etc/ log/ ... +[nginx-dir] +recipe = slapos.cookbook:mkdirectory +srv = ${directory:srv}/nginx +etc = ${directory:etc}/nginx +log = ${directory:log}/nginx + +[nginx-ssl-dir] +recipe = slapos.cookbook:mkdirectory +ssl = ${nginx-dir:etc}/ssl +# contains https key +mode = 0700 + +# self-signed certificate for https +[nginx-generate-certificate] +# NOTE there is slapos.cookbook:certificate_authority.request but it requires +# to start whole service and has up to 60 seconds latency to generate +# certificate. We only need to run 1 command to do it... +recipe = plone.recipe.command +stop-on-error = true +cert_file = ${nginx-ssl-dir:ssl}/gitlab_backend.crt +key_file = ${nginx-ssl-dir:ssl}/gitlab_backend.key + +command = + test -e ${:key_file} || \ + {{ openssl_bin }} req -newkey rsa -batch -new -x509 -days 3650 -nodes \ + -keyout ${:key_file} -out ${:cert_file} +update-command = ${:command} + + +[nginx] +srv = ${nginx-dir:srv} +etc = ${nginx-dir:etc} +log = ${nginx-dir:log} +ssl = ${nginx-ssl-dir:ssl} + +cert_file = ${nginx-generate-certificate:cert_file} +key_file = ${nginx-generate-certificate:key_file} + + +[nginx-symlinks] +# (nginx wants <prefix>/logs to be there from start - else it issues alarm to the log) +recipe = cns.recipe.symlink +symlink = ${nginx:log} = ${nginx:srv}/logs + +[service-nginx] +recipe = slapos.cookbook:wrapper +wrapper-path = ${directory:service}/nginx +command-line = {{ nginx_bin }} -p ${nginx:srv} -c ${nginx.conf:rendered} +depend = + ${nginx-symlinks:recipe} + ${promise-nginx:recipe} + ${logrotate-entry-nginx:recipe} + + +[promise-nginx] +<= promise-byurl +url = ${backend-info:url}/static.css + +[logrotate-entry-nginx] +<= logrotate-entry +log = ${nginx:log}/*.log + + + +############# +# cron # +############# +[cron-dir] +recipe = slapos.cookbook:mkdirectory +cron.d = ${directory:etc}/cron.d +crontabs= ${directory:srv}/cron/crontabs +cronstamps = ${directory:var}/cron/cronstamps +log = ${directory:log}/cron + +[service-cron] +recipe = slapos.cookbook:cron +binary = ${directory:service}/crond +cron-entries = ${cron-dir:cron.d} +crontabs = ${cron-dir:crontabs} +cronstamps = ${cron-dir:cronstamps} +catcher = ${cron-simplelogger:wrapper} + +dcrond-binary = {{ dcron_bin }} + +depends = + ${logrotate-entry-cron:recipe} + +# "mailer" that cron uses to emit messages to logfile +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = ${directory:bin}/${:_buildout_section_name_} +log = ${cron-dir:log}/cron.log + + +# base entry for clients who registers to cron +[cron-entry] +recipe = slapos.cookbook:cron.d +# name = <section-name>.strip_prefix('cron-entry-') +# XXX len() is not available in !py! - 11 hardcoded +name = !py!'${:_buildout_section_name_}' [11:] +# NOTE _not_ ${service-cron:cron-entries} - though the value is the same we do +# not want service-cron to be instantiated just if a cron-entry is registered. +cron-entries = ${cron-dir:cron.d} + +# cron logs are also rotated +[logrotate-entry-cron] +<= logrotate-entry +log = ${cron-dir:log}/*.log + + +####################################### +# logrotate base for all services # +####################################### +[logrotate-dir] +recipe = slapos.cookbook:mkdirectory +srv = ${directory:srv}/logrotate +entries = ${directory:etc}/logrotate.d + +[logrotate] +recipe = slapos.cookbook:logrotate +wrapper = ${directory:bin}/${:_buildout_section_name_} +conf = ${directory:etc}/logrotate.conf +logrotate-entries = ${logrotate-dir:entries} +state-file = ${logrotate-dir:srv}/logrotate.status + +logrotate-binary = {{ logrotate_bin }} +gzip-binary = {{ gzip_bin }} +gunzip-binary = {{ gunzip_bin }} + +depend = ${cron-entry-logrotate:recipe} + + +# base entry for clients who registers to logrotate +[logrotate-entry] +recipe = slapos.cookbook:logrotate.d +logrotate-entries = ${logrotate:logrotate-entries} +# name = <section-name>.strip_prefix('logrotate-entry-') +# XXX len is not available in !py! - 16 hardcoded +name = !py!'${:_buildout_section_name_}'[16:] +# NOTE frequency is hardcoded to `daily` in slapos.cookbook:logrotate.d +# NOTE backup is also used to add custom logrotate options (hack) +backup = ... +# TODO settle whether we need/want olddir or not + noolddir +# override create emitted by slapos.cookbook:logrotate.d + nocreate +# do not move log file and this way we do not need to signal its program to +# reopen the log. There are a lot of bugs when on such reopen / restart / +# graceful-restart something bad happens. Even if copytruncate is a bit racy +# and can loose some data, it is better to keep the system the stable way. + copytruncate + + +# hook logrotate into cron +[cron-entry-logrotate] +<= cron-entry +time = daily +command = ${logrotate:wrapper} + + + +# 6. on-reinstantiate actions + +# NOTE here we only recompile assets. Other on-reinstantiate actions, which +# require pg and redis running, are performed as part of unicorn service - +# right before its startup (see gitlab-unicorn-startup). +[on-reinstantiate] +recipe = plone.recipe.command +stop-on-error = true +rake = ${gitlab-rake:wrapper-path} +# run command on every reinstantiation +update-command = ${:command} + +command = + ${:rake} assets:clean && + ${:rake} assets:precompile && + true diff --git a/software/gitlab/instance.cfg.in b/software/gitlab/instance.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..987d29e249329b6a77d21f2ad7266fad719b91f7 --- /dev/null +++ b/software/gitlab/instance.cfg.in @@ -0,0 +1,65 @@ +# GitLab "switch-softwaretype" instance +[buildout] +parts = switch-softwaretype + +# std stuff for slapos instance +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + + +[switch-softwaretype] +recipe = slapos.cookbook:softwaretype +default = $${instance-gitlab.cfg:rendered} +# TODO -export, -import, -pull-backup + + +[instance-gitlab.cfg] +recipe = slapos.recipe.template:jinja2 +mode = 0644 +template= ${instance-gitlab.cfg.in:target} +rendered= $${buildout:directory}/instance-gitlab.cfg +context = + import os os + import pwd pwd + import multiprocessing multiprocessing + + key eggs_directory buildout:eggs-directory + key develop_eggs_directory buildout:develop-eggs-directory + raw gitlab_repository_location ${gitlab-repository:location} + raw gitlab_shell_repository_location ${gitlab-shell-repository:location} + +# program binaries + raw bash_bin ${bash:location}/bin/bash + raw bundler_4gitlab ${bundler-4gitlab:bundle} + raw curl_bin ${curl:location}/bin/curl + raw dcron_bin ${dcron-output:crond} + raw git ${git:location}/bin/git + raw git_location ${git:location} + raw gitlab_workhorse ${gitlab-workhorse:location}/gitlab-workhorse + raw gunzip_bin ${gzip:location}/bin/gunzip + raw gzip_bin ${gzip:location}/bin/gzip + raw logrotate_bin ${logrotate:location}/usr/sbin/logrotate + raw nginx_bin ${nginx-output:nginx} + raw nginx_mime_types ${nginx-output:mime} + raw openssl_bin ${openssl-output:openssl} + raw postgresql_location ${postgresql92:location} + raw redis_binprefix ${redis28:location}/bin + raw ruby_location ${bundler-4gitlab:ruby-location} + raw watcher_sigkill ${watcher-sigkill:rendered} + +# config files + raw config_ru_in ${config.ru.in:target} + raw database_yml_in ${database.yml.in:target} + raw gitconfig_in ${gitconfig.in:target} + raw gitlab_parameters_cfg ${gitlab-parameters.cfg:target} + raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target} + raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target} + raw gitlab_yml_in ${gitlab.yml.in:target} + raw macrolib_cfg_in ${macrolib.cfg.in:target} + raw nginx_conf_in ${nginx.conf.in:target} + raw nginx_gitlab_http_conf_in ${nginx-gitlab-http.conf.in:target} + raw rack_attack_rb_in ${rack_attack.rb.in:target} + raw resque_yml_in ${resque.yml.in:target} + raw smtp_settings_rb_in ${smtp_settings.rb.in:target} + raw unicorn_rb_in ${unicorn.rb.in:target} diff --git a/software/gitlab/macrolib.cfg.in b/software/gitlab/macrolib.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..ba09e17bca43de76f59bcd1231538ef6509e788d --- /dev/null +++ b/software/gitlab/macrolib.cfg.in @@ -0,0 +1,18 @@ +{# common macros for gitlab instance #} + +{# cfg(name) -> instance_parameter:configuration.<name> #} +{% macro cfg(name) %}{{ instance_parameter[str("configuration." + name)] }}{% endmacro %} + +{# cfg_bool(name) - like cfg(name), but returns 'true'/'' + NOTE macros can return only strings - that's why '' is used for false #} +{% macro cfg_bool(name) %}{{ 'true' if (cfg(name).lower() in ('true', 'yes')) else '' }}{% endmacro %} + + +{# deduce whether to use https from external url + ( here - becasue we cannot use jinja2 logic in instance-gitlab.cfg.in to + process instance parameters ) #} +{% set external_url = urlparse.urlparse(cfg('external_url')) %} +{% set cfg_https = (true if external_url.scheme == 'https' else false) %} + +{# for convenience #} +{% set fqdn = external_url.hostname %} diff --git a/software/gitlab/software.cfg b/software/gitlab/software.cfg new file mode 100644 index 0000000000000000000000000000000000000000..47e49de9e8162a4c7ce12112f7ad1a3f9d1404da --- /dev/null +++ b/software/gitlab/software.cfg @@ -0,0 +1,271 @@ +# GitLab software-release +[buildout] +extends = + ../../stack/slapos.cfg + ../../component/ruby/buildout.cfg + ../../component/golang/buildout.cfg + ../../component/postgresql/buildout.cfg + ../../component/redis/buildout.cfg + ../../component/cmake/buildout.cfg + ../../component/icu/buildout.cfg + ../../component/pkgconfig/buildout.cfg + ../../component/nodejs/buildout.cfg + ../../component/openssl/buildout.cfg + ../../component/nginx/buildout.cfg + +# for instance + ../../component/bash/buildout.cfg + ../../component/curl/buildout.cfg + ../../component/gzip/buildout.cfg + ../../component/dcron/buildout.cfg + ../../component/logrotate/buildout.cfg + +parts = + ruby2.1 + golang15 + git + postgresql92 + redis28 + cmake + icu + pkgconfig + nginx-output + + gitlab-shell/vendor + gitlab/vendor/bundle + gitlab-workhorse + +# for instance + instance.cfg + + slapos-cookbook + eggs + + bash + curl + watcher-sigkill + gzip + dcron-output + logrotate + + +############################ +# Software compilation # +############################ + +# rubygemsrecipe with fixed url and this way pinned rubygems version +[rubygemsrecipe] +recipe = rubygemsrecipe +url = https://rubygems.org/rubygems/rubygems-2.5.1.zip + + +# bundler, that we'll use to +# - install gems for gitlab +# - run gitlab services / jobs (via `bundle exec ...`) +[bundler-4gitlab] +<= rubygemsrecipe +ruby-location = ${ruby2.1:location} +ruby-executable = ${:ruby-location}/bin/ruby +gems = bundler==1.11.2 + +# bin installed here +bundle = ${buildout:bin-directory}/bundle + +# install together with dependencies of gitlab, which we cannot specify using +# --with-... gem option +# ( reason: rubygemsrecipe hardcodes PATH inside generated bin/* and it is +# impossible to adjust it later ) +# +# bundle exec <smth> ; <smth> starts with `#!/usr/bin/env ruby` as rubygems +# Rugged needs: cmake, pkgconfig +# execjs needs: nodejs +# rails needs db client program on path: psql +# gitlab wants to check redis version via running: redis-cli +environment = + PATH = ${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql92:location}/bin:${redis28:location}/bin:%(PATH)s + + +# gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories +# pinned to exact commit +[git-repository] +recipe = slapos.recipe.build:gitclone +git-executable = ${git:location}/bin/git + +[gitlab-repository] +<= git-repository +#repository = https://gitlab.com/gitlab-org/gitlab-ce.git +repository = https://lab.nexedi.com/kirr/gitlab-ce.git +# 8.2.X + NXD patches: +revision = v8.2.3-9-g79c127e6e068a619c53a8c22f1db8c1e28ec87d2 +location = ${buildout:parts-directory}/gitlab + +[gitlab-shell-repository] +<= git-repository +repository = https://gitlab.com/gitlab-org/gitlab-shell.git +# gitlab 8.2 wants gitlab-shell 2.6.8 +# 2.6.8 + NXD patches +revision = v2.6.8-2-g216d7e15fe06917198891a895f762ba84fdcc4d4 +location = ${buildout:parts-directory}/gitlab-shell + +[gitlab-workhorse-repository] +<= git-repository +#repository = https://gitlab.com/gitlab-org/gitlab-workhorse.git +repository = https://lab.nexedi.com/kirr/gitlab-workhorse.git +# 0.4.X + NXD patches for blob download speedup +# (https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/17) +revision = 0.4.1-23-g2beb8c9539433f072e3db540f91f75894ca6b1b0 +location = ${buildout:parts-directory}/gitlab-workhorse + + + +# build needed-by-gitlab gems via bundler +[gitlab/vendor/bundle] +recipe = slapos.recipe.cmmi +path = ${gitlab-repository:location} +bundle = ${bundler-4gitlab:bundle} + +configure-command = cd ${:path} && + ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location} && + ${:bundle} config --local build.pg --with-pg-config=${postgresql92:location}/bin/pg_config + +make-binary = +make-targets= cd ${:path} && + ${:bundle} install --deployment --without development test mysql kerberos + + +# build needed-by-gitlab-shell gems via bundler +# ( there is not vendor/ dir in gitlab-shell, so to avoid having buildout error +# on mkdir vendor/bundle, this part name is just /vendor ) +[gitlab-shell/vendor] +recipe = slapos.recipe.cmmi +path = ${gitlab-shell-repository:location} +bundle = ${bundler-4gitlab:bundle} + +configure-command = true +make-binary = +make-targets= cd ${:path} && + ${:bundle} install --deployment --without development test + + +# build gitlab-workhorse +[gitlab-workhorse] +recipe = slapos.recipe.cmmi +path = ${gitlab-workhorse-repository:location} + +configure-command = : +make-targets= ${:_buildout_section_name_} + +environment = + PATH=${golang15:location}/bin:%(PATH)s + + +############################### +# Trampoline for instance # +############################### + +# eggs for instance.cfg +[eggs] +recipe = zc.recipe.egg +eggs = + plone.recipe.command + cns.recipe.symlink + + +[instance.cfg] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance.cfg.in +output = ${buildout:directory}/instance.cfg +md5sum = b40cd8824b978da867404d8955b06c18 + +[watcher-sigkill] +recipe = slapos.recipe.template:jinja2 +template= ${:_profile_base_location_}/${:_buildout_section_name_}.in +rendered= ${buildout:bin-directory}/${:_buildout_section_name_} +mode = 0755 +md5sum = 2986dcb006dc9e8508ff81f646656131 +context = + section bash bash + + +# macro: download a file named as section name +# +# [filename] +# <= download-file +# md5sum = ... +[download-file] +recipe = slapos.recipe.build:download +url = ${:_profile_base_location_}/${:_buildout_section_name_} +destination = ${buildout:directory}/${:_buildout_section_name_} + +# like download-file, but download from template/<filename> +[download-template] +<= download-file +url = ${:_profile_base_location_}/template/${:_buildout_section_name_} + + +[config.ru.in] +<= download-template +md5sum = bb12852c28079f40a0751f7f3559e2a6 + +[database.yml.in] +<= download-template +md5sum = ee656cfd96e1c82df167f68bb5773291 + +[gitconfig.in] +<= download-template +md5sum = f4cb11e8bca379e016b062d0db859b74 + +[gitlab-parameters.cfg] +<= download-file +md5sum = bc98ec10209bc53f6a49888b1a2b9382 + +[gitlab-shell-config.yml.in] +<= download-template +md5sum = ea351e16b47f0008f61211eb2d7685e2 + +[gitlab-unicorn-startup.in] +<= download-file +md5sum = 2716afaa9445c0c429c6b211356ebe8f + +[gitlab.yml.in] +<= download-template +md5sum = cc32f5053dd2a2461aa5952a5b925310 + +[instance-gitlab.cfg.in] +<= download-file +md5sum = dfd2b14f846eda999fe9d12108d513b4 + +[macrolib.cfg.in] +<= download-file +md5sum = a56a44e96f65f5ed20211bb6a54279f4 + +[nginx-gitlab-http.conf.in] +<= download-template +md5sum = 590da2b00cd198c7bc261c3d893bc199 + +[nginx.conf.in] +<= download-template +md5sum = f1a6e2bce3f28a2243fed49d1e1601df + +[rack_attack.rb.in] +<= download-template +md5sum = 16503c029159ea6db7d0fb5ab67093a3 + +[resque.yml.in] +<= download-template +md5sum = 7d9cba658f9315cd058dfc74db943a66 + +[smtp_settings.rb.in] +<= download-template +md5sum = c7c09c241b5fa8163e4995260be52604 + +[unicorn.rb.in] +<= download-template +md5sum = 9bdca16362fe19c727bca38383e57068 + + +[versions] +cns.recipe.symlink = 0.2.3 +plone.recipe.command = 1.1 +rubygemsrecipe = 0.2.2 +slapos.recipe.template = 2.9 diff --git a/software/gitlab/template/config.ru.in b/software/gitlab/template/config.ru.in new file mode 100644 index 0000000000000000000000000000000000000000..64e8eafdc5eded804e294a6f29d3280d5721db6e --- /dev/null +++ b/software/gitlab/template/config.ru.in @@ -0,0 +1,27 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config.ru +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-rails-config.ru.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +# This file is used by Rack-based servers to start the application. + +{% from 'macrolib.cfg.in' import cfg with context %} + +if defined?(Unicorn) + require 'unicorn' + + if ENV['RAILS_ENV'] == 'production' || ENV['RAILS_ENV'] == 'staging' + # Unicorn self-process killer + require 'unicorn/worker_killer' + + # Max memory size (RSS) per worker + use Unicorn::WorkerKiller::Oom, ({{ cfg('unicorn_worker_memory_limit_min') }}), ({{ cfg('unicorn_worker_memory_limit_max') }}) + end +end + +require ::File.expand_path('../config/environment', __FILE__) + +map ENV['RAILS_RELATIVE_URL_ROOT'] || "/" do + run Gitlab::Application +end diff --git a/software/gitlab/template/database.yml.in b/software/gitlab/template/database.yml.in new file mode 100644 index 0000000000000000000000000000000000000000..c2ca900b5b8222ced4e7b6e3cd20f4ba0d3662bb --- /dev/null +++ b/software/gitlab/template/database.yml.in @@ -0,0 +1,27 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/database.yml.erb +# (last updated for 8.2.3+ce.0-0-g8eda093) + +{% from 'macrolib.cfg.in' import cfg with context %} + +production: + adapter: postgresql + encoding: unicode + {# collation is mainly for mysql + collation: <%= @db_collation %> + #} + database: {{ pgsql.dbname }} + pool: {{ cfg('db_pool') }} + {# XXX is it ok to use superuser, even if the whole database is only for gitlab? #} + username: '{{ pgsql.superuser }}' + {# we have no password - access is via unix socket #} + password: + host: '{{ pgsql["pgdata-directory"] }}' + port: + socket: + {# not needed for unix socket + sslmode: <%= single_quote(@db_sslmode) %> + sslrootcert: <%= single_quote(@db_sslrootcert) %> + #} diff --git a/software/gitlab/template/gitconfig.in b/software/gitlab/template/gitconfig.in new file mode 100644 index 0000000000000000000000000000000000000000..4d48f1712837082efabfb02f83fcb3ecd8e5083b --- /dev/null +++ b/software/gitlab/template/gitconfig.in @@ -0,0 +1,24 @@ +{{ autogenerated }} +# global git configuration for GitLab +# see: +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitconfig.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) +# +{% from 'macrolib.cfg.in' import cfg with context %} + +# don't waste memory when packing (each thread uses own work memory) +# besides it packs better with 1 thread +[pack] + threads = 1 + +# don't allow corrupt/broken objects to go in +[receive] + fsckObjects = true + + +[user] + name = {{ cfg('email_display_name') }} + email = {{ cfg('email_from') }} +[core] + autocrlf = input diff --git a/software/gitlab/template/gitlab-shell-config.yml.in b/software/gitlab/template/gitlab-shell-config.yml.in new file mode 100644 index 0000000000000000000000000000000000000000..d88b02f0dbda7111395d6f9903d56ef5737fbfeb --- /dev/null +++ b/software/gitlab/template/gitlab-shell-config.yml.in @@ -0,0 +1,70 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-shell-config.yml.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +# GitLab user. git by default +user: {{ backend_info.user }} + +# Url to gitlab instance. Used for api calls. Should end with a slash. +gitlab_url: "http+unix://{{ urllib.quote_plus(unicorn.socket) }}/" + +http_settings: +{# we don't need any + <%= @http_settings.to_json if @http_settings %> +#} +# user: someone +# password: somepass +# ca_file: /etc/ssl/cert.pem +# ca_path: /etc/pki/tls/certs +# self_signed_cert: false + +# Repositories path +# Give the canonicalized absolute pathname, +# REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!! +# Check twice that none of the components is a symlink, including "/home". +repos_path: "{{ gitlab.repositories }}" + +# File used as authorized_keys for gitlab user +# NOTE not used in slapos version (all access via https only) +auth_file: "{{ gitlab.var }}/sshkeys-notused" + +# File that contains the secret key for verifying access to GitLab. +# Default is .gitlab_shell_secret in the root directory. +secret_file: "{{ gitlab_shell.secret }}" + + +# Redis settings used for pushing commit notices to gitlab +redis: + bin: {{ redis_binprefix }}/redis-cli + host: {# <%= @redis_host %> #} + port: {# <%= @redis_port %> #} + socket: {{ service_redis.unixsocket }} +{# we don't use password for redis +<% if @redis_password %> + pass: <%= @redis_password %> +<% end %> +#} + database: {# <%= @redis_database %> #} + namespace: resque:gitlab + +# Log file. +# Default is gitlab-shell.log in the root directory. +log_file: "{{ gitlab_shell.log }}/gitlab-shell.log" + +# Log level. INFO by default +log_level: + +# Audit usernames. +# Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but +# incurs an extra API call on every gitlab-shell command. +audit_usernames: + +# Enable git-annex support +# git-annex allows managing files with git, without checking the file contents into git +# See https://git-annex.branchable.com/ for documentation +# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup +# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex +# For CentOS: sudo yum install epel-release && sudo yum install git-annex +git_annex_enabled: diff --git a/software/gitlab/template/gitlab.yml.in b/software/gitlab/template/gitlab.yml.in new file mode 100644 index 0000000000000000000000000000000000000000..8f85e9641007a3de884550c77424533c85ab665d --- /dev/null +++ b/software/gitlab/template/gitlab.yml.in @@ -0,0 +1,426 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +{% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %} + +production: &base + # + # 1. GitLab app settings + # ========================== + + ## GitLab settings + gitlab: + ## Web server settings (note: host is the FQDN, do not include http://) + {% set default_port = {'http': 80, 'https': 443} %} + host: {{ external_url.hostname }} + port: {{ external_url.port or default_port[external_url.scheme] }} + https: {{ cfg_https }} + + {# ssh is disabled completely in slapos version + # Uncommment this line below if your ssh host is different from HTTP/HTTPS one + # (you'd obviously need to replace ssh.host_example.com with your own host). + # Otherwise, ssh host will be set to the `host:` value above + ssh_host: <%= @gitlab_ssh_host %> + #} + + # WARNING: See config/application.rb under "Relative url support" for the list of + # other files that need to be changed for relative url support + # relative_url_root: /gitlab + + # Uncomment and customize if you can't use the default user to run GitLab (default: 'git') + user: {{ backend_info.user }} + + ## Date & Time settings + time_zone: '{{ cfg("time_zone") }}' + + ## Email settings + # Uncomment and set to false if you need to disable email sending from GitLab (default: true) + email_enabled: {{ cfg('email_enabled') }} + # Email address used in the "From" field in mails sent by GitLab + email_from: {{ cfg('email_from') }} + email_display_name: {{ cfg('email_display_name') }} + email_reply_to: {{ cfg('email_reply_to') }} + + # Email server smtp settings are in [a separate file](initializers/smtp_settings.rb.sample). + + ## User settings + default_can_create_group: {{ cfg('default_can_create_group') }} # default: true + username_changing_enabled: {{ cfg('username_changing_enabled') }} # default: true - User can change her username/namespace + ## Default theme + ## BASIC = 1 + ## MARS = 2 + ## MODERN = 3 + ## GRAY = 4 + ## COLOR = 5 + default_theme: {{ cfg('default_theme') }} # default: 2 + + {# we do not need to restrict visibility levels + # Restrict setting visibility levels for non-admin users. + # The default is to allow all levels. + restricted_visibility_levels: <%= @gitlab_restricted_visibility_levels unless @gitlab_restricted_visibility_levels.nil? %> + #} + + {# for now we are ok with default issue-closing pattern + ## Automatic issue closing + # If a commit message matches this regular expression, all issues referenced from the matched text will be closed. + # This happens when the commit is pushed or merged into the default branch of a project. + # When not specified the default issue_closing_pattern as specified below will be used. + # Tip: you can test your closing pattern at http://rubular.com + issue_closing_pattern: <%= single_quote(@gitlab_issue_closing_pattern) %> + #} + + ## Default project features settings + default_projects_features: + issues: {{ cfg('default_projects_features.issues') }} + merge_requests: {{ cfg('default_projects_features.merge_requests') }} + wiki: {{ cfg('default_projects_features.wiki') }} + snippets: {{ cfg('default_projects_features.snippets') }} + visibility_level: '{{ cfg("default_projects_features.visibility_level") }}' # can be "private" | "internal" | "public" + builds: false {# builds not supported yet <%= @gitlab_default_projects_features_builds %> #} + + ## Webhook settings + # Number of seconds to wait for HTTP response after sending webhook HTTP POST request (default: 10) + webhook_timeout: {{ cfg('webhook_timeout') }} + + {# default is just ok + ## Repository downloads directory + # When a user clicks e.g. 'Download zip' on a project, a temporary zip file is created in the following directory. + # The default is 'tmp/repositories' relative to the root of the Rails app. + repository_downloads_path: <%= @gitlab_repository_downloads_path %> + #} + + {# we do not support reply by email + ## Reply by email + # Allow users to comment on issues and merge requests by replying to notification emails. + # For documentation on how to set this up, see http://doc.gitlab.com/ce/incoming_email/README.html + incoming_email: + enabled: <%= @incoming_email_enabled %> + + # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to. + # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`. + address: <%= single_quote(@incoming_email_address) %> + + # Email account username + # With third party providers, this is usually the full email address. + # With self-hosted email servers, this is usually the user part of the email address. + user: <%= single_quote(@incoming_email_email) %> + # Email account password + password: <%= single_quote(@incoming_email_password) %> + + # IMAP server host + host: <%= single_quote(@incoming_email_host) %> + # IMAP server port + port: <%= @incoming_email_port %> + # Whether the IMAP server uses SSL + ssl: <%= @incoming_email_ssl %> + # Whether the IMAP server uses StartTLS + start_tls: <%= @incoming_email_start_tls %> + + # The mailbox where incoming mail will end up. Usually "inbox". + mailbox: <%= single_quote(@incoming_email_mailbox_name) %> + #} + + {# we do not support build artifacts + ## Build Artifacts + artifacts: + enabled: <%= @artifacts_enabled %> + # The location where Build Artifacts are stored (default: shared/artifacts). + storage_path: <%= @artifacts_path %> + #} + + {# we do not support LFS + ## Git LFS + lfs: + enabled: <%= @lfs_enabled %> + # The location where LFS objects are stored (default: shared/lfs-objects). + storage_path: <%= @lfs_storage_path %> + #} + + ## Gravatar + ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html + gravatar: + {# default is just ok + # gravatar urls: possible placeholders: %{hash} %{size} %{email} + plain_url: <%= single_quote(@gravatar_plain_url) %> # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon + ssl_url: <%= single_quote(@gravatar_ssl_url) %> # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon + #} + + + # + # 2. GitLab CI settings + # ========================== + + {# we do not support CI + gitlab_ci: + # Default project notifications settings: + # + # Send emails only on broken builds (default: true) + all_broken_builds: <%= @gitlab_ci_all_broken_builds %> + # + # Add pusher to recipients list (default: false) + add_pusher: <%= @gitlab_ci_add_pusher || @gitlab_ci_add_committer %> + + # The location where build traces are stored (default: builds/). Relative paths are relative to Rails.root + builds_path: <%= @builds_directory %> + #} + + # + # 3. Auth settings + # ========================== + + ## LDAP settings + # You can inspect a sample of the LDAP users with login access by running: + # bundle exec rake gitlab:ldap:check RAILS_ENV=production + ldap: + enabled: false + {# just disabled + enabled: <%= @ldap_enabled %> + <% if @ldap_servers.any? %> + servers: + <% @ldap_servers.each do |provider_id, settings| %> + <%= provider_id %>: <%= settings.to_json %> + <% end %> + <% else %> + host: <%= single_quote(@ldap_host) %> + port: <%= @ldap_port %> + uid: <%= single_quote(@ldap_uid) %> + method: <%= single_quote(@ldap_method) %> # "tls" or "ssl" or "plain" + bind_dn: <%= single_quote(@ldap_bind_dn) %> + password: <%= single_quote(@ldap_password) %> + active_directory: <%= @ldap_active_directory %> + allow_username_or_email_login: <%= @ldap_allow_username_or_email_login %> + base: <%= single_quote(@ldap_base) %> + user_filter: <%= single_quote(@ldap_user_filter) %> + + ## EE only + group_base: <%= single_quote(@ldap_group_base) %> + admin_group: <%= single_quote(@ldap_admin_group) %> + sync_ssh_keys: <%= single_quote(@ldap_sync_ssh_keys) %> + sync_time: <%= @ldap_sync_time %> + <% end %> + #} + + ## Kerberos settings + kerberos: + enabled: false + {# just disabled + # Allow the HTTP Negotiate authentication method for Git clients + enabled: <%= @kerberos_enabled %> + + # Kerberos 5 keytab file. The keytab file must be readable by the GitLab user, + # and should be different from other keytabs in the system. + # (default: use default keytab from Krb5 config) + keytab: <%= @kerberos_keytab %> + + # The Kerberos service name to be used by GitLab. + # (default: accept any service name in keytab file) + service_principal_name: <%= @kerberos_service_principal_name %> + + # Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails. + # To support both Basic and Negotiate methods with older versions of Git, configure + # nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines + # to dedicate this port to Kerberos authentication. (default: false) + use_dedicated_port: <%= @kerberos_use_dedicated_port %> + port: <%= @kerberos_port %> + https: <%= @kerberos_https %> + #} + + + ## OmniAuth settings + omniauth: + enabled: false + {# just disabled + # Allow login via Twitter, Google, etc. using OmniAuth providers + enabled: <%= @omniauth_enabled %> + + # Uncomment this to automatically sign in with a specific omniauth provider's without + # showing GitLab's sign-in page (default: show the GitLab sign-in page) + auto_sign_in_with_provider: <%= @omniauth_auto_sign_in_with_provider %> + + # CAUTION! + # This allows users to login without having a user account first (default: false). + # User accounts will be created automatically when authentication was successful. + allow_single_sign_on: <%= @omniauth_allow_single_sign_on %> + # Locks down those users until they have been cleared by the admin (default: true). + block_auto_created_users: <%= @omniauth_block_auto_created_users %> + # Look up new users in LDAP servers. If a match is found (same uid), automatically + # link the omniauth identity with the LDAP account. (default: false) + auto_link_ldap_user: <%= @omniauth_auto_link_ldap_user %> + + + ## Auth providers + # Uncomment the following lines and fill in the data of the auth provider you want to use + # If your favorite auth provider is not listed you can use others: + # see https://github.com/gitlabhq/gitlab-public-wiki/wiki/Custom-omniauth-provider-configurations + # The 'app_id' and 'app_secret' parameters are always passed as the first two + # arguments, followed by optional 'args' which can be either a hash or an array. + # Documentation for this is available at http://doc.gitlab.com/ce/integration/omniauth.html + providers: + # - { name: 'google_oauth2', app_id: 'YOUR APP ID', + # app_secret: 'YOUR APP SECRET', + # args: { access_type: 'offline', approval_prompt: '' } } + # - { name: 'twitter', app_id: 'YOUR APP ID', + # app_secret: 'YOUR APP SECRET'} + # - { name: 'github', app_id: 'YOUR APP ID', + # app_secret: 'YOUR APP SECRET', + # args: { scope: 'user:email' } } +<% @omniauth_providers.each do |provider| %> + - <%= provider.to_json %> +<% end %> + #} + + {# default ($RAILS_ROOT/shared/) is just ok + # Shared file storage settings + shared: + path: <%= @shared_path %> + #} + + # + # 4. Advanced settings + # ========================== + + # GitLab Satellites + # Important: keep the satellites.path setting until GitLab 9.0 at + # least. This setting is fed to 'rm -rf' in + # db/migrate/20151023144219_remove_satellites.rb + satellites: + # Relative paths are relative to Rails.root (default: tmp/repo_satellites/) + path: /dev/null + timeout: 0 + + ## Backup settings + backup: + path: "{{ gitlab.backup }}" # Relative paths are relative to Rails.root (default: tmp/backups/) + {# default permission is ok + archive_permissions: <%= @backup_archive_permissions %> # Permissions for the resulting backup.tar file (default: 0600) + #} + keep_time: {{ cfg('backup_keep_time') }} # default: 0 (forever) (in seconds) + {# default to backup all schemas is just ok + pg_schema: <%= @backup_pg_schema %> # default: nil, it means that all schemas will be backed up + #} + upload: + {# we don't want to upload backup anywhere by gitlab builtin mechanisms + # Fog storage connection settings, see http://fog.io/storage/ . + connection: <%= @backup_upload_connection.to_json if @backup_upload_connection %> + # The remote 'directory' to store your backups. For S3, this would be the bucket name. + remote_directory: <%= single_quote(@backup_upload_remote_directory) %> + multipart_chunk_size: <%= @backup_multipart_chunk_size %> + encryption: <%= @backup_encryption %> + #} + + + ## GitLab Shell settings + gitlab_shell: + path: {{ gitlab_shell_work.location }} + + # REPOS_PATH MUST NOT BE A SYMLINK!!! + repos_path: {{ gitlab.repositories }} + hooks_path: {{ gitlab_shell_work.location }}/hooks/ + secret_file: {{ gitlab_shell.secret }} + + # Git over HTTP + upload_pack: true + receive_pack: true + + {# Git over SSH is disabled elsewhere (so we don't care about ssh_port) + # If you use non-standard ssh port you need to specify it + ssh_port: <%= @gitlab_shell_ssh_port %> + #} + + + ## Git settings + # CAUTION! + # Use the default values unless you really know what you are doing + git: + bin_path: {{ git }} + # The next value is the maximum memory size grit can use + # Given in number of bytes per git object (e.g. a commit) + # This value can be increased if you have very large commits + max_size: {{ cfg('git_max_size') }} + # Git timeout to read a commit, in seconds + timeout: {{ cfg('git_timeout') }} + + + # + # 5. Extra customization + # ========================== + + extra: + {# we do not use google analytics + <% if @extra_google_analytics_id %> + ## Google analytics. Uncomment if you want it + google_analytics_id: <%= single_quote(@extra_google_analytics_id) %> + <% end %> + #} + + {# we do not use piwik + <% if @extra_piwik_url %> + ## Piwik analytics. + piwik_url: <%= single_quote(@extra_piwik_url) %> + piwik_site_id: <%= single_quote(@extra_piwik_site_id) %> + <% end %> + #} + + {# we are ok (for now) with default rack-attack git settings + rack_attack: + git_basic_auth: <%= @rack_attack_git_basic_auth.to_json if @rack_attack_git_basic_auth %> + #} + + + ## Site ICP License + # XXX unquote needed only for slapos.core earlier than + # https://lab.nexedi.com/nexedi/slapos.core/commit/347d33d6 + # for now we have a lot of old slapos.core deployed... + {% if cfg('icp_license') != '' -%} + ICP: {{ urllib.unquote_plus( str(cfg('icp_license')) ).decode('utf-8') }} + {# ICP: '{{ cfg("icp_license") }}' #} + {% endif %} + + +development: + <<: *base + +test: + <<: *base + gravatar: + enabled: true + gitlab: + host: localhost + port: 80 + + # When you run tests we clone and setup gitlab-shell + # In order to setup it correctly you need to specify + # your system username you use to run GitLab + # user: YOUR_USERNAME + satellites: + path: tmp/tests/gitlab-satellites/ + gitlab_shell: + path: tmp/tests/gitlab-shell/ + repos_path: tmp/tests/repositories/ + hooks_path: tmp/tests/gitlab-shell/hooks/ + issues_tracker: + redmine: + title: "Redmine" + project_url: "http://redmine/projects/:issues_tracker_id" + issues_url: "http://redmine/:project_id/:issues_tracker_id/:id" + new_issue_url: "http://redmine/projects/:issues_tracker_id/issues/new" + ldap: + enabled: false + servers: + main: + label: ldap + host: 127.0.0.1 + port: 3890 + uid: 'uid' + method: 'plain' # "tls" or "ssl" or "plain" + base: 'dc=example,dc=com' + user_filter: '' + group_base: 'ou=groups,dc=example,dc=com' + admin_group: '' + sync_ssh_keys: false + +staging: + <<: *base diff --git a/software/gitlab/template/nginx-gitlab-http.conf.in b/software/gitlab/template/nginx-gitlab-http.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..f8750a91da2b8451aca6c09a0ae567223e125555 --- /dev/null +++ b/software/gitlab/template/nginx-gitlab-http.conf.in @@ -0,0 +1,257 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +{% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn with context %} + +## GitLab +## Modified from https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl & https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab +## +## Lines starting with two hashes (##) are comments with information. +## Lines starting with one hash (#) are configuration parameters that can be uncommented. +## +################################## +## CHUNKED TRANSFER ## +################################## +## +## It is a known issue that Git-over-HTTP requires chunked transfer encoding [0] +## which is not supported by Nginx < 1.3.9 [1]. As a result, pushing a large object +## with Git (i.e. a single large file) can lead to a 411 error. In theory you can get +## around this by tweaking this configuration file and either: +## - installing an old version of Nginx with the chunkin module [2] compiled in, or +## - using a newer version of Nginx. +## +## At the time of writing we do not know if either of these theoretical solutions works. +## As a workaround users can use Git over SSH to push large files. +## +## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 +## [1] https://github.com/agentzh/chunkin-nginx-module#status +## [2] https://github.com/agentzh/chunkin-nginx-module +## +################################### +## configuration ## +################################### + +upstream gitlab { + server unix:{{ unicorn.socket }} fail_timeout=0; +} + +upstream gitlab-workhorse { + server unix:{{ gitlab_workhorse.socket }}; +} + +{# not needed for us - the frontend can do the redirection and also + gitlab/nginx speaks HSTS on https port so when we access https port via http + protocol, it gets redirected to https +<% if @https && @redirect_http_to_https %> +## Redirects all HTTP traffic to the HTTPS host +server { +<% @listen_addresses.each do |listen_address| %> + listen <%= listen_address %>:<%= @redirect_http_to_https_port %>; +<% end %> + server_name <%= @fqdn %>; + server_tokens off; ## Don't show the nginx version number, a security best practice + return 301 https://<%= @fqdn %>:<%= @port %>$request_uri; + access_log <%= @log_directory %>/gitlab_access.log gitlab_access; + error_log <%= @log_directory %>/gitlab_error.log; +} +<% end %> +#} + +server { + listen [{{ backend_info.host }}]:{{ backend_info.port }}{% if cfg_https %} ssl spdy{% endif %}; + + {# we don't use: kerbeeros + <% if @kerberos_enabled && @kerberos_use_dedicated_port %> + listen <%= listen_address %>:<%= @kerberos_port %><% if @kerberos_https %> ssl<% end %>; + <% end %> + #} + + server_name {{ fqdn }}; + server_tokens off; ## Don't show the nginx version number, a security best practice + root {{ gitlab_work.location }}/public; + + ## Increase this if you want to upload large attachments + ## Or if you want to accept large git objects over http + client_max_body_size {{ cfg('nginx_client_max_body_size') }}; + + {% if cfg_https %} + ## Strong SSL Security + ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ + ssl on; + ssl_certificate {{ nginx.cert_file }}; + ssl_certificate_key {{ nginx.key_file }}; + {# we don't need - most root CA will be included by default + <% if @ssl_client_certificate %> + ssl_client_certificate <%= @ssl_client_certificate%>; + <% end %> + #} + + # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs + # NOTE(slapos) ^^^ is not relevant for us - we are behind frontend and clients + # directly connects to frontend + ssl_ciphers '{{ cfg("nginx_ssl_ciphers") }}'; + ssl_protocols {{ cfg('nginx_ssl_protocols') }}; + ssl_prefer_server_ciphers {{ cfg('nginx_ssl_prefer_server_ciphers') }}; + ssl_session_cache {{ cfg('nginx_ssl_session_cache') }}; + ssl_session_timeout {{ cfg('nginx_ssl_session_timeout') }}; + + {# we do not use: ssl_dhparam + <% if @ssl_dhparam %> + ssl_dhparam <%= @ssl_dhparam %>; + <% end %> + #} + {% endif %} + + ## Individual nginx logs for this GitLab vhost + access_log {{ nginx.log }}/gitlab_access.log gitlab_access; + error_log {{ nginx.log }}/gitlab_error.log; + + location / { + ## Serve static files from defined root folder. + ## @gitlab is a named location for the upstream fallback, see below. + try_files $uri /index.html $uri.html @gitlab; + } + + location /uploads/ { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + {{ 'gzip off;' if cfg_https else ''}} + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout {{ cfg('nginx_proxy_read_timeout') }}; + proxy_connect_timeout {{ cfg('nginx_proxy_connect_timeout') }}; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + {% if cfg_https %} + proxy_set_header X-Forwarded-Ssl on; + {% endif %} + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }}; + proxy_set_header X-Frame-Options SAMEORIGIN; + + proxy_pass http://gitlab; + } + + ## If a file, which is not found in the root folder is requested, + ## then the proxy passes the request to the upsteam (gitlab unicorn). + location @gitlab { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + {{ 'gzip off;' if cfg_https else ''}} + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout {{ cfg('nginx_proxy_read_timeout') }}; + proxy_connect_timeout {{ cfg('nginx_proxy_connect_timeout') }}; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + {% if cfg_https %} + proxy_set_header X-Forwarded-Ssl on; + {% endif %} + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }}; + proxy_set_header X-Frame-Options SAMEORIGIN; + + proxy_pass http://gitlab; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + location ~ ^/api/v3/projects/.*/repository/archive { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ ^/[\w\.-]+/[\w\.-]+/builds/download { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # Build artifacts should be submitted to this location + location ~ /ci/api/v1/builds/[0-9]+/artifacts { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + + # access to raw blobs -> @gitlab-workhorse + location ~ ^/[\w\.-]+/[\w\.-]+/raw/ { + client_max_body_size 0; + error_page 418 = @gitlab-workhorse; + return 418; + } + + location @gitlab-workhorse { + client_max_body_size 0; + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + {{ 'gzip off;' if cfg_https else ''}} + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout {{ cfg('nginx_proxy_read_timeout') }}; + proxy_connect_timeout {{ cfg('nginx_proxy_connect_timeout') }}; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + {% if cfg_https %} + proxy_set_header X-Forwarded-Ssl on; + {% endif %} + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }}; + + proxy_pass http://gitlab-workhorse; + } + + ## Enable gzip compression as per rails guide: + ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression + ## WARNING: If you are using relative urls remove the block below + ## See config/application.rb under "Relative url support" for the list of + ## other files that need to be changed for relative url support + location ~ ^/(assets)/ { + root {{ gitlab_work.location }}/public; + gzip_static on; # to serve pre-gzipped version + expires max; + add_header Cache-Control public; + } + + + error_page 502 /502.html; + + {# we don't support custom nginx configs + <%= @custom_gitlab_server_config %> + #} +} diff --git a/software/gitlab/template/nginx.conf.in b/software/gitlab/template/nginx.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..47e6aa36b6885e25799cc8c0fab841944a392b13 --- /dev/null +++ b/software/gitlab/template/nginx.conf.in @@ -0,0 +1,50 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +{% from 'macrolib.cfg.in' import cfg with context %} + +# user directive makes sense only when running initially as root +# (and nginx will complain if not and directive given) +# user {{ backend_info.user }}; + +worker_processes {{ cfg('nginx_worker_processes') }}; +error_log stderr; +pid {{ directory.run }}/nginx.pid; + +daemon off; + +events { + worker_connections {{ cfg('nginx_worker_connections') }}; +} + +http { + log_format gitlab_access '{{ cfg("nginx_log_format") }}'; + {# we do not use: ci, mattermost + log_format gitlab_ci_access '<%= @gitlab_ci_access_log_format %>'; + log_format gitlab_mattermost_access '<%= @gitlab_mattermost_access_log_format %>'; + #} + + sendfile {{ cfg('nginx_sendfile') }}; + tcp_nopush {{ cfg('nginx_tcp_nopush') }}; + tcp_nodelay {{ cfg('nginx_tcp_nodelay') }}; + + keepalive_timeout {{ cfg('nginx_keepalive_timeout') }}; + + gzip {{ cfg('nginx_gzip') }}; + gzip_http_version {{ cfg('nginx_gzip_http_version') }}; + gzip_comp_level {{ cfg('nginx_gzip_comp_level') }}; + gzip_proxied {{ cfg('nginx_gzip_proxied') }}; + gzip_types {{ cfg('nginx_gzip_types') }}; + + include {{ nginx_mime_types }}; + + include {{ nginx_gitlab_http_conf }}; + + {# we don't need: ci, mattermost + include <%= @gitlab_ci_http_config %> + include <%= @gitlab_mattermost_http_config %> + #} +} diff --git a/software/gitlab/template/rack_attack.rb.in b/software/gitlab/template/rack_attack.rb.in new file mode 100644 index 0000000000000000000000000000000000000000..072ec7c4bc7ca94b310a32228224944829b311c1 --- /dev/null +++ b/software/gitlab/template/rack_attack.rb.in @@ -0,0 +1,34 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +{% from 'macrolib.cfg.in' import cfg with context %} + +# 1. Rename this file to rack_attack.rb +# 2. Review the paths_to_be_protected and add any other path you need protecting +# + +paths_to_be_protected = [ + "#{Rails.application.config.relative_url_root}/users/password", + "#{Rails.application.config.relative_url_root}/users/sign_in", + "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json", + "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session", + "#{Rails.application.config.relative_url_root}/users", + "#{Rails.application.config.relative_url_root}/users/confirmation", + "#{Rails.application.config.relative_url_root}/unsubscribes/" + +] + +# Create one big regular expression that matches strings starting with any of +# the paths_to_be_protected. +paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ }) + +unless Rails.env.test? + Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req| + if req.post? && req.path =~ paths_regex + req.ip + end + end +end diff --git a/software/gitlab/template/resque.yml.in b/software/gitlab/template/resque.yml.in new file mode 100644 index 0000000000000000000000000000000000000000..c6daf844a478e71933fb7b7a1c22d73e51828526 --- /dev/null +++ b/software/gitlab/template/resque.yml.in @@ -0,0 +1,7 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb +# (last udpdated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +production: unix://{{ redis.unixsocket }} diff --git a/software/gitlab/template/smtp_settings.rb.in b/software/gitlab/template/smtp_settings.rb.in new file mode 100644 index 0000000000000000000000000000000000000000..7ddc82a9e219036edf0784d1b35e74523b036b27 --- /dev/null +++ b/software/gitlab/template/smtp_settings.rb.in @@ -0,0 +1,29 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +{% from 'macrolib.cfg.in' import cfg, cfg_bool with context %} + +{% if cfg_bool('smtp_enable') %} +if Rails.env.production? + Gitlab::Application.config.action_mailer.delivery_method = :smtp + + ActionMailer::Base.smtp_settings = { + address: "{{ cfg('smtp_address') }}", + port: {{ cfg('smtp_port') }}, + user_name: "{{ cfg('smtp_user_name') }}", + password: "{{ cfg('smtp_password') }}", + domain: "{{ cfg('smtp_domain') }}", + authentication: :{{ cfg('smtp_authentication') }}, + enable_starttls_auto: {{ cfg('smtp_enable_starttls_auto') }}, + openssl_verify_mode: '{{ cfg("smtp_openssl_verify_mode") }}' + # ca_path: + # ca_file: + } +end +{% else %} +# SMTP disabled in instance configuration (see `smtp_enable` parameter). +# Mail sending, if enabled (see `email_enabled`), will be done via sendmail. +{% endif %} diff --git a/software/gitlab/template/unicorn.rb.in b/software/gitlab/template/unicorn.rb.in new file mode 100644 index 0000000000000000000000000000000000000000..ed4715e9756f46bc8b83e427219aab08753643f1 --- /dev/null +++ b/software/gitlab/template/unicorn.rb.in @@ -0,0 +1,73 @@ +{{ autogenerated }} +# see: +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example +# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb +# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093) + +{% from 'macrolib.cfg.in' import cfg with context %} + +# What ports/sockets to listen on, and what options for them. +# we listen only on unix socket +listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }} +#listen "127.0.0.1:8888", :tcp_nopush => true + +working_directory '{{ gitlab_work.location }}' + +# What the timeout for killing busy workers is, in seconds +timeout {{ cfg('unicorn_worker_timeout') }} + +# Whether the app should be pre-loaded +preload_app true + +# How many worker processes +worker_processes {{ cfg('unicorn_worker_processes') }} + +# about before_fork / after_fork - see: +# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb +# http://bogomips.org/unicorn.git/tree/examples/unicorn.conf.rb?id=3312aca8#n75 + +# What to do before we fork a worker +before_fork do |server, worker| + # XXX why gitlab does not enable this? + # # the following is highly recomended for Rails + "preload_app true" + # # as there's no need for the master process to hold a connection + # defined?(ActiveRecord::Base) and + # ActiveRecord::Base.connection.disconnect! + + # This allows a new master process to incrementally + # phase out the old master process with SIGTTOU to avoid a + # thundering herd (especially in the "preload_app false" case) + # when doing a transparent upgrade. The last worker spawned + # will then kill off the old master process with a SIGQUIT. + old_pid = "#{server.config[:pid]}.oldbin" + if old_pid != server.pid + begin + sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU + Process.kill(sig, File.read(old_pid).to_i) + rescue Errno::ENOENT, Errno::ESRCH + end + end +end + +# What to do after we fork a worker +after_fork do |server, worker| + # per-process listener ports for debugging/admin/migrations + # addr = "127.0.0.1:#{9293 + worker.nr}" + # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true) + + # XXX why gitlab does not enable this? + # # the following is *required* for Rails + "preload_app true", + # defined?(ActiveRecord::Base) and + # ActiveRecord::Base.establish_connection +end + + +# Where to drop a pidfile +pid '{{ directory.run }}/unicorn.pid' + +# Where stderr gets logged +stderr_path '{{ unicorn.log }}/unicorn_stderr.log' + +# Where stdout gets logged +stdout_path '{{ unicorn.log }}/unicorn_stdout.log' diff --git a/software/gitlab/watcher-sigkill.in b/software/gitlab/watcher-sigkill.in new file mode 100644 index 0000000000000000000000000000000000000000..91a56cdf99b7b0758bce3977354fbfd2dff894ca --- /dev/null +++ b/software/gitlab/watcher-sigkill.in @@ -0,0 +1,43 @@ +#!{{ bash.location }}/bin/bash +# run program under SIGKILL watchdog +# watcher-sigkill <prog> [<progargs> ...] +# +# if the program terminates with SIGKILL - it is restarted after grace period. +# if the program terminates otherwise - whole process terminates. + +if [ "$#" -lt 1 ]; then + echo "Usage: watcher-sigkill <prog> [<progargs> ...]" 1>&2 + exit 1 +fi +prog="$@" + +progpid="" +killexit="137" # = 128 + 9 (exit code of process terminated by SIGKILL) + +# make sure to terminate children, when we exit. +# needed for e.g. when `slapos node stop ...` kills us. +trap 'atexit' EXIT +atexit() { + jobs="$(jobs -p)" + test -n "$jobs" && kill $jobs +} + +# run prog under monitoring +while true; do + echo "run $prog" + $prog & + progpid=$! + echo "wait $progpid" + wait $progpid + status=$? + echo "-> $status" + + # if program terminated not by SIGKILL - exit + if [ "$status" != "$killexit" ] ; then + echo "exit $status" + exit "$status" + fi + + # otherwise sleep a bit and restart + sleep 1 +done