# GitLab instance # NOTE instance/software layout is inspired by gitlab omnibus # NOTE all services are interconnected via unix sockets - because of easier # security and performance reasons (unix has 2x less latency and more # throughput compared to tcp over loopback). [buildout] extends = {{ gitlab_parameters_cfg }} parts = directory publish-instance-info # gitlab-<prog> # ? mailroom {% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %} {% for prog in gitlab_progv %} gitlab-{{ prog }} {% endfor %} gitconfig gitlab-work gitlab-shell-work service-gitlab-workhorse service-unicorn service-sidekiq service-nginx service-postgresql service-redis service-cron on-reinstantiate # std stuff for slapos instance eggs-directory = {{ eggs_directory }} develop-eggs-directory = {{ develop_eggs_directory }} offline = true ################################## # GitLab instance parameters # ################################## [instance-parameter] # std stuff to fetch slapos instance parameters recipe = slapos.cookbook:slapconfiguration computer= ${slap-connection:computer-id} partition=${slap-connection:partition-id} url = ${slap-connection:server-url} key = ${slap-connection:key-file} cert = ${slap-connection:cert-file} # autogenerated gitlab instance parameters <= gitlab-parameters # adjust/override some default settings: # automatically load all available CPUs configuration.unicorn_worker_processes = {{ multiprocessing.cpu_count() + 1 }} configuration.nginx_worker_processes = {{ multiprocessing.cpu_count() }} # gitlab non-native parameters configuration.icp_license = # for convenience [external-url] recipe = slapos.cookbook:urlparse url = ${instance-parameter:configuration.external_url} [backend-info] host = ${instance-parameter:ipv6-random} port = 7777 # whether to use http or https - determined by external url url = ${external-url:scheme}://[${:host}]:${:port} # current slapuserX user = {{ pwd.getpwuid(os.getuid())[0] }} [publish-instance-info] recipe = slapos.cookbook:publish backend_url = ${backend-info:url} ############################# # GitLab instance setup # ############################# # 1. directories [directory] recipe = slapos.cookbook:mkdirectory home = ${buildout:directory} bin = ${:home}/bin etc = ${:home}/etc var = ${:home}/var log = ${:var}/log run = ${:var}/run srv = ${:home}/srv # slapos startup/service/promise scripts live here: startup = ${:etc}/run service = ${:etc}/service promise = ${:etc}/promise promise.slow = ${:promise}.slow # gitlab: etc/ log/ ... [gitlab-dir] recipe = slapos.cookbook:mkdirectory etc = ${directory:etc}/gitlab log = ${directory:log}/gitlab var = ${directory:var}/gitlab tmp = ${:var}/tmp uploads = ${:var}/uploads assets = ${:var}/assets shared = ${:var}/shared artifacts = ${:shared}/artifacts lfs-objects = ${:shared}/lfs-objects builds = ${:var}/builds backup = ${directory:var}/backup [gitlab-repo-dir] recipe = slapos.cookbook:mkdirectory repositories = ${directory:var}/repositories # gitlab wants it to be drwxrws--- # FIXME setting such mode with :mkdirectory is not possible, because mkdir(2) # does & 0777 and also there is umask. So we workaround: [gitlab-repo-xdir] recipe = plone.recipe.command stop-on-error = yes repositories = ${gitlab-repo-dir:repositories} command = chmod 02770 ${:repositories} [gitlab] etc = ${gitlab-dir:etc} log = ${gitlab-dir:log} var = ${gitlab-dir:var} tmp = ${gitlab-dir:tmp} uploads = ${gitlab-dir:uploads} assets = ${gitlab-dir:assets} shared = ${gitlab-dir:shared} artifacts = ${gitlab-dir:artifacts} lfs-objects = ${gitlab-dir:lfs-objects} builds = ${gitlab-dir:builds} backup = ${gitlab-dir:backup} repositories = ${gitlab-repo-xdir:repositories} # gitlab-shell: etc/ log/ gitlab_shell_secret ... [gitlab-shell-dir] recipe = slapos.cookbook:mkdirectory etc = ${directory:etc}/gitlab-shell log = ${directory:log}/gitlab-shell [gitlab-shell] etc = ${gitlab-shell-dir:etc} log = ${gitlab-shell-dir:log} secret = ${secrets:secrets}/gitlab_shell_secret # place to keep all secrets [secrets] recipe = slapos.cookbook:mkdirectory secrets = ${directory:var}/secrets mode = 0700 # 2. configuration files [etc-template] recipe = slapos.recipe.template:jinja2 extensions = jinja2.ext.do mode = 0640 import-list = rawfile macrolib.cfg.in {{ macrolib_cfg_in }} context = raw autogenerated # This file was autogenerated. (DO NOT EDIT - changes will be lost) section instance_parameter instance-parameter section backend_info backend-info import urlparse urlparse raw git {{ git }} ${:context-extra} context-extra = [gitlab-etc-template] <= etc-template rendered= ${gitlab:etc}/${:_buildout_section_name_} [nginx-etc-template] <= etc-template rendered= ${nginx:etc}/${:_buildout_section_name_} [database.yml] <= gitlab-etc-template template= {{ database_yml_in }} context-extra = section pgsql service-postgresql [gitconfig] <= etc-template template= {{ gitconfig_in }} # NOTE put directly into $HOME/ - this way git will pick it up rendered= ${directory:home}/.${:_buildout_section_name_} [gitlab-shell-config.yml] <= etc-template template= {{ gitlab_shell_config_yml_in }} rendered= ${gitlab-shell:etc}/config.yml context-extra = import urllib urllib section gitlab gitlab section gitlab_shell gitlab-shell section unicorn unicorn section service_redis service-redis raw redis_binprefix {{ redis_binprefix }} [gitlab.yml] <= gitlab-etc-template template= {{ gitlab_yml_in }} context-extra = import urllib urllib section gitlab gitlab section gitlab_shell gitlab-shell section gitlab_shell_work gitlab-shell-work [nginx.conf] <= nginx-etc-template template= {{ nginx_conf_in }} context-extra = section directory directory raw nginx_mime_types {{ nginx_mime_types }} raw nginx_gitlab_http_conf ${nginx-gitlab-http.conf:rendered} [nginx-gitlab-http.conf] <= nginx-etc-template template= {{ nginx_gitlab_http_conf_in }} context-extra = section nginx nginx section gitlab_work gitlab-work section gitlab_workhorse gitlab-workhorse [rack_attack.rb] <= gitlab-etc-template template = {{ rack_attack_rb_in }} [resque.yml] <= gitlab-etc-template template= {{ resque_yml_in }} context-extra = section redis service-redis [smtp_settings.rb] <= gitlab-etc-template template= {{ smtp_settings_rb_in }} # contains smtp password mode = 0600 [unicorn.rb] <= gitlab-etc-template template = {{ unicorn_rb_in }} context-extra = section unicorn unicorn section directory directory section gitlab_work gitlab-work # 3. bin/ # gitlab-<prog> [gitlab-bin] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:bin}/${:_buildout_section_name_} parameters-extra = true # NOTE $HOME needed to pick gitconfig environment = BUNDLE_GEMFILE = {{ gitlab_repository_location }}/Gemfile HOME = ${directory:home} RAILS_ENV = production SIDEKIQ_MEMORY_KILLER_MAX_RSS = ${instance-parameter:configuration.sidekiq_memory_killer_max_rss} # NOTE sys.argv[1:] implicitly appended # (by slapos.recipe.librecipe.execute.generic_exec() at runtime) command-line = {{ bundler_4gitlab }} exec sh -c 'cd ${gitlab-work:location} && ${:prog} "$@"' ${:prog} {% for prog in gitlab_progv %} [gitlab-{{ prog }}] <= gitlab-bin prog = {{ prog }} {% endfor %} [gitlab-unicorn-startup] recipe = slapos.recipe.template:jinja2 mode = 0755 template= {{ gitlab_unicorn_startup_in }} rendered= ${directory:bin}/${:_buildout_section_name_} context = raw bash_bin {{ bash_bin }} raw gitlab_rake ${gitlab-rake:wrapper-path} raw gitlab_unicorn ${gitlab-unicorn:wrapper-path} raw psql_bin {{ postgresql_location }}/bin/psql section pgsql service-postgresql raw log_dir ${gitlab:log} section unicorn_rb unicorn.rb section gitlab_work gitlab-work # 4. gitlab- & gitlab-shell- work directories # # Gitlab/Rails operation is tightened that config/ lives inside code, which goes # against having ability to create several instances configured differently # from 1 SR. # # One possibility to overcome this could be to make another Gitlab root # symbolically linked to original SR _and_ several configuration files # symbolically linked to instance place. Unfortunately this does not work - # Ruby determines realpath on module import and Gitlab and Rails lookup config # files relative to imported modules. # # we clone cloned gitlab and add proper links to vendor/bundle and instance # config files. # XXX there is no need for full clone - we only need worktree checkout (a-la `git # worktree add`, but without creating files in original clone) # # This way Gitlab/Rails still think they work in 1 code / 1 instance way, # and we can reuse SR. # XXX better do such tricks with bind mounting, but that requires user namespaces [work-base] recipe = plone.recipe.command stop-on-error = yes location = ${directory:home}/${:_buildout_section_name_} command = # make sure we start from well-defined empty state # (needed e.g. if previous install failed in the middle) rm -rf ${:location} && # init work repository and add `software` remote pointing to main repo in SR software/... {{ git }} init ${:location} && cd ${:location} && {{ git }} remote add software ${:software} && ${:update-command} update-command = cd ${:location} && {{ git }} fetch software && {{ git }} reset --hard `cd ${:software} && {{ git }} rev-parse HEAD` && ${:tune-command} # NOTE there is no need to link/create .gitlab_shell_secret - we set path to it # in gitlab & gitlab-shell configs, and gitlab creates it on its first start [gitlab-work] <= work-base software = {{ gitlab_repository_location }} tune-command = # secret* tmp/ log/ shared/ builds/ rm -f .secret && rm -rf log tmp shared builds && ln -sf ${secrets:secrets}/gitlab_rails_secret .secret && ln -sf ${gitlab:log} log && ln -sf ${gitlab:tmp} tmp && ln -sf ${gitlab:shared} shared && ln -sf ${gitlab:builds} builds && # config/ cd config && ln -sf ${unicorn.rb:rendered} unicorn.rb && ln -sf ${gitlab.yml:rendered} gitlab.yml && ln -sf ${database.yml:rendered} database.yml && ln -sf ${resque.yml:rendered} resque.yml && ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml && # config/initializers/ cd initializers && ln -sf ${rack_attack.rb:rendered} rack_attack.rb && ln -sf ${smtp_settings.rb:rendered} smtp_settings.rb && # public/ cd ../../public && rm -rf uploads assets && ln -sf ${gitlab:uploads} uploads && ln -sf ${gitlab:assets} assets && true # ----//---- for gitlab-shell [gitlab-shell-work] <= work-base software = {{ gitlab_shell_repository_location }} tune-command = ln -sf ${gitlab-shell-config.yml:rendered} config.yml && true # 5. services # [promise-<something>] to generate promise wrapper <something> [promise-wrapper] recipe = slapos.cookbook:wrapper wrapper-path = !py! '${directory:promise}/' + '${:_buildout_section_name_}'[8:] # [promise-<something>] to check <something> by url [promise-byurl] recipe = slapos.cookbook:check_url_available path = !py! '${directory:promise}/' + '${:_buildout_section_name_}'[8:] dash_path = {{ bash_bin }} curl_path = {{ curl_bin }} http_code = 200 ##################### # Postgresql db # ##################### # XXX gitlab-omnibus also tunes: # - shared_buffers # - work_mem # - checkpoint_* # - effective_check_size # - lc_* en_US.UTF-8 -> C (?) [service-postgresql] recipe = slapos.cookbook:postgres bin = {{ postgresql_location }}/bin services= ${directory:service} dbname = gitlabhq_production # NOTE db name must match to what was used in KVM on lab.nexedi.com (restore script grants access to this user) superuser = gitlab-psql # no password - pgsql will listen only on unix sockets (see below) thus access # is protected with filesystem-level permissions. # ( besides, if we use slapos.cookbook:generate.password and do `password = ...` # the password is stored in plain text in .installed and thus becomes insecure ) password= pgdata-directory = ${directory:srv}/postgresql # empty addresses - listen only on unix socket ipv4 = !py!set([]) ipv6 = !py!set([]) ipv6-random = port = depend = ${promise-postgresql:recipe} [promise-postgresql] <= promise-wrapper command-line = {{ postgresql_location }}/bin/psql -h ${service-postgresql:pgdata-directory} -U ${service-postgresql:superuser} -d ${service-postgresql:dbname} -c '\q' # postgresql logs to stdout/stderr - logs are handled by slapos not us # [logrotate-entry-postgresql] ############# # Redis # ############# [redis] recipe = slapos.cookbook:mkdirectory srv = ${directory:srv}/redis log = ${directory:log}/redis [service-redis] recipe = slapos.cookbook:redis.server wrapper = ${directory:service}/redis promise_wrapper = ${directory:promise}/redis server_dir = ${redis:srv} config_file = ${directory:etc}/redis.conf log_file = ${redis:log}/redis.log pid_file = ${directory:run}/redis.pid use_passwd = false unixsocket = ${:server_dir}/redis.socket # port = 0 means "don't listen on TCP at all" - listen only on unix socket ipv6 = ::1 port = 0 server_bin = {{ redis_binprefix }}/redis-server depend = ${logrotate-entry-redis:recipe} # NOTE slapos.cookbook:redis.server setups promise automatically [logrotate-entry-redis] <= logrotate-entry log = ${redis:log}/*.log ######################## # gitlab-workhorse # ######################## [gitlab-workhorse-dir] recipe = slapos.cookbook:mkdirectory srv = ${directory:srv}/gitlab-workhorse [gitlab-workhorse] srv = ${gitlab-workhorse-dir:srv} socket = ${gitlab-workhorse:srv}/gitlab-workhorse.socket [service-gitlab-workhorse] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:service}/gitlab-workhorse command-line = {{ gitlab_workhorse }} -listenNetwork unix -listenAddr ${gitlab-workhorse:socket} -authSocket ${unicorn:socket} -documentRoot ${gitlab-work:location}/public # NOTE for profiling # -pprofListenAddr ... # NOTE environment for: # - git to be available on path # - ruby to be available on path (gitlab-workhorse -> gitlab-shell -> hooks on push) # - gitconfig be found from ~/.gitconfig environment = PATH={{ git_location }}/bin:{{ ruby_location }}/bin:{{ gzip_location }}/bin:{{ bzip2_location}}/bin HOME=${directory:home} depend = ${promise-gitlab-workhorse:recipe} [promise-gitlab-workhorse] <= promise-byurl url = --unix-socket ${gitlab-workhorse:socket} http://localhost/static.css # gitlab-workhorse logs to stdout/stderr - logs are handled by slapos not us # [logrotate-entry-gitlab-workhorse] ###################### # unicorn worker # ###################### [unicorn-dir] recipe = slapos.cookbook:mkdirectory srv = ${directory:srv}/unicorn log = ${directory:log}/unicorn [unicorn] srv = ${unicorn-dir:srv} log = ${unicorn-dir:log} socket = ${:srv}/unicorn.socket [service-unicorn] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:service}/unicorn # NOTE we perform db setup / migrations as part of unicorn startup. # Those operations require PG and Redis to be up and running already, that's # why we do it here. See gitlab-unicorn-startup for details. command-line = ${gitlab-unicorn-startup:rendered} depend = ${promise-unicorn:recipe} ${promise-gitlab-app:recipe} ${promise-gitlab-shell:recipe} ${logrotate-entry-unicorn:recipe} # gitlab is a service "run" under unicorn # gitlab-shell is called by gitlab # -> associate their logs rotation to here ${logrotate-entry-gitlab:recipe} [promise-unicorn] <= promise-byurl url = --unix-socket ${unicorn:socket} http://localhost/ [promise-rakebase] recipe = slapos.cookbook:wrapper # FIXME gitlab-rake is too slow to load and promise timeouts # that's why we instantiate to <promise>.slow/ (and this way promises are not run) wrapper-path = !py!'${directory:promise.slow}/' + '${:_buildout_section_name_}'[8:] rake = ${gitlab-rake:wrapper-path} [promise-gitlab-app] <= promise-rakebase command-line = ${:rake} gitlab:app:check [promise-gitlab-shell] <= promise-rakebase command-line = ${:rake} gitlab:gitlab_shell:check # very slow # rake gitlab:repo:check (fsck all repos) [logrotate-entry-unicorn] <= logrotate-entry log = ${unicorn:log}/*.log [logrotate-entry-gitlab] <= logrotate-entry log = ${gitlab:log}/*.log [logrotate-entry-gitlab-shell] <= logrotate-entry log = ${gitlab-shell:log}/*.log ####################################### # sidekiq background jobs manager # ####################################### [sidekiq-dir] recipe = slapos.cookbook:mkdirectory log = ${directory:log}/sidekiq [sidekiq] log = ${sidekiq-dir:log} # NOTE see queue list here: # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb # (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053) [service-sidekiq] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:service}/sidekiq command-line = # NOTE Sidekiq memory killer just makes sidekiq processes to be SIGKILL # terminated and relies on managing service to restart it. In slapos we don't # have mechanism to set autorestart=true, nor bang/watchdog currently work with # slapproxy, so we do the monitoring ourselves. {{ watcher_sigkill }} ${gitlab-sidekiq:wrapper-path} # XXX -q runner ? (present in gitlab-ce/Procfile but not in omnibus) # XXX -q pages -q elasticsearch ? (present in omnibus but not in gitlab-ce -- those features are gitlab-ee only) # XXX -P ? (pidfile) -e production -r ${gitlab-work:location} -t ${instance-parameter:configuration.sidekiq_shutdown_timeout} -c ${instance-parameter:configuration.sidekiq_concurrency} -L ${sidekiq:log}/sidekiq.log -q post_receive -q mailers -q archive_repo -q system_hook -q project_web_hook -q gitlab_shell -q incoming_email -q common -q default depend = ${promise-sidekiq:recipe} ${logrotate-entry-sidekiq:recipe} [promise-sidekiq] <= promise-rakebase command-line = ${:rake} gitlab:sidekiq:check [logrotate-entry-sidekiq] <= logrotate-entry log = ${sidekiq:log}/*.log ###################### # Nginx frontend # ###################### # srv/nginx/ prefix + etc/ log/ ... [nginx-dir] recipe = slapos.cookbook:mkdirectory srv = ${directory:srv}/nginx etc = ${directory:etc}/nginx log = ${directory:log}/nginx [nginx-ssl-dir] recipe = slapos.cookbook:mkdirectory ssl = ${nginx-dir:etc}/ssl # contains https key mode = 0700 # self-signed certificate for https [nginx-generate-certificate] # NOTE there is slapos.cookbook:certificate_authority.request but it requires # to start whole service and has up to 60 seconds latency to generate # certificate. We only need to run 1 command to do it... recipe = plone.recipe.command stop-on-error = true cert_file = ${nginx-ssl-dir:ssl}/gitlab_backend.crt key_file = ${nginx-ssl-dir:ssl}/gitlab_backend.key command = test -e ${:key_file} || \ {{ openssl_bin }} req -newkey rsa -batch -new -x509 -days 3650 -nodes \ -keyout ${:key_file} -out ${:cert_file} update-command = ${:command} [nginx] srv = ${nginx-dir:srv} etc = ${nginx-dir:etc} log = ${nginx-dir:log} ssl = ${nginx-ssl-dir:ssl} cert_file = ${nginx-generate-certificate:cert_file} key_file = ${nginx-generate-certificate:key_file} [nginx-symlinks] # (nginx wants <prefix>/logs to be there from start - else it issues alarm to the log) recipe = cns.recipe.symlink symlink = ${nginx:log} = ${nginx:srv}/logs [service-nginx] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:service}/nginx command-line = {{ nginx_bin }} -p ${nginx:srv} -c ${nginx.conf:rendered} depend = ${nginx-symlinks:recipe} ${promise-nginx:recipe} ${logrotate-entry-nginx:recipe} [promise-nginx] <= promise-byurl # XXX this depends on gitlab-workhorse being up # (nginx is configured to proxy all requests to gitlab-workhorse) url = ${backend-info:url}/static.css [logrotate-entry-nginx] <= logrotate-entry log = ${nginx:log}/*.log ############# # cron # ############# [cron-dir] recipe = slapos.cookbook:mkdirectory cron.d = ${directory:etc}/cron.d crontabs= ${directory:srv}/cron/crontabs cronstamps = ${directory:var}/cron/cronstamps log = ${directory:log}/cron [service-cron] recipe = slapos.cookbook:cron binary = ${directory:service}/crond cron-entries = ${cron-dir:cron.d} crontabs = ${cron-dir:crontabs} cronstamps = ${cron-dir:cronstamps} catcher = ${cron-simplelogger:wrapper} dcrond-binary = {{ dcron_bin }} depends = ${logrotate-entry-cron:recipe} # "mailer" that cron uses to emit messages to logfile [cron-simplelogger] recipe = slapos.cookbook:simplelogger wrapper = ${directory:bin}/${:_buildout_section_name_} log = ${cron-dir:log}/cron.log # base entry for clients who registers to cron [cron-entry] recipe = slapos.cookbook:cron.d # name = <section-name>.strip_prefix('cron-entry-') # XXX len() is not available in !py! - 11 hardcoded name = !py!'${:_buildout_section_name_}' [11:] # NOTE _not_ ${service-cron:cron-entries} - though the value is the same we do # not want service-cron to be instantiated just if a cron-entry is registered. cron-entries = ${cron-dir:cron.d} # cron logs are also rotated [logrotate-entry-cron] <= logrotate-entry log = ${cron-dir:log}/*.log ####################################### # logrotate base for all services # ####################################### [logrotate-dir] recipe = slapos.cookbook:mkdirectory srv = ${directory:srv}/logrotate entries = ${directory:etc}/logrotate.d [logrotate] recipe = slapos.cookbook:logrotate wrapper = ${directory:bin}/${:_buildout_section_name_} conf = ${directory:etc}/logrotate.conf logrotate-entries = ${logrotate-dir:entries} state-file = ${logrotate-dir:srv}/logrotate.status logrotate-binary = {{ logrotate_bin }} gzip-binary = {{ gzip_bin }} gunzip-binary = {{ gunzip_bin }} depend = ${cron-entry-logrotate:recipe} # base entry for clients who registers to logrotate [logrotate-entry] recipe = slapos.cookbook:logrotate.d logrotate-entries = ${logrotate:logrotate-entries} # name = <section-name>.strip_prefix('logrotate-entry-') # XXX len is not available in !py! - 16 hardcoded name = !py!'${:_buildout_section_name_}'[16:] # NOTE frequency is hardcoded to `daily` in slapos.cookbook:logrotate.d # NOTE backup is also used to add custom logrotate options (hack) backup = ... # TODO settle whether we need/want olddir or not noolddir # override create emitted by slapos.cookbook:logrotate.d nocreate # do not move log file and this way we do not need to signal its program to # reopen the log. There are a lot of bugs when on such reopen / restart / # graceful-restart something bad happens. Even if copytruncate is a bit racy # and can loose some data, it is better to keep the system the stable way. copytruncate # hook logrotate into cron [cron-entry-logrotate] <= cron-entry time = daily command = ${logrotate:wrapper} # 6. on-reinstantiate actions # NOTE here we only recompile assets. Other on-reinstantiate actions, which # require pg and redis running, are performed as part of unicorn service - # right before its startup (see gitlab-unicorn-startup). [on-reinstantiate] recipe = plone.recipe.command stop-on-error = true rake = ${gitlab-rake:wrapper-path} # run command on every reinstantiation update-command = ${:command} command = ${:rake} assets:clean && ${:rake} assets:precompile && true