Skip to content

slapos
slapos
Commit 85f7d7e3 authored by Kirill Smelkov's avatar Kirill Smelkov
4 Browse files
Options

gitlab/nginx: Slapos'ify config and turn nginx into a service 

Go through nginx configuration templates and convert them to jinja2 with
slapos parameters (reminder: names and default values are imported from
omnibus-gitlab 8.2.3+ce.0-0-g8eda093), except commenting out features we
do not want to support (yet ?).

As nginx is a reverse-proxy, i.e. it integrates all internal services
and works as frontend to them, our gitlab service is now ready to listen
and talk to the world over (standard to slapos services backend) IPv6.

Nginx also acts as SSL termination point - for it to work by default we
setup self-signed certificate for the backend, which can be manually
changed to proper certificate if needed. Backend certificate is used
if gitlab is configured to work in HTTPS mode (and frontend certificate
is another story).

NOTE ssl certificate is generated with just `openssl req ...` - yes, there
    is slapos.cookbook:certificate_authority.request but it requires
    to start whole service and has up to 60 seconds latency to generate
    certificate. And we only need to run 1 command to do that...

The features disabled are:

    - http -> https redirection

      not needed for us at nginx level - the frontend can do the
      redirection and also gitlab speaks HSTS on https port so when we access
      https port via http protocol, it gets redirected to https.

    - kerberos
    - ssl_dhparam
    - providing custom nginx configuration via instance parameter

/cc @kazuhiko, @jerome
parent 45127f6d
Hide whitespace changes
Inline Side-by-side
Showing with 205 additions and 66 deletions
software/gitlab/gitlab-parameters.cfg
View file @ 85f7d7e3
......@@ -74,3 +74,32 @@ configuration.unicorn_backlog_socket = 1024
configuration.unicorn_worker_memory_limit_min = 200*(1024**2)
configuration.unicorn_worker_memory_limit_max = 250*(1024**2)
# nginx
configuration.nginx_client_max_body_size = 250m
# NOTE: we don't really need old ciphers - usually we talk directly to frontend only
configuration.nginx_ssl_ciphers = ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4
configuration.nginx_ssl_prefer_server_ciphers = on
configuration.nginx_ssl_protocols = TLSv1 TLSv1.1 TLSv1.2
# the following is gitlab-omnibus default but not nginx's default
configuration.nginx_ssl_session_cache = builtin:1000 shared:SSL:10m
configuration.nginx_ssl_session_timeout = 5m
configuration.nginx_proxy_read_timeout = 300
configuration.nginx_proxy_connect_timeout = 300
# nginx advanced
configuration.nginx_worker_processes = 4
configuration.nginx_worker_connections = 10240
configuration.nginx_log_format = $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
configuration.nginx_sendfile = on
configuration.nginx_tcp_nopush = on
configuration.nginx_tcp_nodelay = on
configuration.nginx_gzip = on
configuration.nginx_gzip_http_version = 1.0
configuration.nginx_gzip_comp_level = 2
configuration.nginx_gzip_proxied = any
configuration.nginx_gzip_types = text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json
configuration.nginx_keepalive_timeout = 65
software/gitlab/instance-gitlab.cfg.in
View file @ 85f7d7e3
......@@ -24,6 +24,7 @@ parts =
service-unicorn
service-sidekiq
service-nginx
service-postgresql
service-redis
......@@ -57,11 +58,21 @@ cert = ${slap-connection:cert-file}
# automatically load all available CPUs
configuration.unicorn_worker_processes = {{ multiprocessing.cpu_count() + 1 }}
configuration.nginx_worker_processes = {{ multiprocessing.cpu_count() }}
# for convenience
[external-url]
recipe = slapos.cookbook:urlparse
url = ${instance-parameter:configuration.external_url}
[backend-info]
host = ${instance-parameter:ipv6-random}
port = 7777
# whether to use http or https - determined by external url
url = ${external-url:scheme}://[${:host}]:${:port}
# current slapuserX
user = {{ pwd.getpwuid(os.getuid())[0] }}
......@@ -207,10 +218,19 @@ context-extra =
[nginx.conf]
<= nginx-etc-template
template= {{ nginx_conf_in }}
context-extra =
section directory directory
raw nginx_mime_types {{ nginx_mime_types }}
Please register or sign in to reply
raw nginx_gitlab_http_conf ${nginx-gitlab-http.conf:rendered}
Please register or sign in to reply
[nginx-gitlab-http.conf]
<= nginx-etc-template
template= {{ nginx_gitlab_http_conf_in }}
context-extra =
section nginx nginx
section gitlab_work gitlab-work
section gitlab_workhorse gitlab-workhorse
section unicorn unicorn
[rack_attack.rb]
<= gitlab-etc-template
......@@ -637,6 +657,74 @@ command-line = ${:rake} gitlab:sidekiq:check
log = ${sidekiq:log}/*.log
######################
# Nginx frontend #
######################
# srv/nginx/ prefix + etc/ log/ ...
[nginx-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/nginx
etc = ${directory:etc}/nginx
log = ${directory:log}/nginx
[nginx-ssl-dir]
recipe = slapos.cookbook:mkdirectory
ssl = ${nginx-dir:etc}/ssl
# contains https key
mode = 0700
# self-signed certificate for https
[nginx-generate-certificate]
# NOTE there is slapos.cookbook:certificate_authority.request but it requires
# to start whole service and has up to 60 seconds latency to generate
# certificate. We only need to run 1 command to do it...
recipe = plone.recipe.command
stop-on-error = true
cert_file = ${nginx-ssl-dir:ssl}/gitlab_backend.crt
key_file = ${nginx-ssl-dir:ssl}/gitlab_backend.key
command =
test -e ${:key_file} || \
{{ openssl_bin }} req -newkey rsa -batch -new -x509 -days 3650 -nodes \
-keyout ${:key_file} -out ${:cert_file}
update-command = ${:command}
[nginx]
srv = ${nginx-dir:srv}
etc = ${nginx-dir:etc}
log = ${nginx-dir:log}
ssl = ${nginx-ssl-dir:ssl}
cert_file = ${nginx-generate-certificate:cert_file}
key_file = ${nginx-generate-certificate:key_file}
[nginx-symlinks]
# (nginx wants <prefix>/logs to be there from start - else it issues alarm to the log)
recipe = cns.recipe.symlink
symlink = ${nginx:log} = ${nginx:srv}/logs
[service-nginx]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/nginx
command-line = {{ nginx_bin }} -p ${nginx:srv} -c ${nginx.conf:rendered}
depend =
${nginx-symlinks:recipe}
${promise-nginx:recipe}
${logrotate-entry-nginx:recipe}
[promise-nginx]
<= promise-byurl
url = ${backend-info:url}/static.css
[logrotate-entry-nginx]
<= logrotate-entry
log = ${nginx:log}/*.log
#############
# cron #
......
software/gitlab/instance.cfg.in
View file @ 85f7d7e3
......@@ -40,6 +40,9 @@ context =
raw gunzip_bin ${gzip:location}/bin/gunzip
raw gzip_bin ${gzip:location}/bin/gzip
raw logrotate_bin ${logrotate:location}/usr/sbin/logrotate
raw nginx_bin ${nginx-output:nginx}
raw nginx_mime_types ${nginx-output:mime}
raw openssl_bin ${openssl-output:openssl}
raw postgresql_location ${postgresql92:location}
raw redis_binprefix ${redis28:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location}
......
software/gitlab/macrolib.cfg.in
View file @ 85f7d7e3
......@@ -13,3 +13,6 @@
process instance parameters ) #}
{% set external_url = urlparse.urlparse(cfg('external_url')) %}
{% set cfg_https = (true if external_url.scheme == 'https' else false) %}
{# for convenience #}
{% set fqdn = external_url.hostname %}
software/gitlab/software.cfg
View file @ 85f7d7e3
......@@ -165,6 +165,7 @@ environment =
recipe = zc.recipe.egg
eggs =
plone.recipe.command
cns.recipe.symlink
[instance.cfg]
......@@ -261,6 +262,7 @@ url = ${:_profile_base_location_}/template/${:_buildout_section_name_}
[versions]
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2
slapos.recipe.template = 2.9
software/gitlab/template/nginx-gitlab-http.conf.in
View file @ 85f7d7e3
......@@ -3,6 +3,8 @@
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb
# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
{% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn with context %}
## GitLab
## Modified from https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl & https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab
##
......@@ -32,13 +34,16 @@
###################################
upstream gitlab {
server unix:<%= @socket %> fail_timeout=0;
server unix:{{ unicorn.socket }} fail_timeout=0;
}
upstream gitlab-workhorse {
server unix:<%= node['gitlab']['gitlab-workhorse']['listen_addr'] %>;
server unix:{{ gitlab_workhorse.socket }};
}
{# not needed for us - the frontend can do the redirection and also
gitlab/nginx speaks HSTS on https port so when we access https port via http
protocol, it gets redirected to https
<% if @https && @redirect_http_to_https %>
## Redirects all HTTP traffic to the HTTPS host
server {
......@@ -52,49 +57,56 @@ server {
error_log <%= @log_directory %>/gitlab_error.log;
}
<% end %>
#}
server {
<% @listen_addresses.each do |listen_address| %>
listen <%= listen_address %>:<%= @listen_port %><% if @https %> ssl spdy<% end %>;
listen [{{ backend_info.host }}]:{{ backend_info.port }}{% if cfg_https %} ssl spdy{% endif %};
{# we don't use: kerbeeros
<% if @kerberos_enabled && @kerberos_use_dedicated_port %>
listen <%= listen_address %>:<%= @kerberos_port %><% if @kerberos_https %> ssl<% end %>;
<% end %>
#}
<% end %>
server_name <%= @fqdn %>;
server_name {{ fqdn }};
server_tokens off; ## Don't show the nginx version number, a security best practice
root /opt/gitlab/embedded/service/gitlab-rails/public;
root {{ gitlab_work.location }}/public;
## Increase this if you want to upload large attachments
## Or if you want to accept large git objects over http
client_max_body_size <%= @client_max_body_size %>;
client_max_body_size {{ cfg('nginx_client_max_body_size') }};
<% if @https %>
{% if cfg_https %}
## Strong SSL Security
## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
ssl on;
ssl_certificate <%= @ssl_certificate %>;
ssl_certificate_key <%= @ssl_certificate_key %>;
ssl_certificate {{ nginx.cert_file }};
ssl_certificate_key {{ nginx.key_file }};
{# we don't need - most root CA will be included by default
<% if @ssl_client_certificate %>
ssl_client_certificate <%= @ssl_client_certificate%>;
<% end %>
#}
# GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
ssl_ciphers '<%= @ssl_ciphers %>';
ssl_protocols <%= @ssl_protocols %>;
ssl_prefer_server_ciphers <%= @ssl_prefer_server_ciphers %>;
ssl_session_cache <%= @ssl_session_cache %>;
ssl_session_timeout <%= @ssl_session_timeout %>;
# NOTE(slapos) ^^^ is not relevant for us - we are behind frontend and clients
# directly connects to frontend
ssl_ciphers '{{ cfg("nginx_ssl_ciphers") }}';
ssl_protocols {{ cfg('nginx_ssl_protocols') }};
ssl_prefer_server_ciphers {{ cfg('nginx_ssl_prefer_server_ciphers') }};
ssl_session_cache {{ cfg('nginx_ssl_session_cache') }};
ssl_session_timeout {{ cfg('nginx_ssl_session_timeout') }};
{# we do not use: ssl_dhparam
<% if @ssl_dhparam %>
ssl_dhparam <%= @ssl_dhparam %>;
<% end %>
<% end %>
#}
{% endif %}
## Individual nginx logs for this GitLab vhost
access_log <%= @log_directory %>/gitlab_access.log gitlab_access;
error_log <%= @log_directory %>/gitlab_error.log;
access_log {{ nginx.log }}/gitlab_access.log gitlab_access;
error_log {{ nginx.log }}/gitlab_error.log;
location / {
## Serve static files from defined root folder.
......@@ -105,21 +117,21 @@ server {
location /uploads/ {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
<%= 'gzip off;' if @https %>
{{ 'gzip off;' if cfg_https else ''}}
## https://github.com/gitlabhq/gitlabhq/issues/694
## Some requests take more than 30 seconds.
proxy_read_timeout <%= @proxy_read_timeout %>;
proxy_connect_timeout <%= @proxy_connect_timeout %>;
proxy_read_timeout {{ cfg('nginx_proxy_read_timeout') }};
proxy_connect_timeout {{ cfg('nginx_proxy_connect_timeout') }};
proxy_redirect off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
<% if @https %>
{% if cfg_https %}
proxy_set_header X-Forwarded-Ssl on;
<% end %>
{% endif %}
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto <%= @https ? "https" : "http" %>;
proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }};
proxy_set_header X-Frame-Options SAMEORIGIN;
proxy_pass http://gitlab;
......@@ -130,21 +142,21 @@ server {
location @gitlab {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
<%= 'gzip off;' if @https %>
{{ 'gzip off;' if cfg_https else ''}}
## https://github.com/gitlabhq/gitlabhq/issues/694
## Some requests take more than 30 seconds.
proxy_read_timeout <%= @proxy_read_timeout %>;
proxy_connect_timeout <%= @proxy_connect_timeout %>;
proxy_read_timeout {{ cfg('nginx_proxy_read_timeout') }};
proxy_connect_timeout {{ cfg('nginx_proxy_connect_timeout') }};
proxy_redirect off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
<% if @https %>
{% if cfg_https %}
proxy_set_header X-Forwarded-Ssl on;
<% end %>
{% endif %}
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto <%= @https ? "https" : "http" %>;
proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }};
proxy_set_header X-Frame-Options SAMEORIGIN;
proxy_pass http://gitlab;
......@@ -198,21 +210,21 @@ server {
client_max_body_size 0;
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
<%= 'gzip off;' if @https %>
{{ 'gzip off;' if cfg_https else ''}}
## https://github.com/gitlabhq/gitlabhq/issues/694
## Some requests take more than 30 seconds.
proxy_read_timeout <%= @proxy_read_timeout %>;
proxy_connect_timeout <%= @proxy_connect_timeout %>;
proxy_read_timeout {{ cfg('nginx_proxy_read_timeout') }};
proxy_connect_timeout {{ cfg('nginx_proxy_connect_timeout') }};
proxy_redirect off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
<% if @https %>
{% if cfg_https %}
proxy_set_header X-Forwarded-Ssl on;
<% end %>
{% endif %}
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto <%= @https ? "https" : "http" %>;
proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }};
proxy_pass http://gitlab-workhorse;
}
......@@ -223,7 +235,7 @@ server {
## See config/application.rb under "Relative url support" for the list of
## other files that need to be changed for relative url support
location ~ ^/(assets)/ {
root /opt/gitlab/embedded/service/gitlab-rails/public;
root {{ gitlab_work.location }}/public;
gzip_static on; # to serve pre-gzipped version
expires max;
add_header Cache-Control public;
......@@ -232,5 +244,7 @@ server {
error_page 502 /502.html;
{# we don't support custom nginx configs
<%= @custom_gitlab_server_config %>
#}
}
software/gitlab/template/nginx.conf.in
View file @ 85f7d7e3
......@@ -4,47 +4,47 @@
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
user <%= node['gitlab']['web-server']['username'] %> <%= node['gitlab']['web-server']['group']%>;
worker_processes <%= @worker_processes %>;
{% from 'macrolib.cfg.in' import cfg with context %}
# user directive makes sense only when running initially as root
# (and nginx will complain if not and directive given)
# user {{ backend_info.user }};
worker_processes {{ cfg('nginx_worker_processes') }};
error_log stderr;
pid nginx.pid;
pid {{ directory.run }}/nginx.pid;
daemon off;
events {
worker_connections <%= @worker_connections %>;
worker_connections {{ cfg('nginx_worker_connections') }};
}
http {
log_format gitlab_access '<%= @gitlab_access_log_format %>';
log_format gitlab_access '{{ cfg("nginx_log_format") }}';
{# we do not use: ci, mattermost
log_format gitlab_ci_access '<%= @gitlab_ci_access_log_format %>';
log_format gitlab_mattermost_access '<%= @gitlab_mattermost_access_log_format %>';
#}
sendfile <%= @sendfile %>;
tcp_nopush <%= @tcp_nopush %>;
tcp_nodelay <%= @tcp_nodelay %>;
keepalive_timeout <%= @keepalive_timeout %>;
gzip <%= @gzip %>;
gzip_http_version <%= @gzip_http_version %>;
gzip_comp_level <%= @gzip_comp_level %>;
gzip_proxied <%= @gzip_proxied %>;
gzip_types <%= @gzip_types.join(' ') %>;
sendfile {{ cfg('nginx_sendfile') }};
tcp_nopush {{ cfg('nginx_tcp_nopush') }};
tcp_nodelay {{ cfg('nginx_tcp_nodelay') }};
include /opt/gitlab/embedded/conf/mime.types;
keepalive_timeout {{ cfg('nginx_keepalive_timeout') }};
<% if @gitlab_http_config %>
include <%= @gitlab_http_config %>;
<% end %>
gzip {{ cfg('nginx_gzip') }};
gzip_http_version {{ cfg('nginx_gzip_http_version') }};
gzip_comp_level {{ cfg('nginx_gzip_comp_level') }};
gzip_proxied {{ cfg('nginx_gzip_proxied') }};
gzip_types {{ cfg('nginx_gzip_types') }};
<% if @gitlab_ci_http_config %>
include <%= @gitlab_ci_http_config %>;
<% end %>
include {{ nginx_mime_types }};
<% if @gitlab_mattermost_http_config %>
include <%= @gitlab_mattermost_http_config %>;
<% end %>
include {{ nginx_gitlab_http_conf }};
<%= @custom_nginx_config %>
{# we don't need: ci, mattermost
include <%= @gitlab_ci_http_config %>
include <%= @gitlab_mattermost_http_config %>
#}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7