Update Release Candidate

component/git/buildout.cfg

@@ -17,8 +17,8 @@ parts =
 
 [git]
 recipe = slapos.recipe.cmmi
-url = https://www.kernel.org/pub/software/scm/git/git-2.9.0.tar.xz
-md5sum = 118ef1e3108ef0b858cd13b74395a59c
+url = https://www.kernel.org/pub/software/scm/git/git-2.9.2.tar.xz
+md5sum = dc5eb7ff2ca4641c6038f51ae49cad0f
 configure-options =
   --with-curl=${curl:location}
   --with-openssl=${openssl:location}

component/postgresql/buildout.cfg

@@ -27,11 +27,13 @@ configure-options =
   --without-ossp-uuid
   --without-libxml
   --without-libxslt
+# build core PostgreSQL + pg_trgm contrib extension for GitLab
+make-targets = install && make -C contrib/pg_trgm/ install
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${perl:location}/libs-c -Wl,-rpath=${perl:location}/libs-c
 
 [postgresql92]
 <= postgresql-common
-url = http://ftp.postgresql.org/pub/source/v9.2.16/postgresql-9.2.16.tar.bz2
-md5sum = a6ced0ccd4ce3279acfef9bac16a95ee
+url = http://ftp.postgresql.org/pub/source/v9.2.17/postgresql-9.2.17.tar.bz2
+md5sum = a75d4a82eae1edda04eda2e60656e74c

software/cloudooo/software.cfg.json

+{
+    "name": "cloudooo",
+    "description": "Clusterised cloudooo",
+    "serialisation": "xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Cloudooo",
+            "request": "instance-cloudooo-input-schema.json",
+            "response": "instance-cloudooo-output-schema.json",
+            "index": 0
+        }
+    }
+}

software/erp5testnode/instance-default.cfg

 [buildout]
 
+
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+extends = ${monitor2-template:rendered}
+
 parts =
-  connection-dict
   testnode
   shell
   shellinabox
   certificate-authority
   ca-shellinabox
   ca-httpd
+  monitor-base
+  monitor-publish
   
-[connection-dict]
+[monitor-publish]
 recipe = slapos.cookbook:publish
 url = http://[$${shellinabox:ipv6}]:$${shellinabox:port}/
 password = $${pwgen:passwd}

software/erp5testnode/instance-erp5testnode-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+
+  "title": "Input Parameters",
+  "properties": {
+    "test-node-title": {
+      "title": "Test Node Title",
+      "description": "Name for the Test Node which will be used at ERP5.",
+      "type": "string"
+    },
+    "node-quantity": {
+      "title": "Quantity of Parallel runs",
+      "description": "QUantity of Parallel tests will be executed on this Node.",
+      "type": "integer"
+    },
+    "test-suite-master-url": {
+      "title": "Task Distribution URL",
+      "description": "Url for the task distributor master on portal_task_distribution",
+      "type": "string"
+    }
+  }
+}

software/erp5testnode/instance-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Apache Frontend instanciation",
+  "properties": {
+    "password": {
+      "description": "Password to access shellinabox.",
+      "type": "string"
+    },
+    "url": {  
+      "description": "IPv6 URL to access the shell in a box.",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/erp5testnode/software.cfg

@@ -8,6 +8,7 @@ extends =
   ../../component/shellinabox/buildout.cfg
   ../../component/pwgen/buildout.cfg
   ../../component/apache/buildout.cfg
+  ../../stack/monitor/buildout.cfg
 
 parts =
   slapos-cookbook
@@ -55,9 +56,9 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-default.cfg
 output = ${buildout:directory}/template-default.cfg
 mode = 0644
-md5sum = 43ca4026ee1aba5d20bb9e1b69d2f114 
+md5sum = 621c84f7adcba394686acbaf44cc1d7d
 
 [versions]
 PyXML = 0.8.5
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 slapos.recipe.template = 2.7

software/erp5testnode/software.cfg.json

+{
+	"name": "ERP5 Test Node",
+	"description": "ERP5 Test Node Runner",
+	"serialisation": "xml",
+	"software-type": {
+		"default": {
+			"title": "Default",
+      "description": "Default setup for ERP5TestNode Instance.",
+			"request": "instance-erp5testnode-input-schema.json",
+			"response": "instance-output-schema.json"
+		}
+	}
+}

software/gitlab/gitlab-parameters.cfg

@@ -6,7 +6,7 @@
 #
 # TODO better autogenerate from ^^^ (?)
 #
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 [gitlab-parameters]
 configuration.external_url              = http://lab.example.com
@@ -101,3 +101,8 @@ configuration.nginx_gzip_comp_level     = 2
 configuration.nginx_gzip_proxied        = any
 configuration.nginx_gzip_types          = text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json
 configuration.nginx_keepalive_timeout   = 65
+
+# TODO allow configuring trusted proxies
+# configuration.nginx_real_ip_trusted_addresses
+# configuration.nginx_real_ip_header
+# configuration.nginx_real_ip_recursive

software/gitlab/gitlab-unicorn-startup.in

@@ -9,6 +9,15 @@ die() {
     exit 1
 }
 
+# run psql on gitlab db
+psql() {
+    {{ psql_bin }}  \
+        -h {{ pgsql['pgdata-directory'] }}  \
+        -U {{ pgsql.superuser }}            \
+        -d {{ pgsql.dbname }}               \
+        "$@"
+}
+
 # 1. what to do when instance is initially setup
 # see
 #   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/database_migrations.rb
@@ -18,12 +27,7 @@ die() {
 # ( first quering PG several times waiting a bit till postgresql is started and ready )
 tpgwait=5
 while true; do
-    pgtables="$({{ psql_bin }}   \
-        -h {{ pgsql['pgdata-directory'] }}  \
-        -U {{ pgsql.superuser }}            \
-        -d {{ pgsql.dbname }}               \
-        -c '\d')" && break
-
+    pgtables="$(psql -c '\d')" && break
     tpgwait=$(( $tpgwait - 1 ))
     test $tpgwait = 0 && die "pg query problem"
     echo "I: PostgreSQL is not ready (yet ?); will retry $tpgwait times..." 1>&2
@@ -31,6 +35,9 @@ while true; do
 done
 echo "I: PostgreSQL ready." 1>&2
 
+# make sure pg_trgm extension is enabled for gitlab db
+psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'   || die "pg_trgm setup failed"
+
 if echo "$pgtables" | grep -q '^No relations found' ; then
     $RAKE db:schema:load db:seed_fu  || die "initial db setup failed"
 fi

software/gitlab/instance-gitlab.cfg.in

@@ -197,10 +197,6 @@ rendered= ${gitlab:etc}/${:_buildout_section_name_}
 rendered= ${nginx:etc}/${:_buildout_section_name_}
 
 
-[config.ru]
-<= gitlab-etc-template
-template = {{ config_ru_in }}
-
 [database.yml]
 <= gitlab-etc-template
 template= {{ database_yml_in }}
@@ -366,12 +362,10 @@ update-command =
 <= work-base
 software = {{ gitlab_repository_location }}
 tune-command =
-# secret* config.ru tmp/ log/ shared/ builds/
+# secret* tmp/ log/ shared/ builds/
     rm -f .secret  &&
-    rm -f config.ru  &&
     rm -rf log tmp shared builds  &&
     ln -sf ${secrets:secrets}/gitlab_rails_secret .secret  &&
-    ln -sf ${config.ru:rendered} config.ru  &&
     ln -sf ${gitlab:log} log  &&
     ln -sf ${gitlab:tmp} tmp  &&
     ln -sf ${gitlab:shared} shared  &&
@@ -633,7 +627,7 @@ log     = ${sidekiq-dir:log}
 # NOTE see queue list here:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 [service-sidekiq]
 recipe  = slapos.cookbook:wrapper
 wrapper-path    = ${directory:service}/sidekiq

software/gitlab/instance.cfg.in

@@ -51,7 +51,6 @@ context =
     raw watcher_sigkill             ${watcher-sigkill:rendered}
 
 # config files
-    raw config_ru_in                ${config.ru.in:target}
     raw database_yml_in             ${database.yml.in:target}
     raw gitconfig_in                ${gitconfig.in:target}
     raw gitlab_parameters_cfg       ${gitlab-parameters.cfg:target}

software/gitlab/software.cfg

@@ -23,7 +23,7 @@ extends =
 
 parts =
     ruby2.1
-    golang15
+    golang16
     git
     postgresql92
     redis28
@@ -113,26 +113,26 @@ git-executable = ${git:location}/bin/git
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-ce.git
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
-# 8.5.X + NXD patches:
-revision = v8.5.3-15-g827d39147af496bcd4696c845e9094a07b5a09d3
+# 8.7.X + NXD patches:
+revision = v8.7.9-10-g7728df66b90483dac467df95948a532783e782c8
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-shell.git
 repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
-# gitlab 8.5 wants gitlab-shell 2.6.10
-# 2.6.10 + no-hooks.old patch
-revision = v2.6.10-13-gcb8f331c955aa780efe10263d0d45b332f030f42
+# gitlab 8.7 wants gitlab-shell 2.7.2
+# 2.7.2 + NXD patches
+revision = v2.6.10-50-gfbca95be784816349abc5930324659151eca50d1
 location = ${buildout:parts-directory}/gitlab-shell
 
 [gitlab-workhorse-repository]
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-workhorse.git
 repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
-# 0.6.4 + NXD patches for blob download speedup
-# (https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/17)
-revision = v0.6.4-3-g32ec77c4db3556dc66e8c863326325f8ec970849
+# gitlab 8.7 wants gitlab-workhorse 0.7.1
+# 0.7.1 + NXD patches
+revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48
 location = ${buildout:parts-directory}/gitlab-workhorse
 
 
@@ -175,7 +175,7 @@ configure-command = :
 make-targets= ${:_buildout_section_name_}
 
 environment =
-  PATH=${golang15:location}/bin:%(PATH)s
+  PATH=${golang16:location}/bin:%(PATH)s
 
 
 ###############################
@@ -194,7 +194,7 @@ eggs    =
 recipe  = slapos.recipe.template
 url     = ${:_profile_base_location_}/instance.cfg.in
 output  = ${buildout:directory}/instance.cfg
-md5sum  = e5e7ddede71265987bb0c534b009aa45
+md5sum  = ef85f02c4f6070c586d773b859a2f4e2
 
 [watcher-sigkill]
 recipe  = slapos.recipe.template:jinja2
@@ -222,37 +222,33 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
 url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
 
 
-[config.ru.in]
-<= download-template
-md5sum  = 91737a2067be80f3ae48157e5a0738b4
-
 [database.yml.in]
 <= download-template
-md5sum  = 76d8f20532d63282ecd3617a3937fcf1
+md5sum  = 61d1d04b9347b3168a1ad7676e4681ef
 
 [gitconfig.in]
 <= download-template
-md5sum  = d8b3611386c4982605edd9a31832ee28
+md5sum  = eb1230fee50067924ba89f4dc6e82fa9
 
 [gitlab-parameters.cfg]
 <= download-file
-md5sum  = b04b72949f0b9b5da96870ffbf8c1ff4
+md5sum  = 3edd435a984b51b94539ea1d4f1b3994
 
 [gitlab-shell-config.yml.in]
 <= download-template
-md5sum  = 9c62aa1bf7396f207a528e39973aa135
+md5sum  = 58c09b1e609f903e483a76fe9e57366c
 
 [gitlab-unicorn-startup.in]
 <= download-file
-md5sum  = 14c5632182d830c03f7788c85d6f4da1
+md5sum  = a9cb347f60aad3465932fd36cd4fe25d
 
 [gitlab.yml.in]
 <= download-template
-md5sum  = 8f1c52ed223ee2c2d07c6c79bcd2f6c0
+md5sum  = 735a78d0733fd6617d3b5f3d91bfae8c
 
 [instance-gitlab.cfg.in]
 <= download-file
-md5sum  = d3158d58e7fea9a656a8ef6e238a919b
+md5sum  = ba83f01fd7a313b984766c50d6e48e54
 
 [macrolib.cfg.in]
 <= download-file
@@ -260,27 +256,27 @@ md5sum  = a56a44e96f65f5ed20211bb6a54279f4
 
 [nginx-gitlab-http.conf.in]
 <= download-template
-md5sum  = be7ddd26b0b0196fde8d699e50e84b2f
+md5sum  = a11b50d2ff2b1fa842ba4aa20041e2fe
 
 [nginx.conf.in]
 <= download-template
-md5sum  = 71b4221cd91e0e1e20a72b97991f2665
+md5sum  = 7da68dba86fff79eb93c27aa1aaf1055
 
 [rack_attack.rb.in]
 <= download-template
-md5sum  = 2cf56b210a906cee80f86cde7112f468
+md5sum  = bc1a7c1e83b7329d97bff6724f2bec3e
 
 [resque.yml.in]
 <= download-template
-md5sum  = 1fa21655a845602f68a901f23a1da89a
+md5sum  = 7c89a730889e3224548d9abe51a2d719
 
 [smtp_settings.rb.in]
 <= download-template
-md5sum  = 07b640122b8c4840e853360f2479d0ac
+md5sum  = d66a424516ffacea34303e2f512a7d94
 
 [unicorn.rb.in]
 <= download-template
-md5sum  = 4f7c14147fd60b4cc9aefe7f64524abe
+md5sum  = 83921db1835d9e81cbbe808631cc40a9
 
 
 [versions]

software/gitlab/template/config.ru.in

-{{ autogenerated }}
-# see:
-# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config.ru
-# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-rails-config.ru.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
-
-# This file is used by Rack-based servers to start the application.
-
-{% from 'macrolib.cfg.in' import cfg  with context %}
-
-if defined?(Unicorn)
-  require 'unicorn'
-
-  if ENV['RAILS_ENV'] == 'production' || ENV['RAILS_ENV'] == 'staging'
-    # Unicorn self-process killer
-    require 'unicorn/worker_killer'
-
-    # Max memory size (RSS) per worker
-    use Unicorn::WorkerKiller::Oom, ({{ cfg('unicorn_worker_memory_limit_min') }}), ({{ cfg('unicorn_worker_memory_limit_max') }})
-  end
-end
-
-require ::File.expand_path('../config/environment',  __FILE__)
-
-map ENV['RAILS_RELATIVE_URL_ROOT'] || "/" do
-  run Gitlab::Application
-end

software/gitlab/template/database.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/database.yml.erb
-# (last updated for 8.5.1+ce.0-1-ge732b39)
+# (last updated for 8.7.9+ce.1-0-gf589ad7)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 
@@ -23,5 +23,6 @@ production:
   socket:
   {# not needed for unix socket
   sslmode: <%= single_quote(@db_sslmode) %>
-  sslrootcert: <%= single_quote(@db_sslrootcert) %>
+  sslrootcert: <%= single_quote(@db_sslrootcert) || single_quote(@db_sslca) %>
+  sslca: <%= single_quote(@db_sslca) || single_quote(@db_sslrootcert) %>
   #}

software/gitlab/template/gitconfig.in

@@ -3,7 +3,7 @@
 # see:
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitconfig.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 #
 {% from 'macrolib.cfg.in' import cfg  with context %}
 
@@ -22,3 +22,5 @@
         email = {{ cfg('email_from') }}
 [core]
         autocrlf = input
+[gc]
+        auto = 0

software/gitlab/template/gitlab-shell-config.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-shell-config.yml.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 # GitLab user. git by default
 user: {{ backend_info.user }}

software/gitlab/template/gitlab.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_https, external_url  with context %}
 
@@ -32,6 +32,16 @@ production: &base
     relative_url_root: <%= @gitlab_relative_url %>
     #}
 
+    # Trusted Proxies
+    # Customize if you have GitLab behind a reverse proxy which is running on a different machine.
+    # Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
+    trusted_proxies:
+    {# TODO support configuring trusted proxies
+<% @trusted_proxies.each do |proxy| %>
+      - <%= proxy %>
+<% end %>
+    #}
+
     # Uncomment and customize if you can't use the default user to run GitLab (default: 'git')
     user: {{ backend_info.user }}
 
@@ -96,7 +106,7 @@ production: &base
     enabled: <%= @incoming_email_enabled %>
 
     # The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to.
-    # The `%{key}` placeholder is added after the user part, after a `+` character, before the `@`.
+    # The placeholder can be omitted but if present, it must appear in the "user" part of the address (before the `@`).
     address: <%= single_quote(@incoming_email_address) %>
 
     # Email account username
@@ -124,7 +134,7 @@ production: &base
   artifacts:
     enabled: <%= @artifacts_enabled %>
     # The location where Build Artifacts are stored (default: shared/artifacts).
-    storage_path: <%= @artifacts_path %>
+    path: <%= @artifacts_path %>
   #}
 
   {# we do not support LFS
@@ -167,7 +177,7 @@ production: &base
     #}
 
 
-  {# XXX cron jobs are disabled for now - we do not support CI and EE features
+  {# XXX cron jobs are disabled for now - we do not support CI and EE features or we are ok with defaults
   ## Auxiliary jobs
   # Periodically executed jobs, to self-heal GitLab, do external synchronizations, etc.
   # Please read here for more information: https://github.com/ondrejbartas/sidekiq-cron#adding-cron-job
@@ -176,6 +186,10 @@ production: &base
     stuck_ci_builds_worker:
       cron: <%= @stuck_ci_builds_worker_cron %>
 
+    # Remove outdated repository archives
+    repository_archive_cache_worker:
+      cron: <%= @repository_archive_cache_worker_cron %>
+
     ##
     # GitLab EE only jobs:
 
@@ -187,11 +201,20 @@ production: &base
     update_all_mirrors_worker:
       cron: <%= @update_all_mirrors_worker_cron %>
 
+    # Update remote mirrors
+    update_all_remote_mirrors_worker:
+      cron: <%= @update_all_remote_mirrors_worker_cron %>
+
     # In addition to refreshing users when they log in,
     # periodically refresh LDAP users membership.
     # NOTE: This will only take effect if LDAP is enabled
     ldap_sync_worker:
       cron: <%= @ldap_sync_worker_cron %>
+
+    # Gitlab Geo nodes notification worker
+    # NOTE: This will only take effect if Geo is enabled
+    geo_bulk_notify_worker:
+      cron: <%= @geo_bulk_notify_worker_cron %>
   #}
 
   #
@@ -223,6 +246,7 @@ production: &base
     enabled: false
     {# just disabled
     enabled: <%= @ldap_enabled %>
+    sync_time: <%= @ldap_sync_time %>
   <% if @ldap_servers.any? %>
     servers:
     <% @ldap_servers.each do |provider_id, settings| %>
@@ -302,6 +326,12 @@ production: &base
     # (default: false)
     auto_link_saml_user: <%= @omniauth_auto_link_saml_user.to_json %>
 
+    # Set different Omniauth providers as external so that all users creating accounts
+    # via these providers will not be able to have access to internal projects. You
+    # will need to use the full name of the provider, like `google_oauth2` for Google.
+    # Refer to the examples below for the full names of the supported providers.
+    # (default: [])
+    external_providers: <%= @omniauth_external_providers.to_json %>
 
     ## Auth providers
     # Uncomment the following lines and fill in the data of the auth provider you want to use

software/gitlab/template/nginx-gitlab-http.conf.in

 {{ autogenerated }}
 # see:
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn  with context %}
 
@@ -66,7 +66,6 @@ server {
 
   server_name {{ fqdn }};
   server_tokens off; ## Don't show the nginx version number, a security best practice
-  root {{ gitlab_work.location }}/public;
 
   ## Increase this if you want to upload large attachments
   ## Or if you want to accept large git objects over http
@@ -100,6 +99,20 @@ server {
   #}
   {% endif %}
 
+  ## Real IP Module Config
+  ## http://nginx.org/en/docs/http/ngx_http_realip_module.html
+  {# TODO support trusted proxies & realip
+  <% if @real_ip_header %>
+  real_ip_header <%= @real_ip_header %>;
+  <% end %>
+  <% if @real_ip_recursive %>
+  real_ip_recursive <%= @real_ip_recursive %>;
+  <% end %>
+  <% @real_ip_trusted_addresses.each do |trusted_address| %>
+  set_real_ip_from <%= trusted_address %>;
+  <% end %>
+  #}
+
   ## Individual nginx logs for this GitLab vhost
   access_log  {{ nginx.log }}/gitlab_access.log gitlab_access;
   error_log   {{ nginx.log }}/gitlab_error.log;
@@ -135,6 +148,15 @@ server {
     proxy_pass http://gitlab-workhorse;
   }
 
+  error_page 404 /404.html;
+  error_page 422 /422.html;
+  error_page 500 /500.html;
+  error_page 502 /502.html;
+  location ~ ^/(404|422|500|502)\.html$ {
+    root {{ gitlab_work.location }}/public;
+    internal;
+  }
+
   {# we don't support custom nginx configs
   <%= @custom_gitlab_server_config %>
   #}

software/gitlab/template/nginx.conf.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 
@@ -41,6 +41,13 @@ http {
 
   include {{ nginx_mime_types }};
 
+  {# we do not do nginx caching:
+     - gitlab-workhorse serves raw blobs fast
+     - we have caches on frontend nodes
+  proxy_cache_path <%= @proxy_cache_path %>;
+  proxy_cache <%= @proxy_cache %>;
+  #}
+
   include {{ nginx_gitlab_http_conf }};
 
   {# we don't need: ci, pages, mattermost

software/gitlab/template/rack_attack.rb.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/resque.yml.in

@@ -2,6 +2,6 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb
-# (last udpdated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last udpdated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 production: unix://{{ redis.unixsocket }}

software/gitlab/template/smtp_settings.rb.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_bool  with context %}
 
@@ -18,6 +18,7 @@ if Rails.env.production?
     domain:                 "{{ cfg('smtp_domain') }}",
     authentication:         :{{ cfg('smtp_authentication') }},
     enable_starttls_auto:   {{ cfg('smtp_enable_starttls_auto') }},
+    # ssl:
     openssl_verify_mode:    '{{ cfg("smtp_openssl_verify_mode") }}'
     # ca_path:
     # ca_file:

software/gitlab/template/unicorn.rb.in

@@ -3,7 +3,7 @@
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
-# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
+# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 
@@ -79,3 +79,9 @@ ENV['RAILS_RELATIVE_URL_ROOT'] = "<%= @relative_url %>"
 
 <%- end %>
 #}
+
+# Min memory size (RSS) per worker
+ENV['GITLAB_UNICORN_MEMORY_MIN'] = ({{ cfg('unicorn_worker_memory_limit_min') }}).to_s
+
+# Max memory size (RSS) per worker
+ENV['GITLAB_UNICORN_MEMORY_MAX'] = ({{ cfg('unicorn_worker_memory_limit_max') }}).to_s

software/kvm/common.cfg

@@ -155,7 +155,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
 mode = 644
-md5sum = ff281bf8a8905632b32254622db105b7
+md5sum = 4c71fbd0f27616d945df9669c57d302e
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-export.cfg.jinja2

@@ -23,14 +23,6 @@ parts +=
 {% if monitor -%}
 # monitor parts
   monitor-base
-  cron-entry-logrotate
-  certificate-authority
-  monitor-conf
-  start-monitor
-  ca-httpd
-  monitor-httpd-promise
-  monitor-httpd-promise-conf
-  monitor-status2rss-cron-entry
 {% endif %}
 
 # Create the exporter executable, which is a simple shell script

software/kvm/instance-kvm-input-schema.json

@@ -198,7 +198,8 @@
     "bootstrap-script-url": {
       "title": "VM bootstrap script URL.",
       "description": "If dedicaced disk image is used, this script will be downloaded and run at first boot of the virtual machine. Set md5sum to URL like: http://example.com/file#MD5SUM. Get file in vm at http://10.0.2.100/vm-bootstrap.",
-      "type": "uri"
+      "type": "string",
+      "format": "uri"
     },
     "data-to-vm": {
       "title": "Text content to send to this virtual machine.",

software/slapos-testing/instance.cfg

@@ -56,6 +56,10 @@ repository = ${slapos.recipe.build-repository:location}
 <= download-source
 repository = ${slapos.recipe.cmmi-repository:location}
 
+[slapos.toolbox]
+<= download-source
+repository = ${slapos.toolbox-repository:location}
+
 [erp5-util]
 <= download-source
 repository = ${erp5-util-repository:location}
@@ -78,6 +82,7 @@ test-list =
   $${slapos.recipe.template:location}
   $${slapos.recipe.build:location}
   $${slapos.recipe.cmmi:location}
+  $${slapos.toolbox:location}
   $${erp5-util:location}
 prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
 environment = environment
@@ -86,7 +91,7 @@ environment = environment
 CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include
 LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${python2.7:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${libxslt:location}/lib -Wl,-rpath=${zlib:location}/lib
 LD_LIBRARY_PATH = ${python2.7:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${libxslt:location}/lib:${zlib:location}/lib
-PYTHONPATH = ${python-setuptools:pythonpath}:${buildout:eggs-directory}
+PYTHONPATH = ${python-setuptools:pythonpath}:${buildout:eggs-directory}:${buildout:develop-eggs-directory}
 LOCAL_IPV4 = $${slap-configuration:ipv4-random}
 
 [sh-environment]

software/slapos-testing/software.cfg

@@ -17,6 +17,7 @@ parts =
   slapos.recipe.template-repository
   slapos.recipe.build-repository
   slapos.recipe.cmmi-repository
+  slapos.toolbox-repository
   erp5-util-repository
   eggs
   phantomjs
@@ -26,6 +27,7 @@ parts =
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
+  ${python-cryptography:egg}
   Jinja2
   erp5.util
   slapos.cookbook
@@ -33,6 +35,7 @@ eggs =
   plone.recipe.command
   slapos.recipe.template
   slapos.recipe.cmmi
+  slapos.toolbox
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
 scripts =
@@ -64,6 +67,10 @@ repository = https://lab.nexedi.com/nexedi/slapos.recipe.build.git
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.recipe.cmmi.git
 
+[slapos.toolbox-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
+
 [erp5-util-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/erp5.git
@@ -71,12 +78,12 @@ repository = https://lab.nexedi.com/nexedi/erp5.git
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 19b76f8a9f12be111e3a6600b337c21b
+md5sum = 1674921ba26e80d5e5266b71271c7ed1
 output =  ${buildout:directory}/template.cfg
 mode = 640
 
 [versions]
-Pygments = 1.6
+Pygments = 2.1.3
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.7

software/slaprunner/common.cfg

@@ -54,7 +54,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = f2d704c269244c4eb842aefbc9ff9201
+md5sum = 5fbdf6f9996d6cb948ba042e9dd6e43e
 mode = 0644
 
 [template-runner-import-script]
@@ -70,7 +70,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-import.cfg.in
 output = ${buildout:directory}/instance-runner-import.cfg
-md5sum = 673c30e5e7f9b7bb543f79465a56e43d
+md5sum = a41ff9e12a2304224704f6f31529879b
 mode = 0644
 
 [template-runner-export-script]
@@ -86,13 +86,13 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-export.cfg.in
 output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = 4b0ab39bc655ae0b865207147cb2e5bf
+md5sum = cab358589975d6f250b6363ecc95aab2
 mode = 0644
 
 [template-resilient]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
-md5sum = 7b3f74b6ab491e907bbdef2ff40a7791
+md5sum = a902b84ac7d1e29a7fdb06cbc7dec150
 filename = instance-resilient.cfg.jinja2
 mode = 0644
 
@@ -110,7 +110,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/nginx_conf.in
 download-only = true
-md5sum = e6c2e1dd2153afefa0805c4065066e4e
+md5sum = 94d83ef3eb89c2d75c8c079ab12b4518
 filename = nginx_conf.in
 mode = 0644
 
@@ -119,7 +119,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/httpd_conf.in
 download-only = true
-md5sum = 21ef5c7c487bdc774d675e33d000975f
+md5sum = 2e8440fa4b589be649a72108faec7745
 filename = httpd_conf.in
 mode = 0644
 
@@ -135,7 +135,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/template/${:filename}
-md5sum = f406b91c5b0261b198a1c1930c195071
+md5sum = ef16446d432e1397182b1654fe920ffb
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = slapos.cfg.in
 download-only = true
@@ -203,6 +203,7 @@ eggs =
   slapos.recipe.build
   slapos.toolbox[flask_auth]
   Gunicorn
+  futures
   ${slapos-cookbook:eggs}
 
 [extra-eggs]

software/slaprunner/httpd_conf.in

@@ -102,7 +102,7 @@ RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
 RewriteCond %{REQUEST_URI} /git-receive-pack$
 
 <LocationMatch "^/git/">
-        SetEnv GIT_PROJECT_ROOT:{{ parameters.project_private_folder }}
+        SetEnv GIT_PROJECT_ROOT {{ parameters.project_private_folder }}
         Order Deny,Allow
         Deny from env=AUTHREQUIRED
 
@@ -113,7 +113,7 @@ RewriteCond %{REQUEST_URI} /git-receive-pack$
 </LocationMatch>
 
 <LocationMatch "^/git-public/">
-        SetEnv GIT_PROJECT_ROOT:{{ parameters.project_public_folder }}
+        SetEnv GIT_PROJECT_ROOT {{ parameters.project_public_folder }}
         Order Deny,Allow
         Deny from env=AUTHREQUIRED
 

software/slaprunner/instance-resilient.cfg.jinja2

@@ -11,7 +11,8 @@
 {% if number_of_instances > 2 %}
   {% set number_of_instances = 2 %}
 {% endif %}
-{% set slaprunner_return = ['url', 'ssh-public-key', 'ssh-url', 'notification-id', 'ip', 'backend_url', 'url', 'ssh_command', 'access_url', '1_info', '2_info', 'webdav_url', 'public_url', 'git_public_url', 'git_private_url'] -%}
+
+{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-command', 'webdav-url', 'public-url', 'git-public-url', 'git-private-url'] -%}
 {% set monitor_return = ['monitor-base-url', 'monitor-url', 'monitor-user', 'monitor-password'] -%}
 {% set monitor_parameter = {'monitor-cors-domains': slapparameter_dict.pop('monitor-cors-domains', "monitor.app.officejs.com")} -%}
 {% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return, 'set-monitor-url': True} -%}
@@ -55,21 +56,20 @@ return =  {{ slaprunner_return | join(' ')}} {{ monitor_return | join(' ') }}
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-1_info = ${request-runner:connection-1_info}
-2_info = ${request-runner:connection-2_info}
-backend_url = ${request-runner:connection-backend_url}
-access_url = ${request-runner:connection-access_url}
+backend-url = ${request-runner:connection-backend-url}
 url = ${request-runner:connection-url}
-ssh_command = ${request-runner:connection-ssh_command}
-webdav_url = ${request-runner:connection-webdav_url}
-public_url = ${request-runner:connection-public_url}
-git_public_url = ${request-runner:connection-git_public_url}
-git_private_url = ${request-runner:connection-git_private_url}
+init-user = ${request-runner:connection-init-user}
+init-password = ${request-runner:connection-init-password}
+ssh-command = ${request-runner:connection-ssh-command}
+webdav-url = ${request-runner:connection-webdav-url}
+public-url = ${request-runner:connection-public-url}
+git-public-url = ${request-runner:connection-git-public-url}
+git-private-url = ${request-runner:connection-git-private-url}
 {% if slapparameter_dict.get('custom-frontend-backend-url') -%}
 custom-frontend-url = ${request-runner:connection-custom-frontend-url}
 {% endif %}
 monitor-base-url = ${request-runner:connection-monitor-base-url}
-monitor_setup_url = {{ monitor_interface_url }}/#page=settings_configurator&url=${request-runner:connection-monitor-url}&username=${request-runner:connection-monitor-user}&password=${request-runner:connection-monitor-password}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${request-runner:connection-monitor-url}&username=${request-runner:connection-monitor-user}&password=${request-runner:connection-monitor-password}
 
 [slap-parameter]
 # Default parameters for distributed deployment

software/slaprunner/instance-runner-export.cfg.in

@@ -23,7 +23,6 @@ parts +=
   symlinks
   shellinabox
   slapos-cfg
-  slapos-repo
   cron-entry-prepare-software
   deploy-instance-parameters
   instance-software
@@ -56,7 +55,6 @@ context =
 monitor-httpd-port = 8437
 # Pass some parameter to dispay in monitoring interface
 instance-configuration =
-  file recovery-code $${recovery-code:storage-path}
   httpdcors cors-domain $${slaprunner-httpd-cors:location} $${httpd-graceful-wrapper:output}
   raw webrunner-url https://$${request-frontend:connection-domain}
 

software/slaprunner/instance-runner-import.cfg.in

@@ -19,7 +19,6 @@ parts +=
   shellinabox
   symlinks
   slapos-cfg
-  slapos-repo
   cron-entry-prepare-software
   deploy-instance-parameters
   instance-software-type

software/slaprunner/instance-runner.cfg

@@ -20,7 +20,6 @@ parts =
   symlinks
   shellinabox
   slapos-cfg
-  slapos-repo
   cron-entry-prepare-software
   deploy-instance-parameters
   instance-software
@@ -120,12 +119,6 @@ sessions = $${buildout:directory}/.sessions
 private-project = $${:home}/.git-private
 public-project = $${:home}/.git-public
 
-#Create password recovery code for slaprunner
-[recovery-code]
-recipe = slapos.cookbook:generate.password
-storage-path = $${directory:etc}/.rcode
-bytes = 8
-
 [slaprunner]
 slaprunner = ${buildout:directory}/bin/slaprunner
 slapos = ${buildout:directory}/bin/slapos
@@ -171,6 +164,9 @@ instance_info_json = $${runnerdirectory:home}/instance_info.json
 path = $${shell:path}
 instance_name = $${slap-parameter:instance-name}
 
+default_repository = $${slap-parameter:slapos-repository}
+default_repository_branch = $${slap-parameter:slapos-reference}
+
 
 #---------------------------
 #--
@@ -311,7 +307,7 @@ global_ip = $${slap-network-information:global-ipv6}
 global_port = $${slap-parameter:slaprunner-httpd-port}
 working_directory = $${slaprunner:working-directory}
 dav_lock = $${directory:var}/WebDavLock
-htpasswd_file = $${monitor-httpd-conf-parameter:htpasswd-file}
+htpasswd_file = $${directory:etc}/.htpasswd
 etc_dir = $${directory:etc}
 var_dir = $${directory:var}
 project_folder = $${directory:project}
@@ -377,7 +373,7 @@ path_pid = $${directory:run}/gunicorn.pid
 
 [gunicorn-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = $${gunicorn:bin_gunicorn} slapos.runner.run:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --log-level error --preload
+command-line = $${gunicorn:bin_gunicorn} slapos.runner.run:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --timeout 200 --threads 3 --log-level error --preload
 error-log-file = gunicorn-error.log
 wrapper-path = $${gunicorn:bin_launcher}
 environment = PATH=$${environ:PATH}:${git:location}/bin/
@@ -458,26 +454,44 @@ dash_path = {{ dash_executable_location }}
 curl_path = {{ curl_executable_location }}
 check-secure = 1
 
+
+[htpasswd]
+recipe = slapos.cookbook:generate.password
+storage-path = $${directory:etc}/.pwd
+bytes = 8
+
+[runner-htpasswd]
+recipe = plone.recipe.command
+stop-on-error = true
+htpasswd-path = $${monitor-directory:etc}/.htpasswd
+command = if [ ! -f "$${:htpasswd-path}" ]; then ${apache:location}//bin/htpasswd -cb $${:htpasswd-path} $${:user} $${:password}; fi
+update-command = $${:command}
+user = admin
+{% if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{% else -%}
+password = $${htpasswd:passwd}
+{% endif -%}
+
 #--------------------------------------
 #--
 #-- Send information to SlapOS Master
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-1_info = On your first run, Use "access_url" to setup you account. Then you can use both "url" or "access_url". Or "backend_url" if you want to use ipv6. Set up your account in the webrunner in order to use webdav, and being able to clone your git repositories from the runner.
-2_info = In order to set up your account, get the recovery-code from the monitoring interface. Use "monitor_setup_url" to configure monitor instance.
-backend_url = $${slaprunner:access-url}
-access_url = $${:url}/login
+backend-url = $${slaprunner:access-url}
 url =  https://$${request-frontend:connection-domain}
-ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
-webdav_url = $${request-httpd-frontend:connection-secure_access}/share/
-public_url =  $${request-httpd-frontend:connection-secure_access}/public/
-git_public_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
-git_private_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
+init-user = $${runner-htpasswd:user}
+init-password = $${runner-htpasswd:password}
+ssh-command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
+webdav-url = $${request-httpd-frontend:connection-secure_access}/share/
+public-url =  $${request-httpd-frontend:connection-secure_access}/public/
+git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
+git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
 monitor-base-url = $${publish:monitor-base-url}
 {% if slapparameter_dict.get('instance-type', '') != 'resilient' -%}
 {% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
-monitor_setup_url = {{ monitor_interface_url }}/#page=settings_configurator&url=$${publish:monitor-url}&username=$${publish:monitor-user}&password=$${publish:monitor-password}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=$${publish:monitor-url}&username=$${publish:monitor-user}&password=$${publish:monitor-password}
 {% else -%}
 monitor-url = $${publish:monitor-url}
 monitor-user = $${publish:monitor-user}
@@ -528,7 +542,7 @@ auto-deploy-instance = true
 autorun = false
 slaprunner-httpd-port = $${:monitor-port}
 # XXX - for backward compatibility, monitor-port was for slaprunner httpd server
-monitor-port = 9684
+monitor-port = 9686
 instance-name =
 monitor-cors-domains = 
 monitor-interface-url = 
@@ -579,14 +593,6 @@ ps1 = "\\w> "
 [environ]
 recipe = collective.recipe.environment
 
-[slapos-repo]
-recipe = slapos.recipe.build:gitclone
-repository = $${slap-parameter:slapos-repository}
-git-executable = ${git:location}/bin/git
-develop = true
-location = $${directory:project}/slapos
-branch = $${slap-parameter:slapos-reference}
-
 [prepare-software]
 recipe = slapos.cookbook:wrapper
 command-line = ${curl:location}/bin/curl -g https://[$${slaprunner:ipv6}]:$${slaprunner:runner_port}/isSRReady --max-time 1 --insecure
@@ -732,9 +738,7 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of
 {% if slapparameter_dict.get('monitor-username', '') -%}
 username = {{ slapparameter_dict['monitor-username'] }}
 {% endif -%}
-{% if slapparameter_dict.get('monitor-password', '') -%}
-password = {{ slapparameter_dict['monitor-password'] }}
-{% endif -%}
+password = $${runner-htpasswd:password}
 {% if slapparameter_dict.get('monitor-url-list', '') -%}
 monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
 {% endif -%}
@@ -742,7 +746,6 @@ monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
 {% if not slapparameter_dict.get('authorized-key', '') -%}
 # Pass some parameter to dispay in monitoring interface
 instance-configuration =
-  file recovery-code $${recovery-code:storage-path}
   httpdcors cors-domain $${slaprunner-httpd-cors:location} $${httpd-graceful-wrapper:output}
   raw webrunner-url https://$${request-frontend:connection-domain}
 {% endif -%}

software/slaprunner/nginx_conf.in

@@ -57,10 +57,14 @@ http {
             proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
             proxy_set_header   X-Forwarded-Host  $http_host;
             proxy_set_header   X-Accel-Mapping   /private/;
+            proxy_connect_timeout 200;
+            proxy_send_timeout    200;
+            proxy_read_timeout    200;
+            send_timeout          200;
 
             proxy_pass http://unix:{{ socket }};
         }
-        location ~ ^(/login|/doLogin|/static|/setAccount|/configAccount|/slapgridResult|/isSRReady) {
+        location ~ ^(/login|/doLogin|/static|/slapgridResult|/isSRReady) {
             proxy_redirect off;
             proxy_set_header   X-Forwarded-Proto $scheme;
             proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;

software/slaprunner/software.cfg

@@ -14,6 +14,7 @@ cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 ecdsa = 0.13
 erp5.util = 0.4.44
+futures = 3.0.5
 gitdb = 0.6.4
 gunicorn = 19.5.0
 prettytable = 0.7.2
@@ -41,3 +42,7 @@ lockfile = 0.12.2
 # Required by:
 # slapos.toolbox==0.56
 paramiko = 2.0.1
+
+# Required by:
+# slapos.toolbox==0.55
+passlib = 1.6.5
\ No newline at end of file

software/slaprunner/template/slapos.cfg.in

@@ -37,6 +37,8 @@ minishell_cwd_file = {{ slaprunner['minishell_cwd_file'] }}
 minishell_history_file = {{ slaprunner['minishell_history_file'] }}
 path = {{ slaprunner['path'] }}
 instance_name = {{ slaprunner['instance_name'] }}
+default_repository = {{ slaprunner['default_repository'] }}
+default_repository_branch = {{ slaprunner['default_repository_branch'] }}
 
 [slapproxy]
 host = {{ slaprunner['ipv4'] }}

stack/monitor/buildout.cfg

@@ -26,12 +26,6 @@ download-only = true
 url = ${:_profile_base_location_}/${:filename}
 mode = 0644
 
-[monitor-web-base]
-<= monitor-download-base
-url = ${:_profile_base_location_}/web/${:filename}
-destination = ${buildout:parts-directory}/monitor-web
-on-update = true
-
 [monitor-template-base]
 <= monitor-download-base
 url = ${:_profile_base_location_}/templates/${:filename}
@@ -46,6 +40,7 @@ on-update = true
 [monitor-eggs]
 recipe = zc.recipe.egg
 eggs =
+  plone.recipe.command
   collective.recipe.template
   cns.recipe.symlink
 
@@ -61,14 +56,9 @@ eggs =
 # Monitor templates files
 [monitor-httpd-conf]
 <= monitor-template-base
-md5sum = 65cf2c8f5cfffa509ec3fc06b37e7509
+md5sum = 40dc51fc156f1ad7eb94be7f3cbf08b4
 filename = monitor-httpd.conf.in
 
-[monitor-service-conf-template]
-<= monitor-template-base
-filename = monitor-service.cfg.in
-md5sum = 5913d2a0096b50537f394a49b762b3e5
-
 [monitor-template-wrapper]
 <= monitor-template-base
 filename = wrapper.in
@@ -100,7 +90,7 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = 9d6f50e616655c1ae11ace419ac2861e  
+md5sum = 7ac78495de73cadafea87f97428e487f
 context =
     key apache_location apache:location
     key gzip_location gzip:location
@@ -111,13 +101,11 @@ context =
     raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
     raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
     raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
-    raw monitor_password_promise_template ${monitor-password-promise:location}/${monitor-password-promise:filename}
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond
     raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate
     raw monitor_httpd_template ${monitor-httpd-conf:location}/${monitor-httpd-conf:filename}
-    raw monitor_service_conf_template ${monitor-service-conf-template:location}/${monitor-service-conf-template:filename}
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
     raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
@@ -131,31 +119,26 @@ depends =
 [monitor2-bin]
 <= monitor-template-script
 filename = monitor.py
-md5sum = 280412f9024030c8cae343c2b07e5130
+md5sum = 5525e7445dab16fd03f4eeccf069b74b
 
 [run-promise-py]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/scripts/run-promise.py
 rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
-md5sum = 641c6917bbd19a7abaebb120ccd4e713
+md5sum = 97148dfbb730cc4f55ed54513ce823e0
 mode = 0755
 context =
   raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
 
-[monitor-password-promise]
-<= monitor-template-script
-filename = monitor-password-promise.py
-md5sum = f7e937d6619eb674f39f34718928d91d
-
 [status2rss-executable]
 <= monitor-template-script
 filename = status2rss.py
-md5sum = f297779d0881f4bd48081506efb492a4
+md5sum = 88e3bf955e1e4eac76a444d50fa4f020
 
 [monitor-globalstate]
 <= monitor-template-script
 filename = globalstate.py
-md5sum = 1ebead4480d62d5e1d2612c2d9f4e1d8
+md5sum = 3377e325baa4ecfcd6eee06945fb69fc
 
 [monitor-collect]
 <= monitor-template-script
@@ -165,12 +148,9 @@ md5sum = 78fbcb56761315bde354fe7914d3c54f
 [monitor-document-edit]
 <= monitor-template-script
 filename = monitor-document.py
-md5sum = f3e557e5d81291a22d6d2837a9e37bd0
+md5sum = 399ff4939b55ff74e6d48bec5a495981
 
+[versions]
+PyRSS2Gen = 1.1
+cns.recipe.symlink = 0.2.3
 
-[make-rss-script]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/make-rss.sh.in
-md5sum = 98c8f6fd81e405b0ad10db07c3776321
-output = ${buildout:directory}/template-make-rss.sh.in
-mode = 0644

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -78,13 +78,8 @@ public = ${directory:monitor}/public
 private = ${directory:monitor}/private
 services = ${directory:services}
 services-conf = ${directory:etc}/monitor.conf.d
-www = ${directory:monitor}/web
-web-dir = ${directory:monitor}/web
 log = ${directory:log}/monitor
 monitor-var = ${directory:var}/monitor
-monitor-password-var = ${monitor-directory:monitor-var}/password
-monitor-password-interface = ${monitor-directory:monitor-password-var}/password/interface
-monitor-status2rss-var = ${monitor-directory:monitor-var}/status2rss
 
 [logrotate-directory]
 recipe = slapos.cookbook:mkdirectory
@@ -135,7 +130,6 @@ public-folder = ${monitor-directory:public}
 private-folder = ${monitor-directory:private}
 webdav-folder = ${monitor-directory:webdav}
 report-folder = ${monitor-directory:reports}
-web-folder = ${monitor-directory:web-dir}
 base-url = ${monitor-instance-parameter:monitor-base-url}
 monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
 service-pid-folder = ${monitor-directory:pids}
@@ -230,7 +224,6 @@ password = ${monitor-instance-parameter:password}
 listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
 port = ${monitor-instance-parameter:monitor-httpd-port}
 pid-file = ${directory:run}/monitor-httpd.pid
-cgid-pid-file = ${directory:run}/cgid.pid
 access-log = ${monitor-directory:log}/monitor-httpd-access.log
 error-log = ${monitor-directory:log}/monitor-httpd-error.log
 cert-file = ${ca-directory:certs}/httpd.crt
@@ -279,7 +272,7 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
 [monitor-status2rss-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX - hard-coded Urls
-command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
+command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}'  --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
 
 wrapper-path = ${directory:bin}/monitor-status2rss.py
 
@@ -324,45 +317,6 @@ check-secure = 1
 dash_path = {{ dash_executable_location }}
 curl_path = {{ curl_executable_location }}
 
-[monitor-httpd-promise-conf]
-recipe = slapos.recipe.template:jinja2
-rendered = ${directory:monitor-promise}/${monitor-httpd-promise:filename}.cfg
-template = {{ monitor_service_conf_template }}
-mode = 0644
-context = section parameter_dict monitor-httpd-promise-conf-parameter
-
-[monitor-httpd-promise-conf-parameter]
-title = Monitor httpd listening
-# frequency 5 minute hour day mounth weekday
-frequency = */5 * * * *
-public-path-list = ${monitor-httpd-conf-parameter:access-log} ${monitor-httpd-conf-parameter:error-log}
-#private-path-list =
-
-[monitor-password-parameter]
-password-changed-once-path = ${directory:var}/monitor-password-changed-once
-
-[monitor-password-promise]
-recipe = slapos.recipe.template:jinja2
-template = {{ monitor_password_promise_template }}
-rendered = ${directory:monitor-promise}/${:filename}
-filename = monitor-password
-mode = 0755
-context =
-  raw python_executable {{ python_executable }}
-  key password_changed_once_path monitor-password-parameter:password-changed-once-path
-
-[monitor-password-promise-conf-parameter]
-title = Monitor password
-frequency = */5 * * * *
-private-path-list = ${monitor-directory:monitor-password-interface}
-
-[monitor-password-promise-conf]
-recipe = slapos.recipe.template:jinja2
-template = {{ monitor_service_conf_template }}
-rendered = ${directory:monitor-promise}/${monitor-password-promise:filename}.cfg
-mode = 0644
-context = section parameter_dict monitor-password-promise-conf-parameter
-
 [publish]
 # XXX depends on monitor-base section
 monitor-base-url = ${monitor-base:base-url}
@@ -376,6 +330,7 @@ monitor-httpd-ipv6 = ${slap-configuration:ipv6-random}
 monitor-httpd-port = 8196
 # XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port
 monitor-base-url = ${monitor-frontend-promise:url}
+# monitor-base-url = ${monitor-httpd-conf-parameter:url}
 root-instance-title = ${slap-configuration:root-instance-title}
 monitor-url-list =
 cors-domains = monitor.app.officejs.com 
@@ -390,6 +345,8 @@ instance-configuration =
 
 configuration-file-path = ${monitor-directory:etc}/monitor_knowledge0.cfg
 
+interface-url = https://monitor.app.officejs.com
+
 [monitor-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -427,7 +384,6 @@ depends =
   ${monitor-httpd-promise:filename}
   ${monitor-status2rss-cron-entry:name}
 
-#[buildout]
-#parts =
-#  monitor-base
-#  publish
+[monitor-publish]
+monitor-base-url = ${publish:monitor-base-url}
+monitor-setup-url = ${monitor-instance-parameter:interface-url}/#page=settings_configurator&url=${publish:monitor-url}&username=$${publish:monitor-user}&password=${publish:monitor-password}

stack/monitor/make-rss.sh.in

-#!${dash-output:dash}
-
-STATUS_DB={{ monitor_parameters['db-path'] }}
-RSS_FILE={{ monitor_parameters['rss-path'] }}
-PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
-STATUS2RSS=${rss-bin:location}/${rss-bin:filename}
-
-$PYTHON $STATUS2RSS "Monitoring RSS feed" "{{ monitor_parameters['url'] }}/{{ monitor_parameters['index-filename'] }}" $STATUS_DB > $RSS_FILE

stack/monitor/scripts/globalstate.py

@@ -53,6 +53,8 @@ def main(args_list):
   statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
   parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
 
+  report_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
   if not os.path.exists(statistic_folder):
     try:
       os.makedirs(statistic_folder)
@@ -66,7 +68,6 @@ def main(args_list):
       glob.glob("%s/*.status.json" % status_folder)
     )
   error = warning = success = 0
-  latest_date = ''
   status = 'OK'
   promise_list = []
   global_state_file = os.path.join(base_folder, 'monitor.global.json')
@@ -84,18 +85,13 @@ def main(args_list):
       success += 1
     elif tmp_json['status'] == 'WARNING':
       warning += 1
-    if tmp_json['start-date'] > latest_date:
-      latest_date = tmp_json['start-date']
     tmp_json['time'] = tmp_json['start-date'].split(' ')[1]
-    del tmp_json['start-date']
     promise_list.append(tmp_json)
 
   if error:
     status = 'ERROR'
   elif warning:
     status = 'WARNING'
-  if not latest_date:
-    latest_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
 
   global_state_dict = dict(
       status=status,
@@ -105,7 +101,7 @@ def main(args_list):
         'warning': warning,
       },
       type='global',
-      date=latest_date,
+      date=report_date,
       _links={"rss_url": {"href": "%s/public/feed" % base_url},
               "public_url": {"href": "%s/share/jio_public/" % base_url},
               "private_url": {"href": "%s/share/jio_private/" % base_url}
@@ -150,7 +146,7 @@ def main(args_list):
   # Public information with the link to private folder
   public_state_dict = dict(
     status=status,
-    date=latest_date,
+    date=report_date,
     _links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
     title=global_state_dict.get('title', '')
   )

stack/monitor/scripts/monitor-document.py

@@ -63,6 +63,9 @@ def httpdCorsDomainWrite(httpd_cors_file, httpd_gracefull_bin, cors_domain):
     try:
       with open(old_httpd_cors_file, 'r') as cors_file:
         if cors_file.read() == cors_domain:
+          if os.path.exists(httpd_cors_file) and (os.stat(httpd_cors_file).st_size > 0
+            or (cors_domain == "" and os.stat(httpd_cors_file).st_size == 0)):
+            # Skip if cors file is not empty
             return True
     except OSError, e:
       print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))

stack/monitor/scripts/monitor-password-promise.py

-#!/usr/bin/env python
-
-password_changed_once_path = "{{ password_changed_once_path }}"
-
-import os
-
-def main():
-  if os.path.exists(password_changed_once_path):
-    print('{"status":"OK"}')
-    return 0
-  print('{"status":"BAD","message":"Password never changed"}')
-  return 1
-
-if __name__ == "__main__":
-  exit(main())

stack/monitor/scripts/monitor.py

@@ -8,10 +8,9 @@ import json
 import ConfigParser
 import traceback
 import argparse
-import time
-import glob
 import urllib2
 import ssl
+import glob
 from datetime import datetime
 
 OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
@@ -39,24 +38,6 @@ def parseArguments():
   parser.add_argument('--config_file',
                       default='monitor.cfg',
                       help='Monitor Configuration file')
-  parser.add_argument('--promise-folder',
-                      action='append', dest='promise_folder_list',
-                      default=[],
-                      help='The path to get promise executable files')
-
-  parser.add_argument('--public-folder',
-                      action='append', dest='public_folder',
-                      help='The path of public folder. All files in this folders will have public acess')
-
-  parser.add_argument('--private-folder',
-                      action='append', dest='private_folder',
-                      help='The path of private folder. All files in this folders will be accessible with password')
-
-  parser.add_argument('--promise-runner',
-                      help='The path of promise runner, use to run promise files')
-
-  parser.add_argument('--wrapper-path',
-                      help='Path of monitor generated promise scripts files.')
 
   return parser.parse_args()
 
@@ -75,6 +56,13 @@ def softConfigGet(config, *args, **kwargs):
   except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
     return None
 
+def createSymlink(source, destination):
+  try:
+    os.symlink(source, destination)
+  except OSError, e:
+    if e.errno != os.errno.EEXIST:
+      raise
+
 class Monitoring(object):
 
   def __init__(self, configuration_file):
@@ -199,14 +187,6 @@ class Monitoring(object):
         self.promise_dict[filename] = {"path": path,
                                   "configuration": ConfigParser.ConfigParser()}
 
-    # get promises configurations
-    #for filename in os.listdir(monitor_promise_folder):
-    #  path = os.path.join(monitor_promise_folder, filename)
-    #  if os.path.isfile(path) and filename[-4:] == ".cfg":
-    #    promise_name = filename[:-4]
-    #    if promise_name in promise_dict:
-    #      loadConfig([path], promise_dict[promise_name]["configuration"])
-
   def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""):
     if destination_folder:
       if source_path_list:
@@ -277,32 +257,30 @@ class Monitoring(object):
     jio_private = os.path.join(self.webdav_folder, 'jio_private')
     mkdirAll(jio_public)
     mkdirAll(jio_private)
-    try:
-      os.symlink(self.public_folder, os.path.join(jio_public, '.jio_documents'))
-    except OSError, e:
-      if e.errno != os.errno.EEXIST:
-        raise
-    try:
-      os.symlink(self.private_folder, os.path.join(jio_private, '.jio_documents'))
-    except OSError, e:
-      if e.errno != os.errno.EEXIST:
-        raise
+
+    createSymlink(self.public_folder,
+                  os.path.join(jio_public, '.jio_documents'))
+    createSymlink(self.private_folder,
+                  os.path.join(jio_private, '.jio_documents'))
 
     self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
+    self.document_folder = os.path.join(self.private_folder, 'documents')
     config_folder = os.path.join(self.config_folder, '.jio_documents')
+
     mkdirAll(self.data_folder)
     mkdirAll(config_folder)
-    try:
-      os.symlink(os.path.join(self.private_folder, 'data'),
+
+    createSymlink(os.path.join(self.private_folder, 'data'),
                   os.path.join(jio_private, 'data'))
-    except OSError, e:
-      if e.errno != os.errno.EEXIST:
-        raise
+    createSymlink(self.config_folder, os.path.join(jio_private, 'config'))
+    createSymlink(self.data_folder, self.document_folder)
+
+    # Cleanup private folder
+    for file in glob.glob("%s/*.history.json" % self.private_folder):
       try:
-      os.symlink(self.config_folder, os.path.join(jio_private, 'config'))
-    except OSError, e:
-      if e.errno != os.errno.EEXIST:
-        raise
+        os.unlink(file)
+      except OSError:
+        print "failed to remove file %s. Ignoring..." % file
 
   def makeConfigurationFiles(self):
     config_folder = os.path.join(self.config_folder, '.jio_documents')
@@ -352,12 +330,12 @@ class Monitoring(object):
     opml_content += OPML_OUTLINE_FEED % {'title': self.title,
         'html_url': self.public_url + '/feed',
         'xml_url': self.public_url + '/feed',
-        'global_url': "%s/jio_public/" % self.webdav_url}
+        'global_url': "%s/jio_private/" % self.webdav_url}
     for feed_url in feed_url_list:
       opml_content += OPML_OUTLINE_FEED % {'title': self.getMonitorTitleFromUrl(feed_url + "/share/jio_public/"),
         'html_url': feed_url + '/public/feed',
         'xml_url': feed_url + '/public/feed',
-        'global_url': "%s/share/jio_public/" % feed_url}
+        'global_url': "%s/share/jio_private/" % feed_url}
 
     opml_content += OPML_END
 
@@ -376,37 +354,6 @@ class Monitoring(object):
     with open(file_path, 'w') as flog:
       flog.write(content)
 
-  def generateMonitorHalJson(self):
-    monitor_link_dict = {"webdav": {"href": self.webdav_url},
-                          "public": {"href": "%s/public" % self.webdav_url},
-                          "private": {"href": "%s/private" % self.webdav_url},
-                          "rss": {"href": "%s/feed" % self.public_url},
-                          "jio_public": {"href": "%s/jio_public/" % self.webdav_url},
-                          "jio_private": {"href": "%s/jio_private/" % self.webdav_url}
-                        }
-    if self.title:
-      self.monitor_dict["title"] = self.title
-    if self.monitor_url_list:
-      monitor_link_dict["related_monitor"] = [{"href": url}
-                                  for url in self.monitor_url_list]
-    self.monitor_dict["_links"] = monitor_link_dict
-    if self.promise_items:
-      service_list = []
-      for service_name, promise in self.promise_items:
-        service_config = promise["configuration"]
-        tmp = softConfigGet(service_config, "service", "title")
-        service_dict = {}
-        service_dict["id"] = service_name
-        service_dict["_links"] = {"status": {"href": "%s/public/%s.status.json" % (self.webdav_url, service_name)}}  # hardcoded
-        if tmp:
-          service_dict["title"] = tmp
-        service_list.append(service_dict)
-
-      self.monitor_dict["_embedded"] = {"service": service_list}
-
-    with open(self.monitor_hal_json, "w") as fp:
-      json.dump(self.monitor_dict, fp)
-
   def generateReportCronEntries(self):
     cron_line_list = []
 
@@ -429,7 +376,7 @@ class Monitoring(object):
           '--promise_script "%s"' % report_script,
           '--promise_name "%s"' % report_name,
           '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
-          '--history_folder "%s"' % self.report_folder,
+          '--history_folder "%s"' % self.data_folder,
           '--instance_name "%s"' % self.title,
           '--hosting_name "%s"' % self.root_title,
           '--promise_type "report"']
@@ -484,11 +431,6 @@ class Monitoring(object):
       if service_name in service_name_list:
         service_name_list.pop(service_name_list.index(service_name))
 
-      """wrapper_path = os.path.join(self.wraper_folder, service_name)
-      with open(wrapper_path, "w") as fp:
-        fp.write("#!/bin/sh\n%s" % command)  # XXX hardcoded, use dash, sh or bash binary!
-      os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )"""
-
     if service_name_list != []:
       # XXX Some service was removed, delete his status file so monitor will not consider his status anymore
       for service_name in service_name_list:
@@ -532,10 +474,6 @@ class Monitoring(object):
                                       private_path_list.split(),
                                       service_name)
 
-    # generate monitor.json
-    self.monitor_dict = {}
-    self.generateMonitorHalJson()
-
     # Generate OPML file
     self.generateOpmlFile(self.monitor_url_list,
       os.path.join(self.public_folder, 'feeds'))
@@ -551,11 +489,13 @@ class Monitoring(object):
 
     # Rotate monitor data files
     option_list = [
-      'daily', 'nocreate', 'noolddir', 'rotate 5',
+      'daily', 'nocreate', 'olddir %s' % self.data_folder, 'rotate 5',
       'nocompress', 'extension .json', 'dateext',
       'dateformat -%Y-%m-%d', 'notifempty'
     ]
-    file_list = ["%s/*.data.json" % self.data_folder]
+    file_list = [
+      "%s/*.data.json" % self.private_folder,
+      "%s/*.data.json" % self.data_folder]
     self.generateLogrotateEntry('monitor.data', file_list, option_list)
 
     # Rotate public history status file, delete data of previous days

stack/monitor/scripts/rsstoopml.py

-#!/usr/bin/env python
-
-import json
-import os
-import time
-from datetime import datetime
-
-OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
-<!-- OPML generated by SlapOS -->
-<opml version="1.1">
-	<head>
-		<title>SlapOS Monitoring Status Lists</title>
-		<dateCreated>%(creation_date)s</dateCreated>
-		<dateModified>%(mondification_date)s</dateModified>
-	</head>
-	<body>
-	  <outline text="%(outline_title)s">"""
-OPML_END = """	  </outline>
-  </body>
-</opml>"""
-
-OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" />'
-
-def main(config_file, output_file):
-  feed_url_list = []
-  if os.path.exists(output_file):
-    creation_date = datetime.fromtimestamp(os.path.getctime(output_file)).utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-    modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-  else:
-    creation_date = modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-  with open(config_file, 'r') as fconfig:
-    feed_url_list = json.loads(fconfig.read())
-
-  opml_content = OPML_START
-
-	for feed_line in feed_url_list:
-		opml_content += OPML_OUTLINE_FEED % {'title': feed_line['title'], 'html_url': feed_line['url'], 'xml_url': feed_line['url']}
-
-	opml_content += OPML_END
-
-  with open(output_file, 'w') as wfile:
-    wfile.write(opml_content)
-
-if __name__ == "__main__":
-  if len(sys.argv) < 3:
-    print("Usage: %s <rss_conf_file> <output_path>" % sys.argv[0])
-    sys.exit(2)
-	config_file = sys.argv[1]
-	output_file = sys.argv[2]
-	main(config_file, output_file)
\ No newline at end of file

stack/monitor/scripts/run-promise.py

@@ -122,6 +122,12 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
         }
         f_history.write(json.dumps(data_dict))
     else:
+      # Remove useless informations
+      status_dict.pop('hosting_subscription', '')
+      status_dict.pop('title', '')
+      status_dict.pop('instance', '')
+      status_dict.pop('type', '')
+
       with open (history_file, mode="r+") as f_history:
         f_history.seek(0,2)
         position = f_history.tell() -2

stack/monitor/scripts/status2rss.py

 import sys
 import os
 import json
-import datetime
+from datetime import datetime
 import base64
 import hashlib
 import PyRSS2Gen
@@ -16,6 +16,8 @@ def parseArguments():
                       help='Path where to get *.status.json files which contain result of promises.')
   parser.add_argument('--output',
                       help='The Path of file where feed file will be saved.')
+  parser.add_argument('--feed_url',
+                      help='Url of this feed file.')
   parser.add_argument('--public_url',
                       help='Monitor Instance public URL.')
   parser.add_argument('--private_url',
@@ -36,6 +38,7 @@ def main():
   parser = parseArguments()
 
   rss_item_list = []
+  report_date = datetime.utcnow()
   for filename in os.listdir(parser.items_folder):
     if filename.endswith(".status.json"):
       filepath = os.path.join(parser.items_folder, filename)
@@ -46,7 +49,7 @@ def main():
         print "Failed to load json file: %s" % filepath
         continue
       description = result_dict.get('message', '')
-      event_time = datetime.datetime.fromtimestamp(result_dict['change-time'])
+      event_time = datetime.fromtimestamp(result_dict['change-time'])
       rss_item = PyRSS2Gen.RSSItem(
         categories = [result_dict['status']],
         source = PyRSS2Gen.Source(result_dict['title'], parser.public_url),
@@ -55,7 +58,7 @@ def main():
         description = "%s: %s\n%s" % (event_time, result_dict['status'], description),
         link = parser.private_url,
         pubDate = event_time,
-        guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, result_dict['status'])))
+        guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (parser.hosting_name, result_dict['title'])))
       )
       rss_item_list.append(rss_item)
 
@@ -64,11 +67,12 @@ def main():
   sorted(rss_item_list, key=getKey)
   rss_feed = PyRSS2Gen.RSS2 (
     title = parser.instance_name,
-    link = parser.public_url,
+    link = parser.feed_url,
     description = parser.hosting_name,
-    lastBuildDate = datetime.datetime.utcnow(),
+    lastBuildDate = report_date,
     items = rss_item_list
     )
+
   with open(parser.output, 'w') as frss:
     frss.write(rss_feed.to_xml())
 

stack/monitor/templates/monitor-httpd.conf.in

@@ -11,7 +11,7 @@ ServerAdmin someone@email
 Listen [{{ parameter_dict.get('listening-ip') }}]:{{ parameter_dict.get('port') }}
 Define MonitorPort
 </IfDefine>
-DocumentRoot "{{ directory.get('webdav') }}"
+DocumentRoot "{{ directory.get('private') }}"
 ErrorLog "{{ parameter_dict.get('error-log') }}"
 LoadModule unixd_module modules/mod_unixd.so
 LoadModule access_compat_module modules/mod_access_compat.so
@@ -19,7 +19,6 @@ LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule authn_core_module modules/mod_authn_core.so
 LoadModule authz_host_module modules/mod_authz_host.so
 LoadModule mime_module modules/mod_mime.so
-LoadModule cgid_module modules/mod_cgid.so
 LoadModule dir_module modules/mod_dir.so
 LoadModule ssl_module modules/mod_ssl.so
 LoadModule alias_module modules/mod_alias.so
@@ -55,31 +54,12 @@ SSLEngine   On
 
 Include {{ parameter_dict.get('httpd-cors-config-file') }}
 Header set Vary Origin
-Header set Cache-Control "private, max-age=0, must-revalidate"
-Header set Access-Control-Max-Age "0"
-Header unset ETag
-Header set Pragma "no-cache"
+Header set Cache-Control "private, max-age=40"
+Header set Access-Control-Max-Age "40"
 Header set Access-Control-Allow-Credentials "true"
 Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST"
 Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization"
 
-{% if parameter_dict.has_key('monitor-url-list') -%}
-RewriteEngine on
-SSLProxyEngine on
-ProxyPreserveHost On
-SSLProxyVerify none
-SSLProxyCheckPeerCN off
-SSLProxyCheckPeerName off
-{% set index=1 -%}
-{% set monitor_url_list = parameter_dict.get('monitor-url-list').split('\n') -%}
-{% for url in monitor_url_list -%}
-{%   if url.strip() -%}
-RewriteRule /monitor{{ index }}/(.*)  {{ url }}/$1 [L,P]
-{%     set index = index + 1 -%}
-{%   endif -%}
-{% endfor -%}
-{% endif -%}
-
 DavLockDB {{ directory.get('monitor-var') }}/DavLock
 Alias /share {{ directory.get('webdav') }}
 <Directory {{ directory.get('webdav') }}>
@@ -101,22 +81,6 @@ Alias /share {{ directory.get('webdav') }}
     </Limit>
 </LocationMatch>
 
-ScriptSock {{ parameter_dict.get('cgid-pid-file') }}
-<Directory {{ directory.get('www') }}>
-  SSLVerifyDepth    1
-  SSLRequireSSL
-  SSLOptions        +StrictRequire
-  # XXX: security????
-  DirectoryIndex index.html
-  Options FollowSymLinks
-  AllowOverride All
-  Order Deny,Allow
-  AuthType Basic
-  AuthName "Private access"
-  AuthUserFile "{{ parameter_dict.get('htpasswd-file') }}"
-  Require valid-user
-</Directory>
-
 Alias /private {{ directory.get('private') }}/
 <Directory {{ directory.get('private') }}>
   Order Deny,Allow
@@ -140,24 +104,6 @@ Alias /public {{ directory.get('public') }}/
   Allow from all
 </Directory>
 
-Alias /cgi-bin {{ directory.get('cgi-bin') }}
-<Directory {{ directory.get('cgi-bin') }}>
-  # XXX security ???
-  Order Deny,Allow
-  Deny from all
-  <Files "*.cgi">
-    Order Deny,Allow
-    Deny from env=AUTHREQUIRED
-    AuthType Basic
-    AuthName "Private access"
-    AuthUserFile "{{ parameter_dict.get('htpasswd-file') }}"
-    Require valid-user
-  </Files>
-  Options +ExecCGI
-  AddHandler cgi-script .cgi
-  Options Indexes FollowSymLinks
-  Satisfy all
-</Directory>
 {% if parameter_dict.get('httpd-include-file', '') -%}
 Include {{ parameter_dict.get('httpd-include-file') }}
 {% endif -%}

stack/monitor/templates/monitor-service.cfg.in

-[service]
-{% for key, value in parameter_dict.items() -%}
-{{ key }} = {{ value.strip().replace("\n", "\n  ") }}
-{% endfor -%}