re6stnet: clean-up & backup

See merge request !940

software/re6stnet/apache.conf.in

@@ -8,7 +8,6 @@ LoadModule version_module modules/mod_version.so
 LoadModule proxy_module modules/mod_proxy.so
 LoadModule proxy_http_module modules/mod_proxy_http.so
 LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
-LoadModule ssl_module modules/mod_ssl.so
 LoadModule mime_module modules/mod_mime.so
 #LoadModule dav_module modules/mod_dav.so
 #LoadModule dav_fs_module modules/mod_dav_fs.so
@@ -31,17 +30,6 @@ ErrorLog "{{ error_log }}"
 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
 CustomLog "{{ access_log }}" combined
 
-{% if uri_scheme == 'https'  -%}
-# SSL Configuration
-SSLCertificateFile {{ certificate }}
-SSLCertificateKeyFile {{ key }}
-SSLRandomSeed startup builtin
-SSLRandomSeed connect builtin
-SSLProtocol all -SSLv2 -SSLv3
-SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
-SSLHonorCipherOrder on
-{% endif -%}
-
 <Directory />
   Options FollowSymLinks
   AllowOverride None
@@ -50,10 +38,5 @@ SSLHonorCipherOrder on
 
 Listen {{ ipv6 }}:{{ apache_port }}
 <VirtualHost *:{{ apache_port }}>
-{% if uri_scheme == 'https'  -%}
-  SSLEngine On
-  SSLProxyEngine On
-{% endif -%}
-
   ProxyPass / http://{{ re6st_ipv4 }}:{{ re6st_port }}/
 </VirtualHost>

software/re6stnet/buildout.hash.cfg

@@ -14,24 +14,16 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 610fc6fd0444d3bab3fca4478572749a
+md5sum = 01938aa1683b7994f814fff6d964b9ba
 
 [template-re6stnet]
 filename = instance-re6stnet.cfg.in
-md5sum = 002f7405f565c82219b0d4b92790ff8c
+md5sum = 4e4475c8ef80a3a53aa63a2ae444586d
 
 [template-apache-conf]
 filename = apache.conf.in
-md5sum = 2ed3c4e9b9d58d2e57cda227bdd454d2
+md5sum = 3d55f7c9c4fc7279f06bfe6313a78a4b
 
 [template-re6st-registry-conf]
 filename = re6st-registry.conf.in
-md5sum = b0f0facfea82a4481f4fb5b0b263e09a
-
-[template-wrapper]
-filename = wrapper.in
-md5sum = 7633bdfc0e638ff9979f963fbbca8f13
-
-[template-registry-run]
-filename = registry-run.in
-md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6
+md5sum = feb4b3318f37414d1bf3d16a03aec93d

software/re6stnet/instance-re6stnet-input-schema.json

@@ -78,6 +78,11 @@
       "description": "Prevent tunnelling accross borders of listed countries",
       "type": "string",
       "default": ""
+    },
+    "backup-repository": {
+      "title": "Git backup repository",
+      "description": "URL of Git repository where backups are pushed. Nothing is pushed if empty.",
+      "type": "string"
     }
   }
 }

software/re6stnet/instance-re6stnet.cfg.in

-
+{% set bin_directory = parameter_dict['bin-directory'] -%}
 {% set python_bin = parameter_dict['python-executable'] -%}
-{% set re6st_registry = parameter_dict['re6st-registry'] -%}
-{% set re6stnet = parameter_dict['re6stnet'] -%}
 {% set publish_dict = {} -%}
-{% set part_list = [] -%}
 {% set ipv6 = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
-{% set uri_scheme = slapparameter_dict.get('uri-scheme', 'http') -%}
-{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -19,35 +14,16 @@ log = ${:var}/log
 services = ${:etc}/service
 script = ${:etc}/run
 run = ${:var}/run
-ca-dir = ${:etc}/ssl
-requests = ${:ca-dir}/requests
-private = ${:ca-dir}/private
-certs = ${:ca-dir}/certs
-newcerts = ${:ca-dir}/newcerts
-crl = ${:ca-dir}/crl
-re6st = ${:srv}/res6stnet
 
 [re6stnet-dirs]
 recipe = slapos.cookbook:mkdirectory
-registry = ${directory:re6st}/registry
+data = ${directory:srv}/re6stnet
 log = ${directory:log}/re6stnet
 conf = ${directory:etc}/re6stnet
 ssl = ${:conf}/ssl
 token = ${:conf}/token
 run = ${directory:run}/re6stnet
 
-[certificate-authority]
-recipe = slapos.cookbook:certificate_authority
-openssl-binary = {{ openssl_bin }}/openssl
-ca-dir = ${directory:ca-dir}
-requests-directory = ${directory:requests}
-wrapper = ${directory:services}/certificate_authority
-ca-private = ${directory:private}
-ca-certs = ${directory:certs}
-ca-newcerts = ${directory:newcerts}
-ca-crl = ${directory:crl}
-
-
 [apache-conf]
 recipe = slapos.recipe.template:jinja2
 template = {{ parameter_dict['template-apache-conf'] }}
@@ -64,56 +40,33 @@ context =
   key access_log :access-log
   key error_log :error-log
   key pid_file :pid-file
-  raw certificate ${directory:certs}/apache.crt
-  raw key ${directory:private}/apache.key
   raw ipv6 {{ ipv6 }}
-  raw uri_scheme {{ uri_scheme }}
 
-{% set apache_wrapper = '${directory:services}/httpd' -%}
-{% if uri_scheme == 'https' -%}
-{% set apache_wrapper = '${directory:bin}/httpd_raw' -%}
-{% endif -%}
 [apache-httpd]
 recipe = slapos.cookbook:wrapper
-wrapper-path = {{ apache_wrapper }}
-command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
-
-{% if uri_scheme == 'https' %}
-[apache-ca]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-executable = ${apache-httpd:wrapper-path}
-wrapper = ${directory:bin}/httpd
-key-file = ${certificate-authority:ca-private}/apache.key
-cert-file = ${certificate-authority:ca-certs}/apache.crt
-
-[{{ section('apache-ca-service') }}]
-recipe = slapos.cookbook:wrapper
-command-line = ${apache-ca:wrapper}
 wrapper-path = ${directory:services}/httpd
-hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
-{% endif %}
+command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
 
 [apache-httpd-graceful]
 recipe = slapos.recipe.template:jinja2
-template = {{ parameter_dict['template-wrapper'] }}
 rendered = ${directory:script}/httpd-graceful
-mode = 0700
-context =
-  raw content {{ parameter_dict['apache-location'] }}/bin/httpd -Sf ${apache-conf:rendered}; if [ $? -eq 0 ]; then kill -USR1 $(cat ${apache-conf:pid-file}); fi
-  raw dash {{ dash_binary }}
+template = inline:{{'{{content}}'}}
+context = key content :script
+script =
+  #!/bin/sh -e
+  {{ parameter_dict['apache-location'] }}/bin/httpd -Sf ${apache-conf:rendered}
+  {{ bin_directory }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
 
 [logrotate-apache]
 < = logrotate-entry-base
 name = apache
 log = ${apache-conf:error-log} ${apache-conf:access-log}
-post = test ! -s ${apache-conf:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
+post = test ! -s ${apache-conf:pid-file} || {{ bin_directory }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
 
 [re6st-registry-conf-dict]
 port = 9201
 ipv4 = {{ ipv4 }}
-ipv6 = {{ ipv6 }}
-db = ${re6stnet-dirs:registry}/registry.db
+db = ${re6stnet-dirs:data}/registry.db
 ca = ${re6stnet-dirs:ssl}/re6stnet.crt
 key = ${re6stnet-dirs:ssl}/re6stnet.key
 dh = ${re6stnet-dirs:ssl}/dh.pem
@@ -135,18 +88,35 @@ same-country = {{ slapparameter_dict.get('same-country', '') }}
 [re6st-registry-conf]
 recipe = slapos.recipe.template:jinja2
 template = {{ parameter_dict['template-re6st-registry-conf'] }}
-rendered = ${directory:etc}/re6st-registry.conf
+rendered = ${re6stnet-dirs:conf}/registry.conf
 context = section parameter_dict re6st-registry-conf-dict
 
+depends = ${re6st-compat:recipe}
+[re6st-compat]
+recipe = slapos.recipe.build
+update =
+  import errno, os
+  res6stnet = self.buildout['directory']['srv'] + '/res6stnet'
+  try:
+    os.rename(res6stnet + '/registry', self.buildout['re6stnet-dirs']['data'])
+  except OSError as e:
+    if e.errno != errno.ENOENT:
+      raise
+  else:
+    os.rmdir(res6stnet)
+
 [re6st-registry-wrapper]
-recipe = slapos.recipe.template:jinja2
-template = {{ parameter_dict['template-registry-run'] }}
-rendered = ${directory:services}/re6st-registry
-pid-file = ${directory:run}/registry.pid
-context =
-  key pid_file :pid-file
-  raw re6st_command {{ re6st_registry }}
-  key re6st_conf re6st-registry-conf:rendered
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:services}/re6st-registry
+pidfile = ${directory:run}/registry.pid
+command-line = {{ bin_directory }}/re6st-registry @${re6st-registry-conf:rendered}
+
+[cron-entry-re6st-backup]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = re6stnet-backup
+time = hourly
+command = {{ parameter_dict['re6stnet-backup'] }} ${logrotate-directory:backup}/re6stnet ${re6st-registry-conf-dict:db} {{ slapparameter_dict.get('backup-repository', '') }}
 
 [re6st-registry]
 recipe = slapos.cookbook:re6stnet.registry
@@ -167,7 +137,7 @@ key-file = ${re6st-registry-conf-dict:key}
 cert-file = ${re6st-registry-conf-dict:ca}
 dh-file = ${re6st-registry-conf-dict:dh}
 
-slave-instance-list = ${slap-parameter:slave_instance_list}
+slave-instance-list = {{ slapparameter_dict.get('slave_instance_list', '{}') }}
 
 environment =
   PATH={{ openssl_bin }}
@@ -181,14 +151,14 @@ command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper}
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = re6stnet-check-token
-frequency = */5 * * * *
+time = *:0/5
 command = {{ python_bin }} ${re6st-registry:manager-wrapper}
 
 [logrotate-entry-re6stnet]
 < = logrotate-entry-base
 name = re6stnet
 log = ${re6st-registry-conf-dict:logfile}
-post = test ! -s ${re6st-registry-wrapper:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${re6st-registry-wrapper:pid-file} -s USR1
+post = [ ! -s ${re6st-registry-wrapper:pidfile} ] || {{ bin_directory }}/slapos-kill --pidfile ${re6st-registry-wrapper:pidfile} -s USR1
 
 [port-redirection]
 recipe = slapos.recipe.template:jinja2
@@ -214,7 +184,7 @@ name = apache-re6st-registry.py
 config-hostname = ${apache-conf:ipv6}
 config-port = ${apache-conf:port}
 
-{% do publish_dict.__setitem__('re6stry-url', uri_scheme ~ '://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
+{% do publish_dict.__setitem__('re6stry-url', 'http://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
 {% do publish_dict.__setitem__('re6stry-local-url',  'http://${re6st-registry:ipv4}:${re6st-registry:port}/') -%}
 {% do publish_dict.__setitem__('slave-amount',  '${re6st-registry:slave-amount}') -%}
 [publish]
@@ -230,12 +200,12 @@ extends =
   {{ logrotate_cfg }}
 
 parts =
-  certificate-authority
   logrotate-apache
   logrotate-entry-re6stnet
   re6stnet-manage
   cron-entry-logrotate
   cron-entry-re6st-manage
+  cron-entry-re6st-backup
   apache-httpd
   apache-httpd-graceful
   publish
@@ -244,12 +214,3 @@ parts =
   re6st-registry-promise
   apache-registry-promise
   monitor-base
-# Complete parts with sections
-  {{ part_list | join('\n  ') }}
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-[slap-parameter]
-slave_instance_list = {}

software/re6stnet/instance.cfg.in

@@ -16,27 +16,21 @@ recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
 extra-context =
 context =
-    key develop_eggs_directory buildout:develop-eggs-directory
-    key eggs_directory buildout:eggs-directory
     key ipv6_set slap-configuration:ipv6
     key ipv4_set slap-configuration:ipv4
     key slapparameter_dict slap-configuration:configuration
     key computer_id slap-configuration:computer
     raw logrotate_cfg {{ template_logrotate_base }}
-    raw dash_binary {{ dash_location }}/bin/dash
     raw openssl_bin {{ openssl_location}}/bin
     ${:extra-context}
 
 [dynamic-template-re6stnet-parameters]
 bin-directory = {{ bin_directory }}
 python-executable  = {{ python_with_eggs }}
-re6st-registry = {{ bin_directory }}/re6st-registry
-re6stnet = {{ bin_directory }}/re6stnet
 template-apache-conf = {{ template_apache_conf }}
-template-wrapper = {{ template_wrapper }}
 apache-location = {{ apache_location }}
+re6stnet-backup = {{ re6stnet_backup }}
 template-re6st-registry-conf = {{ template_re6st_registry_conf }}
-template-registry-run = {{ template_registry_run }}
 
 [dynamic-template-re6stnet]
 < = jinja2-template-base
@@ -48,7 +42,9 @@ extra-context =
     raw monitor2_template_rendered {{ monitor2_template_rendered }}
 
 [switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
+recipe = slapos.cookbook:switch-softwaretype
 
-default = ${dynamic-template-re6stnet:rendered}
+default = dynamic-template-re6stnet:rendered
 registry = ${:default}
+# BBB
+RootSoftwareInstance = ${:default}

software/re6stnet/re6st-registry.conf.in

 port  {{ parameter_dict['port'] }}
 4  {{ parameter_dict['ipv4'] }}
 authorized-origin  {{ parameter_dict['ipv4'] }}
-# 6  {{ parameter_dict['ipv6'] }}
+6  ''
 db  {{ parameter_dict['db'] }}
 ca  {{ parameter_dict['ca'] }}
 key  {{ parameter_dict['key'] }}

software/re6stnet/registry-run.in

-#!/bin/bash
-
-echo $$ > {{ pid_file }}
-exec {{ re6st_command }} @{{ re6st_conf }}

software/re6stnet/software.cfg

@@ -2,13 +2,10 @@
 
 extends =
   buildout.hash.cfg
-  ../../component/dash/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/gzip/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/apache/buildout.cfg
-  ../../component/pycurl/buildout.cfg
+  ../../component/git/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../stack/slapos.cfg
 
@@ -17,10 +14,22 @@ extends =
 
 parts +=
   slapos-cookbook
-  dash
   template
 
-[re6st-eggs]
+[re6stnet-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/re6stnet.git
+git-executable = ${git:location}/bin/git
+
+[re6stnet-setup-env]
+PATH = ${git:location}/bin:%(PATH)s
+
+[re6stnet-develop]
+recipe = zc.recipe.egg:develop
+setup = ${re6stnet-repository:location}
+environment = re6stnet-setup-env
+
+[re6stnet]
 recipe = zc.recipe.egg
 interpreter = python-${:_buildout_section_name_}
 eggs =
@@ -28,6 +37,33 @@ eggs =
   ${python-cryptography:egg}
   pyOpenSSL
   re6stnet
+initialization =
+  import os
+  path = os.environ.get('PATH', '')
+  os.environ['PATH'] = '${git:location}/bin' + (path and ':' + path)
+depends = ${re6stnet-develop:recipe}
+
+[re6stnet-backup]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/re6st-backup
+template =
+  inline:#!/bin/sh -e
+  PATH={{git_location}}/bin:{{sqlite3_location}}/bin:$PATH
+  cd "$1" || {
+    rm -rf "$1.new"
+    git init --bare "$1.new"
+    rm "$1.new"/description "$1.new"/hooks/*
+    mv "$1.new" "$1"
+    cd "$1"
+  }
+  h=`sqlite3 "$2" .dump |git hash-object --stdin -w`
+  git update-index --add --cacheinfo 0644 "$h" registry.sql
+  git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \
+    -c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm ''
+  [ ! "$3" ] || git push --mirror "$3"
+context =
+  key git_location git:location
+  key sqlite3_location sqlite3:location
 
 [download-base]
 recipe = slapos.recipe.build:download
@@ -43,17 +79,15 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key apache_location apache:location
-    key dash_location dash:location
     key logrotate_location logrotate:location
     key openssl_location openssl:location
+    key re6stnet_backup re6stnet-backup:rendered
     key template_apache_conf template-apache-conf:target
     key template_re6stnet template-re6stnet:target
     key template_re6st_registry_conf template-re6st-registry-conf:target
     key template_logrotate_base template-logrotate-base:rendered
-    key template_wrapper template-wrapper:target
-    key template_registry_run template-registry-run:target
     key monitor2_template_rendered monitor2-template:rendered
-    raw python_with_eggs ${buildout:bin-directory}/${re6st-eggs:interpreter}
+    raw python_with_eggs ${buildout:bin-directory}/${re6stnet:interpreter}
 
 [template-re6stnet]
 < = download-base
@@ -64,15 +98,5 @@ context =
 [template-re6st-registry-conf]
 < = download-base
 
-[template-wrapper]
-< = download-base
-
-[template-registry-run]
-< = download-base
-
 [versions]
-re6stnet = 0.551
-
-# Required by:
-# re6stnet==0.533
 miniupnpc = 1.9

software/re6stnet/test/test.py

@@ -29,7 +29,6 @@ import os
 import requests
 import json
 
-from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
 setUpModule, Re6stnetTestCase = makeModuleSetUpAndTestCaseClass(
@@ -57,35 +56,3 @@ class TestPortRedirection(Re6stnetTestCase):
             'srcPort': 9201,
             'destPort': 9201,
         }, portredir_config[0])
-
-
-
-class ServicesTestCase(Re6stnetTestCase):
-
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {'uri-scheme': 'https'}
-
-  def test_hashes(self):
-    hash_files = [
-        'software_release/buildout.cfg',
-    ]
-    expected_process_names = [
-        'httpd-{hash}-on-watch',
-    ]
-
-    with self.slap.instance_supervisor_rpc as supervisor:
-      process_names = [
-          process['name'] for process in supervisor.getAllProcessInfo()
-      ]
-
-    hash_files = [
-        os.path.join(self.computer_partition_root_path, path)
-        for path in hash_files
-    ]
-
-    for name in expected_process_names:
-      h = generateHashFromFiles(hash_files)
-      expected_process_name = name.format(hash=h)
-
-      self.assertIn(expected_process_name, process_names)

software/re6stnet/wrapper.in

-#!{{ dash }}
-
-{{ content }}