software/slaprunner: auto-restart services on SR upgrade.

/reviewed-on nexedi/slapos!409

software/erp5testnode/testsuite/slaprunner/README.md

+# slaprunner test
+
+This software release is simply to run the test suite from `../slaprunner/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-SLAPRUNNER-TEST` in test result module.
+

software/erp5testnode/testsuite/slaprunner/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 6fa1420739f58e722631564b08727060
+

software/erp5testnode/testsuite/slaprunner/instance.cfg.in

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/slaprunner/test
+
+
+# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries ( not sure they are needed )
+environment =
+  PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/slaprunner/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.slaprunner-setup]
+<= setup-develop-egg
+egg = slapos.test.slaprunner
+setup = ${slapos-repository:location}/software/slaprunner/test/
+
+[erp5.util-setup]
+<= setup-develop-egg
+egg = erp5.util[testnode]
+setup = ${erp5.util-repository:location}
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.slaprunner-setup:egg}
+  ${erp5.util-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite
+# later we can stop fetching it from git and just use egg
+[erp5.util-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/erp5.git
+revision = 69013fa0fb67501089c776ab5e75d7bbf2e0e3bc
+
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.slaprunner =
+erp5.util =
+#erp5.util = 0.4.51
+
+pyasn1 = 0.4.2
+slapos.recipe.template = 4.3
+pysftp = 0.2.9

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 713db528880282d568278f09458d2aab
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = e12255a8c946b3eb8c6373fff481339f
+md5sum = cd855670076979919c0fd00cc0f5938c
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -4,6 +4,8 @@ parts =
   nginx-launcher
   certificate-authority
   ca-nginx
+  certificate-authority-service
+  ca-nginx-service
   logrotate-entry-nginx
   gunicorn-launcher
   gunicorn-graceful
@@ -16,10 +18,13 @@ parts =
   runner-sshd-graceful
   runner-sshd-promise
   runner-sshkeys-authority
+  runner-sshkeys-authority-service
   runner-sshkeys-sshd
+  runner-sshkeys-sshd-service
   runtestsuite
   symlinks
   shellinabox
+  shellinabox-service
   slapos-cfg
   cron-entry-prepare-software
   deploy-instance-parameters
@@ -202,8 +207,9 @@ default_repository_branch = $${slap-parameter:slapos-reference}
 [slaprunner-supervisord-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX hardcoded locations
-command-line = $${buildout:directory}/bin/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
+command-line = $${directory:bin}/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
 wrapper-path = $${directory:services}/slaprunner-supervisord
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 
 [test-runner]
@@ -224,7 +230,7 @@ arguments = --server_url=$${slap-connection:server-url} --key_file=$${slap-conne
 command-line = ${buildout:directory}/bin/slaprunnertest $${:arguments}
 wrapper-path = $${directory:bin}/runTestSuite
 environment = PATH=$${shell-environment:path}
-  RUNNER_CONFIG=$${slapos-cfg:rendered} 
+  RUNNER_CONFIG=$${slapos-cfg:rendered}
 
 # Deploy openssh-server
 [runner-sshd-port]
@@ -287,9 +293,15 @@ keys = $${directory:sshkeys}/runner-keys/
 recipe = slapos.cookbook:sshkeys_authority
 request-directory = $${runner-sshkeys-directory:requests}
 keys-directory = $${runner-sshkeys-directory:keys}
-wrapper = $${directory:services}/runner_sshkeys_authority
+wrapper = $${directory:bin}/runner_sshkeys_authority
 keygen-binary = ${openssh:location}/bin/ssh-keygen
 
+[runner-sshkeys-authority-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${runner-sshkeys-authority:wrapper}
+wrapper-path = $${directory:services}/runner-sshkeys-authority
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
 [runner-sshkeys-sshd]
 <= runner-sshkeys-authority
 recipe = slapos.cookbook:sshkeys_authority.request
@@ -298,7 +310,13 @@ type = rsa
 executable = $${runner-sshd-server:output}
 public-key = $${runner-sshd-raw-server:rsa-keyfile}.pub
 private-key = $${runner-sshd-raw-server:rsa-keyfile}
-wrapper = $${directory:services}/runner-sshd
+wrapper = $${directory:bin}/runner-sshd
+
+[runner-sshkeys-sshd-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${runner-sshkeys-sshd:wrapper}
+wrapper-path = $${directory:services}/runner-sshd
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [runner-sshd-add-authorized-key]
 recipe = slapos.cookbook:dropbear.add_authorized_key
@@ -411,6 +429,7 @@ access-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global
 wait-for-files =
   $${ca-nginx:cert-file}
   $${ca-nginx:key-file}
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [logrotate-entry-apache-httpd]
 <= logrotate-entry-base
@@ -461,6 +480,7 @@ wrapper-path = $${gunicorn:bin_launcher}
 environment = PATH=$${shell-environment:path}
   RUNNER_CONFIG=$${slaprunner:slapos.cfg}
   LANG=en_GB.UTF-8
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [gunicorn-graceful]
 recipe = slapos.cookbook:wrapper
@@ -476,7 +496,7 @@ recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${openssl:location}/bin/openssl
 ca-dir = $${directory:ca-dir}
 requests-directory = $${cadirectory:requests}
-wrapper = $${directory:services}/certificate_authority
+wrapper = $${directory:bin}/certificate_authority
 ca-private = $${cadirectory:private}
 ca-certs = $${cadirectory:certs}
 ca-newcerts = $${cadirectory:newcerts}
@@ -496,10 +516,22 @@ recipe = slapos.cookbook:certificate_authority.request
 key-file = $${cadirectory:certs}/nginx_frontend.key
 cert-file = $${cadirectory:certs}/nginx_frontend.crt
 executable = $${nginx-launcher:rendered}
-wrapper = $${directory:services}/nginx-frontend
+wrapper = $${directory:bin}/nginx-frontend
 # Put domain name
 name = example.com
 
+[ca-nginx-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${directory:bin}/nginx-frontend
+wrapper-path = $${directory:services}/nginx-frontend
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
+[certificate-authority-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${directory:bin}/certificate_authority
+wrapper-path = $${directory:services}/certificate_authority
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
 #--------------------
 #--
 #-- Request frontend
@@ -527,14 +559,14 @@ check-secure = 1
 [request-httpd-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
-# XXX - Unfortunately, we still call webrunner httpd frontend "Monitor Frontend" otherwise 
+# XXX - Unfortunately, we still call webrunner httpd frontend "Monitor Frontend" otherwise
 # buildout will ignore previous frontend that was created and create a new one (in case of upgrade)
 name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
 config-url = $${apache-httpd:access-url}
-config-domain = 
+config-domain =
 return = secure_access domain
 
 [httpd-frontend-promise]
@@ -638,8 +670,8 @@ slaprunner-httpd-port = $${:monitor-port}
 # XXX - for backward compatibility, monitor-port was for slaprunner httpd server
 monitor-port = 9686
 instance-name =
-monitor-cors-domains = 
-monitor-interface-url = 
+monitor-cors-domains =
+monitor-interface-url =
 # XXX - define a new port for monitor here and use monitor-port for backward compatibility
 monitor-httpd-port = 8386
 
@@ -667,7 +699,7 @@ recipe = slapos.recipe.template:jinja2
 # We cannot use slapos.cookbook:wrapper here because this recipe escapes too much
 socket = $${directory:run}/siab.sock
 mode = 0700
-rendered = $${directory:services}/shellinaboxd
+rendered = $${directory:bin}/shellinaboxd
 template = inline:
   #!/bin/sh
   exec ${shellinabox:location}/bin/shellinaboxd \
@@ -676,6 +708,12 @@ template = inline:
     --unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \
     --service "/:$(id -u):$(id -g):HOME:$${shell-environment:shell} -l"
 
+[shellinabox-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${directory:bin}/shellinaboxd
+wrapper-path = $${directory:services}/shellinaboxd
+hash-files = $${buildout:directory}/software_release/buildout.cfg
+
 [shell-environment]
 shell = ${bash:location}/bin/bash
 path = ${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin:${mosh:location}/bin:${bash:location}/bin:$${buildout:directory}/bin/:/usr/bin:/bin/
@@ -743,7 +781,7 @@ context =
     raw shell $${shell-environment:shell}
     key instance_name slap-parameter:instance-name
     key workdir runnerdirectory:home
-    
+
 #---------------------------
 #--
 #-- supervisord managing slaprunner automation features
@@ -808,6 +846,7 @@ context =
 recipe = slapos.cookbook:wrapper
 command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
 wrapper-path = $${directory:services}/supervisord
+hash-files = $${buildout:directory}/software_release/buildout.cfg
 
 [logrotate-entry-supervisord]
 <= logrotate-entry-base
@@ -826,7 +865,7 @@ path = $${directory:promises}/supervisord
 hostname = $${slaprunner:ipv4}
 port = $${supervisord:port}
 
-# XXX Monitor 
+# XXX Monitor
 [monitor-instance-parameter]
 monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
 {% if slapparameter_dict.get('name', '') -%}

software/slaprunner/test/README.md

+Tests for slaprunner Software Release

software/slaprunner/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.slaprunner'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' slaprunner",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+          'slapos.core',
+          'slapos.libnetworkcache',
+          'erp5.util',
+          'supervisor',
+          'psutil',
+      ],
+      zip_safe=True,
+      test_suite='test',
+      )

software/slaprunner/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import shutil
+import urlparse
+import tempfile
+import requests
+import socket
+import StringIO
+import subprocess
+import json
+
+import psutil
+
+import utils
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class InstanceTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+class ServicesTestCase(InstanceTestCase):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      'slaprunner-supervisord-{hash}-on-watch',
+      'runner-sshkeys-authority-{hash}-on-watch',
+      'runner-sshd-{hash}-on-watch',
+      'slaprunner-httpd-{hash}-on-watch',
+      'gunicorn-{hash}-on-watch',
+      'nginx-frontend-{hash}-on-watch',
+      'certificate_authority-{hash}-on-watch',
+      'shellinaboxd-{hash}-on-watch',
+      'supervisord-{hash}-on-watch',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)