slapos-master: Port latest tests for slapos into slapos-master

  This is a partial port, just to have more tests that
  weren't afected by the patches for keep apache as entry-point
  of the backend.

software/slapos-master/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 84f099cc9852c4f53a075dccbb3880f0
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 67022177ad0f08511af426888cf2738e
+md5sum = c7c0bb9abbd0f8cc6c7956d83a61c4b3
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -2,6 +2,7 @@
 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
+{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
 {% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
@@ -38,7 +39,7 @@ mode = 644
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
 {% set zope_virtualhost_monster_backend_dict = {} %}
-{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
+{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
 {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
 {% for family_name, parameter_id_list in sorted(
   slapparameter_dict['zope-family-dict'].iteritems()) -%}
@@ -59,19 +60,19 @@ mode = 644
 {%     set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
 {%     if test_runner_address_list -%}
 {%       set test_runner_backend_mapping = {} %}
-{%       set test_runner_apache_url_list = [] %}
+{%       set test_runner_balancer_url_list = [] %}
 {%       set test_runner_external_port = next_port() %}
 {%       for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
 {%         do test_runner_backend_mapping.__setitem__(
                 'unit_test_' ~ i,
                 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
-{%         do test_runner_apache_url_list.append(
+{%         do test_runner_balancer_url_list.append(
                 'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
 {%       endfor %}
 {%       do zope_virtualhost_monster_backend_dict.__setitem__(
               (ipv4, test_runner_external_port),
               ( ssl_authentication, test_runner_backend_mapping ) ) -%}
-{%       do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
+{%       do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
 {%     endif -%}
 {%   endfor -%}
 
@@ -123,6 +124,23 @@ caucase-key = ${directory:apache-conf}/apache-caucase.pem
 ca-cert =  ${directory:apache-conf}/ca.crt
 crl = ${directory:apache-conf}/crl.pem
 
+[simplefile]
+< = jinja2-template-base
+template = inline:{{ '{{ content }}' }}
+
+{% macro simplefile(section_name, file_path, content, mode='') -%}
+{%   set content_section_name = section_name ~ '-content' -%}
+[{{  content_section_name }}]
+content = {{ dumps(content) }}
+
+[{{  section(section_name) }}]
+< = simplefile
+rendered = {{ file_path }}
+context = key content {{content_section_name}}:content
+mode = {{ mode }}
+{%- endmacro %}
+
+
 [apache-ssl]
 {% if ssl_parameter_dict.get('key') -%}
 key = ${apache-ssl-key:rendered}
@@ -273,8 +291,6 @@ template = inline:
   {% endfor %}
 
 [apachedex-parameters]
-# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
-# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
 apache-log-list = ${apache-conf-parameter-dict:access-log}
 configuration = ${monitor-apachedex-report-config:rendered}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}

software/slapos-master/test/test/__init__.py

@@ -30,11 +30,21 @@ import os
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
-setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+
+_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
 
 
+setup_module_executed = False
+def setUpModule():
+  # slapos.testing.testcase's only need to be executed once
+  global setup_module_executed
+  if not setup_module_executed:
+    _setUpModule()
+  setup_module_executed = True
+
+
 class ERP5InstanceTestCase(SlapOSInstanceTestCase):
   """ERP5 base test case
   """

software/slapos-master/test/test/test_erp5.py

@@ -41,31 +41,50 @@ import requests
 
 from . import ERP5InstanceTestCase
 from . import setUpModule
-setUpModule  # pyflakes
+setUpModule # pyflakes
 
 
 class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
-  def _checkERP5IsReachable(self, url):
+
+  def _checkERP5IsReachable(self, base_url, site_id, verify):
+    # We access ERP5 trough a "virtual host", which should make
+    # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
+    # as base.
+    virtual_host_url = urlparse.urljoin(
+        base_url,
+        '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
+        .format(site_id))
+
     # What happens is that instantiation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
-    # instantiated, but zope is still busy creating the site and haproxy
-    # replies with 503 Service Unavailable when zope is not started yet, with
-    # 404 when erp5 site is not created, with 500 when mysql is not yet
-    # reachable, so we retry in a loop until we get a succesful response.
-    for i in range(1, 60):
-      # XXX can we get CA from caucase already ?
-      r = requests.get(url, verify=False)
-      if r.status_code != requests.codes.ok:
-        delay = i * 2
-        self.logger.warn(
-          "ERP5 was not available, sleeping for %ds and retrying", delay)
-        time.sleep(delay)
-        continue
-      r.raise_for_status()
-      break
-
+    # instantiated, but zope is still busy creating the site and haproxy replies
+    # with 503 Service Unavailable when zope is not started yet, with 404 when
+    # erp5 site is not created, with 500 when mysql is not yet reachable, so we
+    # configure this requests session to retry.
+    # XXX we should probably add a promise instead
+    session = requests.Session()
+    session.mount(
+        base_url,
+        requests.adapters.HTTPAdapter(
+            max_retries=requests.packages.urllib3.util.retry.Retry(
+                total=60,
+                backoff_factor=.5,
+                status_forcelist=(404, 500, 503))))
+
+    r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
+    self.assertEqual(r.status_code, requests.codes.found)
+    # access on / are redirected to login form, with virtual host preserved
+    self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
+
+    # login page can be rendered and contain the text "ERP5"
+    r = session.get(
+        urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
+        verify=verify,
+        allow_redirects=False,
+    )
+    self.assertEqual(r.status_code, requests.codes.ok)
     self.assertIn("ERP5", r.text)
 
   def test_published_family_default_v6_is_reachable(self):
@@ -73,15 +92,18 @@ class TestPublishedURLIsReachableMixin(object):
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+      param_dict['family-default-v6'],
+      param_dict['site-id'],
+      verify=False)
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
-
+      param_dict['family-default'],
+      param_dict['site-id'],
+      verify=False)
 
 class TestDefaultParameters(
   ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
@@ -99,6 +121,30 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wsgi': False})}
 
+class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 Jupyter notebook
+  """
+  __partition_reference__ = 'jupyter'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'jupyter': {'enable': True}})}
+
+  def test_jupyter_notebook_is_reachable(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+
+    self.assertEqual(
+      'https://[%s]:8888/tree' % self._ipv6_address,
+      param_dict['jupyter-url']
+    )
+
+    result = requests.get(
+      param_dict['jupyter-url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [requests.codes.found, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
+
 
 class TestApacheBalancerPorts(ERP5InstanceTestCase):
   """Instantiate with two zope families, this should create for each family:
@@ -132,19 +178,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     self.assertTrue(parsed.port)
 
   def test_published_family_parameters(self):
-    # when we request two families, we have two published family-{family_name}
-    # URLs
+    # when we request two families, we have two published family-{family_name} URLs
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       self.checkValidHTTPSURL(
           param_dict['family-{family_name}'.format(family_name=family_name)])
       self.checkValidHTTPSURL(
-          param_dict['family-{family_name}-v6'.format(
-            family_name=family_name)])
+          param_dict['family-{family_name}-v6'.format(family_name=family_name)])
 
   def test_published_test_runner_url(self):
-    # each family's also a list of test test runner URLs, by default 3 per
-    # family
+    # each family's also a list of test test runner URLs, by default 3 per family
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       family_test_runner_url_list = param_dict[
@@ -154,8 +197,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
         self.checkValidHTTPSURL(url)
 
   def test_zope_listen(self):
-    # we requested 3 zope in family1 and 5 zopes in family2, we should have 8
-    # zope running.
+    # we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
     self.assertEqual(
@@ -190,46 +232,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     ])
 
 
-class TestDisableTestRunner(
-  ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
-  """Test ERP5 can be instantiated without test runner.
-  """
-  __partition_reference__ = 'distr'
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {'_': json.dumps({'test-runner': {'enabled': False}})}
-
-  def test_no_runUnitTestScript(self):
-    """No runUnitTest script should be generated in any partition.
-    """
-    # self.computer_partition_root_path is the path of root partition.
-    # we want to assert that no scripts exist in any partition.
-    bin_programs = map(
-      os.path.basename,
-      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
-
-    self.assertTrue(bin_programs)  # just to check the glob was correct.
-    self.assertNotIn('runUnitTest', bin_programs)
-    self.assertNotIn('runTestSuite', bin_programs)
-
-  def test_no_apache_testrunner_port(self):
-    # Apache only listen on two ports, there is no apache ports allocated for
-    # test runner
-    with self.slap.instance_supervisor_rpc as supervisor:
-      all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [p for p in all_process_info if p['name'] == 'apache']
-    apache_process = psutil.Process(process_info['pid'])
-    self.assertEqual(
-        sorted([socket.AF_INET, socket.AF_INET6]),
-        sorted(
-            c.family
-            for c in apache_process.connections()
-            if c.status == 'LISTEN'
-        ))
-
-
-class TestZopeNodeParameterOverride(
-  ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test override zope node parameters
   """
   __partition_reference__ = 'override'
@@ -244,7 +247,7 @@ class TestZopeNodeParameterOverride(
         "server": {},
         "cache-size-bytes": "20MB",
         "cache-size-bytes!": [
-          ("bb-0", 1 << 20),
+          ("bb-0", 1<<20),
           ("bb-.*", "500MB"),
         ],
         "pool-timeout": "10m",
@@ -315,7 +318,7 @@ class TestZopeNodeParameterOverride(
     partition = self.getComputerPartitionPath('zope-bb')
     for zope in xrange(5):
       checkConf({
-          "cache-size-bytes": "500MB" if zope else 1 << 20,
+          "cache-size-bytes": "500MB" if zope else 1<<20,
         }, {
           "cache-size": None,
         })

software/slapos-master/test/test/test_wcfs.py

+# Copyright (C) 2021  Nexedi SA and Contributors.
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import json
+import os.path
+import unittest
+
+from slapos.grid.utils import md5digest
+
+from . import ERP5InstanceTestCase, setUpModule as _setUpModule
+from .test_erp5 import TestPublishedURLIsReachableMixin
+
+# skip tests when software release is built with wendelin.core 1.
+def setUpModule():
+  _setUpModule()
+
+  cls = ERP5InstanceTestCase
+  if not os.path.exists(
+    os.path.join(
+      cls.slap.software_directory,
+      md5digest(cls.getSoftwareURL()),
+      'bin', 'wcfs')):
+    raise unittest.SkipTest("built with wendelin.core 1")
+
+
+class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test Wendelin Core File System
+  """
+  __partition_reference__ = 'wcfs'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'wcfs': {'enable': True}})}
+
+  def test_wcfs_accessible(self):
+    """Verify that wcfs filesystem is basically accessible.
+
+       - we can read .wcfs/zurl
+       - its content is equal to published `serving-zurl`
+    """
+    zurl = json.loads(
+             self.getComputerPartition('wcfs').getConnectionParameter('_')
+           )['serving-zurl']
+
+    mntpt = lookupMount(zurl)
+    zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
+    self.assertEqual(zurl_, zurl)
+
+
+# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
+def lookupMount(zurl):
+  for line in readfile('/proc/mounts').splitlines():
+    # <zurl> <mountpoint> fuse.wcfs ...
+    zurl_, mntpt, typ, _ = line.split(None, 3)
+    if typ != 'fuse.wcfs':
+      continue
+    if zurl_ == zurl:
+      return mntpt
+  raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
+
+# readfile returns content of file @path.
+def readfile(path):
+  with open(path, 'r') as f:
+    return f.read()