Commit cc1407d2 authored by Rafael Monnerat's avatar Rafael Monnerat

slapos-master: Port latest tests for slapos into slapos-master

  This is a partial port, just to have more tests that
  weren't afected by the patches for keep apache as entry-point
  of the backend.
parent c7dc30a6
Pipeline #16728 failed with stage
...@@ -18,7 +18,7 @@ md5sum = 84f099cc9852c4f53a075dccbb3880f0 ...@@ -18,7 +18,7 @@ md5sum = 84f099cc9852c4f53a075dccbb3880f0
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = 67022177ad0f08511af426888cf2738e md5sum = c7c0bb9abbd0f8cc6c7956d83a61c4b3
[template-apache-backend-conf] [template-apache-backend-conf]
filename = apache-backend.conf.in filename = apache-backend.conf.in
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
{% set part_list = [] -%} {% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set ssl_parameter_dict = slapparameter_dict['ssl'] -%} {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
{% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%} {% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
{# {#
XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
...@@ -38,7 +39,7 @@ mode = 644 ...@@ -38,7 +39,7 @@ mode = 644
{% set haproxy_dict = {} -%} {% set haproxy_dict = {} -%}
{% set apache_dict = {} -%} {% set apache_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %} {% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #} {% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%} {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% for family_name, parameter_id_list in sorted( {% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%} slapparameter_dict['zope-family-dict'].iteritems()) -%}
...@@ -59,19 +60,19 @@ mode = 644 ...@@ -59,19 +60,19 @@ mode = 644
{% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %} {% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
{% if test_runner_address_list -%} {% if test_runner_address_list -%}
{% set test_runner_backend_mapping = {} %} {% set test_runner_backend_mapping = {} %}
{% set test_runner_apache_url_list = [] %} {% set test_runner_balancer_url_list = [] %}
{% set test_runner_external_port = next_port() %} {% set test_runner_external_port = next_port() %}
{% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %} {% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
{% do test_runner_backend_mapping.__setitem__( {% do test_runner_backend_mapping.__setitem__(
'unit_test_' ~ i, 'unit_test_' ~ i,
'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %} 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
{% do test_runner_apache_url_list.append( {% do test_runner_balancer_url_list.append(
'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %} 'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
{% endfor %} {% endfor %}
{% do zope_virtualhost_monster_backend_dict.__setitem__( {% do zope_virtualhost_monster_backend_dict.__setitem__(
(ipv4, test_runner_external_port), (ipv4, test_runner_external_port),
( ssl_authentication, test_runner_backend_mapping ) ) -%} ( ssl_authentication, test_runner_backend_mapping ) ) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%} {% do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
...@@ -123,6 +124,23 @@ caucase-key = ${directory:apache-conf}/apache-caucase.pem ...@@ -123,6 +124,23 @@ caucase-key = ${directory:apache-conf}/apache-caucase.pem
ca-cert = ${directory:apache-conf}/ca.crt ca-cert = ${directory:apache-conf}/ca.crt
crl = ${directory:apache-conf}/crl.pem crl = ${directory:apache-conf}/crl.pem
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[apache-ssl] [apache-ssl]
{% if ssl_parameter_dict.get('key') -%} {% if ssl_parameter_dict.get('key') -%}
key = ${apache-ssl-key:rendered} key = ${apache-ssl-key:rendered}
...@@ -273,8 +291,6 @@ template = inline: ...@@ -273,8 +291,6 @@ template = inline:
{% endfor %} {% endfor %}
[apachedex-parameters] [apachedex-parameters]
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log} apache-log-list = ${apache-conf-parameter-dict:access-log}
configuration = ${monitor-apachedex-report-config:rendered} configuration = ${monitor-apachedex-report-config:rendered}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }} promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
......
...@@ -30,11 +30,21 @@ import os ...@@ -30,11 +30,21 @@ import os
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
setup_module_executed = False
def setUpModule():
# slapos.testing.testcase's only need to be executed once
global setup_module_executed
if not setup_module_executed:
_setUpModule()
setup_module_executed = True
class ERP5InstanceTestCase(SlapOSInstanceTestCase): class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case """ERP5 base test case
""" """
......
This diff is collapsed.
...@@ -47,25 +47,44 @@ setUpModule # pyflakes ...@@ -47,25 +47,44 @@ setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object): class TestPublishedURLIsReachableMixin(object):
"""Mixin that checks that default page of ERP5 is reachable. """Mixin that checks that default page of ERP5 is reachable.
""" """
def _checkERP5IsReachable(self, url):
def _checkERP5IsReachable(self, base_url, site_id, verify):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url = urlparse.urljoin(
base_url,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.format(site_id))
# What happens is that instantiation just create the services, but does not # What happens is that instantiation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is # wait for ERP5 to be initialized. When this test run ERP5 instance is
# instantiated, but zope is still busy creating the site and haproxy # instantiated, but zope is still busy creating the site and haproxy replies
# replies with 503 Service Unavailable when zope is not started yet, with # with 503 Service Unavailable when zope is not started yet, with 404 when
# 404 when erp5 site is not created, with 500 when mysql is not yet # erp5 site is not created, with 500 when mysql is not yet reachable, so we
# reachable, so we retry in a loop until we get a succesful response. # configure this requests session to retry.
for i in range(1, 60): # XXX we should probably add a promise instead
# XXX can we get CA from caucase already ? session = requests.Session()
r = requests.get(url, verify=False) session.mount(
if r.status_code != requests.codes.ok: base_url,
delay = i * 2 requests.adapters.HTTPAdapter(
self.logger.warn( max_retries=requests.packages.urllib3.util.retry.Retry(
"ERP5 was not available, sleeping for %ds and retrying", delay) total=60,
time.sleep(delay) backoff_factor=.5,
continue status_forcelist=(404, 500, 503))))
r.raise_for_status()
break r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
self.assertEqual(r.status_code, requests.codes.found)
# access on / are redirected to login form, with virtual host preserved
self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
# login page can be rendered and contain the text "ERP5"
r = session.get(
urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
verify=verify,
allow_redirects=False,
)
self.assertEqual(r.status_code, requests.codes.ok)
self.assertIn("ERP5", r.text) self.assertIn("ERP5", r.text)
def test_published_family_default_v6_is_reachable(self): def test_published_family_default_v6_is_reachable(self):
...@@ -73,15 +92,18 @@ class TestPublishedURLIsReachableMixin(object): ...@@ -73,15 +92,18 @@ class TestPublishedURLIsReachableMixin(object):
""" """
param_dict = self.getRootPartitionConnectionParameterDict() param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable( self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id'])) param_dict['family-default-v6'],
param_dict['site-id'],
verify=False)
def test_published_family_default_v4_is_reachable(self): def test_published_family_default_v4_is_reachable(self):
"""Tests the IPv4 URL published by the root partition is reachable. """Tests the IPv4 URL published by the root partition is reachable.
""" """
param_dict = self.getRootPartitionConnectionParameterDict() param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable( self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default'], param_dict['site-id'])) param_dict['family-default'],
param_dict['site-id'],
verify=False)
class TestDefaultParameters( class TestDefaultParameters(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin): ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
...@@ -99,6 +121,30 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin): ...@@ -99,6 +121,30 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
def getInstanceParameterDict(cls): def getInstanceParameterDict(cls):
return {'_': json.dumps({'wsgi': False})} return {'_': json.dumps({'wsgi': False})}
class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Jupyter notebook
"""
__partition_reference__ = 'jupyter'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'jupyter': {'enable': True}})}
def test_jupyter_notebook_is_reachable(self):
param_dict = self.getRootPartitionConnectionParameterDict()
self.assertEqual(
'https://[%s]:8888/tree' % self._ipv6_address,
param_dict['jupyter-url']
)
result = requests.get(
param_dict['jupyter-url'], verify=False, allow_redirects=False)
self.assertEqual(
[requests.codes.found, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
class TestApacheBalancerPorts(ERP5InstanceTestCase): class TestApacheBalancerPorts(ERP5InstanceTestCase):
"""Instantiate with two zope families, this should create for each family: """Instantiate with two zope families, this should create for each family:
...@@ -132,19 +178,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase): ...@@ -132,19 +178,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.assertTrue(parsed.port) self.assertTrue(parsed.port)
def test_published_family_parameters(self): def test_published_family_parameters(self):
# when we request two families, we have two published family-{family_name} # when we request two families, we have two published family-{family_name} URLs
# URLs
param_dict = self.getRootPartitionConnectionParameterDict() param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'): for family_name in ('family1', 'family2'):
self.checkValidHTTPSURL( self.checkValidHTTPSURL(
param_dict['family-{family_name}'.format(family_name=family_name)]) param_dict['family-{family_name}'.format(family_name=family_name)])
self.checkValidHTTPSURL( self.checkValidHTTPSURL(
param_dict['family-{family_name}-v6'.format( param_dict['family-{family_name}-v6'.format(family_name=family_name)])
family_name=family_name)])
def test_published_test_runner_url(self): def test_published_test_runner_url(self):
# each family's also a list of test test runner URLs, by default 3 per # each family's also a list of test test runner URLs, by default 3 per family
# family
param_dict = self.getRootPartitionConnectionParameterDict() param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'): for family_name in ('family1', 'family2'):
family_test_runner_url_list = param_dict[ family_test_runner_url_list = param_dict[
...@@ -154,8 +197,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase): ...@@ -154,8 +197,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.checkValidHTTPSURL(url) self.checkValidHTTPSURL(url)
def test_zope_listen(self): def test_zope_listen(self):
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8 # we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
# zope running.
with self.slap.instance_supervisor_rpc as supervisor: with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo() all_process_info = supervisor.getAllProcessInfo()
self.assertEqual( self.assertEqual(
...@@ -190,46 +232,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase): ...@@ -190,46 +232,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
]) ])
class TestDisableTestRunner( class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated without test runner.
"""
__partition_reference__ = 'distr'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'test-runner': {'enabled': False}})}
def test_no_runUnitTestScript(self):
"""No runUnitTest script should be generated in any partition.
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = map(
os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
self.assertNotIn('runTestSuite', bin_programs)
def test_no_apache_testrunner_port(self):
# Apache only listen on two ports, there is no apache ports allocated for
# test runner
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
))
class TestZopeNodeParameterOverride(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters """Test override zope node parameters
""" """
__partition_reference__ = 'override' __partition_reference__ = 'override'
...@@ -244,7 +247,7 @@ class TestZopeNodeParameterOverride( ...@@ -244,7 +247,7 @@ class TestZopeNodeParameterOverride(
"server": {}, "server": {},
"cache-size-bytes": "20MB", "cache-size-bytes": "20MB",
"cache-size-bytes!": [ "cache-size-bytes!": [
("bb-0", 1 << 20), ("bb-0", 1<<20),
("bb-.*", "500MB"), ("bb-.*", "500MB"),
], ],
"pool-timeout": "10m", "pool-timeout": "10m",
...@@ -315,7 +318,7 @@ class TestZopeNodeParameterOverride( ...@@ -315,7 +318,7 @@ class TestZopeNodeParameterOverride(
partition = self.getComputerPartitionPath('zope-bb') partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5): for zope in xrange(5):
checkConf({ checkConf({
"cache-size-bytes": "500MB" if zope else 1 << 20, "cache-size-bytes": "500MB" if zope else 1<<20,
}, { }, {
"cache-size": None, "cache-size": None,
}) })
......
This diff is collapsed.
# Copyright (C) 2021 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import json
import os.path
import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase, setUpModule as _setUpModule
from .test_erp5 import TestPublishedURLIsReachableMixin
# skip tests when software release is built with wendelin.core 1.
def setUpModule():
_setUpModule()
cls = ERP5InstanceTestCase
if not os.path.exists(
os.path.join(
cls.slap.software_directory,
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test Wendelin Core File System
"""
__partition_reference__ = 'wcfs'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wcfs': {'enable': True}})}
def test_wcfs_accessible(self):
"""Verify that wcfs filesystem is basically accessible.
- we can read .wcfs/zurl
- its content is equal to published `serving-zurl`
"""
zurl = json.loads(
self.getComputerPartition('wcfs').getConnectionParameter('_')
)['serving-zurl']
mntpt = lookupMount(zurl)
zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
self.assertEqual(zurl_, zurl)
# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
def lookupMount(zurl):
for line in readfile('/proc/mounts').splitlines():
# <zurl> <mountpoint> fuse.wcfs ...
zurl_, mntpt, typ, _ = line.split(None, 3)
if typ != 'fuse.wcfs':
continue
if zurl_ == zurl:
return mntpt
raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
return f.read()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment