diff --git a/software/gitlab/buildout.hash.cfg b/software/gitlab/buildout.hash.cfg index 458bd3a7c81af54fcddf3fb9c562c25f39295072..fb4a78e839a7aea66c046ce15b3a1be937d1e541 100644 --- a/software/gitlab/buildout.hash.cfg +++ b/software/gitlab/buildout.hash.cfg @@ -54,7 +54,7 @@ md5sum = d769ea27820e932c596c35bbbf3f2902 [instance-gitlab.cfg.in] _update_hash_filename_ = instance-gitlab.cfg.in -md5sum = e17bfe96bb9a0f4666d90c877a663e88 +md5sum = 35bb9f1d8f4fd6675bd768d8a7e1253c [instance-gitlab-export.cfg.in] _update_hash_filename_ = instance-gitlab-export.cfg.in diff --git a/software/gitlab/instance-gitlab.cfg.in b/software/gitlab/instance-gitlab.cfg.in index 3ba414227a404b03cf72f2d722183376c2ec6440..5089b7956bcdd93723ed91f8357d32891eaac687 100644 --- a/software/gitlab/instance-gitlab.cfg.in +++ b/software/gitlab/instance-gitlab.cfg.in @@ -916,8 +916,6 @@ rake = ${gitlab-rake:wrapper-path} # run command on every reinstantiation update-command = ${:command} -# https://gitlab.com/gitlab-org/gitlab-foss/issues/38457 -# we need to manually install ajv@^4.0.0 with yarn to fix the bug 'yarn check failed!' command = ${:rake} gitlab:assets:clean && ${:rake} gettext:compile RAILS_ENV=production && diff --git a/software/theia/buildout.hash.cfg b/software/theia/buildout.hash.cfg index d8549e9e5349847e700949abd9492247e686a038..25f0583ed76dea368b33fe985b2abafd30b84fb4 100644 --- a/software/theia/buildout.hash.cfg +++ b/software/theia/buildout.hash.cfg @@ -15,11 +15,11 @@ [instance-theia] _update_hash_filename_ = instance-theia.cfg.jinja.in -md5sum = 23b498618bce83a6eb8df0470417f59e +md5sum = b4e87cff99a8521e6d0b911e3ef35b30 [instance] _update_hash_filename_ = instance.cfg.in -md5sum = 5aab12790cdb1981cb0caf00d389a227 +md5sum = 837eb2786f185ddb5a28d29e271652f7 [instance-import] _update_hash_filename_ = instance-import.cfg.jinja.in diff --git a/software/theia/instance-theia.cfg.jinja.in b/software/theia/instance-theia.cfg.jinja.in index 3f74e978406641af7706f27e9c381874d2e2b5d8..5d4273fc5f3af48f90db02683567fa07c9af97c1 100644 --- a/software/theia/instance-theia.cfg.jinja.in +++ b/software/theia/instance-theia.cfg.jinja.in @@ -53,7 +53,11 @@ pidfiles = $${:var}/run statefiles = $${:var}/state services = $${:etc}/service +{% if parameter_dict['testing-short-embedded-instance-path'] %} +runner = $${:home}/r +{% else %} runner = $${:srv}/runner +{% endif %} backup = $${:srv}/backup/theia project = $${:srv}/project @@ -567,8 +571,13 @@ ipv4 = {{ ipv4_random }} ipv6 = {{ slap_resource.get('ipv6-range-network') or ipv6_theia }} port = $${slapos-standalone-port:port} base-directory = $${directory:runner} +{% if parameter_dict['testing-short-embedded-instance-path'] %} +software-root = $${directory:runner}/s +instance-root = $${directory:runner}/i +{% else %} software-root = $${directory:runner}/software instance-root = $${directory:runner}/instance +{% endif %} local-software-release-root = $${directory:home} slapos-bin = ${buildout:bin-directory}/slapos slapos-configuration = $${directory:runner}/etc/slapos.cfg diff --git a/software/theia/instance.cfg.in b/software/theia/instance.cfg.in index 0610e0ee30b1d1b28ee4117d4787bcf3fc384462..e1c047453b2ebc351ded2e94ece4954cd714b72c 100644 --- a/software/theia/instance.cfg.in +++ b/software/theia/instance.cfg.in @@ -55,7 +55,8 @@ default-parameters = "additional-frontend-sr": "$${:frontend-sr}", "additional-frontend-sr-type": "default", "additional-frontend-guid": null, - "monitor-httpd-port": 8386 + "monitor-httpd-port": 8386, + "testing-short-embedded-instance-path": null } frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg diff --git a/software/theia/test/project_tests.py b/software/theia/test/project_tests.py index 7dab1ac996f1335c3c9ac6b55ca2473e0bcbfda8..3436d02ad757783ba654ae81f7973e2574a4632e 100644 --- a/software/theia/test/project_tests.py +++ b/software/theia/test/project_tests.py @@ -24,7 +24,6 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## -from __future__ import unicode_literals import gzip import json @@ -32,13 +31,12 @@ import os import re import subprocess import time -import unittest import shutil import requests import tempfile from datetime import datetime, timedelta -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from mimetypes import guess_type from json.decoder import JSONDecodeError @@ -67,6 +65,21 @@ def setUpModule(): ) +class TestTheiaResilienceWithShortPaths(test_resiliency.TestTheiaResilience): + """TestTheiaResilience, but with shorter paths for embedded slapos, to + overcome OS limit with the length of unix sockets or #! "shebang" lines. + """ + @classmethod + def getInstanceParameterDict(cls): + return dict( + super().getInstanceParameterDict(), + **{'testing-short-embedded-instance-path': 'true'}) + + @classmethod + def _getSlapos(cls, instance_type='export'): + return cls.getPartitionPath(instance_type, 'r', 'bin', 'slapos') + + class ERP5Mixin(object): _test_software_url = erp5_software_release_url _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL) @@ -125,7 +138,7 @@ class ERP5Mixin(object): instance_type, 'srv', 'runner', 'instance', partition, *paths) -class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience): +class TestTheiaResilienceERP5(ERP5Mixin, TestTheiaResilienceWithShortPaths): test_instance_max_retries = 12 backup_max_tries = 480 backup_wait_interval = 60 @@ -242,7 +255,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience): out = subprocess.check_output((mysql_bin, 'erp5', '-e', query), universal_newlines=True) self.assertIn(self._erp5_new_title, out, 'Mariadb catalog is not properly restored') -class TestTheiaResiliencePeertube(test_resiliency.TestTheiaResilience): + +class TestTheiaResiliencePeertube(TestTheiaResilienceWithShortPaths): test_instance_max_retries = 12 backup_max_tries = 480 backup_wait_interval = 60 @@ -446,9 +460,10 @@ class TestTheiaResiliencePeertube(test_resiliency.TestTheiaResilience): def _getPeertubePartitionPath(self, instance_type, servicename, *paths): partition = self._getPeertubePartition(servicename) return self.getPartitionPath( - instance_type, 'srv', 'runner', 'instance', partition, *paths) + instance_type, 'r', 'i', partition, *paths) + -class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): +class TestTheiaResilienceGitlab(TestTheiaResilienceWithShortPaths): test_instance_max_retries = 50 # puma takes time to be ready backup_max_tries = 480 backup_wait_interval = 60 @@ -467,7 +482,7 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): stderr=subprocess.STDOUT, text=True, ) - print(out) + self.logger.info("_getGitlabConnectionParameters output: %s", out) return json.loads(self._connection_parameters_regex.search(out).group(0).replace("'", '"')) def test_twice(self): @@ -499,7 +514,7 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): # Create a new project print("Gitlab create a project") - path = '/api/v3/projects' + path = '/api/v4/projects' parameter_dict = {'name': 'sample-test', 'namespace': 'open'} # Token can be set manually headers = {"PRIVATE-TOKEN" : 'SLurtnxPscPsU-SDm4oN'} @@ -508,14 +523,14 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): # Check the project is exist print("Gitlab check project is exist") - path = '/api/v3/projects' - response = requests.get(backend_url + path, headers=headers, verify=False) + path = '/api/v4/projects' + response = requests.get(backend_url + path, params={'search': 'sample-test'}, headers=headers, verify=False) try: projects = response.json() except JSONDecodeError: self.fail("No json file returned! Maybe your Gitlab URL is incorrect.") - # Only one project exist + # Only one project matches the search self.assertEqual(len(projects), 1) # The project name is sample-test, which we created above. self.assertIn("sample-test", projects[0]['name_with_namespace']) @@ -543,12 +558,14 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): output = subprocess.check_output(('git', 'push', 'origin'), cwd=repo_path, universal_newlines=True) # Do a fake periodically update - # Compute backup date in the near future - soon = (datetime.now() + timedelta(minutes=4)) + # Compute backup date in the future + # During slapos node instance, the static assets are recompiled, which takes a lot + # of time, so we give it at least 20 minutes. + soon = (datetime.now() + timedelta(minutes=20)) frequency = "%d * * * *" % soon.minute params = 'backup_frequency=%s' % frequency - # Update Peertube parameters + # Update Gitlab parameters print('Requesting Gitlab with parameters %s' % params) self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params) @@ -557,8 +574,8 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): self.callSlapos('node', 'restart', 'all') # Wait until after the programmed backup date, and a bit more - t = (soon - datetime.now()).total_seconds() - time.sleep(t + 240) + t = ((soon - datetime.now()) + timedelta(minutes=10)).total_seconds() + time.sleep(t) self.callSlapos('node', 'status') os.chdir(self.temp_clone_dir) @@ -583,9 +600,9 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): # Check the project is exist print("Gitlab check project is exist") - path = '/api/v3/projects' + path = '/api/v4/projects' headers = {"PRIVATE-TOKEN" : 'SLurtnxPscPsU-SDm4oN'} - response = requests.get(backend_url + path, headers=headers, verify=False) + response = requests.get(backend_url + path, params={'search': 'sample-test'}, headers=headers, verify=False) try: projects = response.json() except JSONDecodeError: @@ -623,4 +640,4 @@ class TestTheiaResilienceGitlab(test_resiliency.TestTheiaResilience): def _getGitlabPartitionPath(self, instance_type, servicename, *paths): partition = self._getGitlabPartition(servicename) return self.getPartitionPath( - instance_type, 'srv', 'runner', 'instance', partition, *paths) + instance_type, 'r', 'i', partition, *paths) diff --git a/software/theia/test/test_resiliency.py b/software/theia/test/test_resiliency.py index 6a75d79c5ef91deecb7156b9147e2a8339f8decc..7660f44e436b96c7d6fa8ce8554f4d6a4884e57a 100644 --- a/software/theia/test/test_resiliency.py +++ b/software/theia/test/test_resiliency.py @@ -72,34 +72,42 @@ def setUpModule(): class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase): @classmethod def _processEmbeddedInstance(cls, retries=0, instance_type='export'): - for _ in range(retries): + for retry in range(retries): try: - output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT) + output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT, text=True) except subprocess.CalledProcessError: continue - print(output) + cls.logger.info("_processEmbeddedInstance retry=%s output=%s", retry, output) break else: if retries: # Sleep a bit as an attempt to workaround monitoring boostrap not being ready print("Wait before running slapos node instance one last time") time.sleep(120) - cls.checkSlapos('node', 'instance', instance_type=instance_type) + try: + cls.checkSlapos('node', 'instance', instance_type=instance_type, text=True) + except subprocess.CalledProcessError as e: + cls.logger.error(e.output, exc_info=True) + raise @classmethod def _processEmbeddedSoftware(cls, retries=0, instance_type='export'): - for _ in range(retries): + for retry in range(retries): try: - output = cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT) + output = cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT, text=True) except subprocess.CalledProcessError: continue - print(output) + cls.logger.info("_processEmbeddedSoftware retry=%s output=%s", retry, output) break else: if retries: print("Wait before running slapos node software one last time") time.sleep(120) - cls.checkSlapos('node', 'software', instance_type=instance_type) + try: + cls.checkSlapos('node', 'software', instance_type=instance_type, text=True) + except subprocess.CalledProcessError as e: + cls.logger.error(e.output, exc_info=True) + raise @classmethod def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'): diff --git a/stack/resilient/buildout.hash.cfg b/stack/resilient/buildout.hash.cfg index 002423fee3313221a924730eb7c1c67cf86d6aa9..fb5ae7d76c66efe3fb1a304454f5311238e05e71 100644 --- a/stack/resilient/buildout.hash.cfg +++ b/stack/resilient/buildout.hash.cfg @@ -30,7 +30,7 @@ md5sum = 1b8645835f04081861266436505fd28f [template-replicated] filename = template-replicated.cfg.in -md5sum = 67c863b15dbfa937babdbd620f95c1ff +md5sum = 743012b9e8d318712187621867613bd4 [template-parts] filename = template-parts.cfg.in diff --git a/stack/resilient/template-replicated.cfg.in b/stack/resilient/template-replicated.cfg.in index 658cfbe10b1e132b35453e99fafd94b0d6cffffa..294704d7db0acbb217b0b937b0f63b142ed3fa80 100644 --- a/stack/resilient/template-replicated.cfg.in +++ b/stack/resilient/template-replicated.cfg.in @@ -87,7 +87,7 @@ return = ssh-public-key resilient-ssh-url notification-url ip takeover-url takeo pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push -{% for extra_parameter_for_pseudo_replicating_instance in ["software-root", "buildout-shared-folder"] %} +{% for extra_parameter_for_pseudo_replicating_instance in ["software-root", "buildout-shared-folder", "testing-short-embedded-instance-path"] %} {% if slapparameter_dict.get(extra_parameter_for_pseudo_replicating_instance) %} config-{{ extra_parameter_for_pseudo_replicating_instance }} = {{ slapparameter_dict.get(extra_parameter_for_pseudo_replicating_instance) }} {% endif %}