Update Release Candidate

component/babeld/buildout.cfg

@@ -6,7 +6,7 @@ recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/babeld.git
 branch = master
 git-executable = ${git:location}/bin/git
-revision = hmac-nxd2
+revision = hmac-nxd3
 
 [babeld]
 recipe = slapos.recipe.cmmi

component/fluent-bit/0001-slapos-don-t-try-to-install-system-startup-script.patch

+From 90950c10cf6bfcba777a5b22cdcd3b54b4043d35 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
+Date: Tue, 20 Sep 2022 23:10:16 +0900
+Subject: [PATCH] slapos: don't try to install system startup script
+
+in slapos context we don't write to system directories when installing
+packages
+---
+ src/CMakeLists.txt | 21 ---------------------
+ 1 file changed, 21 deletions(-)
+
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 630f875bf..388c72ab1 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -477,27 +477,6 @@ if(FLB_BINARY)
+       DESTINATION "${FLB_INSTALL_BINDIR}")
+   endif()
+ 
+-  # Detect init system, install upstart, systemd or init.d script
+-  if(IS_DIRECTORY /lib/systemd/system)
+-    set(FLB_SYSTEMD_SCRIPT "${PROJECT_SOURCE_DIR}/init/${FLB_OUT_NAME}.service")
+-    configure_file(
+-      "${PROJECT_SOURCE_DIR}/init/systemd.in"
+-      ${FLB_SYSTEMD_SCRIPT}
+-      )
+-    install(FILES ${FLB_SYSTEMD_SCRIPT} COMPONENT binary DESTINATION /lib/systemd/system)
+-    install(DIRECTORY DESTINATION ${FLB_INSTALL_CONFDIR} COMPONENT binary)
+-  elseif(IS_DIRECTORY /usr/share/upstart)
+-    set(FLB_UPSTART_SCRIPT "${PROJECT_SOURCE_DIR}/init/${FLB_OUT_NAME}.conf")
+-    configure_file(
+-      "${PROJECT_SOURCE_DIR}/init/upstart.in"
+-      ${FLB_UPSTART_SCRIPT}
+-      )
+-    install(FILES ${FLB_UPSTART_SCRIPT} COMPONENT binary DESTINATION /etc/init)
+-    install(DIRECTORY DESTINATION COMPONENT binary ${FLB_INSTALL_CONFDIR})
+-  else()
+-    # FIXME: should we support Sysv init script ?
+-  endif()
+-
+   if(FLB_SYSTEM_WINDOWS)
+     install(FILES
+       "${PROJECT_SOURCE_DIR}/conf/fluent-bit-win32.conf"
+-- 
+2.37.0

component/fluent-bit/buildout.cfg

+[buildout]
+extends =
+  ../cmake/buildout.cfg
+  ../bison/buildout.cfg
+  ../flex/buildout.cfg
+  ../patch/buildout.cfg
+  
+[fluent-bit]
+recipe = slapos.recipe.cmmi
+url = https://github.com/fluent/fluent-bit/archive/refs/tags/v1.9.8.tar.gz
+md5sum = a0973b739f1d1f0daaa503b077760cd7
+configure-command = 
+  ${cmake:location}/bin/cmake
+configure-options = 
+  -DFLB_CONFIG_YAML=Off .
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
+environment =
+  PATH=${bison:location}/bin:${flex:location}/bin:%(PATH)s
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/0001-slapos-don-t-try-to-install-system-startup-script.patch#c071d1aefe0adbc6c96bbdf92d9dbb29
+patch-binary = ${patch:location}/bin/patch

software/fluent-bit/buildout.hash.cfg

 [instance-profile]
 filename = instance.cfg.in
-md5sum = 21ea6fe8ebb6b7baf7e26c0f4c13ebcd
+md5sum = 6a32b81db4ea9f2f63b5a92e8a134d58

software/fluent-bit/instance.cfg.in

 [buildout]
 parts =
-  fluentbit-service
+  fluent-bit-service
   
 eggs-directory = {{ buildout['eggs-directory'] }}
 develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
@@ -17,8 +17,8 @@ promise = ${:etc}/promise/
 log = ${:var}/log
 bin = ${:home}/bin
 
-[fluentbit-service]
+[fluent-bit-service]
 recipe  = slapos.cookbook:wrapper
-command-line = {{ fluentbit_source_location }}/build/bin/fluent-bit -i cpu -o stdout
+command-line = {{ fluent_bit_location }}/bin/fluent-bit -i cpu -o stdout
 wrapper-path = ${directory:service}/fluentbit-service
 output = $${:wrapper-path}

software/fluent-bit/software.cfg

@@ -2,13 +2,11 @@
 extends =
   buildout.hash.cfg
   ../../stack/slapos.cfg
-  ../../component/cmake/buildout.cfg
-  ../../component/bison/buildout.cfg
-  ../../component/flex/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/fluent-bit/buildout.cfg
 
 parts =
   slapos-cookbook
-  fluent-bit
   instance-profile
 
 [instance-profile]
@@ -17,16 +15,4 @@ template = ${:_profile_base_location_}/instance.cfg.in
 rendered = ${buildout:directory}/instance.cfg
 context =
   section buildout buildout
-  key fluentbit_source_location fluent-bit:location
-
-[fluent-bit]
-recipe = slapos.recipe.cmmi
-url = https://github.com/fluent/fluent-bit/archive/refs/tags/v1.9.7.tar.gz
-environment =
-  PATH=${bison:location}/bin:${flex:location}/bin:${fluent-bit:location}:%(PATH)s
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-configure-command =
-  cd build && ${cmake:location}/bin/cmake .. -DFLB_CONFIG_YAML=Off .
-make-targets =
-make-binary =
-  cd build && make
+  key fluent_bit_location fluent-bit:location

software/ors-amarisoft/buildout.hash.cfg

@@ -24,7 +24,7 @@ md5sum = c9994f2ef03edd7d6773307a6385c47c
 
 [template-lte-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
-md5sum = c450b8c307d20738a98bcb4f6ffeee16
+md5sum = 34026b07bdd850dd3fb3ffff5377a0c2
 
 [template-lte-gnb-epc]
 _update_hash_filename_ = instance-gnb-epc.jinja2.cfg
@@ -36,7 +36,7 @@ md5sum = b17674f523adce9b0dda942c1493674d
 
 [template-lte-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg
-md5sum = 9732653a0e2c7fe91d80da80762430d2
+md5sum = 7acdbad30c130faa3bce49b04b2c0c2f
 
 [template-lte-mme]
 _update_hash_filename_ = instance-mme.jinja2.cfg

software/ors-amarisoft/instance-enb.jinja2.cfg

@@ -75,6 +75,8 @@ inline =
   #!/bin/sh
 {% if not slapparameter_dict.get("testing", False) %}
   sudo /opt/amarisoft/rm-tmp-lte | true;
+  sudo /opt/amarisoft/init-sdr | true;
+  sudo /opt/amarisoft/init-enb | true;
   {{ enb }}/lteenb ${directory:etc}/enb.cfg >> ${directory:log}/enb-output.log 2>> ${directory:log}/enb-output.log
 {% endif %}
 

software/ors-amarisoft/instance-gnb.jinja2.cfg

@@ -76,6 +76,8 @@ inline =
   #!/bin/sh
 {% if not slapparameter_dict.get("testing", False) %}
   sudo /opt/amarisoft/rm-tmp-lte | true;
+  sudo /opt/amarisoft/init-sdr | true;
+  sudo /opt/amarisoft/init-enb | true;
   {{ enb }}/lteenb ${directory:etc}/gnb.cfg >> ${directory:log}/gnb-output.log 2>> ${directory:log}/gnb-output.log
 {% endif %}
 

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 6ddb028e7427913fdcac34b8e727734c
+md5sum = 56ad229756d2d57971c583104370dcc7
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
@@ -23,7 +23,7 @@ md5sum = e211c439571e2900f9f35482c9638d06
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 23c3df4a889ebfa9f0a94e873e95ad3b
+md5sum = a343818079d4fc106594e5850cc1853a
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
@@ -35,7 +35,7 @@ md5sum = ad9499e7355ded4975ad313442cecb7a
 
 [slapos-standalone-script]
 _update_hash_filename_ = slapos_standalone_script.py.jinja
-md5sum = ef5b73648513caf46573f3302953790f
+md5sum = 6792d29057db35ea69b01e72a5c913e3
 
 [theia-common]
 _update_hash_filename_ = theia_common.py

software/theia/instance-import.cfg.jinja.in

@@ -34,6 +34,11 @@ name = Import {{ parameter_dict['additional-frontend-name'] }}
 abstract-socket-path = $${directory:home}/standalone-import-ready
 
 
+# Disable frontend request forwarding in the 'import' instance
+[slapos-standalone-script]
+forward-frontend-requests = disabled
+
+
 # Change port ranges to avoid race conditions on port allocation
 [frontend-instance-port]
 minimum = 3200

software/theia/instance-theia.cfg.jinja.in

@@ -532,8 +532,10 @@ context =
   key embedded_request_exitcode_file :embedded-request-exitcode-file
   key embedded_instance_config embedded-instance-config:output
   key home_path directory:home
+  key forward_frontend_requests :forward-frontend-requests
   section slap_connection slap-connection
   section slapos_standalone_config slapos-standalone-config
+forward-frontend-requests = enabled
 url = ${slapos-standalone-script:output}
 
 [slapos-standalone]

software/theia/slapos_standalone_script.py.jinja

@@ -51,6 +51,7 @@ def signal_handler(signum, frame):
 
 @contextlib.contextmanager
 def setupStandalone():
+{%- if forward_frontend_requests != "disabled" %}
   partition_forward_configuration = (
     slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
       master_url={{ repr(slap_connection['server-url']) }},
@@ -63,6 +64,9 @@ def setupStandalone():
       ),
     ),
   )
+{%- else %}
+  partition_forward_configuration = ()
+{%- endif %}
   shared_parts = {{ repr(shared_part_list) }}
   shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
   standalone = slapos.slap.standalone.StandaloneSlapOS(

software/theia/test/test_resiliency.py

@@ -30,11 +30,14 @@ import errno
 import os
 import re
 import shutil
+import sqlite3
 import subprocess
 import time
 
 import requests
 
+from slapos.proxy.db_version import DB_VERSION
+
 from slapos.testing.testcase import SlapOSNodeCommandError, installSoftwareUrlList
 
 from test import TheiaTestCase, ResilientTheiaMixin, theia_software_release_url
@@ -479,7 +482,26 @@ class TakeoverMixin(ExportAndImportMixin):
     return resp.text
 
 
-class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase):
+class TheiaSyncMixin(ResilienceMixin, TakeoverMixin):
+  def _doSync(self, max_tries=None, wait_interval=None):
+    max_tries = max_tries or self.backup_max_tries
+    wait_interval = wait_interval or self.backup_wait_interval
+
+    start = time.time()
+
+    # Call exporter script instead of waiting for cron job
+    # XXX Accelerate cron frequency instead ?
+    exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
+    transaction_id = str(int(time.time()))
+    subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
+
+    takeover_url, _ = self._getTakeoverUrlAndPassword()
+
+    # Wait for takoever to be ready
+    self._waitTakeoverReady(takeover_url, start, max_tries, wait_interval)
+
+
+class TestTheiaResilience(TheiaSyncMixin, ResilientTheiaTestCase):
   test_instance_max_retries = 0
   backup_max_tries = 70
   backup_wait_interval = 10
@@ -505,20 +527,6 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
     # Remember content of ~/etc in the import theia
     self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
 
-  def _doSync(self):
-    start = time.time()
-
-    # Call exporter script instead of waiting for cron job
-    # XXX Accelerate cron frequency instead ?
-    exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
-    transaction_id = str(int(time.time()))
-    subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
-
-    takeover_url, _ = self._getTakeoverUrlAndPassword()
-
-    # Wait for takoever to be ready
-    self._waitTakeoverReady(takeover_url, start, self.backup_max_tries, self.backup_wait_interval)
-
   def _checkSync(self):
     # Check that ~/etc still contains everything it did before
     etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
@@ -554,3 +562,35 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
     # Check that the test instance is properly redeployed
     # This checks the promises of the test instance
     self._processEmbeddedInstance(self.test_instance_max_retries)
+
+
+class TestTheiaFrontendForwarding(TheiaSyncMixin, ResilientTheiaTestCase):
+  backup_max_tries = 100
+  backup_wait_interval = 20
+
+  html5as_url = os.path.abspath(
+    os.path.join(
+      os.path.dirname(__file__), '..', '..', 'html5as', 'software.cfg'))
+
+  def _prepareExport(self):
+    # Deploy an embedded html5as
+    self._deployEmbeddedSoftware(self.html5as_url, 'html5as', 1)
+
+  def _checkSync(self):
+    proxy_relpath = os.path.join('srv', 'runner', 'var', 'proxy.db')
+    query = "SELECT rowid, partition_reference FROM forwarded_partition_request%s" % DB_VERSION
+
+    # Check that theia0 forwards frontend requests
+    with sqlite3.connect(self.getPartitionPath('export', proxy_relpath)) as db:
+      rows = db.execute(query).fetchall()
+      self.assertIn("slappart0_HTML5AS frontend", (row[1] for row in rows))
+
+    # Check that theia1 does not forward frontend requests
+    # i.e that there were no new insertions in the database since it was cloned
+    # by ensuring the rowids are still the same
+    with sqlite3.connect(self.getPartitionPath('import', proxy_relpath)) as db:
+      self.assertEqual(db.execute(query).fetchall(), rows)
+
+  def _doTakeover(self):
+    # do nothing
+    pass