Merge remote-tracking branch 'upstream/master' into zope4py2

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 6ddb028e7427913fdcac34b8e727734c
+md5sum = 56ad229756d2d57971c583104370dcc7
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
@@ -23,7 +23,7 @@ md5sum = e211c439571e2900f9f35482c9638d06
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 23c3df4a889ebfa9f0a94e873e95ad3b
+md5sum = a343818079d4fc106594e5850cc1853a
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
@@ -35,7 +35,7 @@ md5sum = ad9499e7355ded4975ad313442cecb7a
 
 [slapos-standalone-script]
 _update_hash_filename_ = slapos_standalone_script.py.jinja
-md5sum = ef5b73648513caf46573f3302953790f
+md5sum = 6792d29057db35ea69b01e72a5c913e3
 
 [theia-common]
 _update_hash_filename_ = theia_common.py

software/theia/instance-import.cfg.jinja.in

@@ -34,6 +34,11 @@ name = Import {{ parameter_dict['additional-frontend-name'] }}
 abstract-socket-path = $${directory:home}/standalone-import-ready
 
 
+# Disable frontend request forwarding in the 'import' instance
+[slapos-standalone-script]
+forward-frontend-requests = disabled
+
+
 # Change port ranges to avoid race conditions on port allocation
 [frontend-instance-port]
 minimum = 3200

software/theia/instance-theia.cfg.jinja.in

@@ -532,8 +532,10 @@ context =
   key embedded_request_exitcode_file :embedded-request-exitcode-file
   key embedded_instance_config embedded-instance-config:output
   key home_path directory:home
+  key forward_frontend_requests :forward-frontend-requests
   section slap_connection slap-connection
   section slapos_standalone_config slapos-standalone-config
+forward-frontend-requests = enabled
 url = ${slapos-standalone-script:output}
 
 [slapos-standalone]

software/theia/slapos_standalone_script.py.jinja

@@ -51,6 +51,7 @@ def signal_handler(signum, frame):
 
 @contextlib.contextmanager
 def setupStandalone():
+{%- if forward_frontend_requests != "disabled" %}
   partition_forward_configuration = (
     slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
       master_url={{ repr(slap_connection['server-url']) }},
@@ -63,6 +64,9 @@ def setupStandalone():
       ),
     ),
   )
+{%- else %}
+  partition_forward_configuration = ()
+{%- endif %}
   shared_parts = {{ repr(shared_part_list) }}
   shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
   standalone = slapos.slap.standalone.StandaloneSlapOS(

software/theia/test/test_resiliency.py

@@ -30,11 +30,14 @@ import errno
 import os
 import re
 import shutil
+import sqlite3
 import subprocess
 import time
 
 import requests
 
+from slapos.proxy.db_version import DB_VERSION
+
 from slapos.testing.testcase import SlapOSNodeCommandError, installSoftwareUrlList
 
 from test import TheiaTestCase, ResilientTheiaMixin, theia_software_release_url
@@ -479,7 +482,26 @@ class TakeoverMixin(ExportAndImportMixin):
     return resp.text
 
 
-class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase):
+class TheiaSyncMixin(ResilienceMixin, TakeoverMixin):
+  def _doSync(self, max_tries=None, wait_interval=None):
+    max_tries = max_tries or self.backup_max_tries
+    wait_interval = wait_interval or self.backup_wait_interval
+
+    start = time.time()
+
+    # Call exporter script instead of waiting for cron job
+    # XXX Accelerate cron frequency instead ?
+    exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
+    transaction_id = str(int(time.time()))
+    subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
+
+    takeover_url, _ = self._getTakeoverUrlAndPassword()
+
+    # Wait for takoever to be ready
+    self._waitTakeoverReady(takeover_url, start, max_tries, wait_interval)
+
+
+class TestTheiaResilience(TheiaSyncMixin, ResilientTheiaTestCase):
   test_instance_max_retries = 0
   backup_max_tries = 70
   backup_wait_interval = 10
@@ -505,20 +527,6 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
     # Remember content of ~/etc in the import theia
     self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
 
-  def _doSync(self):
-    start = time.time()
-
-    # Call exporter script instead of waiting for cron job
-    # XXX Accelerate cron frequency instead ?
-    exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
-    transaction_id = str(int(time.time()))
-    subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
-
-    takeover_url, _ = self._getTakeoverUrlAndPassword()
-
-    # Wait for takoever to be ready
-    self._waitTakeoverReady(takeover_url, start, self.backup_max_tries, self.backup_wait_interval)
-
   def _checkSync(self):
     # Check that ~/etc still contains everything it did before
     etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
@@ -554,3 +562,35 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
     # Check that the test instance is properly redeployed
     # This checks the promises of the test instance
     self._processEmbeddedInstance(self.test_instance_max_retries)
+
+
+class TestTheiaFrontendForwarding(TheiaSyncMixin, ResilientTheiaTestCase):
+  backup_max_tries = 100
+  backup_wait_interval = 20
+
+  html5as_url = os.path.abspath(
+    os.path.join(
+      os.path.dirname(__file__), '..', '..', 'html5as', 'software.cfg'))
+
+  def _prepareExport(self):
+    # Deploy an embedded html5as
+    self._deployEmbeddedSoftware(self.html5as_url, 'html5as', 1)
+
+  def _checkSync(self):
+    proxy_relpath = os.path.join('srv', 'runner', 'var', 'proxy.db')
+    query = "SELECT rowid, partition_reference FROM forwarded_partition_request%s" % DB_VERSION
+
+    # Check that theia0 forwards frontend requests
+    with sqlite3.connect(self.getPartitionPath('export', proxy_relpath)) as db:
+      rows = db.execute(query).fetchall()
+      self.assertIn("slappart0_HTML5AS frontend", (row[1] for row in rows))
+
+    # Check that theia1 does not forward frontend requests
+    # i.e that there were no new insertions in the database since it was cloned
+    # by ensuring the rowids are still the same
+    with sqlite3.connect(self.getPartitionPath('import', proxy_relpath)) as db:
+      self.assertEqual(db.execute(query).fetchall(), rows)
+
+  def _doTakeover(self):
+    # do nothing
+    pass