slapos/proxy: support forwarding requests as a partition

In SlapOS, both users or partitions can requests partitions. With the
multimaster support of slapos proxy, all requests where made as a user,
but in the case of recursive slapos - where a partition from the "outer"
slapos includes an "inner" slapos, it makes sense to forward partitions
requests as the partition in the "outer" slapos - this way when this
partition is destroyed all partitions that might have been requested are
also destroyed.

To request as partition, the multi-master entry must define two extra
keys:
 - computer, that can be optained by $${slap-configuration:computer}
from instance buildout
 - partition, that can be optained by $${slap-configuration:partition}

When these are not set, the request will be made as a user, like it was
the case before.

We also change the test to unset SLAPGRID_INSTANCE_ROOT because this
implementation has side effect - the environment variable is set and
never unset. Without this, test fail when running the full test suite
because a previous test was leaking SLAPGRID_INSTANCE_ROOT. This is
definitely something we'll have to improve later.

slapos/proxy/views.py

@@ -563,18 +563,37 @@ def forwardRequestToExternalMaster(master_url, request_form):
     slap.initializeConnection(master_url)
 
   partition_reference = unicode2str(request_form['partition_reference'])
+  filter_kw = loads(request_form['filter_xml'].encode('utf-8'))
+  partition_parameter_kw = loads(request_form['partition_parameter_xml'].encode('utf-8'))
+
+  app.logger.info("Forwarding request of %s to %s", partition_reference, master_url)
+  app.logger.debug("request_form: %s", request_form)
+
   # Store in database
   execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
              {'partition_reference':partition_reference, 'master_url': master_url})
 
-  new_request_form = request_form.copy()
-  filter_kw = loads(new_request_form['filter_xml'].encode('utf-8'))
+  if master_entry.get('computer') and master_entry.get('partition'):
+    app.logger.debug("requesting from partition %s", master_entry)
+    # XXX ComputerPartition.request and OpenOrder.request have different signatures
+    partition = slap.registerComputerPartition(
+        master_entry['computer'],
+        master_entry['partition'],
+    ).request(
+        software_release=request_form['software_release'],
+        software_type=request_form.get('software_type', ''),
+        partition_reference=partition_reference,
+        shared=loads(request_form['shared_xml'].encode('utf-8')),
+        partition_parameter_kw=partition_parameter_kw,
+        filter_kw=filter_kw,
+        state=loads(request_form['state'].encode('utf-8')),
+    )
+  else:
     filter_kw['source_instance_id'] = partition_reference
-
     partition = slap.registerOpenOrder().request(
         software_release=request_form['software_release'],
-      partition_reference=request_form['partition_reference'],
-      partition_parameter_kw=loads(request_form['partition_parameter_xml'].encode('utf-8')),
+        partition_reference=partition_reference,
+        partition_parameter_kw=partition_parameter_kw,
         software_type=request_form.get('software_type', ''),
         filter_kw=filter_kw,
         state=loads(request_form['state'].encode('utf-8')),

slapos/tests/test_slapproxy.py

@@ -78,6 +78,7 @@ class BasicMixin(object):
     logging.basicConfig(level=logging.DEBUG)
     self.setFiles()
     self.startProxy()
+    os.environ.pop('SLAPGRID_INSTANCE_ROOT', None)
 
   def createSlapOSConfigurationFile(self):
     with open(self.slapos_cfg, 'w') as f:
@@ -1455,6 +1456,25 @@ database_uri = %(tempdir)s/lib/external_proxy.db
     self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML',
                                                      data=request_dict)
 
+  def external_proxy_create_requested_partition(self):
+    # type: () -> None
+    """Create an already requested partition as slappart0, so that we can
+    request from this partition.
+    """
+    external_slap = slapos.slap.slap()
+    external_slap.initializeConnection(self.external_master_url)
+    external_slap.registerSupply().supply(
+        'https://example.com/dummy/software.cfg',
+         computer_guid=self.external_computer_id,
+    )
+    partition = external_slap.registerOpenOrder().request(
+        'https://example.com/dummy/software.cfg',
+        'instance',
+    )
+    # XXX this has to match what is set in slapos_multimaster.cfg.in
+    self.assertEqual('external_computer', partition.slap_computer_id)
+    self.assertEqual('slappart0', partition.slap_computer_partition_id)
+
   def _checkInstanceIsFowarded(self, name, partition_parameter_kw, software_release):
     """
     Test there is no instance on local proxy.
@@ -1580,6 +1600,45 @@ database_uri = %(tempdir)s/lib/external_proxy.db
     self.assertEqual(self.external_software_release, partition.getSoftwareRelease())
     self.assertEqual({}, partition.getConnectionParameterDict())
 
+  def testForwardRequestFromPartition(self):
+    """
+    Test that instance request is forwarded and requested from computer partition.
+    """
+    dummy_parameter_dict = {'foo': 'bar'}
+    instance_reference = 'MyFirstInstance'
+    self.format_for_number_of_partitions(1)
+    self.external_proxy_format_for_number_of_partitions(2)
+    self.external_proxy_create_requested_partition()
+
+    partition = self.request(
+        'https://example.com/request/from/partition/software.cfg',
+        None,
+        instance_reference,
+        'slappart0',
+        partition_parameter_kw=dummy_parameter_dict,
+    )
+
+    instance_parameter_dict = partition.getInstanceParameterDict()
+    instance_parameter_dict.pop('timestamp')
+    self.assertEqual(dummy_parameter_dict, instance_parameter_dict)
+    self.assertEqual('https://example.com/request/from/partition/software.cfg', partition.getSoftwareRelease())
+    self.assertEqual({}, partition.getConnectionParameterDict())
+
+    with sqlite3.connect(os.path.join(
+        self._tempdir,
+        'lib',
+        'external_proxy.db',
+    )) as db:
+      requested_by = slapos.proxy.views.execute_db(
+          "partition", "select reference, requested_by from %s", db=db)
+    self.assertEqual([{
+        'reference': 'slappart0',
+        'requested_by': None
+    }, {
+        'reference': 'slappart1',
+        'requested_by': 'slappart0'
+    }], requested_by)
+
   def testRequestToCurrentMaster(self):
     """
     Explicitely ask deployment of an instance to current master

slapos/tests/test_slapproxy/slapos_multimaster.cfg.in

@@ -27,3 +27,19 @@ software_release_list =
 software_release_list =
   http://mywebsite.me/exteral_software_release.cfg
 
+# Request as a computer partition, so that requested partitions are linked
+# to the partition requesting them.
+[multimaster/https://slap.example.com]
+key = /path/to/cert.key
+cert = /path/to/cert.cert
+computer = COMP-12345
+partition = slappart1
+software_release_list =
+  https://example.com/software.cfg
+
+[multimaster/http://%(external_proxy_host)s:%(external_proxy_port)s/]
+# No certificate here: it is http.
+computer = external_computer
+partition = slappart0
+software_release_list =
+  https://example.com/request/from/partition/software.cfg