Add multimaster support to slapproxy.

slapos/proxy/__init__.py

@@ -30,6 +30,7 @@
 
 import logging
 
+from slapos.proxy.views import app
 
 def _generateSoftwareProductListFromString(software_product_list_string):
   """
@@ -52,33 +53,48 @@ def _generateSoftwareProductListFromString(software_product_list_string):
 class ProxyConfig(object):
   def __init__(self, logger):
     self.logger = logger
+    self.multimaster = {}
+    self.software_product_list = []
 
   def mergeConfig(self, args, configp):
-    # Set options parameters
+    # Set arguments parameters (from CLI) as members of self
     for option, value in args.__dict__.items():
       setattr(self, option, value)
 
-    # Merge the arguments and configuration
-    for section in ("slapproxy", "slapos"):
+    for section in configp.sections():
       configuration_dict = dict(configp.items(section))
-      for key in configuration_dict:
-        if not getattr(self, key, None):
-          setattr(self, key, configuration_dict[key])
-
+      if section in ("slapproxy", "slapos"):
+        # Merge the arguments and configuration as member of self
+        for key in configuration_dict:
+          if not getattr(self, key, None):
+            setattr(self, key, configuration_dict[key])
+      elif section.startswith('multimaster/'):
+        # Merge multimaster configuration if any
+        # XXX: check for duplicate SR entries
+        for key, value in configuration_dict.iteritems():
+          if key == 'software_release_list':
+            # Split multi-lines values
+            configuration_dict[key] = [line.strip() for line in value.strip().split('\n')]
+        self.multimaster[section.split('multimaster/')[1]] = configuration_dict
 
   def setConfig(self):
     if not self.database_uri:
       raise ValueError('database-uri is required.')
+    # XXX: check for duplicate SR entries.
+    self.software_product_list = _generateSoftwareProductListFromString(
+        getattr(self, 'software_product_list', ''))
+
 
+def setupFlaskConfiguration(conf):
+  app.config['computer_id'] = conf.computer_id
+  app.config['DATABASE_URI'] = conf.database_uri
+  app.config['software_product_list'] = conf.software_product_list
+  app.config['multimaster'] = conf.multimaster
 
 def do_proxy(conf):
-  from slapos.proxy.views import app
   for handler in conf.logger.handlers:
     app.logger.addHandler(handler)
   app.logger.setLevel(logging.INFO)
-  app.config['computer_id'] = conf.computer_id
-  app.config['DATABASE_URI'] = conf.database_uri
-  app.config['software_product_list'] = \
-    _generateSoftwareProductListFromString(
-      getattr(conf, 'software_product_list', ""))
-  app.run(host=conf.host, port=int(conf.port))
+  setupFlaskConfiguration(conf)
+  app.run(host=conf.host, port=int(conf.port), threaded=True)
+

slapos/proxy/schema.sql

@@ -45,3 +45,7 @@ CREATE TABLE IF NOT EXISTS partition_network%(version)s (
   netmask VARCHAR(255)
 );
 
+CREATE TABLE IF NOT EXISTS forwarded_partition_request%(version)s (
+  partition_reference VARCHAR(255), -- a.k.a source_instance_id
+  master_url VARCHAR(255)
+);

slapos/proxy/views.py

@@ -29,10 +29,13 @@
 ##############################################################################
 
 from lxml import etree
+import random
 import sqlite3
+import string
 from slapos.slap.slap import Computer, ComputerPartition, \
     SoftwareRelease, SoftwareInstance, NotFoundError
 from slapos.proxy.db_version import DB_VERSION
+import slapos.slap
 
 from flask import g, Flask, request, abort
 import xml_marshaller
@@ -112,14 +115,16 @@ def partitiondict2partition(partition):
   return slap_partition
 
 
-def execute_db(table, query, args=(), one=False, db_version=None, log=False):
+def execute_db(table, query, args=(), one=False, db_version=None, log=False, db=None):
+  if not db:
+    db = g.db
   if not db_version:
     db_version = DB_VERSION
   query = query % (table + db_version,)
   if log:
     print query
   try:
-    cur = g.db.execute(query, args)
+    cur = db.execute(query, args)
   except:
     app.logger.error('There was some issue during processing query %r on table %r with args %r' % (query, table, args))
     raise
@@ -161,7 +166,7 @@ def _upgradeDatabaseIfNeeded():
   # If version of current database is not old, do nothing
   if current_schema_version == DB_VERSION:
     return
- 
+
   schema = app.open_resource('schema.sql')
   schema = schema.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
   g.db.cursor().executescript(schema)
@@ -330,9 +335,7 @@ def supplySupply():
 
 @app.route('/requestComputerPartition', methods=['POST'])
 def requestComputerPartition():
-  parsed_form_dict = parseRequestComputerPartitionForm(request.form)
-  # By default, ALWAYS request instance on default computer
-  parsed_form_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
+  parsed_request_dict = parseRequestComputerPartitionForm(request.form)
 
   # Is it a slave instance?
   slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode())
@@ -341,29 +344,36 @@ def requestComputerPartition():
   if slave:
     # XXX: change schema to include a simple "partition_reference" which
     # is name of the instance. Then, no need to do complex search here.
-    slave_reference = parsed_form_dict['partition_id'] + '_' + parsed_form_dict['partition_reference']
-    requested_computer_id = parsed_form_dict['filter_kw']['computer_guid']
+    slave_reference = parsed_request_dict['partition_id'] + '_' + parsed_request_dict['partition_reference']
+    requested_computer_id = parsed_request_dict['filter_kw'].get('computer_guid', app.config['computer_id'])
     matching_partition = getAllocatedSlaveInstance(slave_reference, requested_computer_id)
   else:
-    matching_partition = getAllocatedInstance(parsed_form_dict['partition_reference'])
+    matching_partition = getAllocatedInstance(parsed_request_dict['partition_reference'])
 
   if matching_partition:
     # Then the instance is already allocated, just update it
     # XXX: split request and request slave into different update/allocate functions and simplify.
+    # By default, ALWAYS request instance on default computer
+    parsed_request_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
     if slave:
-      software_instance = requestSlave(**parsed_form_dict)
+      software_instance = requestSlave(**parsed_request_dict)
     else:
-      software_instance = requestNotSlave(**parsed_form_dict)
+      software_instance = requestNotSlave(**parsed_request_dict)
   else:
     # Instance is not yet allocated: try to do it.
-    # XXX Insert here multimaster allocation
+    external_master_url = isRequestToBeForwardedToExternalMaster(parsed_request_dict)
+    if external_master_url:
+      return forwardRequestToExternalMaster(external_master_url, request.form)
     # XXX add support for automatic deployment on specific node depending on available SR and partitions on each Node.
-    # Note: only deploy on default node if SLA not specified
+    # Note: It only deploys on default node if SLA not specified
     # XXX: split request and request slave into different update/allocate functions and simplify.
+
+    # By default, ALWAYS request instance on default computer
+    parsed_request_dict['filter_kw'].setdefault('computer_guid', app.config['computer_id'])
     if slave:
-      software_instance = requestSlave(**parsed_form_dict)
+      software_instance = requestSlave(**parsed_request_dict)
     else:
-      software_instance = requestNotSlave(**parsed_form_dict)
+      software_instance = requestNotSlave(**parsed_request_dict)
 
   return dumps(software_instance)
 
@@ -378,15 +388,123 @@ def parseRequestComputerPartitionForm(form):
   parsed_dict['partition_id'] = form.get('computer_partition_id', '').encode()
   parsed_dict['partition_parameter_kw'] = loads(form.get('partition_parameter_xml', EMPTY_DICT_XML).encode())
   parsed_dict['filter_kw'] = loads(form.get('filter_xml', EMPTY_DICT_XML).encode())
-  # Note: currently ignored on for slave instance (slave instances
+  # Note: currently ignored for slave instance (slave instances
   # are always started).
   parsed_dict['requested_state'] = loads(form.get('state').encode())
 
   return parsed_dict
 
+run_id = ''.join([random.choice(string.ascii_letters + string.digits) for n in xrange(32)])
+def checkIfMasterIsCurrentMaster(master_url):
+  """
+  Because there are several ways to contact this server, we can't easily check
+  in a request() if master_url is ourself or not. So we contact master_url,
+  and if it returns an ID we know: it is ourself
+  """
+  # Dumb way: compare with listening host/port
+  host = request.host
+  port = request.environ['SERVER_PORT']
+  if master_url == 'http://%s:%s/' % (host, port):
+    return True
+
+  # Hack way: call ourself
+  slap = slapos.slap.slap()
+  slap.initializeConnection(master_url)
+  try:
+    master_run_id = slap._connection_helper.GET('/getRunId')
+  except:
+    return False
+  if master_run_id == run_id:
+    return True
+  return False
+
+@app.route('/getRunId', methods=['GET'])
+def getRunId():
+  return run_id
+
+def checkMasterUrl(master_url):
+  """
+  Check if master_url doesn't represent ourself, and check if it is whitelisted
+  in multimaster configuration.
+  """
+  if not master_url:
+    return False
+
+  if checkIfMasterIsCurrentMaster(master_url):
+    # master_url is current server: don't forward
+    return False
+
+  master_entry = app.config.get('multimaster').get(master_url, None)
+  # Check if this master is known
+  if not master_entry:
+    # Check if it is ourself
+    if not master_url.startswith('https') and checkIfMasterIsCurrentMaster(master_url):
+      return False
+    app.logger.warning('External SlapOS Master URL %s is not listed in multimaster list.' % master_url)
+    abort(404)
+
+  return True
+
+def isRequestToBeForwardedToExternalMaster(parsed_request_dict):
+    """
+    Check if we HAVE TO forward the request.
+    Several cases:
+     * The request specifies a master_url in filter_kw
+     * The software_release of the request is in a automatic forward list
+    """
+    master_url = parsed_request_dict['filter_kw'].get('master_url')
+
+    if checkMasterUrl(master_url):
+      # Don't allocate the instance locally, but forward to specified master
+      return master_url
+
+    software_release = parsed_request_dict['software_release']
+    for mutimaster_url, mutimaster_entry in app.config.get('multimaster').iteritems():
+      if software_release in mutimaster_entry['software_release_list']:
+        # Don't allocate the instance locally, but forward to specified master
+        return mutimaster_url
+    return None
+
+def forwardRequestToExternalMaster(master_url, request_form):
+  """
+  Forward instance request to external SlapOS Master.
+  """
+  master_entry = app.config.get('multimaster').get(master_url, {})
+  key_file = master_entry.get('key')
+  cert_file = master_entry.get('cert')
+  if master_url.startswith('https') and (not key_file or not cert_file):
+    app.logger.warning('External master %s configuration did not specify key or certificate.' % master_url)
+    abort(404)
+  if master_url.startswith('https') and not master_url.startswith('https') and (key_file or cert_file):
+    app.logger.warning('External master %s configurqtion specifies key or certificate but is using plain http.' % master_url)
+    abort(404)
+
+  slap = slapos.slap.slap()
+  if key_file:
+    slap.initializeConnection(master_url, key_file=key_file, cert_file=cert_file)
+  else:
+    slap.initializeConnection(master_url)
+
+  partition_reference = request_form['partition_reference'].encode()
+  # Store in database
+  execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
+             {'partition_reference':partition_reference, 'master_url': master_url})
+
+  new_request_form = request_form.copy()
+  filter_kw = loads(new_request_form['filter_xml'].encode())
+  filter_kw['source_instance_id'] = partition_reference
+  new_request_form['filter_xml'] = dumps(filter_kw)
+
+  partition = loads(slap._connection_helper.POST('/requestComputerPartition', new_request_form))
+
+  # XXX move to other end
+  partition._master_url = master_url
+
+  return dumps(partition)
+
 def getAllocatedInstance(partition_reference):
   """
-  Look for existence of instance, if so return the 
+  Look for existence of instance, if so return the
   corresponding partition dict, else return None
   """
   args = []
@@ -398,7 +516,7 @@ def getAllocatedInstance(partition_reference):
 
 def getAllocatedSlaveInstance(slave_reference, requested_computer_id):
   """
-  Look for existence of instance, if so return the 
+  Look for existence of instance, if so return the
   corresponding partition dict, else return None
   """
   args = []

slapos/tests/slapproxy/__init__.py

 # -*- coding: utf-8 -*-
+# vim: set et sts=2:
 ##############################################################################
 #
-# Copyright (c) 2012 Vifib SARL and Contributors.
+# Copyright (c) 2012, 2013, 2014 Vifib SARL and Contributors.
 # All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
@@ -31,7 +32,9 @@ import ConfigParser
 import os
 import logging
 import shutil
+import subprocess
 import tempfile
+import time
 import unittest
 import xml_marshaller
 from xml_marshaller.xml_marshaller import loads, dumps
@@ -69,14 +72,7 @@ class BasicMixin(object):
     self.setFiles()
     self.startProxy()
 
-  def setFiles(self):
-    """
-    Set environment to run slapproxy
-    """
-    self.slapos_cfg = os.path.join(self._tempdir, 'slapos.cfg')
-    self.proxy_db = os.path.join(self._tempdir, 'lib', 'proxy.db')
-    self.proxyaddr = 'http://127.0.0.1:8080/'
-    self.computer_id = 'computer'
+  def createSlapOSConfigurationFile(self):
     open(self.slapos_cfg, 'w').write("""[slapos]
 software_root = %(tempdir)s/opt/slapgrid
 instance_root = %(tempdir)s/srv/slapgrid
@@ -87,6 +83,16 @@ host = 127.0.0.1
 port = 8080
 database_uri = %(tempdir)s/lib/proxy.db
 """ % {'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr})
+
+  def setFiles(self):
+    """
+    Set environment to run slapproxy
+    """
+    self.slapos_cfg = os.path.join(self._tempdir, 'slapos.cfg')
+    self.proxy_db = os.path.join(self._tempdir, 'lib', 'proxy.db')
+    self.proxyaddr = 'http://localhost:80/'
+    self.computer_id = 'computer'
+    self.createSlapOSConfigurationFile()
     for directory in ['opt', 'srv', 'lib']:
       path = os.path.join(self._tempdir, directory)
       os.mkdir(path)
@@ -101,10 +107,8 @@ database_uri = %(tempdir)s/lib/proxy.db
     conf.mergeConfig(ProxyOption(self.proxy_db), configp)
     conf.setConfig()
     views.app.config['TESTING'] = True
-    views.app.config['computer_id'] = self.computer_id
-    views.app.config['DATABASE_URI'] = self.proxy_db
-    views.app.config['HOST'] = conf.host
-    views.app.config['port'] = conf.port
+    slapos.proxy.setupFlaskConfiguration(conf)
+    
     self.app_config = views.app.config
     self.app = views.app.test_client()
 
@@ -850,7 +854,280 @@ class TestMultiNodeSupport(MasterMixin):
     except slapos.slap.NotFoundError:
       self.fail('Could not fetch informations for registered computer.')
 
+class TestMultiMasterSupport(MasterMixin):
+  """
+  Test multimaster support in slapproxy.
+  """
+  external_software_release = 'http://mywebsite.me/exteral_software_release.cfg'
+  software_release_not_in_list = 'http://mywebsite.me/exteral_software_release_not_listed.cfg'
+
+  def setUp(self):
+    # XXX don't use lo
+    self.external_proxy_host = '127.0.0.1'
+    self.external_proxy_port = 8281
+    self.external_master_url = 'http://%s:%s' % (self.external_proxy_host, self.external_proxy_port)
+    self.external_computer_id = 'external_computer'
+
+    super(TestMultiMasterSupport, self).setUp()
+
+    self.db = sqlite3.connect(self.proxy_db)
+    self.external_slapproxy_configuration_file_location = os.path.join(
+        self._tempdir, 'external_slapos.cfg')
+    self.createExternalProxyConfigurationFile()
+    self.startExternalProxy()
+
+  def tearDown(self):
+    self.external_proxy_process.kill()
+    super(TestMultiMasterSupport, self).tearDown()
+
+  def createExternalProxyConfigurationFile(self):
+    open(self.external_slapproxy_configuration_file_location, 'w').write("""[slapos]
+computer_id = %(external_computer_id)s
+[slapproxy]
+host = 127.0.0.1
+port = %(port)s
+database_uri = %(tempdir)s/lib/external_proxy.db
+""" % {
+    'tempdir': self._tempdir,
+    'port': self.external_proxy_port,
+    'external_computer_id': self.external_computer_id
+    })
+
+  def startExternalProxy(self):
+    """
+    Start external slapproxy
+    """
+    # XXX use current dev version, not standard one installed through package
+    self.external_proxy_process = subprocess.Popen(['slapos', 'proxy', 'start', '--cfg', self.external_slapproxy_configuration_file_location ])
+    # Wait a bit for proxy to be started
+    time.sleep(0.5)
+    self.external_proxy_slap = slapos.slap.slap()
+    self.external_proxy_slap.initializeConnection(self.external_master_url)
+
+  def createSlapOSConfigurationFile(self):
+    """
+    Overwrite default slapos configuration file to enable specific multimaster
+    behaviours.
+    """
+    configuration = pkg_resources.resource_stream(
+        'slapos.tests.slapproxy', 'slapos_multimaster.cfg.in'
+    ).read() % {
+        'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr,
+        'external_proxy_host': self.external_proxy_host,
+        'external_proxy_port': self.external_proxy_port
+    }
+    open(self.slapos_cfg, 'w').write(configuration)
+
+  def external_proxy_supply(self, url, computer_id=None):
+    if computer_id is None:
+      computer_id = self.external_computer_id
+    self.external_proxy_slap.registerSupply().supply(url, computer_id)
+
+  def external_proxy_add_free_partition(self, partition_amount, computer_id=None):
+    """
+    Will simulate a slapformat first run
+    and create "partition_amount" partitions
+    """
+    if not computer_id:
+      computer_id = self.external_computer_id
+    computer_dict = {
+        'reference': computer_id,
+        'address': '123.456.789',
+        'netmask': 'fffffffff',
+        'partition_list': [],
+    }
+    for i in range(partition_amount):
+      partition_example = {
+          'reference': 'slappart%s' % i,
+          'address_list': [
+              {'addr': '1.2.3.4', 'netmask': '255.255.255.255'},
+              {'addr': '4.3.2.1', 'netmask': '255.255.255.255'}
+           ],
+           'tap': {'name': 'tap0'},
+      }
+      computer_dict['partition_list'].append(partition_example)
+
+    request_dict = {
+        'computer_id': self.computer_id,
+        'xml': xml_marshaller.xml_marshaller.dumps(computer_dict),
+    }
+    self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML',
+                  parameter_dict=request_dict)
+
+  def _checkInstanceIsFowarded(self, name, partition_parameter_kw, software_release):
+    """
+    Test there is no instance on local proxy.
+    Test there is instance on external proxy.
+    Test there is instance reference in external table of databse of local proxy.
+    """
+    # Test it has been correctly added to local database
+    forwarded_instance_list = slapos.proxy.views.execute_db('forwarded_partition_request', 'SELECT * from %s', db=self.db)
+    self.assertEqual(len(forwarded_instance_list), 1)
+    forwarded_instance = forwarded_instance_list[0]
+    self.assertEqual(forwarded_instance['partition_reference'], name)
+    self.assertEqual(forwarded_instance['master_url'], self.external_master_url)
+
+    # Test there is nothing allocated locally
+    computer = loads(self.app.get(
+        '/getFullComputerInformation?computer_id=%s' % self.computer_id
+    ).data)
+    self.assertEqual(
+        computer._computer_partition_list[0]._software_release_document,
+        None
+    )
+
+    # Test there is an instance allocated in external master
+    external_slap = slapos.slap.slap()
+    external_slap.initializeConnection(self.external_master_url)
+    external_computer = external_slap.registerComputer(self.external_computer_id)
+    external_partition = external_computer.getComputerPartitionList()[0]
+    for k, v in partition_parameter_kw.iteritems():
+      self.assertEqual(
+          external_partition.getInstanceParameter(k),
+          v
+      )
+    self.assertEqual(
+        external_partition._software_release_document._software_release,
+        software_release
+    )
+
+  def _checkInstanceIsAllocatedLocally(self, name, partition_parameter_kw, software_release):
+    """
+    Test there is one instance on local proxy.
+    Test there NO is instance reference in external table of databse of local proxy.
+    Test there is not instance on external proxy.
+    """
+    # Test it has NOT been added to local database
+    forwarded_instance_list = slapos.proxy.views.execute_db('forwarded_partition_request', 'SELECT * from %s', db=self.db)
+    self.assertEqual(len(forwarded_instance_list), 0)
+
+    # Test there is an instance allocated locally
+    computer = loads(self.app.get(
+        '/getFullComputerInformation?computer_id=%s' % self.computer_id
+    ).data)
+    partition = computer._computer_partition_list[0]
+    for k, v in partition_parameter_kw.iteritems():
+      self.assertEqual(
+          partition.getInstanceParameter(k),
+          v
+      )
+    self.assertEqual(
+        partition._software_release_document._software_release,
+        software_release
+    )
+
+    # Test there is NOT instance allocated in external master
+    external_slap = slapos.slap.slap()
+    external_slap.initializeConnection(self.external_master_url)
+    external_computer = external_slap.registerComputer(self.external_computer_id)
+    external_partition = external_computer.getComputerPartitionList()[0]
+    self.assertEqual(
+        external_partition._software_release_document,
+        None
+    )
+
+  def testForwardToMasterInList(self):
+    """
+    Test that explicitely asking a master_url in SLA causes
+    proxy to forward request to this master.
+    """
+    dummy_parameter_dict = {'foo': 'bar'}
+    instance_reference = 'MyFirstInstance'
+    self.add_free_partition(1)
+    self.external_proxy_supply(self.external_software_release)
+    self.external_proxy_add_free_partition(1)
+
+    filter_kw = {'master_url': self.external_master_url}
+    partition = self.request(self.software_release_not_in_list, None, instance_reference, 'slappart0',
+                             filter_kw=filter_kw, partition_parameter_kw=dummy_parameter_dict)
+
+    self._checkInstanceIsFowarded(instance_reference, dummy_parameter_dict, self.software_release_not_in_list)
+    self.assertEqual(
+        partition._master_url,
+        self.external_master_url
+    )
+
+  def testForwardToMasterNotInList(self):
+    """
+    Test that explicitely asking a master_url in SLA causes
+    proxy to refuse to forward if this master_url is not whitelisted
+    """
+    self.add_free_partition(1)
+    self.external_proxy_supply(self.external_software_release)
+    self.external_proxy_add_free_partition(1)
+
+    filter_kw = {'master_url': self.external_master_url + 'bad'}
+    rv = self._requestComputerPartition(self.software_release_not_in_list, None, 'MyFirstInstance', 'slappart0', filter_kw=filter_kw)
+    self.assertEqual(rv._status_code, 404)
+
+  def testForwardRequest_SoftwareReleaseList(self):
+    """
+    Test that instance request is automatically forwarded
+    if its Software Release matches list.
+    """
+    dummy_parameter_dict = {'foo': 'bar'}
+    instance_reference = 'MyFirstInstance'
+    self.add_free_partition(1)
+    self.external_proxy_supply(self.external_software_release)
+    self.external_proxy_add_free_partition(1)
+
+    partition = self.request(self.external_software_release, None, instance_reference, 'slappart0',
+                             partition_parameter_kw=dummy_parameter_dict)
+
+    self._checkInstanceIsFowarded(instance_reference, dummy_parameter_dict, self.external_software_release)
+
+  def testRequestToCurrentMaster(self):
+    """
+    Explicitely ask deployment of an instance to current master
+    """
+    self.add_free_partition(1)
+    self.external_proxy_add_free_partition(1)
+    instance_reference = 'MyFirstInstance'
+
+    dummy_parameter_dict = {'foo': 'bar'}
+
+    filter_kw = {'master_url': self.proxyaddr}
+    self.request(self.software_release_not_in_list, None, instance_reference, 'slappart0',
+                 filter_kw=filter_kw, partition_parameter_kw=dummy_parameter_dict)
+    self._checkInstanceIsAllocatedLocally(instance_reference, dummy_parameter_dict, self.software_release_not_in_list)
+
+  def testRequestExplicitelyOnExternalMasterThenRequestAgain(self):
+    """
+    Request an instance that will get forwarded to another an instance.
+    Test that subsequent request without SLA doesn't forward
+    """
+    dummy_parameter_dict = {'foo': 'bar'}
+
+    self.testForwardToMasterInList()
+    partition = self.request(self.software_release_not_in_list, None, 'MyFirstInstance', 'slappart0', partition_parameter_kw=dummy_parameter_dict)
+    self.assertEqual(
+        getattr(partition, '_master_url', None),
+        None
+    )
+
+    # Test it has not been removed from local database (we keep track)
+    forwarded_instance_list = slapos.proxy.views.execute_db('forwarded_partition_request', 'SELECT * from %s', db=self.db)
+    self.assertEqual(len(forwarded_instance_list), 1)
+
+    # Test there is an instance allocated locally
+    computer = loads(self.app.get(
+        '/getFullComputerInformation?computer_id=%s' % self.computer_id
+    ).data)
+    partition = computer._computer_partition_list[0]
+    for k, v in dummy_parameter_dict.iteritems():
+      self.assertEqual(
+          partition.getInstanceParameter(k),
+          v
+      )
+    self.assertEqual(
+        partition._software_release_document._software_release,
+        self.software_release_not_in_list
+    )
+
 
+# XXX: when testing new schema version, 
+# rename to "TestMigrateVersion10ToLatest" and test accordingly.
+# Of course, also test version 11 to latest (should be 12).
 class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, TestMultiNodeSupport):
   """
   Test that old database version are automatically migrated without failure

slapos/tests/slapproxy/slapos_multimaster.cfg.in

+[slapos]
+software_root = %(tempdir)s/opt/slapgrid
+instance_root = %(tempdir)s/srv/slapgrid
+master_url = %(proxyaddr)s
+computer_id = computer
+
+[slapproxy]
+host = 127.0.0.1
+port = 8080
+database_uri = %(tempdir)s/lib/proxy.db
+
+# Here goes the list of slapos masters that slapproxy can contact
+# Each section beginning by multimaster is a different SlapOS Master, represented by arbitrary name.
+# For each section, you need to specify the URL of the SlapOS Master.
+# For each section, you can specify if needed the location of key/certificate used to authenticate to this slapOS Master.
+# For each section, you can specify a list of Software Releases. Any instance request matching this Softwrare Release will be automatically forwarded to this SlapOS Master and will not be allocated locally.
+[multimaster/https://slap.vifib.com]
+key = /path/to/cert.key
+cert = /path/to/cert.cert
+# XXX add wildcard support for SR list.
+software_release_list =
+  http://something.io/software.cfg
+  /some/arbitrary/local/unix/path
+
+[multimaster/http://%(external_proxy_host)s:%(external_proxy_port)s]
+# No certificate here: it is http.
+software_release_list =
+  http://mywebsite.me/exteral_software_release.cfg
+