Merge branch 'params-non-ascii' into 'master'

Teach slapproxy to handle non-ascii instance parameters

Several fixes to make slapproxy accept creating instances with non-ascii parameters. My use-case it to instantiate gitlab with ICP, e.g. this way:

    ICP = 沪ICP备14008524号

kirr/gitlab-ce@1e11409c  
kirr/slapos@66919597

/cc @rafael, @alain.takoudjou, @kazuhiko, @jerome, @arnau

See merge request !3

slapos/cli/proxy_show.py

@@ -37,6 +37,7 @@ import sqlite3
 from slapos.cli.config import ConfigCommand
 from slapos.proxy import ProxyConfig
 from slapos.proxy.db_version import DB_VERSION
+from slapos.util import sqlite_connect
 
 
 class ProxyShowCommand(ConfigCommand):
@@ -191,7 +192,7 @@ def log_network(logger, conn):
 
 def do_show(conf):
     conf.logger.debug('Using database: %s', conf.database_uri)
-    conn = sqlite3.connect(conf.database_uri)
+    conn = sqlite_connect(conf.database_uri)
     conn.row_factory = sqlite3.Row
 
     conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest())

slapos/collect/db.py

@@ -27,11 +27,12 @@
 #
 ##############################################################################
 
-import sqlite3
 import os
 from time import strftime
 import datetime
 
+from slapos.util import sqlite_connect
+
 class Database:
 
   database_name = "collector.db"
@@ -125,7 +126,7 @@ class Database:
     self._bootstrap()
 
   def connect(self):
-    self.connection = sqlite3.connect(self.uri)
+    self.connection = sqlite_connect(self.uri)
     self.cursor = self.connection.cursor()
 
   def commit(self):

slapos/proxy/views.py

@@ -30,12 +30,12 @@
 
 from lxml import etree
 import random
-import sqlite3
 import string
 from slapos.slap.slap import Computer, ComputerPartition, \
     SoftwareRelease, SoftwareInstance, NotFoundError
 from slapos.proxy.db_version import DB_VERSION
 import slapos.slap
+from slapos.util import sqlite_connect
 
 from flask import g, Flask, request, abort
 import xml_marshaller
@@ -50,10 +50,19 @@ class UnauthorizedError(Exception):
   pass
 
 
+# cast everything to string, utf-8 encoded
+def to_str(v):
+  if isinstance(v, str):
+    return v
+  if not isinstance(v, unicode):
+    v = unicode(v)
+  return v.encode('utf-8')
+
+
 def xml2dict(xml):
   result_dict = {}
   if xml is not None and xml != '':
-    tree = etree.fromstring(xml.encode('utf-8'))
+    tree = etree.fromstring(to_str(xml))
     for element in tree.iter(tag=etree.Element):
       if element.tag == 'parameter':
         key = element.get('id')
@@ -70,7 +79,7 @@ def dict2xml(dictionary):
   instance = etree.Element('instance')
   for parameter_id, parameter_value in dictionary.iteritems():
     # cast everything to string
-    parameter_value = str(parameter_value)
+    parameter_value = unicode(parameter_value)
     etree.SubElement(instance, "parameter",
                      attrib={'id': parameter_id}).text = parameter_value
   return etree.tostring(instance,
@@ -136,7 +145,7 @@ def execute_db(table, query, args=(), one=False, db_version=None, log=False, db=
 
 
 def connect_db():
-  return sqlite3.connect(app.config['DATABASE_URI'])
+  return sqlite_connect(app.config['DATABASE_URI'])
 
 def _getTableList():
   return g.db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY Name").fetchall()

slapos/tests/slapproxy/__init__.py

@@ -45,6 +45,7 @@ import slapos.proxy
 import slapos.proxy.views as views
 import slapos.slap
 import slapos.slap.slap
+from slapos.util import sqlite_connect
 
 import sqlite3
 import pkg_resources
@@ -462,6 +463,15 @@ class TestRequest(MasterMixin):
         self.request('http://sr//', None, 'MyFirstInstance', 'slappart2').__dict__,
         self.request('http://sr//', None, 'frontend', 'slappart2').__dict__)
 
+  def test_request_with_nonascii_parameters(self):
+    """
+    Verify that request with non-ascii parameters is correctly accepted
+    """
+    self.add_free_partition(1)
+    request = self.request('http://sr//', None, 'myinstance', 'slappart0',
+                           partition_parameter_kw={'text': u'Привет Мир!'})
+    self.assertIsInstance(request, slapos.slap.ComputerPartition)
+
 
 class TestSlaveRequest(MasterMixin):
   """
@@ -904,7 +914,7 @@ class TestMultiMasterSupport(MasterMixin):
 
     super(TestMultiMasterSupport, self).setUp()
 
-    self.db = sqlite3.connect(self.proxy_db)
+    self.db = sqlite_connect(self.proxy_db)
     self.external_slapproxy_configuration_file_location = os.path.join(
         self._tempdir, 'external_slapos.cfg')
     self.createExternalProxyConfigurationFile()
@@ -1181,7 +1191,7 @@ class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, T
     super(TestMigrateVersion10To11, self).setUp()
     schema = pkg_resources.resource_stream('slapos.tests.slapproxy', 'database_dump_version_10.sql')
     schema = schema.read() % dict(version='11')
-    self.db = sqlite3.connect(self.proxy_db)
+    self.db = sqlite_connect(self.proxy_db)
     self.db.cursor().executescript(schema)
     self.db.commit()
 

slapos/util.py

@@ -30,6 +30,7 @@
 import errno
 import os
 import subprocess
+import sqlite3
 
 
 def mkdir_p(path, mode=0o700):
@@ -94,3 +95,9 @@ def string_to_boolean(string):
     return False
   else:
     raise ValueError('%s is neither True nor False.' % string)
+
+
+def sqlite_connect(dburi):
+  conn = sqlite3.connect(dburi)
+  conn.text_factory = str       # allow 8-bit strings
+  return conn