Update Release Candidate

component/coturn/buildout.cfg

+[buildout]
+extends =
+  ../sqlite3/buildout.cfg
+  ../openssl/buildout.cfg
+  ../libevent/buildout.cfg
+parts = coturn
+
+[coturn]
+recipe = slapos.recipe.cmmi
+url = https://github.com/coturn/coturn/archive/4.5.1.1.tar.gz
+md5sum = ef20628b026d666be24df056f20a86ea
+environment =
+  PATH=${sqlite3:location}/bin:%(PATH)s
+  CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
+  CFLAGS=-I${libevent2:location}/include
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib

component/kumo/buildout.cfg

@@ -14,7 +14,6 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/downloads/etolabo/kumofs/kumofs-0.4.13.tar.gz
 md5sum = 46148e9536222d0ad2ef36777c55714d
-pre-configure-hook = ${:_profile_base_location_}/kumo-hooks.py#958a595a02de75624728f8d65e39d800:pre_configure_hook
 patches =
   ${:_profile_base_location_}/kumofs-0.4.13_ipv6support_multiiplistenfix.patch#53af9f1f1375940841c589a6cbe11425
   ${:_profile_base_location_}/kumofs-0.4.13_fix_gcc-4.9_ftbfs.patch#c09e04c620ce11c3fdd4afc3459cd355

component/kumo/kumo-hooks.py

-import os
-import sys
-import traceback
-from shutil import copy
-from subprocess import Popen, PIPE
-
-CONFIGURE_PATH = os.path.join('configure')
-CONFIGURE_BACKUP_PATH = CONFIGURE_PATH + '_disabled'
-# Fake configure, generating a fake Makefile which will create a marker file
-# instead of actually installing anything.
-# This is needed (?) to fetch --prefix from configure parameters, so we know
-# where to tell Makefile to put the dummy file.
-FAKE_CONFIGURE = '''#!%(python)s -S
-import os
-import sys
-print 'Configuration is disabled on this host because %%s'
-print 'Original configure file available at %(backup)s'
-prefix = None
-next = False
-for arg in sys.argv:
-    if next:
-        prefix = arg
-        break
-    if arg.startswith('--prefix'):
-        if arg.startswith('--prefix='):
-            _, prefix = arg.split('=', 1)
-            break
-        next = True
-if prefix is None:
-    raise '--prefix parameter not found'
-# Generate Makefile with proper prefix
-open('Makefile', 'w').write("""all:
-\techo 'make disabled, see configure'
-
-install:
-\ttouch %%%%s""" %%%% (
-  os.path.join(prefix, 'BUILD_DISABLED_BY_BUILDOUT'),
-))
-sys.exit(0)
-''' % {
-    'backup': CONFIGURE_BACKUP_PATH,
-    'python': sys.executable,
-}
-
-def pre_configure_hook(options, buildout):
-    gcc_executable = os.getenv('CC', 'gcc')
-    try:
-        gcc = Popen([gcc_executable, '-v'], stdout=PIPE, stderr=PIPE,
-            close_fds=True)
-    except OSError, (errno, _):
-        if errno == 2:
-            # No gcc installed, nothing to check
-            pass
-        else:
-            print 'Unexpected failure trying to detect gcc version'
-            traceback.print_exc()
-    else:
-        gcc.wait()
-        # Considered innocent until proven guilty.
-        error = None
-        for line in '\n'.join((gcc.stdout.read(), gcc.stderr.read())).splitlines():
-            if line.startswith('gcc version'):
-                if '4.1.1' in line and 'prerelease' in line:
-                    # There is a bug in 4.1.1 prerelease (ie, as of mandriva
-                    # 2007.0) g++ preventing kumo compilation from succeeding.
-                    error = 'broken GCC version: %s' % (line, )
-                break
-        else:
-            print >>sys.stderr, 'GCC version could not be detected, ' \
-                'building anyway'
-        if error is not None:
-            print 'Disabling build, with reason:', error
-            # Copy to preserver permission
-            copy(CONFIGURE_PATH, CONFIGURE_BACKUP_PATH)
-            open(CONFIGURE_PATH, 'w').write(FAKE_CONFIGURE % (error, ))
-

component/libevent/buildout.cfg

 [buildout]
 parts = libevent
+extends =
+  ../openssl/buildout.cfg
 
 [libevent]
 recipe = slapos.recipe.cmmi
@@ -7,3 +9,11 @@ url = http://www.monkey.org/~provos/libevent-1.4.13-stable.tar.gz
 md5sum = 0b3ea18c634072d12b3c1ee734263664
 configure-options =
   --disable-static
+
+[libevent2]
+<= libevent
+url = https://github.com/libevent/libevent/releases/download/release-2.1.10-stable/libevent-2.1.10-stable.tar.gz
+md5sum = 999caf86f52943af2363bc8077f00167
+environment =
+  CPPFLAGS=-I${openssl:location}/include
+  LDFLAGS=-L${openssl:location}/lib
\ No newline at end of file

component/perl/buildout.cfg

@@ -93,7 +93,7 @@ make-binary=
   ${:extra-env} PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" make
 
 # this post-make-hook is same for all users of the macro.
-post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#d012f7ba3300b14b2c6b173351d410be:post_make_hook
+post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#f28c45a0f473ae050ca3ebaed5c39a4e:post_make_hook
 
 perl_location = ${perl:location}
 

component/perl/perl-CPAN-package-create-wrapper.py

@@ -17,7 +17,7 @@ def post_make_hook(options, buildout, environmet):
 export PERL5LIB="{site_perl}:$PERL5LIB"
 exec {perl_location}/bin/perl "$@"
 '''.format(**locals()))
-  os.chmod(perl_wrapper_path, 0755)
+  os.chmod(perl_wrapper_path, 0o755)
 
   # create a wrapper for each scripts installed in perl-bin
   for script_path in glob.glob(os.path.join(prefix, 'perl-bin', '*')):
@@ -28,4 +28,4 @@ exec {perl_location}/bin/perl "$@"
 export PERL5LIB="{site_perl}:$PERL5LIB"
 exec {perl_location}/bin/perl {script_path} "$@"
 '''.format(**locals()))
-      os.chmod(wrapper_path, 0755)
+      os.chmod(wrapper_path, 0o755)

component/phantomjs/buildout.cfg

@@ -2,7 +2,6 @@
 extends =
   ../fontconfig/buildout.cfg
   ../libexpat/buildout.cfg
-  ../dash/buildout.cfg
 parts =
   phantomjs
 
@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686
 x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
 
 script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+  if not self.options.get('url'):
+    self.options['url'], self.options['md5sum'] = \
+      self.options[guessPlatform()].split(' ')
+  extract_dir = self.extract(self.download(self.options['url'],
+                             self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
   wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
-  wrapper = open(wrapper_location, 'w')
-  wrapper.write("""#!${dash:location}/bin/dash
+  with open(wrapper_location, 'w') as wrapper:
+    wrapper.write("""#!/bin/sh
   cd %(location)s
   export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/phantomjs $*""")
-  wrapper.flush()
-  wrapper.close()
-  os.chmod(wrapper_location, 0755)
+  exec %(location)s/bin/phantomjs "$@"
+  """)
+  os.chmod(wrapper_location, 0o755)

slapos/recipe/apachephpconfigure/__init__.py

@@ -24,6 +24,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
+
 from slapos.recipe.librecipe import GenericBaseRecipe
 import zc.buildout
 import sys
@@ -101,17 +103,17 @@ class Recipe(GenericBaseRecipe):
 
     # TODO factor
     if delete != []:
-      print "Creating lampconfigure with 'delete' arguments"
+      print("Creating lampconfigure with 'delete' arguments")
       command = argument + delete
     if rename != []:
       for parameters in rename:
-        print "Creating lampconfigure with 'rename' arguments"
+        print("Creating lampconfigure with 'rename' arguments")
         command = argument + rename
     if chmod != []:
-      print "Creating lampconfigure with 'chmod' arguments"
+      print("Creating lampconfigure with 'chmod' arguments")
       command = argument + chmod
     if data != []:
-      print "Creating lampconfigure with 'run' arguments"
+      print("Creating lampconfigure with 'run' arguments")
       command = argument + data
 
 

slapos/recipe/bonjourgrid/__init__.py

@@ -94,7 +94,7 @@ class Recipe(GenericBaseRecipe):
       dict(ip_address=self.options['ipv6'].strip(),
             project=project,
             middleware=type)))
-    os.chmod(config_info_file, 0744)
+    os.chmod(config_info_file, 0o744)
     path_list.append(config_info)
 
   update = install

slapos/recipe/certificate_authority/__init__.py

@@ -26,11 +26,11 @@
 ##############################################################################
 import os
 import hashlib
-import ConfigParser
+from six.moves import configparser
 import tempfile
 
 from slapos.recipe.librecipe import GenericBaseRecipe
-from certificate_authority import popenCommunicate
+from .certificate_authority import popenCommunicate
 
 class Recipe(GenericBaseRecipe):
 
@@ -119,7 +119,7 @@ class Request(Recipe):
       open(certificate, 'w').write(cert_content)
       request_needed = False
     else:
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       parser.add_section('certificate')
       parser.set('certificate', 'name', name)
       parser.set('certificate', 'key_file', key)

slapos/recipe/certificate_authority/certificate_authority.py

+from __future__ import print_function
+
 import os
 import subprocess
 import time
-import ConfigParser
+from six.moves import configparser
 import uuid
 
 
@@ -95,12 +97,12 @@ class CertificateAuthority:
 
   def checkRequestDir(self):
     for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
       if self._checkCertificate(parser.get('certificate', 'name'),
           parser.get('certificate', 'key_file'), parser.get('certificate',
             'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
+        print('Created certificate %r' % parser.get('certificate', 'name'))
 
 def runCertificateAuthority(*args):
   ca = CertificateAuthority(*args)

slapos/recipe/condor/__init__.py

@@ -29,7 +29,6 @@ import os
 import subprocess
 import zc.buildout
 import filecmp
-import urlparse
 import shutil
 import re
 import json
@@ -130,9 +129,9 @@ class Recipe(GenericBaseRecipe):
 
     #create condor binary launcher for slapos
     if not os.path.exists(self.wrapper_bin):
-      os.makedirs(self.wrapper_bin, int('0744', 8))
+      os.makedirs(self.wrapper_bin, int('0o744', 8))
     if not os.path.exists(self.wrapper_sbin):
-      os.makedirs(self.wrapper_sbin, int('0744', 8))
+      os.makedirs(self.wrapper_sbin, int('0o744', 8))
     #generate script for each file in prefix/bin
     for binary in os.listdir(self.prefix+'/bin'):
       wrapper_location = os.path.join(self.wrapper_bin, binary)
@@ -153,7 +152,7 @@ class Recipe(GenericBaseRecipe):
       wrapper.write(content)
       wrapper.close()
       path_list.append(wrapper_location)
-      os.chmod(wrapper_location, 0744)
+      os.chmod(wrapper_location, 0o744)
 
     #generate script for each file in prefix/sbin
     for binary in os.listdir(self.prefix+'/sbin'):
@@ -175,7 +174,7 @@ class Recipe(GenericBaseRecipe):
       wrapper.write(content)
       wrapper.close()
       path_list.append(wrapper_location)
-      os.chmod(wrapper_location, 0744)
+      os.chmod(wrapper_location, 0o744)
 
     #generate script for start condor
     wrapper = self.createPythonScript(
@@ -228,7 +227,7 @@ class AppSubmit(GenericBaseRecipe):
         for file in file_list:
           if file and (file.startswith('http') or file.startswith('ftp')):
             file_list[file] = self.download(file_list[file])
-          os.chmod(file_list[file], 0600)
+          os.chmod(file_list[file], 0o600)
       else:
         app_list[app]['files'] = {}
 
@@ -236,11 +235,11 @@ class AppSubmit(GenericBaseRecipe):
       if executable and (executable.startswith('http') or executable.startswith('ftp')):
         app_list[app]['executable'] = self.download(executable,
                                       app_list[app]['executable-name'])
-        os.chmod(app_list[app]['executable-name'], 0700)
+        os.chmod(app_list[app]['executable-name'], 0o700)
       submit_file = app_list[app].get('description-file', '')
       if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
         app_list[app]['description-file'] = self.download(submit_file, 'submit')
-        os.chmod(app_list[app]['description-file'], 0600)
+        os.chmod(app_list[app]['description-file'], 0o600)
 
     return app_list
 

slapos/recipe/davstorage/__init__.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 import subprocess
-import httplib
+from six.moves import http_client as httplib
 import base64
 import os
 import shutil

slapos/recipe/dcron.py

@@ -29,6 +29,8 @@ import os
 from slapos.recipe.librecipe import GenericBaseRecipe
 from zc.buildout import UserError
 
+from six.moves import map
+
 class Recipe(GenericBaseRecipe):
 
   def install(self):
@@ -124,7 +126,7 @@ def systemd_to_cron(spec):
     x = spec[i]
     if x != '*':
       for x in x.split(','):
-        x = map(int, x.split('/', 1))
+        x = list(map(int, x.split('/', 1)))
         a = x[0] - y
         if 0 <= a < z:
           if len(x) == 1:

slapos/recipe/erp5_promise/__init__.py

@@ -26,7 +26,7 @@
 ##############################################################################
 
 from slapos.recipe.librecipe import GenericBaseRecipe
-import ConfigParser
+from six.moves import configparser
 
 class Recipe(GenericBaseRecipe):
   """
@@ -34,7 +34,7 @@ class Recipe(GenericBaseRecipe):
   """
 
   def install(self):
-    promise_parser = ConfigParser.RawConfigParser()
+    promise_parser = configparser.RawConfigParser()
     for section_name, option_id_list in (
           ('portal_templates', (
             ('repository', 'bt5-repository-url'),

slapos/recipe/erp5testnode/__init__.py

@@ -24,10 +24,10 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import ConfigParser
+from six.moves import configparser
+import io
 import json
 import os
-import StringIO
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -40,13 +40,13 @@ class Recipe(GenericBaseRecipe):
     CONFIG['PATH'] = os.environ['PATH']
 
     if self.options['instance-dict']:
-      config_instance_dict = ConfigParser.ConfigParser()
+      config_instance_dict = configparser.ConfigParser()
       config_instance_dict.add_section('instance_dict')
       instance_dict = json.loads(self.options['instance-dict'])
 
       for k ,v in instance_dict.iteritems():
         config_instance_dict.set('instance_dict', k, v)
-      value = StringIO.StringIO()
+      value = io.StringIO()
       config_instance_dict.write(value)
       CONFIG['instance_dict'] = value.getvalue()
 

slapos/recipe/free_port.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 
-import ConfigParser
+from six.moves import configparser
 import os
 import netaddr
 import socket
@@ -48,7 +48,7 @@ class Recipe(object):
     # If this check isn't done, a new port would be picked for every upgrade
     # of the software release
     try:
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       if os.path.exists(buildout['buildout']['installed']):
         with open(buildout['buildout']['installed']) as config_file:
           parser.readfp(config_file)
@@ -59,7 +59,7 @@ class Recipe(object):
         if port != '0':
           self.options['port'] = port
           return
-    except (IOError, ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+    except (IOError, configparser.NoSectionError, configparser.NoOptionError):
       pass
 
     # Otherwise, let's find one

slapos/recipe/generic_cloudooo/__init__.py

@@ -24,9 +24,11 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from functools import cmp_to_key
 import zc.buildout
 from slapos.recipe.librecipe import GenericBaseRecipe
 
+@cmp_to_key
 def compareMimetypeEntryPair(a, b):
   """
     Like comparing strings, but here the star `*` is stronger than any other
@@ -115,7 +117,7 @@ class Recipe(GenericBaseRecipe):
       if l and not l.isspace()
     ]
     mimetype_entry_list.extend(default_mimetype_entry_list)
-    mimetype_entry_list.sort(compareMimetypeEntryPair)
+    mimetype_entry_list.sort(key=compareMimetypeEntryPair)
     conversion_server_dict['MIMETYPE_ENTRY_LIST'] = \
       "\n".join(["  " + l for l in mimetype_entry_list])
     config_file = self.createFile(self.options['configuration-file'],

slapos/recipe/lamp/__init__.py

@@ -31,7 +31,7 @@ import pkg_resources
 import zc.buildout
 import sys
 import zc.recipe.egg
-import urlparse
+from six.moves.urllib.parse import urlparse
 
 # Warning : this recipe is deprecated and has been replaced by apachephp.
 
@@ -264,7 +264,7 @@ class Request(BaseRecipe):
     mysql = self.request(self.options['mariadb-software-url'],
       software_type, 'MariaDB Server', partition_parameter_kw=parameters
     ).getConnectionParameter('url')
-    mysql_parsed = urlparse.urlparse(mysql)
+    mysql_parsed = urlparse(mysql)
 
     mysql_host, mysql_port = mysql_parsed.hostname, mysql_parsed.port
     if mysql_parsed.scheme == 'mysqls': # Listen over stunnel

slapos/recipe/libcloud/__init__.py

@@ -57,7 +57,7 @@ class Recipe(BaseSlapRecipe):
       try:
         self.slave_partition_configuration_dict_list.append(
             self._installSlavePartition(slave_partition))
-      except SlavePartitionError, e:
+      except SlavePartitionError as e:
         self.logger.warning('Slave Parttion %r not installed, issue: %r'%(
           slave_partition.getId(), e))
     # Installs wrappers

slapos/recipe/librecipe/__init__.py

@@ -34,13 +34,13 @@ import stat
 import netaddr
 import time
 import re
-import urlparse
+from six.moves.urllib.parse import urlunparse
 import json
 
 # Use to do from slapos.recipe.librecipe import GenericBaseRecipe
-from generic import GenericBaseRecipe
-from genericslap import GenericSlapRecipe
-from filehash import filehash
+from .generic import GenericBaseRecipe
+from .genericslap import GenericSlapRecipe
+from .filehash import filehash
 
 # Utility functions to (de)serialise live python objects in order to send them
 # to master.
@@ -324,7 +324,7 @@ class BaseSlapRecipe:
     if port is not None:
       netloc += ':%s' % port
 
-    url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    url = urlunparse((scheme, netloc, path, params, query, fragment))
 
     return url
 

slapos/recipe/librecipe/execute.py

+from __future__ import print_function
+
 import sys
 import os
 import signal
@@ -5,6 +7,8 @@ import subprocess
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
+import six
+
 def _wait_files_creation(file_list):
   # Establish a list of directory and subfiles.
   # and test existence before watching, so that we don't miss an event.
@@ -14,7 +18,7 @@ def _wait_files_creation(file_list):
     directories[dirname][filename] = os.path.lexists(f)
 
   def all_files_exists():
-    return all(all(files.itervalues()) for files in directories.itervalues())
+    return all(all(six.itervalues(files)) for files in six.itervalues(directories))
 
   with INotify() as inotify:
     watchdescriptors = {inotify.add_watch(dirname,
@@ -101,7 +105,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
 child_pg = None
 
 def sig_handler(sig, frame):
-  print 'Received signal %r, killing children and exiting' % sig
+  print('Received signal %r, killing children and exiting' % sig)
   if child_pg is not None:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)
@@ -116,7 +120,7 @@ def execute_with_signal_translation(args):
   child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
   child_pg = child.pid
   try:
-    print 'Process %r started' % (args, )
+    print('Process %r started' % (args, ))
     signal.pause()
   finally:
     os.killpg(child_pg, signal.SIGHUP)

slapos/recipe/librecipe/filehash.py

@@ -24,6 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
 import hashlib
 import shutil
 import os
@@ -91,6 +92,6 @@ if __name__ == '__main__':
   if len(sys.argv) == 1:
     raise ValueError("Not enough command line arguments")
   if len(sys.argv) == 2:
-    print sys.argv[1], '-', pathhash(sys.argv[1])
+    print(sys.argv[1], '-', pathhash(sys.argv[1]))
   else:
-    print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1])
+    print(sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]))

slapos/recipe/librecipe/generic.py

@@ -35,8 +35,12 @@ import inspect
 import re
 import shutil
 import stat
-import urllib
-import urlparse
+from six.moves.urllib.parse import quote
+import itertools
+import six
+from six.moves import map
+from six.moves.urllib.parse import urlunparse
+
 
 import pkg_resources
 import zc.buildout
@@ -90,7 +94,7 @@ class GenericBaseRecipe(object):
     """Options Hook method. This method can be overriden in child classes"""
     return
 
-  def createFile(self, name, content, mode=0600):
+  def createFile(self, name, content, mode=0o600):
     """Create a file with content
 
     The parent directory should exists, else it would raise IOError"""
@@ -117,7 +121,7 @@ class GenericBaseRecipe(object):
       f.write(content)
     return os.path.abspath(name)
 
-  def createExecutable(self, name, content, mode=0700):
+  def createExecutable(self, name, content, mode=0o700):
     return self.createFile(name, content, mode)
 
   def addLineToFile(self, filepath, line, encoding='utf8'):
@@ -148,9 +152,9 @@ class GenericBaseRecipe(object):
       module, function = function
     path, filename = os.path.split(os.path.abspath(name))
 
-    assert not isinstance(args, (basestring, dict)), args
-    args = map(repr, args)
-    args += map('%s=%r'.__mod__, kw.iteritems())
+    assert not isinstance(args, (six.string_types, dict)), args
+    args = itertools.chain(map(repr, args),
+                           map('%s=%r'.__mod__, six.iteritems(kw)))
 
     return zc.buildout.easy_install.scripts(
       [(filename, module, function)], self._ws, sys.executable,
@@ -173,12 +177,12 @@ class GenericBaseRecipe(object):
     lines = ['#!/bin/sh']
 
     if env:
-      for k, v in sorted(env.iteritems()):
+      for k, v in sorted(six.iteritems(env)):
         lines.append('export %s=%s' % (k, shlex.quote(v)))
 
     lines.append('exec')
 
-    args = map(shlex.quote, args)
+    args = list(map(shlex.quote, args))
     args.append('"$@"')
     for arg in args:
       if len(lines[-1]) < 40:
@@ -188,9 +192,9 @@ class GenericBaseRecipe(object):
         lines.append('\t' + arg)
 
     lines.append('')
-    return self.createFile(path, '\n'.join(lines), 0700)
+    return self.createFile(path, '\n'.join(lines), 0o700)
 
-  def createDirectory(self, parent, name, mode=0700):
+  def createDirectory(self, parent, name, mode=0o700):
     path = os.path.join(parent, name)
     if not os.path.exists(path):
       os.mkdir(path, mode)
@@ -240,9 +244,9 @@ class GenericBaseRecipe(object):
     netloc = ''
     if auth is not None:
       auth = tuple(auth)
-      netloc = urllib.quote(str(auth[0])) # Login
+      netloc = quote(str(auth[0])) # Login
       if len(auth) > 1:
-        netloc += ':%s' % urllib.quote(auth[1]) # Password
+        netloc += ':%s' % quote(auth[1]) # Password
       netloc += '@'
 
     # host is an ipv6 address whithout brackets
@@ -254,7 +258,7 @@ class GenericBaseRecipe(object):
     if port is not None:
       netloc += ':%s' % port
 
-    url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    url = urlunparse((scheme, netloc, path, params, query, fragment))
 
     return url
 

slapos/recipe/librecipe/genericslap.py

@@ -27,7 +27,7 @@
 from slapos import slap
 import time
 
-from generic import GenericBaseRecipe
+from .generic import GenericBaseRecipe
 
 CONNECTION_CACHE = {}
 

slapos/recipe/mkdirectory.py

@@ -27,6 +27,7 @@
 import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
+import six
 
 class Recipe(GenericBaseRecipe):
 
@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe):
     self.mode = int(self.directory.pop('mode', '0777'), 8)
 
   def install(self):
-    for path in sorted(self.directory.itervalues()):
+    for path in sorted(six.itervalues(self.directory)):
       if path and not os.path.isdir(path):
         os.makedirs(path, self.mode)
     # WARNING: This recipe is currently used to create directories that will

slapos/recipe/nosqltestbed/__init__.py

@@ -25,6 +25,8 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import sys
 import pkg_resources
 from logging import Formatter
@@ -41,7 +43,7 @@ class NoSQLTestBed(BaseSlapRecipe):
 
       testbed = plugin_class()
     except:
-      print Formatter().formatException(sys.exc_info())
+      print(Formatter().formatException(sys.exc_info()))
       return None
 
     software_type = self.parameter_dict.get('slap_software_type', 'default')

slapos/recipe/pbs.py

@@ -25,12 +25,14 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import json
 import os
 import subprocess
 import sys
 import textwrap
-import urlparse
+from six.moves.urllib.parse import urlparse
 
 from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
@@ -208,7 +210,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
       # This behavior has been removed to accelerate deployment of the
       # Software Release. The buildout, instead of failing, can process
       # other sections, which will return parameters to the main instance faster
-    parsed_url = urlparse.urlparse(url)
+    parsed_url = urlparse(url)
 
     slave_type = entry['type']
     if not slave_type in ['pull', 'push']:
@@ -216,7 +218,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     slave_id = entry['notification-id']
 
-    print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
+    print('Processing PBS slave %s with type %s' % (slave_id, slave_type))
 
     path_list.append(self.createPythonScript(
       os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),

slapos/recipe/postgres/__init__.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 
-import md5
+import hashlib
 import os
 import subprocess
 import textwrap
@@ -195,7 +195,7 @@ class Recipe(GenericBaseRecipe):
         password = self.options['password']
 
         # encrypt the password to avoid storing in the logs
-        enc_password = 'md5' + md5.md5(password+user).hexdigest()
+        enc_password = 'md5' + hashlib.md5(password+user).hexdigest()
 
         self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
 

slapos/recipe/publish.py

@@ -24,9 +24,11 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
 import zc.buildout
 from slapos.recipe.librecipe import wrap
 from slapos.recipe.librecipe import GenericSlapRecipe
+import six
 
 CONNECTION_PARAMETER_STRING = 'connection-'
 
@@ -77,9 +79,9 @@ class PublishSection(GenericSlapRecipe):
     for section in self.options['section-list'].strip().split():
       section = section.strip()
       options = self.buildout[section].copy()
-      for k, v in options.iteritems():
+      for k, v in six.iteritems(options):
         if k.startswith(CONNECTION_PARAMETER_STRING):
-          print k, v
+          print(k, v)
           publish_dict[k.lstrip(CONNECTION_PARAMETER_STRING)] = v
     self.setConnectionDict(publish_dict)
     return []

slapos/recipe/publish_early.py

@@ -25,8 +25,9 @@
 #
 ##############################################################################
 
-from collections import defaultdict
+from collections import OrderedDict
 from .librecipe import unwrap, wrap, GenericSlapRecipe
+import six
 
 def volatileOptions(options, volatile):
   def copy():
@@ -65,6 +66,9 @@ class Recipe(GenericSlapRecipe):
   the published value 'foo' if it exists. If its __init__ modifies 'x', the new
   value is published. To prevent [gen-foo] from being accessed too early, 'x'
   is then removed and the value can only be accessed with ${publish-early:foo}.
+  Init sections are processed in the order of first appearance in the '-init'
+  section, so that a init section can access a value that is generated by a
+  previous one (above, [gen-bar] can access ${publish-early:foo}).
 
   Generated values don't end up in the buildout installed file, which is good
   if they're secret. Note however that buildout won't detect if values change
@@ -75,13 +79,16 @@ class Recipe(GenericSlapRecipe):
   """
   def __init__(self, buildout, name, options):
     GenericSlapRecipe.__init__(self, buildout, name, options)
-    init = defaultdict(dict)
+    init = OrderedDict()
     for line in options['-init'].splitlines():
       if line:
         k, v = line.split()
         if k not in options:
           section, v = v.split(':')
+          try:
             init[section][k] = v
+          except KeyError:
+            init[section] = {k: v}
     if init:
       self.slap.initializeConnection(self.server_url, self.key_file,
         self.cert_file)
@@ -109,9 +116,9 @@ class Recipe(GenericSlapRecipe):
       publish = False
       publish_dict = {}
       try:
-        for init_section, init in init.iteritems():
+        for init_section, init in six.iteritems(init):
           override = {}
-          for k, v in init.iteritems():
+          for k, v in six.iteritems(init):
             try:
               override[v] = published_dict[k]
             except KeyError:
@@ -120,9 +127,9 @@ class Recipe(GenericSlapRecipe):
           init_section = buildout[init_section]
           assert buildout.Options is Options
           new = {}
-          for k, v in init.iteritems():
+          for k, v in six.iteritems(init):
             try:
-              publish_dict[k] = new[v] = init_section.pop(v)
+              options[k] = publish_dict[k] = new[v] = init_section.pop(v)
             except KeyError:
               pass
           if new != override:
@@ -138,6 +145,5 @@ class Recipe(GenericSlapRecipe):
       publish += publish_dict
       publish_dict['-publish'] = ' '.join(publish)
       volatileOptions(options, list(publish_dict))
-      options.update(publish_dict)
 
   install = update = lambda self: None

slapos/recipe/random.py

@@ -172,7 +172,7 @@ class Password(object):
           raise
 
       fd = os.open(self.storage_path,
-        os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0600)
+        os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0o600)
       try:
         os.write(fd, self.passwd)
       finally:

slapos/recipe/re6stnet/__init__.py

@@ -123,7 +123,7 @@ class Recipe(GenericBaseRecipe):
         token_dict[reference] = new_token
         to_add_dict[reference] = new_token
 
-    for reference in token_dict.keys():
+    for reference in list(token_dict):
       if not reference in reference_list:
         # This slave instance is destroyed ?
         to_remove_dict[reference] = token_dict.pop(reference)

slapos/recipe/redis/__init__.py

@@ -81,5 +81,5 @@ def promise(host, port, unixsocket):
     r = Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
     r.publish("Promise-Service","SlapOS Promise")
     r.connection_pool.disconnect()
-  except Exception, e:
+  except Exception as e:
     sys.exit(e)

slapos/recipe/request.py

@@ -33,6 +33,8 @@ from slapos.slap import SoftwareProductCollection
 import slapos.recipe.librecipe.generic as librecipe
 import traceback
 
+import six
+
 SOFTWARE_PRODUCT_NAMESPACE = "product."
 DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
@@ -110,10 +112,10 @@ class Recipe(object):
       raise UserError("'config' & 'sla' options are obsolete."
                       " Clean up your software release.")
     filter_kw = {k[4:]: v
-      for k, v in options.iteritems()
+      for k, v in six.iteritems(options)
       if k.startswith('sla-') and v}
     partition_parameter_kw = self._filterForStorage({k[7:]: v
-      for k, v in options.iteritems()
+      for k, v in six.iteritems(options)
       if k.startswith('config-')})
     slave = options.get('slave', 'false').lower() in \
       librecipe.GenericBaseRecipe.TRUE_VALUES
@@ -196,7 +198,7 @@ class Recipe(object):
       except KeyError:
         if self.failed is None:
           self.failed = param
-      if isinstance(value, unicode):
+      if six.PY2 and isinstance(value, unicode):
         value = value.encode('UTF-8')
       options['connection-%s' % param] = value
 
@@ -310,12 +312,12 @@ class RequestEdge(Recipe):
 
       self.request_dict[country] = Recipe(buildout, name, local_options)
       # "Bubble" all connection parameters
-      for option, value in local_options.iteritems():
+      for option, value in six.iteritems(local_options):
         if option.startswith(CONNECTION_PARAMETER_STRING):
           self.options['%s-%s' % (option, country)] = value
 
   def install(self):
-    for country, request in self.request_dict.iteritems():
+    for country, request in six.iteritems(self.request_dict):
       request.install()
     return []
 

slapos/recipe/sheepdogtestbed/__init__.py

@@ -26,8 +26,6 @@
 ##############################################################################
 
 import os
-import urllib
-import urllib2
 import pkg_resources
 from slapos.recipe.librecipe import BaseSlapRecipe
 

slapos/recipe/slapconfiguration.py

@@ -31,7 +31,8 @@ import os
 
 import slapos.slap
 from slapos.recipe.librecipe import unwrap
-from ConfigParser import RawConfigParser
+import six
+from six.moves.configparser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
 from slapos.util import mkdir_p
 from slapos import format as slapformat
@@ -115,7 +116,7 @@ class Recipe(object):
                                       buildout['buildout']['directory'])
 
       match = self.OPTCRE_match
-      for key, value in parameter_dict.iteritems():
+      for key, value in six.iteritems(parameter_dict):
           if match(key) is not None:
               continue
           options['configuration.' + key] = value
@@ -157,11 +158,10 @@ class Recipe(object):
               options[his_key.replace('_', '-')] = value
       # Get Instance and root instance title or return UNKNOWN if not set
       options['instance-title'] = parameter_dict.pop('instance_title',
-                                            'UNKNOWN Instance').encode('UTF-8')
+                                            'UNKNOWN Instance')
       options['root-instance-title'] = parameter_dict.pop('root_instance_title',
-                                            'UNKNOWN').encode('UTF-8')
-      options['instance-guid'] = computer_partition.getInstanceGuid() \
-          .encode('UTF-8')
+                                            'UNKNOWN')
+      options['instance-guid'] = computer_partition.getInstanceGuid()
 
       ipv4_set = set()
       v4_add = ipv4_set.add
@@ -204,9 +204,9 @@ class Recipe(object):
 
       # also export single ip values for those recipes that don't support sets.
       if ipv4_set:
-          options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
+          options['ipv4-random'] = min(ipv4_set)
       if ipv6_set:
-          options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
+          options['ipv6-random'] = min(ipv6_set)
 
       storage_home = options.get('storage-home')
       storage_dict = {}
@@ -240,7 +240,7 @@ class Recipe(object):
           # be very careful with overriding master's information
           for key, value in flatten_dict(partition_params).items():
             if key not in options:
-              if isinstance(value, unicode):
+              if six.PY2 and isinstance(value, unicode):
                 value = value.encode('UTF-8')
               options[key] = value
       # print out augmented options to see what we are passing
@@ -265,8 +265,11 @@ class JsonDump(Recipe):
   def __init__(self, buildout, name, options):
     parameter_dict = self.fetch_parameter_dict(options)
     self._json_output = options['json-output']
-    with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fout:
-      fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
+    # XXX: do not touch file if there's no change to avoid excessive IO
+    #      (see https://lab.nexedi.com/nexedi/slapos.recipe.template/commit/14d26bc8c77a1940f389026bdbd3a9b229b241f4
+    #       for an example to fix this)
+    with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fout:
+      json.dump(parameter_dict, fout, indent=2, sort_keys=True)
 
     def install(self):
         return [self._json_output]

slapos/recipe/stunnel/__init__.py

@@ -36,7 +36,7 @@ def kill(pid_file, sig=signal.SIGUSR1):
       pid = int(f.read().strip())
     try:
       os.kill(pid, sig)
-    except OSError, e:
+    except OSError as e:
       if e.errno != errno.ESRCH: # No such process
         raise e
       os.unlink(pid_file)

slapos/recipe/wrapper.py

@@ -76,9 +76,9 @@ class Recipe(GenericBaseRecipe):
       import hashlib
       hasher = hashlib.md5()
       for path in file_list:
-        with open(path, 'r') as afile:
+        with open(path, 'rb') as afile:
           buf = afile.read()
-        hasher.update("%s\n" % len(buf))
+        hasher.update(b"%u\n" % len(buf))
         hasher.update(buf)
       hash = hasher.hexdigest()
       return hash

slapos/recipe/xwiki/__init__.py

@@ -69,7 +69,7 @@ export JAVA_OPTS="${JAVA_OPTS} -Djava.awt.headless=true"
     bindir = os.path.join(tomcat_home, 'bin')
     for f in os.listdir(bindir):
       if f.endswith('.sh'):
-        os.chmod(os.path.join(bindir, f), 0755)
+        os.chmod(os.path.join(bindir, f), 0o755)
     tomcat_wrapper = self.createRunningWrapper('xwiki', """#!/bin/sh
 export JRE_HOME=%(java_home)s
 exec %(catalina)s run

slapos/recipe/zabbixagent/__init__.py

@@ -33,7 +33,6 @@ import hashlib
 import sys
 import zc.buildout
 import zc.recipe.egg
-import ConfigParser
 
 class Recipe(BaseSlapRecipe):
   def installLogrotate(self):

slapos/test/recipe/test_pbs.py

@@ -5,6 +5,8 @@ import sys
 import tempfile
 import unittest
 
+import six
+
 
 class PBSTest(unittest.TestCase):
 
@@ -22,7 +24,7 @@ class PBSTest(unittest.TestCase):
     def test_push(self):
         recipe = self.new_recipe()
 
-        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+        with tempfile.NamedTemporaryFile('w+') as rdiff_wrapper:
             recipe.wrapper_push(remote_schema='TEST_REMOTE_SCHEMA',
                                 local_dir='TEST_LOCAL_DIR',
                                 remote_dir='TEST_REMOTE_DIR',
@@ -35,7 +37,7 @@ class PBSTest(unittest.TestCase):
     def test_pull(self):
         recipe = self.new_recipe()
 
-        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+        with tempfile.NamedTemporaryFile('w+') as rdiff_wrapper:
             recipe.wrapper_pull(remote_schema='TEST_REMOTE_SCHEMA',
                                 local_dir='TEST_LOCAL_DIR',
                                 remote_dir='TEST_REMOTE_DIR',
@@ -104,22 +106,22 @@ class PBSTest(unittest.TestCase):
 
         recipe._install()
 
-        self.assertItemsEqual(os.listdir(promises_directory),
+        six.assertCountEqual(self, os.listdir(promises_directory),
                               ['ssh-to-pulltest', 'ssh-to-pushtest'])
 
-        self.assertItemsEqual(os.listdir(wrappers_directory),
+        six.assertCountEqual(self, os.listdir(wrappers_directory),
                               ['pulltest_raw', 'pulltest', 'pushtest_raw', 'pushtest'])
 
-        self.assertItemsEqual(os.listdir(directory),
+        six.assertCountEqual(self, os.listdir(directory),
                               ['TEST_NAME'])
 
-        self.assertItemsEqual(os.listdir(feeds_directory),
+        six.assertCountEqual(self, os.listdir(feeds_directory),
                               ['pulltest', 'pushtest'])
 
-        self.assertItemsEqual(os.listdir(run_directory),
+        six.assertCountEqual(self, os.listdir(run_directory),
                               [])
 
-        self.assertItemsEqual(os.listdir(cron_directory),
+        six.assertCountEqual(self, os.listdir(cron_directory),
                               ['pulltest', 'pushtest'])
 
         shutil.rmtree(promises_directory)

slapos/test/recipe/test_plugin.py

@@ -3,6 +3,7 @@ from slapos.recipe import promise_plugin
 from slapos.test.utils import makeRecipe
 from pprint import pformat
 import stat, json
+import six
 
 class TestPromisePlugin(unittest.TestCase):
 
@@ -68,7 +69,7 @@ in multi line
     with open(self.output) as f:
       content = f.read()
     self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
-    self.assertIn('extra_config_dict = { }', content)
+    self.assertIn('extra_config_dict = %s' % ('{}' if six.PY3 else '{ }'), content)
 
 
   def test_bad_parameters(self):
@@ -99,7 +100,7 @@ in multi line
     with self.assertRaises(ValueError) as p:
       recipe.install()
 
-    self.assertEqual(p.exception.message, "Import path %r is not a valid" % self.options['import'])
+    self.assertEqual(str(p.exception), "Import path %r is not a valid" % self.options['import'])
 
   def test_bad_content(self):
     self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"'
@@ -110,5 +111,5 @@ in multi line
     with self.assertRaises(ValueError) as p:
       recipe.install()
 
-    self.assertEqual(p.exception.message, "Promise content %r is not valid" % self.options['content'])
+    self.assertEqual(str(p.exception), "Promise content %r is not valid" % self.options['content'])
 

slapos/test/recipe/test_re6stnet.py

@@ -6,6 +6,8 @@ import tempfile
 import unittest
 from slapos.slap.slap import NotFoundError, ConnectionError
 
+import six
+
 
 class Re6stnetTest(unittest.TestCase):
 
@@ -88,7 +90,7 @@ class Re6stnetTest(unittest.TestCase):
     
     recipe.generateCertificate()
     
-    self.assertItemsEqual(os.listdir(self.ssl_dir),
+    six.assertCountEqual(self, os.listdir(self.ssl_dir),
                           ['cert.key', 'cert.crt', 'dh.pem'])
     
     last_time = time.ctime(os.stat(self.options['key-file'])[7])
@@ -143,10 +145,10 @@ class Re6stnetTest(unittest.TestCase):
     
     token_dict = recipe.loadJsonFile(token_file)
     self.assertEqual(len(token_dict), 2)
-    self.assertTrue(token_dict.has_key('SOFTINST-58770'))
-    self.assertTrue(token_dict.has_key('SOFTINST-58778'))
+    self.assertIn('SOFTINST-58770', token_dict)
+    self.assertIn('SOFTINST-58778', token_dict)
         
-    self.assertItemsEqual(os.listdir(self.token_dir),
+    six.assertCountEqual(self, os.listdir(self.token_dir),
                           ['SOFTINST-58770.add', 'SOFTINST-58778.add'])
 
     first_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58770.add'))
@@ -173,7 +175,7 @@ class Re6stnetTest(unittest.TestCase):
     
     self.assertEqual(len(token_dict), 1)
     self.assertEqual(token_dict['SOFTINST-58770'], first_add)
-    self.assertItemsEqual(os.listdir(self.token_dir),
+    six.assertCountEqual(self, os.listdir(self.token_dir),
                           ['SOFTINST-58770.add', 'SOFTINST-58778.remove'])
     
     second_remove = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.remove'))
@@ -193,7 +195,7 @@ class Re6stnetTest(unittest.TestCase):
     
     token_content = recipe.readFile(token_file)
     self.assertEqual(token_content, '{}')
-    self.assertItemsEqual(os.listdir(self.options['token-dir']), [])
+    six.assertCountEqual(self, os.listdir(self.options['token-dir']), [])
 
     self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
 

slapos/test/utils.py

@@ -2,7 +2,7 @@
 """
 import sys
 import os.path
-from ConfigParser import ConfigParser
+from zc.buildout.configparser import parse
 
 import logging
 
@@ -52,18 +52,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None):
   buildout_cfg = os.path.join(base_directory, 'buildout.cfg')
 
   if os.path.exists(buildout_cfg):
-    parser = ConfigParser()
-    parser.readfp(open(buildout_cfg))
-    if parser.has_option('buildout', 'eggs-directory'):
-      # when buildout_cfg is an instance buildout (like in SLAPOS-EGG-TEST),
+    with open(buildout_cfg) as f:
+      parsed_cfg = parse(f, buildout_cfg)
+
+    # When buildout_cfg is an instance buildout (like in SLAPOS-EGG-TEST),
     # there's a ${buildout:eggs-directory} we can use.
-      eggs_directory = parser.get('buildout', 'eggs-directory')
-      develop_eggs_directory = parser.get('buildout', 'develop-eggs-directory')
-    else:
-      # when when buildout_cfg is a software buildout, we can only guess the
+    # When buildout_cfg is a software buildout, we can only guess the
     # standard eggs directories.
-      eggs_directory = os.path.join(base_directory, 'eggs')
-      develop_eggs_directory = os.path.join(base_directory, 'develop-eggs')
+    eggs_directory = parsed_cfg['buildout'].get(
+      'eggs-directory', os.path.join(base_directory, 'eggs'))
+    develop_eggs_directory = parsed_cfg['buildout'].get(
+      'develop-eggs-directory', os.path.join(base_directory, 'develop-eggs'))
 
     logging.getLogger(__name__).info(
         'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s',

software/neoppod/software-common.cfg

@@ -147,6 +147,11 @@ template =
       --datadir="$datadir.new"
     mv -v "$datadir.new" "$datadir"
   }
+  {%- if environ is defined %}
+  {%-   for variable in environ.splitlines() %}
+  {{ variable }} \
+  {%-   endfor %}
+  {%- endif %}
   exec '${mariadb:location}/bin/mysqld' \
     --defaults-file='{{defaults_file}}' \
     "$@"

software/slapos-master/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 7f3318aec3b682c3379609ccdd659411
+md5sum = 001affafc204b638615deea04c95cfdf
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -213,7 +213,7 @@ config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
 config-neo-name = {{ server_dict.keys()[0] }}
-config-neo-masters = ${neo-0-final:connection-masters}
+config-neo-masters = ${publish-early:neo-masters}
 {%   else -%}
 config-zodb-zeo = ${request-zodb:connection-storage-dict}
 config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
@@ -411,10 +411,6 @@ return = site_url
 <= monitor-publish
 recipe = slapos.cookbook:publish.serialised
 -extends = publish-early
-{% if 'neo' in storage_dict -%}
-neo-masters = ${neo-0-final:connection-masters}
-neo-admins = ${neo-0-final:connection-admins}
-{% endif  -%}
 {% if zope_address_list_id_dict -%}
 {#
 Pick any published hosts-dict, they are expected to be identical - and there is
@@ -431,7 +427,6 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {%   endfor -%}
 {% endif -%}
 
-
 [publish-early]
 recipe = slapos.cookbook:publish-early
 -init =
@@ -447,6 +442,8 @@ recipe = slapos.cookbook:publish-early
 {%- endif %}
 {%- if neo %}
   neo-cluster gen-neo-cluster:name
+  neo-admins neo-cluster:admins
+  neo-masters neo-cluster:masters
 {%-  if neo[0] %}
 neo-cluster = {{ dumps(neo[0]) }}
 {%-  endif %}

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 81b01eb048e0db93bf0152504bf8a04d
+md5sum = 3d12778d8dc910180ff3a7fcae8b37f9

software/slapos-sr-testing/instance.cfg

@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
 working-dir = $${buildout:directory}/tmp
 
 [test-list]
-path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup}
+path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup}
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper

software/slapos-sr-testing/software.cfg

@@ -97,6 +97,11 @@ setup = ${slapos-repository:location}/software/jupyter/test/
 egg = slapos.test.nextcloud
 setup = ${slapos-repository:location}/software/nextcloud/test/
 
+[slapos.test.turnserver-setup]
+<= setup-develop-egg
+egg = slapos.test.nextcloud
+setup = ${slapos-repository:location}/software/turnserver/test/
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -121,6 +126,7 @@ eggs =
   ${slapos.test.slaprunner-setup:egg}
   ${slapos.test.jupyter-setup:egg}
   ${slapos.test.nextcloud-setup:egg}
+  ${slapos.test.turnserver-setup:egg}
   ${backports.lzma:egg}
 
 entry-points =

software/slapos-testing/software-py3.cfg

+[buildout]
+extends =
+  ../../component/python3/buildout.cfg
+  software.cfg
+python = python3.5
+
+[nghttp2]
+environment =
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python3.5:location}/bin:%(PATH)s
+
+[supervisor-repository]
+<= git-clone-repository
+repository = https://github.com/Supervisor/supervisor.git
+
+[supervisor-develop]
+recipe = zc.recipe.egg:develop
+egg = supervisor
+setup = ${supervisor-repository:location}
+
+[eggs]
+recipe = zc.recipe.egg
+eggs += ${supervisor-develop:egg}
+
+[versions]
+supervisor =

software/turnserver/instance-turnserver.cfg.jinja2.in

+{% set part_list = [] -%}
+{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+run = ${:var}/run
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promises = ${:etc}/promise
+plugins = ${:etc}/plugin
+ssl = ${:etc}/ssl
+
+[file-base]
+recipe = slapos.recipe.template:jinja2
+template = inline:{{ '{{ content }}' }}
+
+
+{% macro simplefile(section_name, file_path, content, mode='') -%}
+{%   set content_section_name = section_name ~ '-content' -%}
+[{{  content_section_name }}]
+content = {{ dumps(content) }}
+[{{  section_name }}]
+< = file-base
+rendered = {{ file_path }}
+context = key content {{ content_section_name }}:content
+mode = {{ mode }}
+{% do part_list.append(section_name) -%}
+{%- endmacro %}
+
+{% if slapparameter_dict.get('ssl-key') and slapparameter_dict.get('ssl-crt') -%}
+{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', slapparameter_dict.get('ssl-crt')) }}
+{{ simplefile('ssl-key', '${turnserver-ssl:key}', slapparameter_dict.get('ssl-key'), 600) }}
+{% else -%}
+{%  do part_list.append('gen-certificate') -%}
+[gen-certificate]
+recipe = plone.recipe.command
+command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${turnserver-ssl:key}" -out "${turnserver-ssl:certificate}"
+{% endif -%}
+
+[turnserver-ssl]
+recipe = plone.recipe.command
+certificate = ${directory:ssl}/cert.pem
+key = ${directory:ssl}/key.pem
+dhparam = ${directory:ssl}/dhparam.pem
+command =
+  if [ ! -s "${directory:ssl}//dhparam.pem" ]; then
+    "{{ parameter_dict['openssl'] }}/bin/openssl" dhparam -out ${:dhparam} 4096
+  fi
+
+[gen-secret]
+recipe = plone.recipe.command
+secret-file = ${directory:etc}/.turnsecret
+command =
+  if [ ! -s "${:secret-file}" ]; then
+    cat <<EOF > ${:secret-file}
+    [turnserver]
+    secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
+    EOF
+  fi
+  chmod 600 ${:secret-file}
+
+[read-secret]
+recipe = slapos.cookbook:zero-knowledge.read
+file-path = ${gen-secret:secret-file}
+secret =
+
+{% set turn_port = slapparameter_dict.get('port', 3478) -%}
+{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
+{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
+[turnserver-config]
+recipe = collective.recipe.template
+input = inline:
+  listening-port={{ turn_port }}
+  tls-listening-port={{ turn_tls_port }}
+  fingerprint
+  lt-cred-mech
+  use-auth-secret
+  static-auth-secret=${read-secret:secret}
+  listening-ip={{ listining_ip }}
+{% if slapparameter_dict.get('external-ip', '') %}
+  external-ip={{ slapparameter_dict['external-ip'] }}
+{% endif %}
+  server-name={{ server_name }}
+  realm={{ server_name }}
+  total-quota=100
+  bps-capacity=0
+  stale-nonce=600
+  cert=${turnserver-ssl:certificate}
+  pkey=${turnserver-ssl:key}
+  dh-file=${turnserver-ssl:dhparam}
+  cipher-list="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5"
+  no-loopback-peers
+  no-multicast-peers
+  mobility
+  no-tlsv1
+  no-tlsv1_1
+  no-stdout-log
+  log-file=${directory:log}/turnserver.log
+  userdb=${directory:srv}/turndb
+  pidfile=${directory:run}/turnserver.pid
+  verbose
+output = ${directory:etc}/turnserver.conf
+mode = 644
+
+[turnserver-wrapper]
+recipe = slapos.cookbook:wrapper
+command-line = {{ parameter_dict['turnserver-location'] }}/bin/turnserver
+               -c ${turnserver-config:output}
+wrapper-path = ${directory:services}/turnserver
+hash-files = ${buildout:directory}/software_release/buildout.cfg
+
+[promise-check-turnserver-port]
+<= monitor-promise-base
+module = check_port_listening
+name = turnserver-port-listening.py
+config-hostname = {{ listining_ip }}
+config-port = {{ turn_port }}
+
+[promise-check-turnserver-tls-port]
+<= monitor-promise-base
+module = check_port_listening
+name = turnserver-tls-port-listening.py
+config-hostname = {{ listining_ip }}
+config-port = {{ turn_tls_port }}
+
+[publish-connection-information]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+turn-url = {{ server_name ~ ':' ~ turn_port }}
+turn-tls-url = {{ server_name ~ ':' ~ turn_tls_port }}
+secret = ${read-secret:secret}
+
+[buildout]
+
+extends = {{ template_monitor }}
+
+parts =
+  publish-connection-information
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
+# turn server
+  turnserver-wrapper
+  promise-check-turnserver-tls-port
+  promise-check-turnserver-port
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
\ No newline at end of file

software/turnserver/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+
+# std stuff for slapos instance
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default  = $${dynamic-template-turnserver:rendered}
+RootSoftwareInstance = $${:default}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = $${buildout:directory}/$${:filename}
+extensions = jinja2.ext.do
+mode = 0644
+extra-context =
+context =
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key ipv4 slap-configuration:ipv4
+  key ipv6 slap-configuration:ipv6
+  key global_ipv4_prefix network-information:global-ipv4-network
+  key slapparameter_dict slap-configuration:configuration
+  key computer_id slap-configuration:computer
+  raw template_monitor ${monitor2-template:rendered}
+  raw logrotate_cfg ${template-logrotate-base:rendered}
+  $${:extra-context}
+
+
+[dynamic-template-turnserver-parameters]
+openssl = ${openssl:location}
+turnserver-location = ${coturn:location}
+
+
+[dynamic-template-turnserver]
+<= jinja2-template-base
+template = ${template-turnserver:location}/${template-turnserver:filename}
+filename = instance-turnserver.cfg
+extra-context =
+  section parameter_dict dynamic-template-turnserver-parameters

software/turnserver/software.cfg

+[buildout]
+
+extends =
+  ../../component/coturn/buildout.cfg
+  ../../component/openssl/buildout.cfg
+#  ../../component/6tunnel/buildout.cfg
+  ../../component/socat/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../stack/slapos.cfg
+
+
+parts +=
+  slapos-cookbook
+  coturn
+  instance-cfg
+
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+mode = 644
+
+[instance-cfg]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+md5sum = d027a2dccaf15ae6e7d3a28cc02d70c3
+
+[template-turnserver]
+<= download-base
+filename = instance-turnserver.cfg.jinja2.in
+md5sum = 02bddf180519f7649d3b1f997a496eed
+
+[versions]
+slapos.recipe.template = 4.3
\ No newline at end of file

software/turnserver/test/README.md

+
+
+Tests for TurnServer Software Release
\ No newline at end of file

software/turnserver/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.turnserver'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for TurnServer Software Release",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+          'slapos.core',
+          'slapos.libnetworkcache',
+          'erp5.util',
+          'supervisor',
+          'psutil',
+      ],
+      zip_safe=True,
+      test_suite='test',
+      )
\ No newline at end of file

software/turnserver/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import subprocess
+import json
+import glob
+import ConfigParser
+
+import utils
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('SLAPOS_TEST_DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+def subprocess_status_output(*args, **kwargs):
+  prc = subprocess.Popen(
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT,
+    *args,
+    **kwargs)
+  out, err = prc.communicate()
+  return prc.returncode, out
+
+class InstanceTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+
+
+class ServicesTestCase(InstanceTestCase):
+
+  @staticmethod
+  def generateHash(file_list):
+    import hashlib
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_process_list(self):
+    hash_list = [
+      'software_release/buildout.cfg',
+    ]
+    expected_process_names = [
+      'bootstrap-monitor',
+      'turnserver-{hash}-on-watch',
+      'certificate_authority-{hash}-on-watch',
+      'crond-{hash}-on-watch',
+      'monitor-httpd-{hash}-on-watch',
+      'monitor-httpd-graceful',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_name_list = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_file_list = [os.path.join(self.computer_partition_root_path, path)
+                      for path in hash_list]
+
+    for name in expected_process_names:
+      h = ServicesTestCase.generateHash(hash_file_list)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_name_list)
+
+  def test_default_deployment(self):
+    partition_path_list = glob.glob(os.path.join(self.instance_path, '*'))
+    instance_folder = None
+    for partition_path in partition_path_list:
+      if os.path.exists(os.path.join(partition_path, 'etc/turnserver.conf')):
+        instance_folder = partition_path
+        break
+
+    secret_file = os.path.join(instance_folder, 'etc/.turnsecret')
+    self.assertTrue(os.path.exists(instance_folder))
+    self.assertTrue(os.path.exists(secret_file))
+    config = ConfigParser.ConfigParser()
+    config.readfp(open(secret_file))
+    secret = config.get('turnserver', 'secret')
+
+    expected_config = """listening-port=3478
+tls-listening-port=5349
+fingerprint
+lt-cred-mech
+use-auth-secret
+static-auth-secret=%(secret)s
+listening-ip=%(ipv4)s
+server-name=turn.example.com
+realm=turn.example.com
+total-quota=100
+bps-capacity=0
+stale-nonce=600
+cert=%(instance_path)s/etc/ssl/cert.pem
+pkey=%(instance_path)s/etc/ssl/key.pem
+dh-file=%(instance_path)s/etc/ssl/dhparam.pem
+cipher-list="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5"
+no-loopback-peers
+no-multicast-peers
+mobility
+no-tlsv1
+no-tlsv1_1
+no-stdout-log
+log-file=%(instance_path)s/var/log/turnserver.log
+userdb=%(instance_path)s/srv/turndb
+pidfile=%(instance_path)s/var/run/turnserver.pid
+verbose""" % {'instance_path': instance_folder, 'secret': secret, 'ipv4': self.config['ipv4_address']}
+
+    with open(os.path.join(instance_folder, 'etc/turnserver.conf')) as f:
+      current_config = f.read().strip()
+
+    self.assertEqual(current_config, expected_config)
+
+  def test_turnserver_promises(self):
+    partition_path_list = glob.glob(os.path.join(self.instance_path, '*'))
+    instance_folder = None
+    for partition_path in partition_path_list:
+      if os.path.exists(os.path.join(partition_path, 'etc/turnserver.conf')):
+        instance_folder = partition_path
+        break
+    self.assertTrue(os.path.exists(instance_folder))
+
+    promise_path_list = glob.glob(os.path.join(instance_folder, 'etc/plugin/*.py'))
+    promise_name_list = [x for x in
+                         os.listdir(os.path.join(instance_folder, 'etc/plugin'))
+                         if not x.endswith('.pyc')]
+    partition_name = os.path.basename(instance_folder.rstrip('/'))
+    self.assertEqual(sorted(promise_name_list),
+                    sorted([
+                      "__init__.py",
+                      "check-free-disk-space.py",
+                      "monitor-http-frontend.py",
+                      "buildout-%s-status.py" % partition_name,
+                      "monitor-bootstrap-status.py",
+                      "monitor-httpd-listening-on-tcp.py",
+                      "turnserver-port-listening.py",
+                      "turnserver-tls-port-listening.py",
+                    ]))
+
+    ignored_plugin_list = [
+      '__init__.py',
+      'monitor-http-frontend.py',
+    ]
+    runpromise_bin = os.path.join(
+      self.software_path, 'bin', 'monitor.runpromise')
+    monitor_conf = os.path.join(instance_folder, 'etc', 'monitor.conf')
+    msg = []
+    status = 0
+    for plugin_path in promise_path_list:
+      plugin_name = os.path.basename(plugin_path)
+      if plugin_name in ignored_plugin_list:
+        continue
+      plugin_status, plugin_result = subprocess_status_output([
+        runpromise_bin,
+        '-c', monitor_conf,
+        '--run-only', plugin_name,
+        '--force',
+        '--check-anomaly'
+      ])
+      status += plugin_status
+      if plugin_status == 1:
+        msg.append(plugin_result)
+      # sanity check
+      if 'Checking promise %s' % plugin_name not in plugin_result:
+        plugin_status = 1
+        msg.append(plugin_result)
+    msg = ''.join(msg).strip()
+    self.assertEqual(status, 0, msg)
+
+class ParametersTestCase(InstanceTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'server-name': "turn.site.com",
+      'port': 3488,
+      'tls-port': 5369,
+      'external-ip': '127.0.0.1',
+      'listening-ip': '127.0.0.1'
+    }
+
+  def test_turnserver_with_parameters(self):
+    partition_path_list = glob.glob(os.path.join(self.instance_path, '*'))
+    instance_folder = None
+    for partition_path in partition_path_list:
+      if os.path.exists(os.path.join(partition_path, 'etc/turnserver.conf')):
+        instance_folder = partition_path
+        break
+
+    secret_file = os.path.join(instance_folder, 'etc/.turnsecret')
+    self.assertTrue(os.path.exists(instance_folder))
+    self.assertTrue(os.path.exists(secret_file))
+    config = ConfigParser.ConfigParser()
+    config.readfp(open(secret_file))
+    secret = config.get('turnserver', 'secret')
+
+    expected_config = """listening-port=%(port)s
+tls-listening-port=%(tls_port)s
+fingerprint
+lt-cred-mech
+use-auth-secret
+static-auth-secret=%(secret)s
+listening-ip=%(ipv4)s
+external-ip=%(external_ip)s
+server-name=%(name)s
+realm=%(name)s
+total-quota=100
+bps-capacity=0
+stale-nonce=600
+cert=%(instance_path)s/etc/ssl/cert.pem
+pkey=%(instance_path)s/etc/ssl/key.pem
+dh-file=%(instance_path)s/etc/ssl/dhparam.pem
+cipher-list="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5"
+no-loopback-peers
+no-multicast-peers
+mobility
+no-tlsv1
+no-tlsv1_1
+no-stdout-log
+log-file=%(instance_path)s/var/log/turnserver.log
+userdb=%(instance_path)s/srv/turndb
+pidfile=%(instance_path)s/var/run/turnserver.pid
+verbose""" % {'instance_path': instance_folder,
+              'secret': secret,
+              'ipv4': '127.0.0.1',
+              'name': 'turn.site.com',
+              'external_ip': '127.0.0.1',
+              'port': 3488,
+              'tls_port': 5369,}
+
+    with open(os.path.join(instance_folder, 'etc/turnserver.conf')) as f:
+      current_config = f.read().strip()
+
+    self.assertEqual(current_config, expected_config)
+
+

software/turnserver/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+import StringIO
+import xmlrpclib
+
+import supervisor.xmlrpc
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+# TODO:
+#  - allow requesting multiple instances ?
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  """Install one slapos instance.
+
+  This test case install software(s) and request one instance during `setUpClass`
+  and destroy the instance during `tearDownClass`.
+
+  Software Release URL, Instance Software Type and Instance Parameters can be defined
+  on the class.
+
+  All tests from the test class will run with the same instance.
+
+  The following class attributes are available:
+
+    * `computer_partition`:  the computer partition instance, implementing
+    `slapos.slap.interface.slap.IComputerPartition`.
+
+    * `computer_partition_root_path`: the path of the instance root directory.
+
+  """
+
+  # Methods to be defined by subclasses.
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    """Return software type for instance, default "default"
+
+    To be defined by subclasses if they need to request instance with specific
+    software type.
+    """
+    return "default"
+
+  # Utility methods.
+  def getSupervisorRPCServer(self):
+    """Returns a XML-RPC connection to the supervisor used by slapos node
+
+    Refer to http://supervisord.org/api.html for details of available methods.
+    """
+    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
+    return xmlrpclib.ServerProxy(
+       'http://slapos-supervisor',
+       transport=supervisor.xmlrpc.SupervisorTransport(
+           None,
+           None,
+           # XXX hardcoded socket path
+           serverurl="unix://{working_directory}/inst/supervisord.socket".format(
+             **self.config)))
+
+  # Unittest methods
+  @classmethod
+  def setUpClass(cls):
+    """Setup the class, build software and request an instance.
+
+    If you have to override this method, do not forget to call this method on
+    parent class.
+    """
+    try:
+      cls.setUpWorkingDirectory()
+      cls.setUpConfig()
+      cls.setUpSlapOSController()
+
+      cls.runSoftwareRelease()
+      # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
+      # cls.supplySoftwares()
+      # cls.installSoftwares()
+
+      cls.runComputerPartition()
+      # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
+      # cls.requestInstances()
+      # cls.createInstances()
+      # cls.requestInstances()
+
+    except Exception:
+      cls.stopSlapOSProcesses()
+      raise
+    cls.instance_path = os.path.join(
+      cls.config['working_directory'],
+      'inst')
+    cls.software_path = os.path.realpath(os.path.join(
+      cls.computer_partition_root_path, 'software_release'))
+
+  @classmethod
+  def tearDownClass(cls):
+    """Tear down class, stop the processes and destroy instance.
+    """
+    cls.stopSlapOSProcesses()
+
+  # Implementation
+  @classmethod
+  def stopSlapOSProcesses(cls):
+    if hasattr(cls, '_process_manager'):
+      cls._process_manager.killPreviousRun()
+
+  @classmethod
+  def setUpWorkingDirectory(cls):
+    """Initialise the directories"""
+    cls.working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    # Supervisord socket name contains the pid number, which is why we add
+    # .xxxxxxx in this check.
+    if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
+      raise RuntimeError('working directory ( {} ) is too deep, try setting '
+              'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
+
+    if not os.path.exists(cls.working_directory):
+      os.mkdir(cls.working_directory)
+
+  @classmethod
+  def setUpConfig(cls):
+    """Create slapos configuration"""
+    cls.config = {
+      "working_directory": cls.working_directory,
+      "slapos_directory": cls.working_directory,
+      "log_directory": cls.working_directory,
+      "computer_id": 'slapos.test',  # XXX
+      'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+      'node_quantity': '3',
+    }
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('SLAPOS_TEST_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['SLAPOS_TEST_IPV6']
+
+    cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
+    cls.config['ipv6_address'] = ipv6_address
+    cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
+      **cls.config)
+
+  @classmethod
+  def setUpSlapOSController(cls):
+    """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
+
+    This is equivalent to:
+
+    slapos proxy start
+    for sr in getSoftwareURLList; do
+      slapos supply $SR $COMP
+    done
+    """
+    cls._process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    cls.slapos_controler = SlapOSControler(
+        cls.working_directory,
+        cls.config
+    )
+
+    slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    cls.software_url_list = cls.getSoftwareURLList()
+    cls.slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=cls._process_manager,
+        reset_software=False,
+        software_path_list=cls.software_url_list)
+
+    # XXX we should check *earlier* if that pidfile exist and if supervisord
+    # process still running, because if developer started supervisord (or bugs?)
+    # then another supervisord will start and starting services a second time
+    # will fail.
+    cls._process_manager.supervisord_pid_file = os.path.join(
+      cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+  @classmethod
+  def runSoftwareRelease(cls):
+    """Run all the software releases that were supplied before.
+
+    This is the equivalent of `slapos node software`.
+
+    The tests will be marked file if software building fail.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    try:
+      cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
+        cls.config, environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.software_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+
+  @classmethod
+  def runComputerPartition(cls):
+    """Instanciate the software.
+
+    This is the equivalent of doing:
+
+    slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
+    slapos node instance
+
+    and return the slapos request instance parameters.
+
+    This can be called by tests to simulate re-request with different parameters.
+    """
+    logger = logging.getLogger()
+    logger.level = logging.DEBUG
+    stream = StringIO.StringIO()
+    stream_handler = logging.StreamHandler(stream)
+    logger.addHandler(stream_handler)
+
+    if cls.getInstanceSoftwareType() != 'default':
+      raise NotImplementedError
+
+    instance_parameter_dict = cls.getInstanceParameterDict()
+    try:
+      cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
+        cls.config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+      stream.seek(0)
+      stream.flush()
+      message = ''.join(stream.readlines()[-100:])
+      assert cls.instance_status_dict['status_code'] == 0, message
+    finally:
+      logger.removeHandler(stream_handler)
+      del stream
+
+    # FIXME: similar to test node, only one (root) partition is really
+    #        supported for now.
+    computer_partition_list = []
+    for i in range(len(cls.software_url_list)):
+      computer_partition_list.append(
+          cls.slapos_controler.slap.registerOpenOrder().request(
+            cls.software_url_list[i],
+            # This is how testnode's SlapOSControler name created partitions
+            partition_reference='testing partition {i}'.format(
+              i=i, **cls.config),
+            partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParameterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        cls.config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+

stack/erp5/buildout.cfg

@@ -294,6 +294,7 @@ context =
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
     key template_my_cnf template-my-cnf:target
+    key template_mysqld_wrapper template-mysqld-wrapper:rendered
     key template_postfix template-postfix:target
     key template_postfix_aliases template-postfix-aliases:target
     key template_postfix_main_cf template-postfix-main-cf:target

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d95e8500bdc72d1f40b97cc414656e7e
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 5a93ff00ee4eb60cc065d51fb437877a
+md5sum = 591fe60e2f615c7690fa5078473b1b0a
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -70,7 +70,7 @@ md5sum = c64f35f825200fe35328641b2b8e0fdd
 
 [template]
 filename = instance.cfg.in
-md5sum = f81f4c9881bf868882cd56fe6a769666
+md5sum = f6c6820f9b3653d0d5c29708606fc591
 
 [monitor-template-dummy]
 filename = dummy.cfg

stack/erp5/instance-mariadb.cfg.in

@@ -126,7 +126,7 @@ command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:maria
 ip = {{ ip }}
 port = {{ port }}
 socket = ${directory:run}/mariadb.sock
-data-directory = ${directory:mariadb-data}
+data-directory = ${directory:srv}/mariadb
 tmp-directory = ${directory:tmp}
 etc-directory = ${directory:etc}
 pid-file = ${directory:run}/mariadb.pid
@@ -178,25 +178,18 @@ mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
 
 [{{ section('mysqld') }}]
 < = jinja2-template-executable
-# Note: all rendering is done when this file is rendered, not when the mysqld
-# section is installed - so I only use jinja2 as a fancy way to write an
-# executable file with partition-dependent but instance-parameters independent
-# content.
-template = inline:#!{{ dash }}
-  '{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \
-    --defaults-file='${my-cnf:rendered}' \
-    --skip-name-resolve \
-    --datadir='${my-cnf-parameters:data-directory}' \
-    --basedir='{{ parameter_dict['mariadb-location'] }}' \
-    && ODBCSYSINI='${my-cnf-parameters:etc-directory}' \
-    LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{{ parameter_dict['unixodbc-location'] }}/lib \
-    {% for variable in slapparameter_dict.get('environment-variables', ()) %}
-    {{ variable }} \
-    {% endfor %}
-    exec '{{ parameter_dict['mariadb-location'] }}/bin/mysqld' \
-    --defaults-file='${my-cnf:rendered}' \
-    "$@"
 rendered = ${directory:services}/mariadb
+template = {{ parameter_dict['template-mysqld-wrapper'] }}
+context =
+  key defaults_file my-cnf:rendered
+  key datadir my-cnf-parameters:data-directory
+  key environ :environ
+environ =
+  ODBCSYSINI='${my-cnf-parameters:etc-directory}'
+  LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
+  {%- for variable in slapparameter_dict.get('environment-variables', ()) %}
+  {{ variable }}
+  {%- endfor %}
 
 [odbc-ini-text]
 text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
@@ -265,7 +258,6 @@ tmp = ${buildout:directory}/tmp
 backup = ${:srv}/backup
 mariadb-backup-full = ${:backup}/mariadb-full
 mariadb-backup-incremental = ${:backup}/mariadb-incremental
-mariadb-data = ${:srv}/mariadb
 mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 log = ${:var}/log
@@ -275,7 +267,7 @@ slowquery = ${monitor-directory:private}/slowquery_digest
 [{{ section('resiliency-exclude-file') }}]
 # Generate rdiff exclude file in case of resiliency
 < = jinja2-template-base
-template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n${directory:mariadb-backup-incremental}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }}
+template = {{ 'inline:{{ "${my-cnf-parameters:data-directory}/**\\n${directory:mariadb-backup-incremental}/**\\n${directory:log}/**\\n${directory:tmp}/**\\n" }}' }}
 rendered = ${directory:srv}/exporter.exclude
 
 [{{ section("resiliency-identity-signature-script")}}]
@@ -295,7 +287,7 @@ rendered = ${directory:bin}/start-clone-from-backup
 context =
   key dash dash:dash
   key client binary-wrap-mysql:wrapper-path
-  key data_directory directory:mariadb-data
+  key data_directory my-cnf-parameters:data-directory
   key pid_file my-cnf-parameters:pid-file
   key server mysqld:rendered
   key update update-mysql:output
@@ -309,7 +301,7 @@ rendered = ${directory:bin}/restore-from-backup
 context =
   key dash dash:dash
   key mysql_executable binary-wrap-mysql:wrapper-path
-  key mariadb_data_directory directory:mariadb-data
+  key mariadb_data_directory my-cnf-parameters:data-directory
   key mariadb_backup_directory directory:mariadb-backup-full
   key pid_file my-cnf-parameters:pid-file
   key binlog_path my-cnf-parameters:binlog-path

stack/erp5/instance.cfg.in

@@ -141,6 +141,7 @@ gzip-location = {{ gzip_location }}
 mariadb-location = {{ mariadb_location }}
 template-my-cnf = {{ template_my_cnf }}
 template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
+template-mysqld-wrapper = {{ template_mysqld_wrapper }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
 mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}

stack/slapos.cfg

@@ -148,7 +148,7 @@ slapos.toolbox = 0.94
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1
-xml-marshaller = 0.9.7
+xml-marshaller = 1.0.2
 paramiko = 2.1.3
 CacheControl = 0.12.5
 msgpack = 0.6.1
@@ -195,7 +195,7 @@ enum34 = 1.1.6
 
 # Required by:
 # slapos.toolbox==0.94
-erp5.util = 0.4.59.1
+erp5.util = 0.4.60
 
 # Required by:
 # slapos.toolbox==0.94
@@ -228,7 +228,7 @@ lockfile = 0.12.2
 # Required by:
 # slapos.core==1.4.26
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
-netifaces = 0.10.4
+netifaces = 0.10.7
 
 # Required by:
 # cryptography==1.8.1