slapos.recipe:wrapper: add py3 support for pidfile and private-tmpfs

These two options were unusable for software releases using python3.

This also adds test coverage for the recipe.

slapos/recipe/librecipe/execute.py

@@ -67,7 +67,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
     else:
       # With chained shebangs, several paths may be inserted at the beginning.
       n = len(args)
-      for i in xrange(1+len(running)-n):
+      for i in six.moves.xrange(1+len(running)-n):
         if args == running[i:n+i]:
           sys.exit("Already running with pid %s." % pid)
     with open(pidfile, 'w') as f:
@@ -91,16 +91,19 @@ def generic_exec(args, extra_environ=None, wait_list=None,
     uid = os.getuid()
     gid = os.getgid()
     unshare(CLONE_NEWUSER |CLONE_NEWNS)
-    with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
-    with open('/proc/self/uid_map',   'wb') as f: f.write('%s %s 1' % (uid, uid))
-    with open('/proc/self/gid_map',   'wb') as f: f.write('%s %s 1' % (gid, gid))
+    with open('/proc/self/setgroups', 'w') as f:
+      f.write('deny')
+    with open('/proc/self/uid_map', 'w') as f:
+      f.write('%s %s 1' % (uid, uid))
+    with open('/proc/self/gid_map', 'w') as f:
+      f.write('%s %s 1' % (gid, gid))
     for size, path in private_tmpfs:
       try:
         os.mkdir(path)
       except OSError as e:
         if e.errno != errno.EEXIST:
           raise
-      mount('tmpfs', path, 'tmpfs', 0, 'size=' + size)
+      mount(b'tmpfs', path.encode(), b'tmpfs', 0, ('size=' + size).encode())
 
   if extra_environ:
     env = os.environ.copy()

slapos/test/recipe/test_re6stnet.py

@@ -52,13 +52,15 @@ class Re6stnetTest(unittest.TestCase):
       return makeRecipe(
             re6stnet.Recipe,
             options=self.options,
-            slap_connection={
+            buildout={
+              'slap-connection': {
                    'computer-id': 'comp-test',
                    'partition-id': 'slappart0',
                    'server-url': 'http://server.com',
                    'software-release-url': 'http://software.com',
                    'key-file': '/path/to/key',
                    'cert-file': '/path/to/cert'
+              }
             },
             name='re6stnet')
 

slapos/test/recipe/test_wrapper.py

+import errno
+import functools
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+
+from slapos.recipe import wrapper
+from slapos.test.utils import makeRecipe
+
+
+class WrapperTestCase(unittest.TestCase):
+  def getOptions(self):
+    raise NotImplementedError()
+
+  def setUp(self):
+    self.buildout_directory = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.buildout_directory)
+    self.getTempPath = functools.partial(os.path.join, self.buildout_directory)
+
+    self.wrapper_path = self.getTempPath('wrapper')
+
+    self.recipe = makeRecipe(
+      wrapper.Recipe,
+      options=self.getOptions(),
+      name="wrapper",
+      buildout={'buildout': {
+        'directory': self.buildout_directory,
+      }})
+
+  def terminate_process(self, process):
+    try:
+      process.terminate()
+    except OSError as e:
+      if e.errno != errno.ESRCH:
+        raise
+    process.wait()
+
+
+class TestSimpleCommandLineWrapper(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': 'echo hello world',
+      'wrapper-path': self.wrapper_path,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      'hello world\n')
+
+
+class TestEscapeCommandLine(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': "echo esca $PE",
+      'wrapper-path': self.wrapper_path,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      "esca $PE\n")
+
+
+class TestEnvironment(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': 'sh -c "echo $FOO"',
+      'wrapper-path': self.wrapper_path,
+      'environment': 'FOO=bar',
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed, universal_newlines=True, env={'FOO': 'foo'})
+    self.assertEqual(output, 'bar\n')
+
+
+class TestHashFiles(WrapperTestCase):
+  def getOptions(self):
+    hashed_file = self.getTempPath('hashed_file')
+    with open(hashed_file, 'w') as f:
+      f.write('hello world')
+    return {
+      'command-line': "cat " + hashed_file,
+      'wrapper-path': self.wrapper_path,
+      'hash-files': hashed_file
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    # 83af3240d992b2165abbd245a3e43368 is hashlib.md5(b'11\nhello world').hexdigest()
+    self.assertEqual(
+      installed, self.wrapper_path + '-83af3240d992b2165abbd245a3e43368')
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      "hello world")
+
+
+class TestPidFile(WrapperTestCase):
+  def getOptions(self):
+    self.pidfile = self.getTempPath('hello.pid')
+    return {
+      'command-line': "/bin/sleep 10",
+      'wrapper-path': self.wrapper_path,
+      'pidfile': self.pidfile
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    for _ in range(20):
+      time.sleep(0.1)
+      if os.path.exists(self.pidfile):
+        break
+
+    with open(self.pidfile) as f:
+      pid = int(f.read())
+    self.assertEqual(process.pid, pid)
+
+    with self.assertRaises(subprocess.CalledProcessError) as ctx:
+      subprocess.check_output(
+        installed, stderr=subprocess.STDOUT, universal_newlines=True)
+    self.assertEqual(
+      ctx.exception.output, 'Already running with pid %s.\n' % pid)
+
+  def test_stale_pidfile_is_ignored(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    with open(self.pidfile, 'w') as f:
+      f.write('1234')
+
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    for _ in range(20):
+      time.sleep(0.1)
+      with open(self.pidfile) as f:
+        pid = int(f.read())
+      if process.pid == pid:
+        break
+    else:
+      self.fail('pidfile not updated', process.stdout.read())
+
+
+class TestWaitForFiles(WrapperTestCase):
+  def getOptions(self):
+    self.waitfile = self.getTempPath('wait')
+    return {
+      'command-line': "/bin/echo done",
+      'wrapper-path': self.wrapper_path,
+      'wait-for-files': self.waitfile,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    # nothing happens when file is not there
+    for _ in range(10):
+      time.sleep(0.1)
+      if process.poll():
+        self.fail(process.stdout.read())
+
+    open(self.waitfile, 'w').close()
+    for _ in range(20):
+      time.sleep(0.1)
+      if process.poll() is not None:
+        self.assertEqual(process.stdout.read(), 'done\n')
+        self.assertEqual(process.returncode, 0)
+        break
+    else:
+      self.fail('process did not start after file was created')
+
+
+class TestPrivateTmpFS(WrapperTestCase):
+  def getOptions(self):
+    self.tmpdir = self.getTempPath('tmpdir')
+    self.tmpfile = self.getTempPath('tmpdir', 'file')
+    self.program = self.getTempPath('program')
+    with open(self.program, 'w') as f:
+      f.write(
+        textwrap.dedent(
+          '''\
+          #!{sys_executable}
+          import os
+          with open({tmpfile!r}, 'w') as f:
+            f.write('ok')
+          with open({tmpfile!r}, 'r') as f:
+            print(f.read())
+        ''').format(sys_executable=sys.executable, tmpfile=self.tmpfile))
+    os.chmod(self.program, 0o700)
+    return {
+      'command-line': self.program,
+      'wrapper-path': self.wrapper_path,
+      'private-tmpfs': '1000 ' + self.tmpdir
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed,
+      universal_newlines=True,
+    )
+    self.assertEqual(output, 'ok\n')
+    self.assertFalse(os.path.exists(self.tmpfile))
+
+
+class TestReserveCPU(WrapperTestCase):
+  def getOptions(self):
+    self.slapos_cpu_exclusive = self.getTempPath('.slapos-cpu-exclusive')
+    self.program = self.getTempPath('program')
+    with open(self.program, 'w') as f:
+      f.write(
+        textwrap.dedent(
+          '''\
+          #!{sys_executable}
+          import os
+          with open({slapos_cpu_exclusive!r}, 'r') as f:
+            print('ok' if int(f.read()) == os.getpid() else 'error')
+        ''').format(
+            sys_executable=sys.executable,
+            slapos_cpu_exclusive=self.slapos_cpu_exclusive,
+          ))
+    os.chmod(self.program, 0o700)
+    return {
+      'command-line': self.program,
+      'wrapper-path': self.wrapper_path,
+      'reserve-cpu': 'true',
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed,
+      universal_newlines=True,
+      env={'HOME': self.buildout_directory})
+    self.assertEqual(output, 'ok\n')

slapos/test/utils.py

@@ -2,18 +2,18 @@
 """
 import os
 import sys
+import six
 
 
-def makeRecipe(recipe_class, options, name='test', slap_connection=None):
-  """Instanciate a recipe of `recipe_class` with `options` with a buildout
-  mapping containing a python and an empty `slapos-connection` mapping, unless
-  provided as `slap_connection`.
+def makeRecipe(recipe_class, options, name='test', buildout=None):
+  """Instantiate a recipe of `recipe_class` with `options` with a `buildout`
+  mapping containing by default a python and an empty slap-connection.
 
   This function expects the test suite to have set SLAPOS_TEST_EGGS_DIRECTORY
   and SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY environment variables, so that the
   test recipe does not need to install eggs again when using working set.
   """
-  buildout = {
+  _buildout = {
     'buildout': {
       'bin-directory': '',
       'find-links': '',
@@ -32,15 +32,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None):
        'software-release-url': '',
      }
   }
-  if slap_connection is not None:
-    buildout['slap-connection'] = slap_connection
 
-  buildout['buildout']['eggs-directory'] = os.environ['SLAPOS_TEST_EGGS_DIRECTORY']
-  buildout['buildout']['develop-eggs-directory'] = os.environ['SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY']
+  _buildout['buildout']['eggs-directory'] = os.environ['SLAPOS_TEST_EGGS_DIRECTORY']
+  _buildout['buildout']['develop-eggs-directory'] = os.environ['SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY']
+
+  if buildout:
+    for section, _options in six.iteritems(buildout):
+      _buildout.setdefault(section, {}).update(**_options)
 
   # Prevent test from accidentally writing to the buildout's eggs
-  buildout['buildout']['newest'] = False
-  buildout['buildout']['offline'] = True
+  _buildout['buildout']['newest'] = False
+  _buildout['buildout']['offline'] = True
 
-  return recipe_class(buildout=buildout, name=name, options=options)
+  return recipe_class(buildout=_buildout, name=name, options=options)