WIP fix namespace packaging

slapos/recipe/__init__.py

@@ -4,171 +4,3 @@ try:
 except ImportError:
     from pkgutil import extend_path
     __path__ = extend_path(__path__, __name__)
-
-import errno, json, logging, os, shutil, stat
-from hashlib import md5
-from zc.buildout import UserError
-from zc.buildout.rmtree import rmtree as buildout_rmtree
-
-def generatePassword(length=8):
-  from random import SystemRandom
-  from string import ascii_lowercase
-  choice = SystemRandom().choice
-  return ''.join(choice(ascii_lowercase) for _ in range(length))
-
-def is_true(value, default=False):
-  return default if value is None else ('false', 'true').index(value)
-
-def make_read_only(path):
-  if not os.path.islink(path):
-    os.chmod(path, os.stat(path).st_mode & 0o555)
-
-def make_read_only_recursively(path):
-  make_read_only(path)
-  for root, dir_list, file_list in os.walk(path):
-    for dir_ in dir_list:
-      make_read_only(os.path.join(root, dir_))
-    for file_ in file_list:
-      make_read_only(os.path.join(root, file_))
-
-def rmtree(path):
-  try:
-    buildout_rmtree(path)
-  except OSError as e:
-    if e.errno == errno.ENOENT:
-      return
-    if e.errno != errno.ENOTDIR:
-      raise
-    os.remove(path)
-
-
-class EnvironMixin(object):
-
-  def __init__(self, allow_none=True):
-    environment = self.options.get('environment')
-    if environment:
-      if '=' in environment:
-        self._environ = env = {}
-        for line in environment.splitlines():
-          line = line.strip()
-          if line:
-            try:
-              k, v = line.split('=', 1)
-            except ValueError:
-              raise UserError('Line %r in environment is incorrect' % line)
-            k = k.rstrip()
-            if k in env:
-              raise UserError('Key %r is repeated' % k)
-            env[k] = v.lstrip()
-      else:
-        self._environ = self.buildout[environment]
-    else:
-      self._environ = None if allow_none else {}
-
-  def __getattr__(self, attr):
-    if attr == 'logger':
-      value = logging.getLogger(self.name)
-    elif attr == 'environ':
-      env = self._environ
-      del self._environ
-      if env is None:
-        value = None
-      else:
-        from os import environ
-        value = environ.copy()
-        for k in sorted(env):
-          value[k] = v = env[k] % environ
-          self.logger.info('[ENV] %s = %s', k, v)
-    else:
-      return self.__getattribute__(attr)
-    setattr(self, attr, value)
-    return value
-
-
-class Shared(object):
-
-  keep_on_error = False
-  mkdir_location = True
-  signature = None
-
-  def __init__(self, buildout, name, options):
-    self.maybe_shared = shared = is_true(options.get('shared'))
-    if shared:
-      # Trigger computation of part signature for shared signature.
-      # From now on, we should not pull new dependencies.
-      # Ignore if buildout is too old.
-      options.get('__buildout_signature__')
-      shared = buildout['buildout'].get('shared-part-list')
-      if shared:
-        profile_base_location = options.get('_profile_base_location_')
-        signature = json.dumps({
-          k: (v.replace(profile_base_location, '${:_profile_base_location_}')
-              if profile_base_location else v)
-          for k, v in options.items()
-          if k != '_profile_base_location_'
-        }, ensure_ascii=False, indent=2, sort_keys=True,
-          # BBB: Python 2 ( https://bugs.python.org/issue16333 )
-          separators=(',', ': '))
-        if not isinstance(signature, bytes): # BBB: Python 3
-          signature = signature.encode()
-        digest = md5(signature).hexdigest()
-        location = None
-        for shared in shared.splitlines():
-          shared = shared.strip().rstrip('/')
-          if shared:
-            location = os.path.join(os.path.join(shared, name), digest)
-            if os.path.exists(location):
-              break
-        if location:
-          self.logger = logging.getLogger(name)
-          self.logger.info('shared at %s', location)
-          self.location = location
-          self.signature = signature
-          return
-    self.location = os.path.join(buildout['buildout']['parts-directory'], name)
-
-  def assertNotShared(self, reason):
-    if self.maybe_shared:
-      raise UserError("When shared=true, " + reason)
-
-  def install(self, install):
-    signature = self.signature
-    location = self.location
-    if signature is not None:
-      path = os.path.join(location, '.buildout-shared.json')
-      if os.path.exists(path):
-        self.logger.info('shared part is already installed')
-        return ()
-    rmtree(location)
-    try:
-      if self.mkdir_location:
-        os.makedirs(location)
-      else:
-        parent = os.path.dirname(location)
-        if parent and not os.path.isdir(parent):
-          os.makedirs(parent)
-      install()
-      try:
-        s = os.stat(location)
-      except OSError as e:
-        if e.errno != errno.ENOENT:
-          raise
-        raise UserError('%r was not created' % location)
-      if self.maybe_shared and not stat.S_ISDIR(s.st_mode):
-        raise UserError('%r is not a directory' % location)
-      if signature is None:
-        return [location]
-      tmp = path + '.tmp'
-      with open(tmp, 'wb') as f:
-        f.write(signature)
-      # XXX: The following symlink is for backward compatibility with old
-      #      'slapos node prune' (slapos.core).
-      os.symlink('.buildout-shared.json', os.path.join(location,
-        '.buildout-shared.signature'))
-      os.rename(tmp, path)
-    except:
-      if not self.keep_on_error:
-        rmtree(location)
-      raise
-    make_read_only_recursively(location)
-    return ()

slapos/recipe/build/__init__.py

@@ -35,7 +35,7 @@ import sys
 import tempfile
 from setuptools import archive_util
 import zc.buildout
-from .. import is_true, rmtree, EnvironMixin, Shared
+from ..utils import is_true, rmtree, EnvironMixin, Shared
 from ..downloadunpacked import unpack_archive
 
 def readElfAsDict(f):

slapos/recipe/build/tests.py

@@ -15,7 +15,8 @@ from zc.buildout.download import check_md5sum, ChecksumError
 from zc.buildout.testing import buildoutTearDown
 from zope.testing import renormalizing
 
-from .. import download, downloadunpacked, gitclone, make_read_only_recursively
+from .. import download, downloadunpacked, gitclone
+from ..utils import make_read_only_recursively
 
 
 optionflags = (doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)

slapos/recipe/download.py

@@ -27,7 +27,7 @@
 import errno
 import os
 from zc.buildout import download, UserError
-from . import is_true, Shared
+from .utils import is_true, Shared
 
 class Recipe(object):
 

slapos/recipe/downloadunpacked.py

@@ -31,7 +31,7 @@ import tarfile
 import tempfile
 from functools import partial
 from setuptools import archive_util
-from . import is_true, EnvironMixin, Shared
+from .utils import is_true, EnvironMixin, Shared
 
 class Recipe(EnvironMixin):
 

slapos/recipe/utils.py

+import errno, json, logging, os, shutil, stat
+from hashlib import md5
+from zc.buildout import UserError
+from zc.buildout.rmtree import rmtree as buildout_rmtree
+
+def generatePassword(length=8):
+  from random import SystemRandom
+  from string import ascii_lowercase
+  choice = SystemRandom().choice
+  return ''.join(choice(ascii_lowercase) for _ in range(length))
+
+def is_true(value, default=False):
+  return default if value is None else ('false', 'true').index(value)
+
+def make_read_only(path):
+  if not os.path.islink(path):
+    os.chmod(path, os.stat(path).st_mode & 0o555)
+
+def make_read_only_recursively(path):
+  make_read_only(path)
+  for root, dir_list, file_list in os.walk(path):
+    for dir_ in dir_list:
+      make_read_only(os.path.join(root, dir_))
+    for file_ in file_list:
+      make_read_only(os.path.join(root, file_))
+
+def rmtree(path):
+  try:
+    buildout_rmtree(path)
+  except OSError as e:
+    if e.errno == errno.ENOENT:
+      return
+    if e.errno != errno.ENOTDIR:
+      raise
+    os.remove(path)
+
+
+class EnvironMixin(object):
+
+  def __init__(self, allow_none=True):
+    environment = self.options.get('environment')
+    if environment:
+      if '=' in environment:
+        self._environ = env = {}
+        for line in environment.splitlines():
+          line = line.strip()
+          if line:
+            try:
+              k, v = line.split('=', 1)
+            except ValueError:
+              raise UserError('Line %r in environment is incorrect' % line)
+            k = k.rstrip()
+            if k in env:
+              raise UserError('Key %r is repeated' % k)
+            env[k] = v.lstrip()
+      else:
+        self._environ = self.buildout[environment]
+    else:
+      self._environ = None if allow_none else {}
+
+  def __getattr__(self, attr):
+    if attr == 'logger':
+      value = logging.getLogger(self.name)
+    elif attr == 'environ':
+      env = self._environ
+      del self._environ
+      if env is None:
+        value = None
+      else:
+        from os import environ
+        value = environ.copy()
+        for k in sorted(env):
+          value[k] = v = env[k] % environ
+          self.logger.info('[ENV] %s = %s', k, v)
+    else:
+      return self.__getattribute__(attr)
+    setattr(self, attr, value)
+    return value
+
+
+class Shared(object):
+
+  keep_on_error = False
+  mkdir_location = True
+  signature = None
+
+  def __init__(self, buildout, name, options):
+    self.maybe_shared = shared = is_true(options.get('shared'))
+    if shared:
+      # Trigger computation of part signature for shared signature.
+      # From now on, we should not pull new dependencies.
+      # Ignore if buildout is too old.
+      options.get('__buildout_signature__')
+      shared = buildout['buildout'].get('shared-part-list')
+      if shared:
+        profile_base_location = options.get('_profile_base_location_')
+        signature = json.dumps({
+          k: (v.replace(profile_base_location, '${:_profile_base_location_}')
+              if profile_base_location else v)
+          for k, v in options.items()
+          if k != '_profile_base_location_'
+        }, ensure_ascii=False, indent=2, sort_keys=True,
+          # BBB: Python 2 ( https://bugs.python.org/issue16333 )
+          separators=(',', ': '))
+        if not isinstance(signature, bytes): # BBB: Python 3
+          signature = signature.encode()
+        digest = md5(signature).hexdigest()
+        location = None
+        for shared in shared.splitlines():
+          shared = shared.strip().rstrip('/')
+          if shared:
+            location = os.path.join(os.path.join(shared, name), digest)
+            if os.path.exists(location):
+              break
+        if location:
+          self.logger = logging.getLogger(name)
+          self.logger.info('shared at %s', location)
+          self.location = location
+          self.signature = signature
+          return
+    self.location = os.path.join(buildout['buildout']['parts-directory'], name)
+
+  def assertNotShared(self, reason):
+    if self.maybe_shared:
+      raise UserError("When shared=true, " + reason)
+
+  def install(self, install):
+    signature = self.signature
+    location = self.location
+    if signature is not None:
+      path = os.path.join(location, '.buildout-shared.json')
+      if os.path.exists(path):
+        self.logger.info('shared part is already installed')
+        return ()
+    rmtree(location)
+    try:
+      if self.mkdir_location:
+        os.makedirs(location)
+      else:
+        parent = os.path.dirname(location)
+        if parent and not os.path.isdir(parent):
+          os.makedirs(parent)
+      install()
+      try:
+        s = os.stat(location)
+      except OSError as e:
+        if e.errno != errno.ENOENT:
+          raise
+        raise UserError('%r was not created' % location)
+      if self.maybe_shared and not stat.S_ISDIR(s.st_mode):
+        raise UserError('%r is not a directory' % location)
+      if signature is None:
+        return [location]
+      tmp = path + '.tmp'
+      with open(tmp, 'wb') as f:
+        f.write(signature)
+      # XXX: The following symlink is for backward compatibility with old
+      #      'slapos node prune' (slapos.core).
+      os.symlink('.buildout-shared.json', os.path.join(location,
+        '.buildout-shared.signature'))
+      os.rename(tmp, path)
+    except:
+      if not self.keep_on_error:
+        rmtree(location)
+      raise
+    make_read_only_recursively(location)
+    return ()

slapos/recipe/vm.py

@@ -32,7 +32,7 @@ from collections import defaultdict
 from contextlib import contextmanager
 from os.path import join
 from zc.buildout import UserError
-from . import EnvironMixin, generatePassword, is_true, rmtree
+from .utils import EnvironMixin, generatePassword, is_true, rmtree
 
 ARCH = os.uname()[4]