Fix shared=true, other bugs, and inconsistencies between recipes; much cleanup
| ... | @@ -5,59 +5,177 @@ except ImportError: | ... | @@ -5,59 +5,177 @@ except ImportError: |
| from pkgutil import extend_path | from pkgutil import extend_path | ||
| __path__ = extend_path(__path__, __name__) | __path__ = extend_path(__path__, __name__) | ||
| import errno, logging, os, shutil | import errno, json, logging, os, shutil, stat | ||
| import zc.buildout | from hashlib import md5 | ||
| from zc.buildout import UserError | |||
| logger = logging.getLogger(__name__) | from zc.buildout.rmtree import rmtree as buildout_rmtree | ||
| def generatePassword(length=8): | def generatePassword(length=8): | ||
| from random import SystemRandom | from random import SystemRandom | ||
| from string import ascii_lowercase | from string import ascii_lowercase | ||
| return ''.join(SystemRandom().sample(ascii_lowercase, length)) | return ''.join(SystemRandom().sample(ascii_lowercase, length)) | ||
| def is_true(value, default=False): | |||
| return default if value is None else ('false', 'true').index(value) | |||
| def make_read_only(path): | |||
| if not os.path.islink(path): | |||
| os.chmod(path, os.stat(path).st_mode & 0o555) | |||
| def make_read_only_recursively(path): | |||
| make_read_only(path) | |||
| for root, dir_list, file_list in os.walk(path): | |||
| for dir_ in dir_list: | |||
| make_read_only(os.path.join(root, dir_)) | |||
| for file_ in file_list: | |||
| make_read_only(os.path.join(root, file_)) | |||
| def rmtree(path): | def rmtree(path): | ||
| try: | try: | ||
| os.remove(path) | buildout_rmtree(path) | ||
| except OSError as e: | except OSError as e: | ||
| if e.errno != errno.EISDIR: | if e.errno == errno.ENOENT: | ||
| return | |||
| if e.errno != errno.ENOTDIR: | |||
| raise | raise | ||
| shutil.rmtree(path) | os.remove(path) | ||
| class EnvironMixin: | class EnvironMixin(object): | ||
| def __init__(self, allow_none=True): | def __init__(self, allow_none=True, compat=False): | ||
| environment = self.options.get('environment', '').strip() | environment = self.options.get('environment') | ||
| if environment: | if environment: | ||
| from os import environ | |||
| if '=' in environment: | if '=' in environment: | ||
| self._environ = env = {} | self._environ = env = {} | ||
| if compat: # for slapos.recipe.cmmi | |||
| environment_section = self.options.get('environment-section') | |||
| if environment_section: | |||
| env.update(self.buildout[environment_section]) | |||
| compat = set(env) | |||
| else: | |||
| compat = () | |||
| for line in environment.splitlines(): | for line in environment.splitlines(): | ||
| line = line.strip() | line = line.strip() | ||
| if line: | if line: | ||
| try: | try: | ||
| k, v = line.split('=', 1) | k, v = line.split('=', 1) | ||
| except ValueError: | except ValueError: | ||
| raise zc.buildout.UserError('Line %r in environment is incorrect' % | raise UserError('Line %r in environment is incorrect' % line) | ||
| line) | k = k.rstrip() | ||
| k = k.strip() | |||
| if k in env: | if k in env: | ||
| raise zc.buildout.UserError('Key %r is repeated' % k) | if k in compat: | ||
| env[k] = v.strip() % environ | compat.remove(k) | ||
| else: | |||
| raise UserError('Key %r is repeated' % k) | |||
| env[k] = v.lstrip() | |||
| else: | else: | ||
| self._environ = dict((k, v.strip() % environ) | self._environ = self.buildout[environment] | ||
| for k, v in self.buildout[environment].items()) | |||
| else: | else: | ||
| self._environ = None if allow_none else {} | self._environ = None if allow_none else {} | ||
| @property | def __getattr__(self, attr): | ||
| def environ(self): | if attr == 'logger': | ||
| if self._environ is not None: | value = logging.getLogger(self.name) | ||
| from os import environ | elif attr == 'environ': | ||
| env = self._environ.copy() | env = self._environ | ||
| for k, v in env.items(): | del self._environ | ||
| logger.info( | if env is None: | ||
| 'Environment %r set to %r' if k in environ else | value = None | ||
| 'Environment %r added with %r', k, v) | else: | ||
| for kw in environ.items(): | from os import environ | ||
| env.setdefault(*kw) | value = environ.copy() | ||
| return env | for k in sorted(env): | ||
| value[k] = v = env[k] % environ | |||
| self.logger.info('[ENV] %s = %s', k, v) | |||
| else: | |||
| return self.__getattribute__(attr) | |||
| setattr(self, attr, value) | |||
| return value | |||
| class Shared(object): | |||
| keep_on_error = False | |||
| mkdir_location = True | |||
| signature = None | |||
| def __init__(self, buildout, name, options): | |||
| self.maybe_shared = shared = is_true(options.get('shared')) | |||
| if shared: | |||
| # Trigger computation of part signature for shared signature. | |||
| # From now on, we should not pull new dependencies. | |||
| # Ignore if buildout is too old. | |||
| options.get('__buildout_signature__') | |||
| shared = buildout['buildout'].get('shared-part-list') | |||
| if shared: | |||
| profile_base_location = options.get('_profile_base_location_') | |||
| signature = json.dumps({ | |||
| k: (v.replace(profile_base_location, '${:_profile_base_location_}') | |||
| if profile_base_location else v) | |||
| for k, v in options.items() | |||
| if k != '_profile_base_location_' | |||
| }, indent=0, sort_keys=True) | |||
| if not isinstance(signature, bytes): # BBB: Python 3 | |||
| signature = signature.encode() | |||
| digest = md5(signature).hexdigest() | |||
| location = None | |||
| for shared in shared.splitlines(): | |||
| shared = shared.strip().rstrip('/') | |||
| if shared: | |||
| location = os.path.join(os.path.join(shared, name), digest) | |||
| if os.path.exists(location): | |||
| break | |||
| if location: | |||
| self.logger = logging.getLogger(name) | |||
| self.logger.info('shared at %s', location) | |||
| self.location = location | |||
| self.signature = signature | |||
| return | |||
| self.location = os.path.join(buildout['buildout']['parts-directory'], name) | |||
| def assertNotShared(self, reason): | |||
| if self.maybe_shared: | |||
| raise UserError("When shared=true, " + reason) | |||
| def install(self, install): | |||
| signature = self.signature | |||
| location = self.location | |||
| if signature is not None: | |||
| path = os.path.join(location, '.buildout-shared.json') | |||
|
|||
| if os.path.exists(path): | |||
| self.logger.info('shared part is already installed') | |||
| return () | |||
| rmtree(location) | |||
| try: | |||
| if self.mkdir_location: | |||
| os.makedirs(location) | |||
| else: | |||
| parent = os.path.dirname(location) | |||
| if not os.path.isdir(parent): | |||
| os.makedirs(parent) | |||
| install() | |||
| try: | |||
| s = os.stat(location) | |||
| except OSError as e: | |||
| if e.errno != errno.ENOENT: | |||
| raise | |||
| raise UserError('%r was not created' % location) | |||
| if self.maybe_shared and not stat.S_ISDIR(s.st_mode): | |||
| raise UserError('%r is not a directory' % location) | |||
| if signature is None: | |||
| return [location] | |||
| tmp = path + '.tmp' | |||
| with open(tmp, 'wb') as f: | |||
| f.write(signature) | |||
| # XXX: The following symlink is for backward compatibility with old | |||
| # 'slapos node prune' (slapos.core). | |||
| os.symlink('.buildout-shared.json', os.path.join(location, | |||
| '.buildout-shared.signature')) | |||
| os.rename(tmp, path) | |||
| except: | |||
| if not self.keep_on_error: | |||
| rmtree(location) | |||
| raise | |||
| make_read_only_recursively(location) | |||
| return () | |||
