Fix shared=true, other bugs, and inconsistencies between recipes; much cleanup
| ... | ... | @@ -5,59 +5,177 @@ except ImportError: |
| from pkgutil import extend_path | ||
| __path__ = extend_path(__path__, __name__) | ||
| import errno, logging, os, shutil | ||
| import zc.buildout | ||
| logger = logging.getLogger(__name__) | ||
| import errno, json, logging, os, shutil, stat | ||
| from hashlib import md5 | ||
| from zc.buildout import UserError | ||
| from zc.buildout.rmtree import rmtree as buildout_rmtree | ||
| def generatePassword(length=8): | ||
| from random import SystemRandom | ||
| from string import ascii_lowercase | ||
| return ''.join(SystemRandom().sample(ascii_lowercase, length)) | ||
| def is_true(value, default=False): | ||
| return default if value is None else ('false', 'true').index(value) | ||
| def make_read_only(path): | ||
| if not os.path.islink(path): | ||
| os.chmod(path, os.stat(path).st_mode & 0o555) | ||
| def make_read_only_recursively(path): | ||
| make_read_only(path) | ||
| for root, dir_list, file_list in os.walk(path): | ||
| for dir_ in dir_list: | ||
| make_read_only(os.path.join(root, dir_)) | ||
| for file_ in file_list: | ||
| make_read_only(os.path.join(root, file_)) | ||
| def rmtree(path): | ||
| try: | ||
| os.remove(path) | ||
| buildout_rmtree(path) | ||
| except OSError as e: | ||
| if e.errno != errno.EISDIR: | ||
| if e.errno == errno.ENOENT: | ||
| return | ||
| if e.errno != errno.ENOTDIR: | ||
| raise | ||
| shutil.rmtree(path) | ||
| os.remove(path) | ||
| class EnvironMixin: | ||
| class EnvironMixin(object): | ||
| def __init__(self, allow_none=True): | ||
| environment = self.options.get('environment', '').strip() | ||
| def __init__(self, allow_none=True, compat=False): | ||
| environment = self.options.get('environment') | ||
| if environment: | ||
| from os import environ | ||
| if '=' in environment: | ||
| self._environ = env = {} | ||
| if compat: # for slapos.recipe.cmmi | ||
| environment_section = self.options.get('environment-section') | ||
| if environment_section: | ||
| env.update(self.buildout[environment_section]) | ||
| compat = set(env) | ||
| else: | ||
| compat = () | ||
| for line in environment.splitlines(): | ||
| line = line.strip() | ||
| if line: | ||
| try: | ||
| k, v = line.split('=', 1) | ||
| except ValueError: | ||
| raise zc.buildout.UserError('Line %r in environment is incorrect' % | ||
| line) | ||
| k = k.strip() | ||
| raise UserError('Line %r in environment is incorrect' % line) | ||
| k = k.rstrip() | ||
| if k in env: | ||
| raise zc.buildout.UserError('Key %r is repeated' % k) | ||
| env[k] = v.strip() % environ | ||
| if k in compat: | ||
| compat.remove(k) | ||
| else: | ||
| raise UserError('Key %r is repeated' % k) | ||
| env[k] = v.lstrip() | ||
| else: | ||
| self._environ = dict((k, v.strip() % environ) | ||
| for k, v in self.buildout[environment].items()) | ||
| self._environ = self.buildout[environment] | ||
| else: | ||
| self._environ = None if allow_none else {} | ||
| @property | ||
| def environ(self): | ||
| if self._environ is not None: | ||
| from os import environ | ||
| env = self._environ.copy() | ||
| for k, v in env.items(): | ||
| logger.info( | ||
| 'Environment %r set to %r' if k in environ else | ||
| 'Environment %r added with %r', k, v) | ||
| for kw in environ.items(): | ||
| env.setdefault(*kw) | ||
| return env | ||
| def __getattr__(self, attr): | ||
| if attr == 'logger': | ||
| value = logging.getLogger(self.name) | ||
| elif attr == 'environ': | ||
| env = self._environ | ||
| del self._environ | ||
| if env is None: | ||
| value = None | ||
| else: | ||
| from os import environ | ||
| value = environ.copy() | ||
| for k in sorted(env): | ||
| value[k] = v = env[k] % environ | ||
| self.logger.info('[ENV] %s = %s', k, v) | ||
| else: | ||
| return self.__getattribute__(attr) | ||
| setattr(self, attr, value) | ||
| return value | ||
| class Shared(object): | ||
| keep_on_error = False | ||
| mkdir_location = True | ||
| signature = None | ||
| def __init__(self, buildout, name, options): | ||
| self.maybe_shared = shared = is_true(options.get('shared')) | ||
| if shared: | ||
| # Trigger computation of part signature for shared signature. | ||
| # From now on, we should not pull new dependencies. | ||
| # Ignore if buildout is too old. | ||
| options.get('__buildout_signature__') | ||
| shared = buildout['buildout'].get('shared-part-list') | ||
| if shared: | ||
| profile_base_location = options.get('_profile_base_location_') | ||
| signature = json.dumps({ | ||
| k: (v.replace(profile_base_location, '${:_profile_base_location_}') | ||
| if profile_base_location else v) | ||
| for k, v in options.items() | ||
| if k != '_profile_base_location_' | ||
| }, indent=0, sort_keys=True) | ||
| if not isinstance(signature, bytes): # BBB: Python 3 | ||
| signature = signature.encode() | ||
| digest = md5(signature).hexdigest() | ||
| location = None | ||
| for shared in shared.splitlines(): | ||
| shared = shared.strip().rstrip('/') | ||
| if shared: | ||
| location = os.path.join(os.path.join(shared, name), digest) | ||
| if os.path.exists(location): | ||
| break | ||
| if location: | ||
| self.logger = logging.getLogger(name) | ||
| self.logger.info('shared at %s', location) | ||
| self.location = location | ||
| self.signature = signature | ||
| return | ||
| self.location = os.path.join(buildout['buildout']['parts-directory'], name) | ||
| def assertNotShared(self, reason): | ||
| if self.maybe_shared: | ||
| raise UserError("When shared=true, " + reason) | ||
| def install(self, install): | ||
| signature = self.signature | ||
| location = self.location | ||
| if signature is not None: | ||
| path = os.path.join(location, '.buildout-shared.json') | ||
|
||
| if os.path.exists(path): | ||
| self.logger.info('shared part is already installed') | ||
| return () | ||
| rmtree(location) | ||
| try: | ||
| if self.mkdir_location: | ||
| os.makedirs(location) | ||
| else: | ||
| parent = os.path.dirname(location) | ||
| if not os.path.isdir(parent): | ||
| os.makedirs(parent) | ||
| install() | ||
| try: | ||
| s = os.stat(location) | ||
| except OSError as e: | ||
| if e.errno != errno.ENOENT: | ||
| raise | ||
| raise UserError('%r was not created' % location) | ||
| if self.maybe_shared and not stat.S_ISDIR(s.st_mode): | ||
| raise UserError('%r is not a directory' % location) | ||
| if signature is None: | ||
| return [location] | ||
| tmp = path + '.tmp' | ||
| with open(tmp, 'wb') as f: | ||
| f.write(signature) | ||
| # XXX: The following symlink is for backward compatibility with old | ||
| # 'slapos node prune' (slapos.core). | ||
| os.symlink('.buildout-shared.json', os.path.join(location, | ||
| '.buildout-shared.signature')) | ||
| os.rename(tmp, path) | ||
| except: | ||
| if not self.keep_on_error: | ||
| rmtree(location) | ||
| raise | ||
| make_read_only_recursively(location) | ||
| return () | ||
