Support multiple paths for shared parts

Use a new option `buildout:shared-path-list` that is now a new line separated list of path, only the last part is used to install new parts, other are read-only.

Also some cleanups in the test suite.

/reviewed-on !9

setup.py

@@ -18,7 +18,7 @@ setup(
         + '\n' +
         read('CHANGELOG.rst')
         + '\n' +
-        read('slapos', 'recipe', 'cmmi', 'README.txt')
+        read('slapos', 'recipe', 'cmmi', 'README.rst')
         + '\n' +
         'Download\n'
         '========\n'

slapos/recipe/cmmi/README.txt → slapos/recipe/cmmi/README.rst

@@ -30,7 +30,7 @@ Supported options
 
     Specify the path in which this package is shared by many other
     packages.
-    Shared-parts should be defined in [buildout] section
+    ``shared-part-list`` should be defined in ``[buildout]`` section
     Shared option is True or False
     The package will be installed on path/name/hash of options.
 
@@ -229,13 +229,13 @@ platform.machine().
 platform could be 'linux', 'cygwin', 'macos', 'sunos', 'freebsd',
 'netbsd', 'unixware' ... which equals a formatted sys.platform.
 
-For example,
+For example::
 
-[bzip2]
-recipe = slapos.recipe.cmmi
+  [bzip2]
+  recipe = slapos.recipe.cmmi
 
-[bzip2:cygwin]
-patches = cygwin-bzip2-1.0.6.src.patch
+  [bzip2:cygwin]
+  patches = cygwin-bzip2-1.0.6.src.patch
 
 All the options in the [part:platform] have high priority level.
 
@@ -261,7 +261,7 @@ effects, recipe will return all the installed files in the prefix
 directory. The own ``prefix`` of part will disable this behaviour.
 
 If the ``buildout`` section has a valid ``prefix`` option, the recipe
-will add it to environmet variables as the following:
+will add it to environmet variables as the following::
 
   PATH=${buildout:prefix}/bin:$PATH
   CPPFLAGS=-I${buildout:prefix} $CPPFLAGS
@@ -818,11 +818,11 @@ make-targets aren't set, if so, string "prefix=xxx" will be appended
 in the make-targets. xxx is the final prefix of this recipe. We call
 it Magic Prefix.
 
-In these options magic prefix can be represented by %(prefix)s:
+In these options magic prefix can be represented by ``%(prefix)s``:
 
-    ``onfigure-command`` ``configure-options``
-    ``make-binary`` ``make-options`` ``make-targets``
-    ``pre-configure`` ``pre-build`` ``pre-install`` ``post-install``
+    ``configure-command``, ``configure-options``,
+    ``make-binary``, ``make-options``, ``make-targets``,
+    ``pre-configure``, ``pre-build``, ``pre-install``, ``post-install``
 
 For example::
 
@@ -831,7 +831,7 @@ For example::
 
 The other part can refer to magic prefix of this part by
 ${part:prefix}, it will return the magic prefix, other than literal
-value in the part section. For example,
+value in the part section. For example::
 
     >>> write('buildout.cfg',
     ... """
@@ -875,7 +875,7 @@ can display "prefix" value in the stdout.
     ... recipe = slapos.recipe.cmmi
     ... url = file://%s/package-0.0.0.tar.gz
     ... configure-command = ./configure
-    ... pre-install = sed -i -e "s/installing package/installing package at \$\$prefix /g" Makefile
+    ... pre-install = sed -i -e "s/installing package/installing package at \\$\\$prefix /g" Makefile
     ... """ % src)
 
     >>> print(system(buildout))
@@ -901,7 +901,7 @@ replaced with the recipe final prefix.
     ... url = file://%s/package-0.0.0.tar.gz
     ... configure-command = ./configure
     ... make-targets = install-lib prefix=%%(prefix)s
-    ... pre-install = sed -i -e "s/installing package/installing package at \$\$prefix /g" Makefile
+    ... pre-install = sed -i -e "s/installing package/installing package at \\$\\$prefix /g" Makefile
     ... """ % src)
 
     >>> print(system(buildout))
@@ -983,16 +983,23 @@ Look, "package" is reinstalled either:
     installing package
 
 Install shared package
-=====================
+======================
 
 Use option ``shared`` to install a shared pacakge.
     >>> import os
-    >>> _ = system('chmod -R u+w %(path)s && rm -rf %(path)s' % dict(path=join(os.path.dirname(__file__), 'shared')))
+    >>> _ = system('chmod -R u+w %(path)s && rm -rf %(path)s' % dict(
+    ...     path=join(os.path.dirname(__file__), 'shared')))
     >>> shared_dir = join(os.path.dirname(__file__), 'shared')
     >>> os.mkdir(shared_dir)
+    >>> _ = system('chmod -R u+w %(path)s && rm -rf %(path)s' % dict(
+    ...     path=join(os.path.dirname(__file__), 'another_shared_dir')))
+    >>> another_shared_dir = join(
+    ...     os.path.dirname(__file__), 'another_shared_dir')
+    >>> os.mkdir(another_shared_dir)
 
 
-If no shared-parts is set, and shared is True, shared feature is not used:
+If no ``shared-part-list`` is set, and ``shared`` is True, ``shared`` feature
+is not used:
     >>> write('buildout.cfg',
     ... """
     ... [buildout]
@@ -1013,10 +1020,12 @@ If no shared-parts is set, and shared is True, shared feature is not used:
     building package
     installing package
 
-If shared-parts is set and shared is True, build package failed, the build directory is removed,
-a build directory__compile__ is left for debugging.
-Also a shell script with the environment variable is created, so that developer can try same build
-process as the recipe tried.
+If ``shared-part-list`` is set and shared is True, build package failed, the
+build directory is removed, a build directory__compile__ is left for
+debugging.
+Also a shell script with the environment variable is created, so that
+developer can try same build process as the recipe tried.
+
     >>> _ = system('mv %s/package-0.0.0.tar.gz  %s/package-0.0.0.tar.gz.bak' % (src, src))
     >>> import tarfile
     >>> from io import BytesIO
@@ -1033,7 +1042,7 @@ process as the recipe tried.
     ... [buildout]
     ... newest = false
     ... parts = package
-    ... shared-parts = %s
+    ... shared-part-list = %s
     ...
     ... [package]
     ... recipe = slapos.recipe.cmmi
@@ -1043,13 +1052,13 @@ process as the recipe tried.
     ...   FOO=bar
     ... """ % (shared_dir, src))
     >>> print(system(buildout)) #doctest:+ELLIPSIS
-    package: shared directory  .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/... set for package
+    package: shared directory /shared/package/FIRST_SHARED_PACKAGE_HASH set for package
     Uninstalling package.
     Installing package.
-    package: Checking whether package is installed at shared path: .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...
+    package: Checking whether package is installed at shared path: /shared/package/FIRST_SHARED_PACKAGE_HASH
     package: [ENV] FOO = bar
-    package: Command './configure --prefix=".../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/..."' returned non-zero exit status 127
-    package: Compilation error. The package is left as is at .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...__compile__ where you can inspect what went wrong.
+    package: Command './configure --prefix="/shared/package/FIRST_SHARED_PACKAGE_HASH"' returned non-zero exit status 127.
+    package: Compilation error. The package is left as is at /shared/package/FIRST_SHARED_PACKAGE_HASH__compile__ where you can inspect what went wrong.
     A shell script slapos.recipe.build.env.sh has been generated. You can source it in your shell to reproduce build environment.
     /bin/sh: 1: ./configure: not found
     While:
@@ -1060,25 +1069,39 @@ process as the recipe tried.
     export FOO="bar"
     ...
 
-If shared-parts is set and shared is True, package will be installed in shared_part/package/a hash of the recipe's configuration options
+If ``shared-part-list`` is set as an option in buildout section and
+``shared`` is True, package will be installed in shared_part/package
+and a hash of the recipe's configuration options.
+
+There can be multiple path listed in ``shared-part-list``, the recipe
+will look in each of these paths if package was already installed and
+if not, it will install the package in the last entry the last entry
+from the list of ``shared-part-list``.
+
     >>> _ = system('mv %s/package-0.0.0.tar.gz.bak  %s/package-0.0.0.tar.gz' % (src, src))
     >>> print(system(buildout)) #doctest:+ELLIPSIS
-    package: shared directory  .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/... set for package
+    package: shared directory /shared/package/FIRST_SHARED_PACKAGE_HASH set for package
     Installing package.
-    package: Checking whether package is installed at shared path: .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...
-    package: Removing already existing directory .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...__compile__
-    configure --prefix=.../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...
+    package: Checking whether package is installed at shared path: /shared/package/FIRST_SHARED_PACKAGE_HASH
+    package: [ENV] FOO = bar
+    package: Removing already existing directory /shared/package/FIRST_SHARED_PACKAGE_HASH__compile__
+    configure --prefix=/shared/package/FIRST_SHARED_PACKAGE_HASH
     building package
     installing package
 
-Do nothing if one package has been installed.
+If package was already installed in any of the ``shared-part-list`` used, it will be
+used instead of installing if one package has been installed.
+
     >>> remove('.installed.cfg')
     >>> write('buildout.cfg',
     ... """
     ... [buildout]
     ... newest = false
     ... parts = package
-    ... shared-parts = %s
+    ... shared-part-list =
+    ...   %s
+    ...   not/exists
+    ...   %s
     ...
     ... [package]
     ... recipe = slapos.recipe.cmmi
@@ -1086,11 +1109,11 @@ Do nothing if one package has been installed.
     ... shared = True
     ... environment =
     ...   FOO=bar
-    ... """ % (shared_dir, src))
+    ... """ % (shared_dir, another_shared_dir, src))
     >>> print(system(buildout)) #doctest:+ELLIPSIS
-    package: shared directory  .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/... set for package
+    package: shared directory /shared/package/FIRST_SHARED_PACKAGE_HASH set for package
     Installing package.
-    package: Checking whether package is installed at shared path: .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...
+    package: Checking whether package is installed at shared path: /shared/package/FIRST_SHARED_PACKAGE_HASH
     package: This shared package has been installed by other package
 
 If options change, reinstall in different location:
@@ -1099,7 +1122,7 @@ If options change, reinstall in different location:
     ... [buildout]
     ... newest = false
     ... parts = package
-    ... shared-parts = %s
+    ... shared-part-list = %s
     ...
     ... [package]
     ... recipe = slapos.recipe.cmmi
@@ -1109,11 +1132,11 @@ If options change, reinstall in different location:
     ... """ % (shared_dir, src))
 
     >>> print(system(buildout)) #doctest:+ELLIPSIS
-    package: shared directory  .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/... set for package
+    package: shared directory /shared/package/ANOTHER_SHARED_PACKAGE_HASH set for package
     Uninstalling package.
     Installing package.
-    package: Checking whether package is installed at shared path: .../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...
-    configure --prefix=.../slapos.recipe.cmmi/slapos/recipe/cmmi/shared/package/...
+    package: Checking whether package is installed at shared path: /shared/package/ANOTHER_SHARED_PACKAGE_HASH
+    configure --prefix=/shared/package/ANOTHER_SHARED_PACKAGE_HASH
     building package
     installing package
 

slapos/recipe/cmmi/__init__.py

@@ -41,13 +41,8 @@ class Recipe(object):
             options.update(platform_options)
 
         shared = ((options.get('shared', '').lower() == 'true') and
-                  buildout['buildout'].get('shared-parts', None))
+                  buildout['buildout'].get('shared-part-list', None))
         if shared:
-           shared_part = buildout['buildout'].get('shared-parts', None)
-           shared = os.path.join(shared_part.strip().rstrip('/'), self.name)
-           if not os.path.exists(shared):
-              os.makedirs(shared)
-
             self._signature = slapos.recipe.downloadunpacked.Signature('.slapos.recipe.cmmi.signature')
             buildout_directory = buildout['buildout']['directory']
             profile_base_location = options.get('_profile_base_location_', '')
@@ -57,7 +52,15 @@ class Recipe(object):
                     v = v.replace(profile_base_location, '${:_profile_base_location_}')
                 self._signature.update(k, v)
 
-           shared = os.path.join(shared, self._signature.hexdigest())
+            shared_parts = [part.strip().rstrip('/')
+                    for part in buildout['buildout']['shared-part-list'].splitlines()
+                    if part.strip()]
+            for shared in shared_parts:
+                shared = os.path.join(os.path.join(shared, self.name), self._signature.hexdigest())
+                if os.path.exists(shared):
+                    break
+            else:
+                shared = os.path.join(os.path.join(shared_parts[-1], self.name), self._signature.hexdigest())
             log.info('shared directory %s set for %s', shared, self.name)
         else:
             shared = ''
@@ -303,7 +306,7 @@ class Recipe(object):
                 # leftovers from a previous failed attempt, removing it.
                 log.warning('Removing already existing directory %s' % compile_dir)
                 shutil.rmtree(compile_dir)
-            os.mkdir(compile_dir)
+            os.makedirs(compile_dir)
             try:
                 self.options.get('md5sum') # so that buildout does not complain "unused option md5sum"
                 opt = self.options.copy()

slapos/recipe/cmmi/tests.py

+from slapos.recipe.downloadunpacked import Signature
 from zope.testing import renormalizing
 import doctest
 import errno
@@ -319,17 +320,47 @@ class NonInformativeTests(unittest.TestCase):
         self.assertTrue(os.path.exists(build_directory))
 
 def test_suite():
+    # Hash used in the test depend on the file:// url of the package, so
+    # we use re-normalizer to replace SHARED_PACKAGE_HASH with the actual
+    # hash for this path.
+    package_url = 'file://%s/package-0.0.0.tar.gz' % os.path.join(
+        os.path.dirname(__file__), 'testdata')
+
+    signature_FIRST_SHARED_PACKAGE_HASH = Signature('.slapos.recipe.cmmi.signature')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('environment', 'FOO=bar')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('recipe', 'slapos.recipe.cmmi')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('shared', 'True')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('url', package_url)
+
+    signature_ANOTHER_SHARED_PACKAGE_HASH = Signature('.slapos.recipe.cmmi.signature')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('change', 'True')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('recipe', 'slapos.recipe.cmmi')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('shared', 'True')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('url', package_url)
+
     suite = unittest.TestSuite((
         doctest.DocFileSuite(
-            'README.txt',
+            'README.rst',
             setUp=setUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
             optionflags=optionflags,
             checker=renormalizing.RENormalizing([
-                (re.compile('--prefix=\S+sample-buildout'),
+                (re.compile(r'--prefix=\S+sample-buildout'),
                  '--prefix=/sample_buildout'),
-                (re.compile('\s/\S+sample-buildout'),
+                (re.compile(r'\s/\S+sample-buildout'),
                  ' /sample_buildout'),
+                (re.compile(r'--prefix=\S+\/shared\/'),
+                 '--prefix=/shared/'),
+               (re.compile(r'\s/\S+\/shared\/'),
+                 ' /shared/'),
+               (re.compile('FIRST_SHARED_PACKAGE_HASH'),
+                signature_FIRST_SHARED_PACKAGE_HASH.hexdigest()),
+               (re.compile('ANOTHER_SHARED_PACKAGE_HASH'),
+                signature_ANOTHER_SHARED_PACKAGE_HASH.hexdigest()),
+                # Normalize subprocess.CalledProcessError message, on python >= 3.6
+                # there's an extra . at the end.
+               (re.compile(r'Command (.*) returned non-zero exit status (\d+)[\.]{0,1}'),
+                  r'Command \1 returned non-zero exit status \2.'),
                 zc.buildout.testing.normalize_path,
                 zc.buildout.testing.not_found,
             ]),