simplify resulting code a bit more and try again to remove warnings

src/zc/buildout/easy_install.py

@@ -1063,7 +1063,8 @@ def _generate_scripts(reqs, working_set, dest, path, scripts, relative_paths,
     return generated
 
 def _relative_path_and_setup(sname, path,
-                             relative_paths=False, indent_level=1):
+                             relative_paths=False, indent_level=1,
+                             omit_os_import=False):
     """Return a string of code of paths and of setup if appropriate.
 
     - sname is the full path to the script name to be created.
@@ -1080,8 +1081,10 @@ def _relative_path_and_setup(sname, path,
             [_relativitize(os.path.normcase(path_item), sname, relative_paths)
              for path_item in path], indent_level=indent_level)
         rpsetup = relative_paths_setup
+        if not omit_os_import:
+            rpsetup = '\n\nimport os\n' + rpsetup
         for i in range(_relative_depth(relative_paths, sname)):
-            rpsetup += "base = os.path.dirname(base)\n"
+            rpsetup += "\nbase = os.path.dirname(base)"
     else:
         spath = _format_paths((repr(p) for p in path),
                               indent_level=indent_level)
@@ -1143,11 +1146,8 @@ def _relativitize(path, script, relative_paths):
         return repr(path)
 
 relative_paths_setup = """
-import os
-
 join = os.path.join
-base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
-"""
+base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))"""
 
 def _write_script(full_name, contents, logged_type):
     """Write contents of script in full_name, logging the action.
@@ -1214,8 +1214,8 @@ sys.path[0:0] = [
 '''
 
 script_template = script_header + '''\
-
 %(relative_paths_setup)s
+
 import sys
 sys.path[0:0] = [
     %(path)s,
@@ -1240,8 +1240,8 @@ def _pyscript(path, dest, executable, rsetup):
     return _write_script(dest, contents, 'interpreter')
 
 py_script_template = script_header + '''\
-
 %(relative_paths_setup)s
+
 import sys
 
 sys.path[0:0] = [
@@ -1384,29 +1384,32 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
     """
     path = _get_path(working_set, extra_paths)
     site_path = os.path.join(dest, 'site.py')
-    path_string, rpsetup = _relative_path_and_setup(
-        site_path, path, relative_paths, indent_level=2)
-    if rpsetup:
-        rpsetup = '\n'.join(
+    egg_path_string, preamble = _relative_path_and_setup(
+        site_path, path, relative_paths, indent_level=2, omit_os_import=True)
+    if preamble:
+        preamble = '\n'.join(
             [(line and '    %s' % (line,) or line)
-             for line in rpsetup.split('\n')])
-    namespace_setup = ''
-    addsitedir = addsitedir_snippet
+             for line in preamble.split('\n')])
+    original_path_setup = ''
     if add_site_packages:
         stdlib, site_paths = _get_system_paths(executable)
-        path_string = ''.join([
-            path_string,
-            (",\n"
-             "        # These are the underlying Python's site-packages.\n"
-             "        "),
-            _format_paths((repr(p) for p in site_paths), 2)])
+        original_path_setup = original_path_snippet % (
+            _format_paths((repr(p) for p in site_paths), 2),)
         distribution = working_set.find(
             pkg_resources.Requirement.parse('setuptools'))
         if distribution is not None:
             # We need to worry about namespace packages.
-            namespace_setup = namespace_add_site_packages_setup % (
-                distribution.location,)
-            addsitedir = addsitedir_namespace_add_site_packages_snippet
+            if relative_paths:
+                location = _relativitize(
+                    distribution.location,
+                    os.path.normcase(os.path.abspath(site_path)),
+                    relative_paths)
+            else:
+                location = repr(distribution.location)
+            preamble += namespace_add_site_packages_setup % (location,)
+            original_path_setup = (
+                addsitedir_namespace_originalpackages_snippet +
+                original_path_setup)
     addsitepackages_marker = 'def addsitepackages('
     enableusersite_marker = 'ENABLE_USER_SITE = '
     successful_rewrite = False
@@ -1420,7 +1423,7 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
                 site.write('False # buildout does not support user sites.\n')
             elif line.startswith(addsitepackages_marker):
                 site.write(addsitepackages_script % (
-                    namespace_setup, rpsetup, path_string, addsitedir))
+                    preamble, egg_path_string, original_path_setup))
                 site.write(line[len(addsitepackages_marker):])
                 successful_rewrite = True
             else:
@@ -1433,49 +1436,41 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
     return site_path
 
 namespace_add_site_packages_setup = '''
-    setuptools_path = %r
+    setuptools_path = %s
     sys.path.append(setuptools_path)
-    known_paths.add(setuptools_path)
+    known_paths.add(os.path.normcase(setuptools_path))
     import pkg_resources'''
 
-addsitedir_snippet = '''
-    for path in paths:
-        addsitedir(path, known_paths)'''
+addsitedir_namespace_originalpackages_snippet = '''
+            for dist in pkg_resources.find_distributions(sitedir, True):
+                pkg_resources.fixup_namespace_packages(dist.location)
+                if dist.has_metadata('namespace_packages.txt'):
+                    for namespace in dist.get_metadata_lines(
+                        'namespace_packages.txt'):
+                        pkg_resources.declare_namespace(namespace)'''
 
-addsitedir_namespace_add_site_packages_snippet = '''
-    dotpth = os.extsep + "pth"
-    for path in paths:
-        # This duplicates addsitedir except for adding the pkg_resources call.
-        sitedir, sitedircase = makepath(path)
-        if not sitedircase in known_paths and os.path.exists(sitedir):
-            sys.path.append(sitedir)
-            pkg_resources.working_set.add_entry(sitedir)
-            known_paths.add(sitedircase)
-        try:
-            names = os.listdir(sitedir)
-        except os.error:
-            continue
-        names = [name for name in names if name.endswith(dotpth)]
-        names.sort()
-        for name in names:
-            addpackage(sitedir, name, known_paths)'''
+original_path_snippet = '''
+    original_paths = [
+        %s
+        ]
+    for path in original_paths:
+        addsitedir(path, known_paths)'''
 
 addsitepackages_script = '''\
 def addsitepackages(known_paths):
-    """Add site packages.
+    """Add site packages, as determined by zc.buildout.
 
-    This function is written by buildout.  See original_addsitepackages,
-    below, for the original version."""%s
-%s    paths = [
-        # Eggs.
+    See original_addsitepackages, below, for the original version."""%s
+    buildout_paths = [
         %s
-        ]%s
-    global addsitepackages
-    addsitepackages = original_addsitepackages
+        ]
+    for path in buildout_paths:
+        sitedir, sitedircase = makepath(path)
+        if not sitedircase in known_paths and os.path.exists(sitedir):
+            sys.path.append(sitedir)
+            known_paths.add(sitedircase)%s
     return known_paths
 
-buildout_addsitepackages = addsitepackages
-
 def original_addsitepackages('''
 
 def _generate_interpreter(name, dest, executable, site_py_dest,
@@ -1483,7 +1478,9 @@ def _generate_interpreter(name, dest, executable, site_py_dest,
     """Write an interpreter script, using the site.py approach."""
     full_name = os.path.join(dest, name)
     site_py_dest_string, rpsetup = _relative_path_and_setup(
-        full_name, [site_py_dest], relative_paths)
+        full_name, [site_py_dest], relative_paths, omit_os_import=True)
+    if rpsetup:
+        rpsetup += "\n"
     if sys.platform == 'win32':
         windows_import = '\nimport subprocess'
         # os.exec* is a mess on Windows, particularly if the path
@@ -1534,12 +1531,10 @@ def _generate_interpreter(name, dest, executable, site_py_dest,
         )
     return _write_script(full_name, contents, 'interpreter')
 
-interpreter_template = script_header + '''\
-
-%(relative_paths_setup)s
+interpreter_template = script_header + '''
 import os
 import sys%(windows_import)s
-
+%(relative_paths_setup)s
 argv = [sys.executable] + sys.argv[1:]
 environ = os.environ.copy()
 path = %(site_dest)s

src/zc/buildout/easy_install.txt

@@ -997,8 +997,7 @@ The interpreter script is simple.  It puts the directory with the
 site.py and sitecustomize.py on the PYTHONPATH and (re)starts Python.
 
     >>> cat(py_path)
-    #!/usr/bin/python2.4 -S
-    <BLANKLINE>
+    #!/usr/bin/python -S
     import os
     import sys
     <BLANKLINE>
@@ -1012,31 +1011,28 @@ site.py and sitecustomize.py on the PYTHONPATH and (re)starts Python.
 
 The site.py file is a modified version of the underlying Python's site.py.
 The most important modification is that it has a different version of the
-addsitepackages function.  It has all of the trickier bits, and sets up the
-Python path, similarly to the behavior of the function it replaces.  The
-following shows the part that buildout inserts.
+addsitepackages function.  It sets up the Python path, similarly to the
+behavior of the function it replaces.  The following shows the part that
+buildout inserts, in the simplest case.
 
     >>> sys.stdout.write('#\n'); cat(site_path)
     ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     #...
     def addsitepackages(known_paths):
-        """Add site packages.
+        """Add site packages, as determined by zc.buildout.
     <BLANKLINE>
-        This function is written by buildout.  See original_addsitepackages,
-        below, for the original version."""
-        paths = [
-            # Eggs.
+        See original_addsitepackages, below, for the original version."""
+        buildout_paths = [
             '/interpreter/eggs/demo-0.3-pyN.N.egg',
             '/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
             ]
-        for path in paths:
-            addsitedir(path, known_paths)
-        global addsitepackages
-        addsitepackages = original_addsitepackages
+        for path in buildout_paths:
+            sitedir, sitedircase = makepath(path)
+            if not sitedircase in known_paths and os.path.exists(sitedir):
+                sys.path.append(sitedir)
+                known_paths.add(sitedircase)
         return known_paths
     <BLANKLINE>
-    buildout_addsitepackages = addsitepackages
-    <BLANKLINE>
     def original_addsitepackages(known_paths):...
 
 Here are some examples of the interpreter in use.
@@ -1079,24 +1075,23 @@ If you provide initialization, it goes in sitecustomize.py.
     bar baz bing shazam
     <BLANKLINE>
 
-If you use relative paths, this affects the interpreter and site.py.
+If you use relative paths, this affects the interpreter and site.py.  (This is
+again the UNIX version; the Windows version uses subprocess instead of
+os.execve.)
 
     >>> reset_interpreter()
     >>> generated = zc.buildout.easy_install.generate_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', relative_paths=interpreter_dir)
     >>> cat(py_path)
-    #!/usr/bin/python2.4 -S
-    <BLANKLINE>
+    #!/usr/bin/python -S
     import os
+    import sys
     <BLANKLINE>
     join = os.path.join
     base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
     base = os.path.dirname(base)
     <BLANKLINE>
-    import os
-    import sys
-    <BLANKLINE>
     argv = [sys.executable] + sys.argv[1:]
     environ = os.environ.copy()
     path = join(base, 'parts/interpreter')
@@ -1112,19 +1107,14 @@ paths join a base to a path, as with the use of this argument in the
     >>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
     #...
     def addsitepackages(known_paths):
-        """Add site packages.
-    <BLANKLINE>
-        This function is written by buildout.  See original_addsitepackages,
-        below, for the original version."""
-    <BLANKLINE>
-        import os
+        """Add site packages, as determined by zc.buildout.
     <BLANKLINE>
+        See original_addsitepackages, below, for the original version."""
         join = os.path.join
         base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
         base = os.path.dirname(base)
         base = os.path.dirname(base)
-        paths = [
-            # Eggs.
+        buildout_paths = [
             join(base, 'eggs/demo-0.3-pyN.N.egg'),
             join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
             ]...
@@ -1152,12 +1142,10 @@ The ``extra_paths`` argument affects the path in site.py.  Notice that
     >>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
     #...
     def addsitepackages(known_paths):
-        """Add site packages.
+        """Add site packages, as determined by zc.buildout.
     <BLANKLINE>
-        This function is written by buildout.  See original_addsitepackages,
-        below, for the original version."""
-        paths = [
-            # Eggs.
+        See original_addsitepackages, below, for the original version."""
+        buildout_paths = [
             '/interpreter/eggs/demo-0.3-pyN.N.egg',
             '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
             '/interpreter/other'
@@ -1213,19 +1201,29 @@ this is very straightforward.
     ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     #...
     def addsitepackages(known_paths):
-        """Add site packages.
+        """Add site packages, as determined by zc.buildout.
     <BLANKLINE>
-        This function is written by buildout.  See original_addsitepackages,
-        below, for the original version."""
-        paths = [
-            # Eggs.
+        See original_addsitepackages, below, for the original version."""
+        buildout_paths = [
             '/interpreter/eggs/demo-0.3-pyN.N.egg',
-            '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
-            # These are the underlying Python's site-packages.
+            '/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
+            ]
+        for path in buildout_paths:
+            sitedir, sitedircase = makepath(path)
+            if not sitedircase in known_paths and os.path.exists(sitedir):
+                sys.path.append(sitedir)
+                known_paths.add(sitedircase)
+        original_paths = [
             ...
-            ]...
+            ]
+        for path in original_paths:
+            addsitedir(path, known_paths)
+        return known_paths
+    <BLANKLINE>
+    def original_addsitepackages(known_paths):...
 
-It simply adds the site-packages after the eggs.
+It simply adds the original paths using addsitedir after the code to add the
+buildout paths.
 
 Here's an example of the new script in use.  Other documents and tests in
 this package give the feature a more thorough workout, but this should
@@ -1271,45 +1269,37 @@ call to another text fixture to create.
     ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     #...
     def addsitepackages(known_paths):
-        """Add site packages.
+        """Add site packages, as determined by zc.buildout.
     <BLANKLINE>
-        This function is written by buildout.  See original_addsitepackages,
-        below, for the original version."""
+        See original_addsitepackages, below, for the original version."""
         setuptools_path = '...setuptools...'
         sys.path.append(setuptools_path)
-        known_paths.add(setuptools_path)
+        known_paths.add(os.path.normcase(setuptools_path))
         import pkg_resources
-        paths = [
-            # Eggs.
+        buildout_paths = [
             '/interpreter/eggs/demo-0.3-pyN.N.egg',
             '/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
             '...setuptools...',
-            '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
-            # These are the underlying Python's site-packages.
-            ...
+            '/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
             ]
-        dotpth = os.extsep + "pth"
-        for path in paths:
-            # This duplicates addsitedir except for adding the pkg_resources call.
+        for path in buildout_paths:
             sitedir, sitedircase = makepath(path)
             if not sitedircase in known_paths and os.path.exists(sitedir):
                 sys.path.append(sitedir)
-                pkg_resources.working_set.add_entry(sitedir)
                 known_paths.add(sitedircase)
-            try:
-                names = os.listdir(sitedir)
-            except os.error:
-                continue
-            names = [name for name in names if name.endswith(dotpth)]
-            names.sort()
-            for name in names:
-                addpackage(sitedir, name, known_paths)
-        global addsitepackages
-        addsitepackages = original_addsitepackages
+                for dist in pkg_resources.find_distributions(sitedir, True):
+                    pkg_resources.fixup_namespace_packages(dist.location)
+                    if dist.has_metadata('namespace_packages.txt'):
+                        for namespace in dist.get_metadata_lines(
+                            'namespace_packages.txt'):
+                            pkg_resources.declare_namespace(namespace)
+        original_paths = [
+            ...
+            ]
+        for path in original_paths:
+            addsitedir(path, known_paths)
         return known_paths
     <BLANKLINE>
-    buildout_addsitepackages = addsitepackages
-    <BLANKLINE>
     def original_addsitepackages(known_paths):...
 
     >>> print call_py(interpreter_path, "import sys; print sys.path")
@@ -1330,6 +1320,66 @@ feature--lines that might be part of the setuptools namespace package
 implementation for system packages, as mentioned above, and that must
 come after processing egg namespaces.
 
+The most complex that this function gets is if you use namespace packages,
+include site-packages, and use relative paths.  For completeness, we'll look
+at that result.
+
+    >>> generated = zc.buildout.easy_install.generate_scripts(
+    ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
+    ...     interpreter='py', add_site_packages=True,
+    ...     relative_paths=interpreter_dir)
+    >>> sys.stdout.write('#\n'); cat(site_path)
+    ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    #...
+    def addsitepackages(known_paths):
+        """Add site packages, as determined by zc.buildout.
+    <BLANKLINE>
+        See original_addsitepackages, below, for the original version."""
+        join = os.path.join
+        base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+        base = os.path.dirname(base)
+        base = os.path.dirname(base)
+        setuptools_path = '...setuptools...'
+        sys.path.append(setuptools_path)
+        known_paths.add(os.path.normcase(setuptools_path))
+        import pkg_resources
+        buildout_paths = [
+            join(base, 'eggs/demo-0.3-pyN.N.egg'),
+            join(base, 'eggs/tellmy.fortune-1.0-pyN.N.egg'),
+            '...setuptools...',
+            join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
+            ]
+        for path in buildout_paths:
+            sitedir, sitedircase = makepath(path)
+            if not sitedircase in known_paths and os.path.exists(sitedir):
+                sys.path.append(sitedir)
+                known_paths.add(sitedircase)
+                for dist in pkg_resources.find_distributions(sitedir, True):
+                    pkg_resources.fixup_namespace_packages(dist.location)
+                    if dist.has_metadata('namespace_packages.txt'):
+                        for namespace in dist.get_metadata_lines(
+                            'namespace_packages.txt'):
+                            pkg_resources.declare_namespace(namespace)
+        original_paths = [
+            ...
+            ]
+        for path in original_paths:
+            addsitedir(path, known_paths)
+        return known_paths
+    <BLANKLINE>
+    def original_addsitepackages(known_paths):...
+
+    >>> print call_py(interpreter_path, "import sys; print sys.path")
+    ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    ['',
+     '/interpreter/parts/interpreter',
+     ...,
+     '...setuptools...',
+     '/interpreter/eggs/demo-0.3-pyN.N.egg',
+     '/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
+     '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
+     ...]
+
 The ``exec_sitecustomize`` argument does the same thing for the
 sitecustomize module--it allows you to include the code from the
 sitecustomize module in the underlying Python if you set the argument to

src/zc/buildout/testing.py

@@ -310,6 +310,16 @@ def buildoutSetUp(test):
         old_wd = os.getcwd()
         os.chdir(buildout)
         make_buildout()
+        # Normally we don't process .pth files in extra-paths.  We want to
+        # in this case so that we can test with setuptools system installs
+        # (--single-version-externally-managed), which use .pth files.
+        initialization = (
+            ('import sys\n'
+             'import site\n'
+             'known_paths = set(sys.path)\n'
+             'site_packages_dir = %r\n'
+             'site.addsitedir(site_packages_dir, known_paths)\n'
+            ) % (site_packages_dir,)) + initialization
         initialization = '\n'.join(
             '  ' + line for line in initialization.split('\n'))
         install_develop(

z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt

@@ -273,11 +273,11 @@ into the sitecustomize.
     Generated interpreter '/sample-buildout/bin/py'.
 
     >>> cat(sample_buildout, 'parts', 'py', 'sitecustomize.py')
-    ... # doctest: +NORMALIZE_WHITESPACE
+    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
     <BLANKLINE>
     # The following is from
     # /executable_buildout/parts/py/sitecustomize.py
-    <BLANKLINE>
+    ...
     import os
     os.environ['zc.buildout'] = 'foo bar baz shazam'
 

z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py

@@ -175,14 +175,14 @@ custom Python.
     Generated interpreter '/sample-buildout/bin/py'.
 
     >>> cat(sample_buildout, 'parts', 'py', 'sitecustomize.py')
-    ... # doctest: +NORMALIZE_WHITESPACE
+    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
     <BLANKLINE>
     import os
     os.environ['zc.recipe.egg'] = 'baLOOba'
     <BLANKLINE>
     # The following is from
     # /executable_buildout/parts/py/sitecustomize.py
-    <BLANKLINE>
+    ...
     import os
     os.environ['zc.buildout'] = 'foo bar baz shazam'
 
@@ -227,15 +227,11 @@ Let's look at the site.py that was generated:
     #...
     def addsitepackages(known_paths):
         "..."
-    <BLANKLINE>
-        import os
-    <BLANKLINE>
         join = os.path.join
         base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
         base = os.path.dirname(base)
         base = os.path.dirname(base)
-        paths = [
-            # Eggs.
+        buildout_paths = [
             '/foo/bar',
             join(base, 'spam')
             ]...