Problem: there is no support for `Requires-Python`

Solution: monkeypatch `setuptools.packageIndex`
with code from `pip._internal`.

Because it depends on `pip._internal`, it is fragile.
As a first defense against breakage, imports are protected.
In case they would fail, buildout still works.

.github/workflows/centos.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/scripts.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
         package: [zest.releaser, pyspf]
 
     steps:

.github/workflows/setuptools-master.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
         package: [zest.releaser, pyspf]
 
     steps:

.travis.yml

 language: minimal
 env:
+    - PYTHON_VER=2.7
     - PYTHON_VER=3.5
     - PYTHON_VER=3.6
     - PYTHON_VER=3.7

CHANGES.rst

@@ -4,6 +4,10 @@ Change History
 3.0.0 (unreleased)
 ==================
 
+- Add support for ``Requires-Python`` metadata.
+  Fragile monkeypatch that relies on ``pip._internal``.
+  Emits a warning when support is disabled due to changes in ``pip``.
+
 - Use ``pip install`` instead of deprecated ``setuptools.easy_install``.
 
 - Patch ``pkg_resources.Distribution`` to make install of unpinned versions quicker.

Makefile

@@ -100,12 +100,14 @@ test: $(VENV)/bin/test
 	$(VENV)/bin/test -c -vvv $(testargs)
 
 all_pythons:
+	$(MAKE) PYTHON_VER=2.7 python
 	$(MAKE) PYTHON_VER=3.5 python
 	$(MAKE) PYTHON_VER=3.6 python
 	$(MAKE) PYTHON_VER=3.7 python
 	$(MAKE) PYTHON_VER=3.8 python
 
 all_test:
+	$(MAKE) PYTHON_VER=2.7 test
 	$(MAKE) PYTHON_VER=3.5 test
 	$(MAKE) PYTHON_VER=3.6 test
 	$(MAKE) PYTHON_VER=3.7 test

dev.py

@@ -16,8 +16,16 @@
 This is different from a normal boostrapping process because the
 buildout egg itself is installed as a develop egg.
 """
+import sys
 
-import os, shutil, sys, subprocess
+if sys.version_info < (2, 7):
+    raise SystemError("Outside Python 2.7, no support for Python 2.x.")
+
+if sys.version_info > (3, ) and sys.version_info < (3, 5):
+    raise SystemError("No support for Python 3.x under 3.5.")
+
+
+import os, shutil, sys, subprocess, tempfile
 
 for d in 'eggs', 'develop-eggs', 'bin', 'parts':
     if not os.path.exists(d):
@@ -30,6 +38,40 @@ if os.path.isfile(bin_buildout):
 if os.path.isdir('build'):
     shutil.rmtree('build')
 
+
+#######################################################################
+def install_pip():
+    print('')
+    print('Install pip')
+    print('')
+    try:
+        from urllib.request import urlopen
+    except ImportError:
+        from urllib2 import urlopen
+
+    tmp = tempfile.mkdtemp(prefix='buildout-dev-')
+    try:
+        get_pip = os.path.join(tmp, 'get-pip.py')
+        with open(get_pip, 'wb') as f:
+           f.write(urlopen('https://bootstrap.pypa.io/get-pip.py').read())
+
+        sys.stdout.flush()
+        if subprocess.call([sys.executable, get_pip]):
+            raise RuntimeError("Failed to install pip.")
+    finally:
+        shutil.rmtree(tmp)
+    print("Restart")
+    sys.stdout.flush()
+    return_code = subprocess.call(
+        [sys.executable] + sys.argv
+    )
+    sys.exit(return_code)
+
+try:
+    import pip
+except ImportError:
+    install_pip()
+
 ######################################################################
 def check_upgrade(package):
     print('')
@@ -76,7 +118,7 @@ pkg_resources.working_set.add_entry('src')
 
 import zc.buildout.easy_install
 zc.buildout.easy_install.scripts(
-    ['zc.buildout'], pkg_resources.working_set , sys.executable, 'bin')
+    ['zc.buildout'], pkg_resources.working_set, sys.executable, 'bin')
 
 ######################################################################
 print('')

src/zc/buildout/patches.py

@@ -56,3 +56,124 @@ def patch_Distribution():
 
 
 patch_Distribution()
+
+
+def patch_PackageIndex():
+    try:
+        from setuptools.package_index import PackageIndex
+        from setuptools.package_index import URL_SCHEME
+        from setuptools.package_index import HREF
+        from setuptools.package_index import distros_for_url
+        from setuptools.package_index import htmldecode
+
+        from pip._internal.index.collector import HTMLPage
+        from pip._internal.index.collector import parse_links
+        from pip._internal.index.collector import _clean_link
+        from pip._internal.index.package_finder import _check_link_requires_python
+        from pip._internal.models.target_python import TargetPython
+        from pip._vendor import six
+        from pip._vendor.six.moves import urllib
+
+        PY_VERSION_INFO = TargetPython().py_version_info
+
+        # method copied over from setuptools 46.1.3
+        def process_url(self, url, retrieve=False):
+            """Evaluate a URL as a possible download, and maybe retrieve it"""
+            if url in self.scanned_urls and not retrieve:
+                return
+            self.scanned_urls[url] = True
+            if not URL_SCHEME(url):
+                self.process_filename(url)
+                return
+            else:
+                dists = list(distros_for_url(url))
+                if dists:
+                    if not self.url_ok(url):
+                        return
+                    self.debug("Found link: %s", url)
+
+            if dists or not retrieve or url in self.fetched_urls:
+                list(map(self.add, dists))
+                return  # don't need the actual page
+
+            if not self.url_ok(url):
+                self.fetched_urls[url] = True
+                return
+
+            self.info("Reading %s", url)
+            self.fetched_urls[url] = True  # prevent multiple fetch attempts
+            tmpl = "Download error on %s: %%s -- Some packages may not be found!"
+            f = self.open_url(url, tmpl % url)
+            if f is None:
+                return
+            if isinstance(f, urllib.error.HTTPError) and f.code == 401:
+                self.info("Authentication error: %s" % f.msg)
+            self.fetched_urls[f.url] = True
+            if 'html' not in f.headers.get('content-type', '').lower():
+                f.close()  # not html, we can't process it
+                return
+
+            base = f.url  # handle redirects
+            page = f.read()
+
+            # --- LOCAL CHANGES MADE HERE: ---
+
+            if isinstance(page, six.text_type):
+                page = page.encode('utf8')
+                charset = 'utf8'
+            else:
+                if isinstance(f, urllib.error.HTTPError):
+                    # Errors have no charset, assume latin1:
+                    charset = 'latin-1'
+                else:
+                    try:
+                        charset = f.headers.get_param('charset') or 'latin-1'
+                    except AttributeError:
+                        # Python 2
+                        charset = f.headers.getparam('charset') or 'latin-1'
+            try:
+                html_page = HTMLPage(page, charset, base, cache_link_parsing=False)
+            except TypeError:
+                html_page = HTMLPage(page, charset, base)
+
+            plinks = list(parse_links(html_page))
+            pip_links = [l.url for l in plinks]
+
+            # --- END OF LOCAL CHANGES ---
+
+            if not isinstance(page, str):
+                # In Python 3 and got bytes but want str.
+                page = page.decode(charset, "ignore")
+            f.close()
+
+            # --- LOCAL CHANGES MADE HERE: ---
+
+            links = []
+            for match in HREF.finditer(page):
+                link = urllib.parse.urljoin(base, htmldecode(match.group(1)))
+                links.append(_clean_link(link))
+
+            # TODO: remove assertion and double index page parsing before releasing.
+            assert set(pip_links) == set(links)
+
+            for link in plinks:
+                if _check_link_requires_python(link, PY_VERSION_INFO):
+                    self.process_url(link.url)
+
+            # --- END OF LOCAL CHANGES ---
+
+            if url.startswith(self.index_url) and getattr(f, 'code', None) != 404:
+                page = self.process_index(url, page)
+
+        setattr(PackageIndex, 'process_url', process_url)
+
+    except ImportError:
+        import logging
+        logger = logging.getLogger('zc.buildout.patches')
+        logger.warning('Requires-Python support missing. \n\n',
+            exc_info=True
+        )
+        return
+
+
+patch_PackageIndex()

src/zc/buildout/testing.py

@@ -25,7 +25,7 @@ except ImportError:
 
 import errno
 import logging
-from multiprocessing import get_context
+import multiprocessing
 import os
 import pkg_resources
 import random
@@ -592,6 +592,20 @@ normalize_open_in_generated_script = (
 
 not_found = (re.compile(r'Not found: [^\n]+/(\w|\.)+/\r?\n'), '')
 
+python27_warning = (re.compile(r'DEPRECATION: Python 2.7 reached the end of its '
+    'life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no '
+    'longer maintained. A future version of pip will drop support for Python '
+    '2.7. More details about Python 2 support in pip, can be found at '
+    'https://pip.pypa.io/en/latest/development/release-process/#python-2-support\n'),
+    '')
+
+python27_warning_2 = (re.compile(r'DEPRECATION: Python 2.7 reached the end of its '
+    'life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no '
+    'longer maintained. pip 21.0 will drop support for Python 2.7 in January 2021. '
+    'More details about Python 2 support in pip, can be found at '
+    'https://pip.pypa.io/en/latest/development/release-process/#python-2-support\n'),
+    '')
+
 # Setuptools now pulls in dependencies when installed.
 adding_find_link = (re.compile(r"Adding find link '[^']+'"
                                r" from setuptools .*\r?\n"), '')
@@ -615,8 +629,11 @@ def run_from_process(target, *args, **kw):
     target(*args, **kw)
 
 def run_in_process(*args, **kwargs):
-    ctx = get_context('fork')
+    try:
+        ctx = multiprocessing.get_context('fork')
         process = ctx.Process(target=run_from_process, args=args, kwargs=kwargs)
+    except AttributeError:
+        process = multiprocessing.Process(target=run_from_process, args=args, kwargs=kwargs)
     process.daemon = True
     process.start()
     process.join(99)

src/zc/buildout/tests.py

@@ -3611,6 +3611,8 @@ def test_suite():
                     zc.buildout.testing.normalize_egg_py,
                     zc.buildout.testing.not_found,
                     zc.buildout.testing.adding_find_link,
+                    zc.buildout.testing.python27_warning,
+                    zc.buildout.testing.python27_warning_2,
                     # (re.compile(r"Installing 'zc.buildout >=\S+"), ''),
                     (re.compile(r'__buildout_signature__ = recipes-\S+'),
                      '__buildout_signature__ = recipes-SSSSSSSSSSS'),
@@ -3748,6 +3750,8 @@ def test_suite():
                 zc.buildout.testing.normalize_open_in_generated_script,
                 zc.buildout.testing.adding_find_link,
                 zc.buildout.testing.not_found,
+                zc.buildout.testing.python27_warning,
+                zc.buildout.testing.python27_warning_2,
                 normalize_bang,
                 normalize_S,
                 (re.compile(r'[-d]  setuptools-\S+[.]egg'), 'setuptools.egg'),
@@ -3793,6 +3797,8 @@ def test_suite():
                 zc.buildout.testing.not_found,
                 zc.buildout.testing.normalize_exception_type_for_python_2_and_3,
                 zc.buildout.testing.adding_find_link,
+                zc.buildout.testing.python27_warning,
+                zc.buildout.testing.python27_warning_2,
                 normalize_bang,
                 (re.compile(r'^(\w+\.)*(Missing\w+: )'), '\2'),
                 (re.compile(r"buildout: Running \S*setup.py"),

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -50,6 +50,8 @@ def test_suite():
                zc.buildout.tests.normalize_bang,
                zc.buildout.tests.normalize_S,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.python27_warning,
+               zc.buildout.testing.python27_warning_2,
                (re.compile(r'[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
                (re.compile(r'[d-]  setuptools-[^-]+-'), 'setuptools-X-'),
@@ -66,6 +68,8 @@ def test_suite():
                zc.buildout.testing.normalize_path,
                zc.buildout.testing.normalize_endings,
                zc.buildout.testing.not_found,
+               zc.buildout.testing.python27_warning,
+               zc.buildout.testing.python27_warning_2,
                (re.compile('__buildout_signature__ = '
                            r'sample-\S+\s+'
                            r'zc.recipe.egg-\S+\s+'
@@ -87,6 +91,8 @@ def test_suite():
                 zc.buildout.testing.normalize_path,
                 zc.buildout.testing.normalize_endings,
                 zc.buildout.testing.not_found,
+                zc.buildout.testing.python27_warning,
+                zc.buildout.testing.python27_warning_2,
                 (re.compile("(d  ((ext)?demo(needed)?|other)"
                             r"-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
                  '\\1V.V.egg'),