easy_install: rough prototype of supporting --hash in version

This currently cause distributions to be downloaded twice, even with a
download cache

src/zc/buildout/easy_install.py

@@ -58,11 +58,11 @@ from setuptools.package_index import HREF, URL_SCHEME, \
 try:
     # Python 3
     from urllib.error import HTTPError
-    from urllib.parse import urljoin
+    from urllib.parse import urljoin, urlparse
 except ImportError:
     # Python 2
     from urllib2 import HTTPError
-    from urlparse import urljoin
+    from urlparse import urljoin, urlparse
 
 default_index_url = os.environ.get(
     'BUILDOUT_TESTING_INDEX_URL',
@@ -380,13 +380,13 @@ class Installer:
                    if name.lower() in line.lower()]
         return '\n  '.join(output)
 
-    def _satisfied(self, req, source=None):
+    def _satisfied(self, req, source=None, hashes=None):
         dists = [dist for dist in self._env[req.project_name] if dist in req]
         if not dists:
             logger.debug('We have no distributions for %s that satisfies %r.',
                          req.project_name, str(req))
 
-            return None, self._obtain(req, source)
+            return None, self._obtain(req, source, hashes)
 
         # Note that dists are sorted from best to worst, as promised by
         # env.__getitem__
@@ -424,7 +424,7 @@ class Installer:
         # any are newer.  We only do this if we're willing to install
         # something, which is only true if dest is not None:
 
-        best_available = self._obtain(req, source)
+        best_available = self._obtain(req, source, hashes)
 
         if best_available is None:
             # That's a bit odd.  There aren't any distros available.
@@ -547,11 +547,12 @@ class Installer:
         finally:
             shutil.rmtree(tmp)
 
-    def _obtain(self, requirement, source=None):
+    def _obtain(self, requirement, source=None, hashes=None):
         # get the non-patched version
         req = str(requirement)
         if PATCH_MARKER in req:
             requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
+            # XXX hashes: maybe we can just put the hash in requirement ??
 
         # initialize out index for this project:
         index = self._index
@@ -582,6 +583,42 @@ class Installer:
                 # There are final dists, so only use those
                 dists = fdists
 
+        # if we are using hashes, filter for dists matching our hashes
+        if hashes is not None:
+
+            def dist_key(dist):
+                # optimization: sort to try first the distributions with the
+                # hash in the URL, so that if there's a matching dist we use
+                # it directly without trying all dists
+                for hash_ in hashes:
+                    # the --hash format uses alg:hash, but in the URL fragments
+                    # it is alg=hash
+                    if hash_.replace(':', '=') in urlparse(dist.location).fragment:
+                        return 0
+                return 1
+
+            # TODO: can we reuse download cache here ????
+            #tmp = self._download_cache
+            #if tmp is None:
+            #    tmp = tempfile.mkdtemp('check_hash')
+
+            downloader = zc.buildout.download.Download()
+            def hash_match(dist):
+                try:
+                    downloaded_filename, is_temp = downloader(dist.location, hashes=hashes)
+                    if is_temp:
+                        os.remove(downloaded_filename)
+                except zc.buildout.download.ChecksumError:
+                    return False
+                return True
+
+            dists.sort(key=dist_key)
+            if hash_match(dists[0]):
+                dists = [dists[0]]
+            else:
+                dists = [dist for dist in dists[1:] if hash_match(dist)]
+
+
         # Now find the best one:
         best = []
         bestv = None
@@ -657,12 +694,12 @@ class Installer:
 
         return dist.clone(location=new_location)
 
-    def _get_dist(self, requirement, ws, for_buildout_run=False):
+    def _get_dist(self, requirement, ws, for_buildout_run=False, hashes=None):
         __doing__ = 'Getting distribution for %r.', str(requirement)
 
         # Maybe an existing dist is already the best dist that satisfies the
         # requirement
-        dist, avail = self._satisfied(requirement)
+        dist, avail = self._satisfied(requirement, hashes=hashes)
 
         if dist is None:
             if self._dest is None:
@@ -720,6 +757,10 @@ class Installer:
                 if tmp != self._download_cache:
                     shutil.rmtree(tmp)
 
+            # TODO hashes if we:
+            #   1. install version1 h1
+            #   2. change to version1 h2 (to get another lower wheel build number)
+            #  ... will it keep using h1 ??? maybe this is not something we can support.
             self._env.scan([self._dest])
             dist = self._env.best_match(requirement, ws)
             logger.info("Got %s.", dist)
@@ -819,6 +860,22 @@ class Installer:
         requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
                         for spec in specs]
 
+        def get_hashes(spec):
+            version = self._versions.get(spec, '')
+            if '--hash' in version:
+                # TODO see pip actual implementation
+                # TODO for test, there can be multiple --hash like with pip
+                hashes = []
+                next_is_hash = False
+                for part in version.split():
+                    if next_is_hash:
+                        hashes.append(part)
+                    next_is_hash = part == '--hash'
+                return ' '.join(hashes)
+            return None    
+
+        hashes_dict = {spec: get_hashes(spec) for spec in specs}
+
         if working_set is None:
             ws = pkg_resources.WorkingSet([])
         else:
@@ -829,7 +886,8 @@ class Installer:
                 self._env.scan(
                     self.build(str(requirement), {}, patch_dict=patch_dict))
             for dist in self._get_dist(requirement, ws,
-                                       for_buildout_run=for_buildout_run):
+                                       for_buildout_run=for_buildout_run,
+                                       hashes=hashes_dict.get(requirement.project_name)):
                 ws.add(dist)
                 self._maybe_add_setuptools(ws, dist)
 
@@ -906,6 +964,8 @@ class Installer:
 
         requirement = self._constrain(pkg_resources.Requirement.parse(spec))
 
+        print("TODO: Why are we here ???") # XXX hash this branch is not covered
+        import pdb; pdb.set_trace()
         dist, avail = self._satisfied(requirement, 1)
         if dist is not None:
             return [dist.location]
@@ -1766,6 +1826,8 @@ def redo_pyc(egg):
                 call_subprocess(args)
 
 def _constrained_requirement(constraint, requirement):
+    if "--hash" in constraint:
+        constraint = constraint.split()[0]
     if constraint[0] not in '<>':
         if constraint.startswith('='):
             assert constraint.startswith('==')
@@ -1865,3 +1927,9 @@ def _move_to_eggs_dir_and_compile(dist, dest):
         # Remember that temporary directories must be removed
         shutil.rmtree(tmp_dest)
     return newloc
+
+
+# TODO hash sudy more:
+#   https://pip.pypa.io/en/stable/cli/pip_install/#requirements-file-format
+#   https://www.python.org/dev/peps/pep-0440/ 
+#   https://pip.pypa.io/en/stable/cli/pip_install/#hash-checking-mode