download: add netrc file support

Like for URL that contain credentials, we still skip auth challenge
because it's faster and:
- we only support one auth scheme (basic)
- netrc provides no way to specify realms, which seem anyway to be
  less and less used (https://stackoverflow.com/q/69303610 reports
  that recent browsers don't display them anymore)

See merge request nexedi/slapos.buildout!25

src/zc/buildout/download.py

@@ -20,18 +20,20 @@ except ImportError:
 
 try:
     # Python 3
-    from urllib.request import Request, splituser, urlopen
+    from urllib.request import Request, splitport, splituser, urlopen
     from urllib.parse import urlparse, urlunparse
 except ImportError:
     # Python 2
     from urlparse import urlparse
     from urlparse import urlunparse
-    from urllib2 import Request, splituser, urlopen
+    from urllib2 import Request, splitport, splituser, urlopen
 
 from zc.buildout.easy_install import realpath
 from base64 import b64encode
 from contextlib import closing
+import errno
 import logging
+import netrc
 import os
 import os.path
 import re
@@ -40,6 +42,23 @@ import tempfile
 import zc.buildout
 from . import bytes2str, str2bytes
 
+class netrc(netrc.netrc):
+
+    def __init__(*args):
+        pass
+
+    def authenticators(self, host):
+        self.__class__, = self.__class__.__bases__
+        try:
+            self.__init__()
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            self.__init__(os.devnull)
+        return self.authenticators(host)
+
+netrc = netrc()
+
 class ChecksumError(zc.buildout.UserError):
     pass
 
@@ -258,13 +277,19 @@ class Download(object):
     def urlretrieve(self, url, tmp_path):
         scheme, netloc, path, params, query, frag = urlparse(url)
         req = url
-        if scheme in ('http', 'https'):
+        while scheme in ('http', 'https'): # not a loop
             auth, host = splituser(netloc)
             if auth:
                 url = urlunparse((scheme, host, path, params, query, frag))
+            else:
+                auth = netrc.authenticators(splitport(host)[0])
+                if not auth:
+                    break
+                auth = '{0}:{2}'.format(*auth)
             req = Request(url)
             req.add_header("Authorization",
                            "Basic " + bytes2str(b64encode(str2bytes(auth))))
+            break
         with closing(urlopen(req)) as src:
             with open(tmp_path, 'wb') as dst:
                 shutil.copyfileobj(src, dst)

src/zc/buildout/download.txt

@@ -166,6 +166,24 @@ True
 Traceback (most recent call last):
 UserError: Error downloading ...: HTTP Error 403: Forbidden
 
+... with netrc:
+
+>>> url = server_url + 'private/foo:bar'
+>>> download(url)
+Traceback (most recent call last):
+UserError: Error downloading ...: HTTP Error 403: Forbidden
+>>> import os, zc.buildout.download
+>>> old_home = os.environ['HOME']
+>>> home = os.environ['HOME'] = tmpdir('test-netrc')
+>>> netrc = join(home, '.netrc')
+>>> write(netrc, 'machine localhost login foo password bar')
+>>> os.chmod(netrc, 0o600)
+>>> zc.buildout.download.netrc.__init__()
+>>> path, is_temp = download(url)
+>>> is_temp; remove(path)
+True
+>>> os.environ['HOME'] = old_home
+
 Downloading using the download cache
 ------------------------------------