check first credentials from netrc, remove lxml

setup.py

@@ -50,7 +50,6 @@ setup(
         'setuptools>=38.2.3',
         'pip',
         'wheel',
-        'lxml',
     ],
     include_package_data = True,
     entry_points = entry_points,

src/zc/buildout/download.py

@@ -44,7 +44,6 @@ import tempfile
 import zc.buildout
 from . import bytes2str, str2bytes
 from .rmtree import rmtree
-from lxml.html import parse as lxmlparse
 
 class netrc(netrc.netrc):
 
@@ -244,7 +243,7 @@ class Download(object):
                 download_url = alternate_url
                 self.urlretrieve(alternate_url, path)
             except GitlabAccessDeniedError:
-                header_dict, laburl  = self._labraw_authproxy(url)
+                laburl, header_dict = self._labraw_authproxy(url)
                 if len(header_dict.keys()) > 0:
                     # gitlab url, try from API
                     self.urlretrieve(laburl, path, headers=header_dict)
@@ -297,14 +296,12 @@ class Download(object):
 
     def _labraw_authproxy(self, url): # -> url'
         header_dict = {}
-        # url should be https://XXX.YYY/namespace/project/[-/]raw/....
-        if not re.match(r"https://[\w\-_\.\:\@\+]+/([\.\w\-\+_]+/[\.\w\-\+_]+/(-/){0,1}raw/)", url):
-            return header_dict, url
         p = urlparse(url)
         pathv = p.path.split('/')
 
-        if p.username == "PRIVATE-TOKEN" and p.password:
-            header_dict["PRIVATE-TOKEN"] = p.password
+        # url path should be /namespace/project/[-/]raw/....
+        if pathv[3:5] != ['-', 'raw'] and pathv[3:4] != ['raw']:
+            return url, header_dict
 
         repo = '/'.join(pathv[1:3])
         # FIXME this does not support refs like y/bstr.
@@ -313,20 +310,33 @@ class Download(object):
         # was doing - try to extract all variants for ref from longest to
         # shortest and stop on the first variant thay yields good result.
         if pathv[3] == '-': # the url is like .../-/raw/...
-          ref  = pathv[5]
-          filepath = '/'.join(pathv[6:])
+            ref  = pathv[5]
+            filepath = '/'.join(pathv[6:])
         else:
-          ref  = pathv[4]
-          filepath = '/'.join(pathv[5:])
+            ref  = pathv[4]
+            filepath = '/'.join(pathv[5:])
+
+        query = urlencode({'ref': ref})
+        auth_list = (
+             netrc.authenticators('%s/%s' % (p.hostname, repo)),    # auth for lab.nexedi.com/namespace/project
+             netrc.authenticators(p.hostname)    # auth for lab.nexedi.com
+        )
+        auth = auth_list[1] if auth_list[0] is None else auth_list[0]
+        if auth is not None:
+            if auth[0] == "private_token":
+                header_dict["PRIVATE-TOKEN"] = auth[1]
+            else:
+                query[auth[0]] = auth[2] # only private_token is supported ?
+        elif p.username == "PRIVATE-TOKEN" and p.password:
+            header_dict["PRIVATE-TOKEN"] = p.password
 
         qrepo     = quote(repo, '')
         qfilepath = quote(filepath, '')
 
         path  = '/api/v4/projects/%s/repository/files/%s/raw' % (qrepo, qfilepath)
-        query = urlencode({'ref': ref})
         netloc = '%s:%s' % (p.hostname, p.port) if p.port else p.hostname
 
-        return header_dict, urlunparse((p.scheme, netloc, path, p.params, query, p.fragment))
+        return urlunparse((p.scheme, netloc, path, p.params, query, p.fragment)), header_dict
 
 
     def urlretrieve(self, url, tmp_path, headers={}):
@@ -340,14 +350,14 @@ class Download(object):
         for k, v in headers.items():
             req.add_header(k, v)
         with closing(urlopen(req)) as src:
-            # Is this a gitlab raw URL ?
-            # Gitlab return to sign in page with code 200 if authentication failed.
-            if re.match(r"https://[\w\-_\.\:\@\+]+/([\.\w\-\+_]+/[\.\w\-\+_]+/(-/){0,1}raw/)", url):
-                parsed = lxmlparse(src)
-                page_title = parsed.find(".//title")
-                if page_title is not None and page_title.text.startswith("Sign in"):
-                    # the content is gitlab Sign in page
-                    raise GitlabAccessDeniedError("You have been redirected to Sign in page")
+            try:
+                # If Access denied to gitlab url, we have response 200 here
+                # and url is to BASE_URL/users/sign_in
+                if src.url.rindex("users/sign_in"):
+                    raise GitlabAccessDeniedError("Redirected to Sign in page")
+            except ValueError:
+                # nothing to do
+                pass
             with open(tmp_path, 'wb') as dst:
                 shutil.copyfileobj(src, dst)
             return tmp_path, src.info()