Commit 79c38fb3 authored by Kirill Smelkov's avatar Kirill Smelkov

Fix handling GeoLite2 URLs

- handle hash in url of slapos.recipe.build:download
  A hash there is instruction to verify checksum of the tarball, not
  part of the real URL.

- fix namever to detect that '-Country' is not version suffix.

Before this patch BOM entry for GeoLite2-Country was emittted as:

    GeoLite2                     Country.tar.gz#dc6224c648350d90f344a0c5c3ca5474 http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#dc6224c648350d90f344a0c5c3ca5474

Noticed while computing BOM(Re6stnet).
parent 9b39f658
...@@ -124,9 +124,17 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo ...@@ -124,9 +124,17 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
addbom(url, '') # XXX detect kind? addbom(url, '') # XXX detect kind?
elif recipe in ('slapos.recipe.build:download', 'slapos.recipe.build:download-unpacked'): elif recipe in ('slapos.recipe.build:download', 'slapos.recipe.build:download-unpacked'):
url = geturl(part)
# strip off hash from the url
vhash = None
h = url.rfind('#')
if h != -1:
url, vhash = url[:h], url[h+1:]
vhash = vhash[:8] # abbreviate
# slapos.recipe.build:download is often used to download .conf files, but sometimes it is used to download e.g. binaries # slapos.recipe.build:download is often used to download .conf files, but sometimes it is used to download e.g. binaries
# skip the part, if we can detect that downloaded item is a configuration file # skip the part, if we can detect that downloaded item is a configuration file
url = geturl(part)
if isconf(url): if isconf(url):
continue continue
...@@ -139,7 +147,7 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo ...@@ -139,7 +147,7 @@ def bom_software(installed_software_path): # -> {} (name,kind) -> PkgInfo
# let's see if maybe its an archive from known place # let's see if maybe its an archive from known place
_ = namever(url, failonerr=False) _ = namever(url, failonerr=False)
if _ is not None: if _ is not None:
addbom(url, '') # XXX detect kind addbom(url, '', vhash) # XXX detect kind
continue continue
raise NotImplementedError('%s uses %s with url that does not look like a .conf, archive or SR file: %s' % (s, recipe, url)) raise NotImplementedError('%s uses %s with url that does not look like a .conf, archive or SR file: %s' % (s, recipe, url))
...@@ -315,7 +323,10 @@ def _namever(url, failonerr): ...@@ -315,7 +323,10 @@ def _namever(url, failonerr):
if m is not None: if m is not None:
name = filename[:m.start()] name = filename[:m.start()]
ver = filename[m.start()+1:] ver = filename[m.start()+1:]
if re.search(r'[0-9]', ver):
return name, ver return name, ver
else:
return filename, None # no version
m = re.search(r'\.v.+$', filename) # jpegsrc.v9d m = re.search(r'\.v.+$', filename) # jpegsrc.v9d
if m is not None: if m is not None:
......
...@@ -39,6 +39,7 @@ from os.path import dirname, exists ...@@ -39,6 +39,7 @@ from os.path import dirname, exists
('https://osdn.net/frs/redir.php?f=ipafonts%2F57330%2FIPAexfont00201.zip', 'IPAexfont', '00201'), ('https://osdn.net/frs/redir.php?f=ipafonts%2F57330%2FIPAexfont00201.zip', 'IPAexfont', '00201'),
('https://osdn.net/frs/redir.php?f=ipafonts%2F51868%2FIPAfont00303.zip', 'IPAfont', '00303'), ('https://osdn.net/frs/redir.php?f=ipafonts%2F51868%2FIPAfont00303.zip', 'IPAfont', '00303'),
('https://osdn.net/frs/redir.php?f=tsukurimashou%2F56948%2Focr-0.2.zip', 'ocr', '0.2'), ('https://osdn.net/frs/redir.php?f=tsukurimashou%2F56948%2Focr-0.2.zip', 'ocr', '0.2'),
('http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz', 'GeoLite2-Country', None),
]) ])
def test_namever(url, nameok, verok): def test_namever(url, nameok, verok):
assert nxdbom.namever(url) == (nameok, verok) assert nxdbom.namever(url) == (nameok, verok)
...@@ -118,6 +119,15 @@ url = /BASE/stack/erp5/instance-mariadb-resiliency-after-import-script.sh.in ...@@ -118,6 +119,15 @@ url = /BASE/stack/erp5/instance-mariadb-resiliency-after-import-script.sh.in
_profile_base_location_ = /BASE _profile_base_location_ = /BASE
""", '') """, '')
# slapos.build.:download with hash in the url
case1("""\
[geolite2-country]
recipe = slapos.recipe.build:download-unpacked
url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#dc6224c648350d90f344a0c5c3ca5474
""", """\
GeoLite2-Country dc6224c6 http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz
""")
for x in ('gcc', 'python', 'ZODB', 'ZEO', 'tempstorage'): for x in ('gcc', 'python', 'ZODB', 'ZEO', 'tempstorage'):
case1("""\ case1("""\
[%s] [%s]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment