Commit 63b17738 authored by Łukasz Nowak's avatar Łukasz Nowak

slapgrid: binary cache only option

/reviewed-on nexedi/slapos.core!173
parent 78f01daf
Pipeline #7389 failed with stage
in 0 seconds
...@@ -162,3 +162,7 @@ upload-to-binary-cache-url-blacklist = ...@@ -162,3 +162,7 @@ upload-to-binary-cache-url-blacklist =
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/heads http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/heads
/ /
# List of URL(s) which are forced to be downloaded from binary cache
#download-from-binary-cache-force-url-list =
# https://lab.node.vifib.com/nexedi/slapos/raw/1.0.
...@@ -112,6 +112,7 @@ class Software(object): ...@@ -112,6 +112,7 @@ class Software(object):
download_binary_dir_url=None, upload_binary_dir_url=None, download_binary_dir_url=None, upload_binary_dir_url=None,
download_from_binary_cache_url_blacklist=None, download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None, upload_to_binary_cache_url_blacklist=None,
download_from_binary_cache_force_url_list=None,
software_min_free_space=None, software_min_free_space=None,
buildout_debug=False, buildout_debug=False,
shared_part_list=''): shared_part_list=''):
...@@ -124,6 +125,9 @@ class Software(object): ...@@ -124,6 +125,9 @@ class Software(object):
if upload_to_binary_cache_url_blacklist is None: if upload_to_binary_cache_url_blacklist is None:
upload_to_binary_cache_url_blacklist = [] upload_to_binary_cache_url_blacklist = []
if download_from_binary_cache_force_url_list is None:
download_from_binary_cache_force_url_list = []
self.url = url self.url = url
self.software_root = software_root self.software_root = software_root
self.software_url_hash = md5digest(self.url) self.software_url_hash = md5digest(self.url)
...@@ -151,6 +155,8 @@ class Software(object): ...@@ -151,6 +155,8 @@ class Software(object):
download_from_binary_cache_url_blacklist download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \ self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist upload_to_binary_cache_url_blacklist
self.download_from_binary_cache_force_url_list = \
download_from_binary_cache_force_url_list
self.software_min_free_space = software_min_free_space self.software_min_free_space = software_min_free_space
def check_free_space(self): def check_free_space(self):
...@@ -175,6 +181,12 @@ class Software(object): ...@@ -175,6 +181,12 @@ class Software(object):
try: try:
tarpath = os.path.join(cache_dir, self.software_url_hash) tarpath = os.path.join(cache_dir, self.software_url_hash)
# Check if we can download from cache # Check if we can download from cache
force_binary_cache = False
for url_match in self.download_from_binary_cache_force_url_list:
if self.url.startswith(url_match):
force_binary_cache = True
self.logger.debug('Binary cache forced for %r because of match %r', self.url, url_match)
break
if (not os.path.exists(self.software_path)) \ if (not os.path.exists(self.software_path)) \
and download_network_cached( and download_network_cached(
self.download_binary_cache_url, self.download_binary_cache_url,
...@@ -190,6 +202,10 @@ class Software(object): ...@@ -190,6 +202,10 @@ class Software(object):
tar.extractall(path=self.software_root) tar.extractall(path=self.software_root)
finally: finally:
tar.close() tar.close()
elif force_binary_cache:
message = 'Binary cache forced for %r, but failed to download, will retry again' % (self.url,)
self.logger.error(message)
raise BuildoutFailedError(message)
else: else:
self._install_from_buildout() self._install_from_buildout()
# Upload to binary cache if possible and allowed # Upload to binary cache if possible and allowed
......
...@@ -188,6 +188,9 @@ def merged_options(args, configp): ...@@ -188,6 +188,9 @@ def merged_options(args, configp):
options["upload-to-binary-cache-url-blacklist"] = [ options["upload-to-binary-cache-url-blacklist"] = [
url.strip() for url in options.get( url.strip() for url in options.get(
"upload-to-binary-cache-url-blacklist", "").split('\n') if url] "upload-to-binary-cache-url-blacklist", "").split('\n') if url]
options["download-from-binary-cache-force-url-list"] = [
url.strip() for url in options.get(
"download-from-binary-cache-force-url-list", "").split('\n') if url]
options['firewall'] = {} options['firewall'] = {}
if configp.has_section('firewall'): if configp.has_section('firewall'):
...@@ -259,6 +262,8 @@ def create_slapgrid_object(options, logger): ...@@ -259,6 +262,8 @@ def create_slapgrid_object(options, logger):
op.get('download-from-binary-cache-url-blacklist', []), op.get('download-from-binary-cache-url-blacklist', []),
upload_to_binary_cache_url_blacklist= upload_to_binary_cache_url_blacklist=
op.get('upload-to-binary-cache-url-blacklist', []), op.get('upload-to-binary-cache-url-blacklist', []),
download_from_binary_cache_force_url_list=
op.get('download-from-binary-cache-force-url-list', []),
upload_cache_url=op.get('upload-cache-url'), upload_cache_url=op.get('upload-cache-url'),
download_binary_dir_url=op.get('download-binary-dir-url'), download_binary_dir_url=op.get('download-binary-dir-url'),
upload_binary_dir_url=op.get('upload-binary-dir-url'), upload_binary_dir_url=op.get('upload-binary-dir-url'),
...@@ -317,6 +322,7 @@ class Slapgrid(object): ...@@ -317,6 +322,7 @@ class Slapgrid(object):
upload_binary_cache_url=None, upload_binary_cache_url=None,
download_from_binary_cache_url_blacklist=None, download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None, upload_to_binary_cache_url_blacklist=None,
download_from_binary_cache_force_url_list=None,
upload_cache_url=None, upload_cache_url=None,
download_binary_dir_url=None, download_binary_dir_url=None,
upload_binary_dir_url=None, upload_binary_dir_url=None,
...@@ -362,6 +368,8 @@ class Slapgrid(object): ...@@ -362,6 +368,8 @@ class Slapgrid(object):
download_from_binary_cache_url_blacklist download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \ self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist upload_to_binary_cache_url_blacklist
self.download_from_binary_cache_force_url_list = \
download_from_binary_cache_force_url_list
self.upload_cache_url = upload_cache_url self.upload_cache_url = upload_cache_url
self.download_binary_dir_url = download_binary_dir_url self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url self.upload_binary_dir_url = upload_binary_dir_url
...@@ -555,6 +563,8 @@ stderr_logfile_backups=1 ...@@ -555,6 +563,8 @@ stderr_logfile_backups=1
self.download_from_binary_cache_url_blacklist, self.download_from_binary_cache_url_blacklist,
upload_to_binary_cache_url_blacklist= upload_to_binary_cache_url_blacklist=
self.upload_to_binary_cache_url_blacklist, self.upload_to_binary_cache_url_blacklist,
download_from_binary_cache_force_url_list=
self.download_from_binary_cache_force_url_list,
upload_cache_url=self.upload_cache_url, upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url, download_binary_dir_url=self.download_binary_dir_url,
upload_binary_dir_url=self.upload_binary_dir_url, upload_binary_dir_url=self.upload_binary_dir_url,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment