From 0690e74303b2a9b793a5a1cbfa49366a60445243 Mon Sep 17 00:00:00 2001 From: Yingjie Xu <yxu@nexedi.com> Date: Thu, 26 Jan 2012 23:32:33 +0100 Subject: [PATCH] Implement download. --- slapos/grid/SlapObject.py | 50 +++++++++++++++++++++++-------------- slapos/grid/networkcache.py | 36 +++++++++++--------------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/slapos/grid/SlapObject.py b/slapos/grid/SlapObject.py index 7cfa79d20..6c8b30730 100644 --- a/slapos/grid/SlapObject.py +++ b/slapos/grid/SlapObject.py @@ -82,25 +82,37 @@ class Software(object): tarname = self.software_url_hash cache_dir = tempfile.mkdtemp() tarpath = os.path.join(cache_dir, tarname) - self._install_from_buildout() - tar = tarfile.open(tarpath, "w:gz") - try: - tar.add(self.software_path, arcname=self.software_url_hash) - finally: - tar.close() - upload_network_cached( - self.software_root, - self.url, - self.software_url_hash, - self.upload_binary_cache_url, - self.upload_binary_dir_url, - tarpath, self.logger, - self.signature_private_key_file, - self.shacache_cert_file, - self.shacache_key_file, - self.shadir_cert_file, - self.shadir_key_file - ) + if os.path.exists(self.software_path): + self._install_from_buildout() + else: + if download_network_cached( + self.download_binary_cache_url, + self.download_binary_dir_url, + self.url, self.software_url_hash, + tarpath, self.logger, + self.signature_certificate_list): + tar = tarfile.open(tarpath) + try: + tar.extractall(path=self.software_root) + finally: + tar.close() + else: + tar = tarfile.open(tarpath, "w:gz") + try: + tar.add(self.software_path, arcname=self.software_url_hash) + finally: + tar.close() + upload_network_cached( + self.software_root, + self.url, self.software_url_hash, + self.upload_binary_cache_url, + self.upload_binary_dir_url, + tarpath, self.logger, + self.signature_private_key_file, + self.shacache_cert_file, + self.shacache_key_file, + self.shadir_cert_file, + self.shadir_key_file) def _install_from_buildout(self): """ Fetches buildout configuration from the server, run buildout with diff --git a/slapos/grid/networkcache.py b/slapos/grid/networkcache.py index 8ca3fdfe7..1346ad27d 100644 --- a/slapos/grid/networkcache.py +++ b/slapos/grid/networkcache.py @@ -21,6 +21,7 @@ import shutil import urlparse import traceback import utils +import json try: try: @@ -53,29 +54,19 @@ def fallback_call(function): @fallback_call -def download_network_cached(dir_url, cache_url, path, url, logger, - signature_certificate_list, md5sum=None): +def download_network_cached(cache_url, dir_url, software_url, key, path, + logger, signature_certificate_list): """Downloads from a network cache provider - If something fail (providor be offline, or hash_string fail), we ignore - network cached files. - return True if download succeeded. """ if not LIBNETWORKCACHE_ENABLED: return False - if not(dir_url and cache_url): + if not(cache_url and dir_url): return False - if md5sum is None: - md5sum = _get_md5_from_url(url) - - - directory_key = get_directory_key(url) - url = os.path.basename(url) if len(signature_certificate_list) == 0: - # convert [] into None in order to call nc nicely signature_certificate_list = None try: nc = NetworkcacheClient(cache_url, dir_url, @@ -84,21 +75,24 @@ def download_network_cached(dir_url, cache_url, path, url, logger, logger.warning('Incompatible version of networkcache, not using it.') return False - logger.info('Downloading %s from network cache.' % url) + logger.info('Downloading %s binary from network cache.' % software_url) try: - file_descriptor = nc.select(directory_key) - + json_entry_list = nc.select_generic(key) + for entry in json_entry_list: + json_information, _ = entry + try: + tags = json.loads(json_information) + sha512 = tags.get('sha512') + file_descriptor = nc.download(sha512) + break + except Exception: + continue f = open(path, 'w+b') try: shutil.copyfileobj(file_descriptor, f) finally: f.close() file_descriptor.close() - - if not check_md5sum(path, md5sum): - logger.info('MD5 checksum mismatch downloading %s' % url) - return False - except (IOError, DirectoryNotFound), e: logger.info('Failed to download from network cache %s: %s' % \ (url, str(e))) -- 2.30.9