Commit 1b59eb89 authored by Lucas Carvalho's avatar Lucas Carvalho

Now slapos.buildout is working again if the new implementation.

After this commit it is possible to download and upload anything to
SHACACHE and SHADIR servers.

It uses slapos.libnetworkcache to handle the download and upload.
parent 9010daa6
...@@ -17,8 +17,10 @@ import hashlib ...@@ -17,8 +17,10 @@ import hashlib
import os import os
import posixpath import posixpath
import re import re
import urllib
import urlparse
from download import check_md5sum from download import check_md5sum
from slapos.tool.libnetworkcache import NetworkcacheClient from slapos.libnetworkcache import NetworkcacheClient, UploadError
_md5_re = re.compile(r'md5=([a-f0-9]+)') _md5_re = re.compile(r'md5=([a-f0-9]+)')
...@@ -34,7 +36,7 @@ def _get_md5_from_url(url): ...@@ -34,7 +36,7 @@ def _get_md5_from_url(url):
def _update_network_cached_log(path, url): def _update_network_cached_log(path, url):
"""Update the networkcached.log file. """Update the networkcached.log file.
Adding the last key and URL which has been retrieved from networkcached server. Adding the last key and URL which has been retrieved from shacache server.
""" """
f = open(path, 'r') f = open(path, 'r')
...@@ -63,7 +65,31 @@ def _update_network_cached_log(path, url): ...@@ -63,7 +65,31 @@ def _update_network_cached_log(path, url):
f.close() f.close()
def download_network_cached(network_cache, path, url, logger, md5sum=None): def select_sha512sum_from_shadir(sha_dir, url, logger):
"""It must return the sha512sum if exists on selected directory. """
# On buildout.cfg the option sha-dir is the complete link including the
# directory key, so we must to extract it to use the libnetworkcache
# correctly.
directory_key = sha_dir.split('/')[-1]
sha_dir_without_directory_key = '/'.join(sha_dir.split('/')[:-1])
try:
nc = NetworkcacheClient(shadir=sha_dir_without_directory_key)
data_list = nc.select(directory_key)
except (IOError, UploadError), e:
logger.info('Could not retrieve the information from SHADIR. %s' % e)
return None
urlmd5 = hashlib.md5(url).hexdigest()
for data in data_list:
information_dict, signature = data
if information_dict.get('urlmd5') == urlmd5:
return information_dict.get('sha512')
return None
def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
"""Download from a network cache provider """Download from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore If something fail (providor be offline, or hash_string fail), we ignore
...@@ -71,68 +97,63 @@ def download_network_cached(network_cache, path, url, logger, md5sum=None): ...@@ -71,68 +97,63 @@ def download_network_cached(network_cache, path, url, logger, md5sum=None):
return True if download succeeded. return True if download succeeded.
""" """
if network_cache in (None, '',): if sha_cache in (None, '',):
# Not able to use network cache # Not able to use network cache
return False return False
if md5sum is None: if md5sum is None:
print "URL ", url
md5sum = _get_md5_from_url(url) md5sum = _get_md5_from_url(url)
# XXX (lucas): This will be removed.
key = hashlib.sha512(url).hexdigest()
network_cached_url = os.path.join(network_cache, key)
logger.info('Downloading from network cache %s' % network_cached_url)
try:
nc = NetworkcacheClient(network_cache)
result = nc.get(key)
if result.status != 200: sha512sum = select_sha512sum_from_shadir(sha_dir, url, logger)
logger.info('Fail to download from network cache: File Not Found.') if sha512sum is None:
return False return False
file_content = result.read() sha_cache_url = os.path.join(sha_cache, sha512sum)
# save the file logger.info('Downloading from network cache %s' % sha_cache_url)
f = open(path, 'w+b')
try: try:
f.write(file_content) nc = NetworkcacheClient(shacache=sha_cache)
finally: nc.download(path, sha512sum)
f.close()
if not check_md5sum(path, md5sum): if not check_md5sum(path, md5sum):
logger.info('MD5 checksum mismatch downloading %r' % \ logger.info('MD5 checksum mismatch downloading %r' % sha_cache_url)
network_cached_url)
return False return False
except IOError, e: except IOError, e:
logger.info('Fail to download from network cache %s: %s' % \ logger.info('Fail to download from network cache %s: %s' % \
(network_cached_url, str(e))) (sha_cache_url, str(e)))
return False return False
return True return True
def upload_network_cached(network_cache, external_url, path, logger): def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
"""Upload file to a network cache server""" """Upload file to a shacache server"""
if network_cache in [None, '']: if sha_cache in [None, ''] or sha_dir in [None, '']:
logger.debug( logger.debug(
'Upload cache ignored, network-cache was not provided') 'Upload cache ignored, shacache or shadir was not provided!')
return False return False
try: # On buildout.cfg the option sha-dir is the complete link including the
# directory key, so we must to extract it to use the libnetworkcache
# correctly.
directory_key = sha_dir.split('/')[-1]
sha_dir_without_directory_key = '/'.join(sha_dir.split('/')[:-1])
f = open(path, 'r') f = open(path, 'r')
try: try:
file_content = f.read()
finally:
f.close()
nc = NetworkcacheClient(network_cache) kw = dict(file_name=os.path.basename(external_url),
result = nc.put(file_content) urlmd5=hashlib.md5(external_url).hexdigest())
if result.status == 200:
nc = NetworkcacheClient(shacache=sha_cache,
shadir=sha_dir_without_directory_key)
data = nc.upload(f, directory_key, **kw)
_update_network_cached_log(path, external_url) _update_network_cached_log(path, external_url)
except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % \
(str(e)))
return False
finally:
f.close()
except (IOError, EOFError), e:
logger.info('Fail to upload cache on %s. %s' % \
(external_url, str(e)))
return True return True
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment