Commit fb3af4fc authored by Lucas Carvalho's avatar Lucas Carvalho

Cleanup the code to use the libnetworkcache.py

parent 66d44d8f
......@@ -14,10 +14,9 @@
import os
import urllib
import posixpath
import base64
import hashlib
from libnetworkcache import NetworkcacheClient
def _get_hash_from_file(path, hash_object):
......@@ -35,26 +34,16 @@ def _get_hash_from_file(path, hash_object):
f.close()
def _get_sha256_from_file(path):
"""
Return the sha256sum from file.
"""
if not os.path.isfile(path):
raise ValueError, 'Could not extract sha256sum.' \
' The path is not a file: %s ' % path
return _get_hash_from_file(path, hashlib.sha256())
def check_sha256sum(path, sha256sum):
"""Tell whether the SHA256sum checksum of the file at path matches
"""
return sha256sum == _get_hash_from_file(path, hashlib.sha256())
def _update_network_cached_log(sha256sum, url):
def _update_network_cached_log(key, url):
"""Update the networkcached.log file.
Adding the last sha256sum and URL which has been retrieved from networkcached server.
Adding the last key and URL which has been retrieved from networkcached server.
"""
# Keep the log
......@@ -71,28 +60,12 @@ def _update_network_cached_log(sha256sum, url):
break
if not flag:
f.seek(0, 2)
f.write('%s %s\n' % (sha256sum, url))
f.write('%s %s\n' % (key, url))
f.truncate(f.tell())
finally:
f.close()
def get_sha256sum_from_networkcached(network_cache, url, logger):
"""
Return the sha256sum if the url exists in networkcached server.
"""
network_cached_url = os.path.join(network_cache,
'get', base64.encodestring(url))
try:
result = urllib.urlopen(network_cached_url)
if int(result.code) == 200:
return result.read().strip()
logger.info('The url is not cached yet: %s' % url)
except IOError, e:
logger.info('An error occurred to get sha256sum of url %s. %s' % \
(url, str(e)))
def download_network_cached(network_cache, path, url, logger):
"""Download from a network cache provider
......@@ -105,16 +78,18 @@ def download_network_cached(network_cache, path, url, logger):
# Not able to use network cache
return False
sha256sum = get_sha256sum_from_networkcached(network_cache, url, logger)
if sha256sum is None:
return False
network_cached_url = os.path.join(network_cache, sha256sum)
key = hashlib.sha256(url).hexdigest()
network_cached_url = os.path.join(network_cache, key)
logger.info('Downloading from network cache %s' % network_cached_url)
try:
path, headers = urllib.urlretrieve(network_cached_url, path)
if not check_sha256sum(path, sha256sum):
logger.info('MD5/SHA256 checksum mismatch downloading %r' % \
nc = NetworkcacheClient(network_cache)
result, path = nc.get(key, path)
if result.status != 200:
logger.info('Fail to download from network cache: File Not Found.')
return False
if not check_sha256sum(path, result.getheader('x-sha256sum')):
logger.info('SHA256 checksum mismatch downloading %r' % \
network_cached_url)
return False
except IOError, e:
......@@ -132,23 +107,16 @@ def upload_network_cached(network_cache, external_url, path, logger):
'Upload cache ignored, network-cache was not provided')
return False
sha256sum = _get_sha256_from_file(path)
try:
f = open(path, 'r')
data = f.read()
url = os.path.join(network_cache, sha256sum)
key = hashlib.sha256(external_url).hexdigest()
try:
result = urllib.urlopen(url, urllib.urlencode({
'data': base64.encodestring(data),
'url': base64.encodestring(external_url)}))
if result.code == 200:
_update_network_cached_log(sha256sum, external_url)
nc = NetworkcacheClient(network_cache)
result = nc.put(key, path)
if result.status == 200:
_update_network_cached_log(key, external_url)
except (IOError, EOFError), e:
logger.info('Fail to upload cache on %s. %s' % \
(external_url, str(e)))
finally:
f.close()
return True
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment