Commit fb3af4fc authored by Lucas Carvalho's avatar Lucas Carvalho

Cleanup the code to use the libnetworkcache.py

parent 66d44d8f
...@@ -14,10 +14,9 @@ ...@@ -14,10 +14,9 @@
import os import os
import urllib
import posixpath import posixpath
import base64
import hashlib import hashlib
from libnetworkcache import NetworkcacheClient
def _get_hash_from_file(path, hash_object): def _get_hash_from_file(path, hash_object):
...@@ -35,26 +34,16 @@ def _get_hash_from_file(path, hash_object): ...@@ -35,26 +34,16 @@ def _get_hash_from_file(path, hash_object):
f.close() f.close()
def _get_sha256_from_file(path):
"""
Return the sha256sum from file.
"""
if not os.path.isfile(path):
raise ValueError, 'Could not extract sha256sum.' \
' The path is not a file: %s ' % path
return _get_hash_from_file(path, hashlib.sha256())
def check_sha256sum(path, sha256sum): def check_sha256sum(path, sha256sum):
"""Tell whether the SHA256sum checksum of the file at path matches """Tell whether the SHA256sum checksum of the file at path matches
""" """
return sha256sum == _get_hash_from_file(path, hashlib.sha256()) return sha256sum == _get_hash_from_file(path, hashlib.sha256())
def _update_network_cached_log(sha256sum, url): def _update_network_cached_log(key, url):
"""Update the networkcached.log file. """Update the networkcached.log file.
Adding the last sha256sum and URL which has been retrieved from networkcached server. Adding the last key and URL which has been retrieved from networkcached server.
""" """
# Keep the log # Keep the log
...@@ -71,28 +60,12 @@ def _update_network_cached_log(sha256sum, url): ...@@ -71,28 +60,12 @@ def _update_network_cached_log(sha256sum, url):
break break
if not flag: if not flag:
f.seek(0, 2) f.seek(0, 2)
f.write('%s %s\n' % (sha256sum, url)) f.write('%s %s\n' % (key, url))
f.truncate(f.tell()) f.truncate(f.tell())
finally: finally:
f.close() f.close()
def get_sha256sum_from_networkcached(network_cache, url, logger):
"""
Return the sha256sum if the url exists in networkcached server.
"""
network_cached_url = os.path.join(network_cache,
'get', base64.encodestring(url))
try:
result = urllib.urlopen(network_cached_url)
if int(result.code) == 200:
return result.read().strip()
logger.info('The url is not cached yet: %s' % url)
except IOError, e:
logger.info('An error occurred to get sha256sum of url %s. %s' % \
(url, str(e)))
def download_network_cached(network_cache, path, url, logger): def download_network_cached(network_cache, path, url, logger):
"""Download from a network cache provider """Download from a network cache provider
...@@ -105,16 +78,18 @@ def download_network_cached(network_cache, path, url, logger): ...@@ -105,16 +78,18 @@ def download_network_cached(network_cache, path, url, logger):
# Not able to use network cache # Not able to use network cache
return False return False
sha256sum = get_sha256sum_from_networkcached(network_cache, url, logger) key = hashlib.sha256(url).hexdigest()
if sha256sum is None: network_cached_url = os.path.join(network_cache, key)
return False
network_cached_url = os.path.join(network_cache, sha256sum)
logger.info('Downloading from network cache %s' % network_cached_url) logger.info('Downloading from network cache %s' % network_cached_url)
try: try:
path, headers = urllib.urlretrieve(network_cached_url, path) nc = NetworkcacheClient(network_cache)
if not check_sha256sum(path, sha256sum): result, path = nc.get(key, path)
logger.info('MD5/SHA256 checksum mismatch downloading %r' % \ if result.status != 200:
logger.info('Fail to download from network cache: File Not Found.')
return False
if not check_sha256sum(path, result.getheader('x-sha256sum')):
logger.info('SHA256 checksum mismatch downloading %r' % \
network_cached_url) network_cached_url)
return False return False
except IOError, e: except IOError, e:
...@@ -132,23 +107,16 @@ def upload_network_cached(network_cache, external_url, path, logger): ...@@ -132,23 +107,16 @@ def upload_network_cached(network_cache, external_url, path, logger):
'Upload cache ignored, network-cache was not provided') 'Upload cache ignored, network-cache was not provided')
return False return False
sha256sum = _get_sha256_from_file(path) key = hashlib.sha256(external_url).hexdigest()
try: try:
f = open(path, 'r') nc = NetworkcacheClient(network_cache)
data = f.read() result = nc.put(key, path)
url = os.path.join(network_cache, sha256sum) if result.status == 200:
try: _update_network_cached_log(key, external_url)
result = urllib.urlopen(url, urllib.urlencode({
'data': base64.encodestring(data), except (IOError, EOFError), e:
'url': base64.encodestring(external_url)})) logger.info('Fail to upload cache on %s. %s' % \
if result.code == 200: (external_url, str(e)))
_update_network_cached_log(sha256sum, external_url)
except (IOError, EOFError), e:
logger.info('Fail to upload cache on %s. %s' % \
(external_url, str(e)))
finally:
f.close()
return True return True
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment