Commit 37b86728 authored by Julien Muchembled's avatar Julien Muchembled

Review API of select() to prepare removal of networkcachehelper

This also fixes a bug returning wrong metadata when several entries exists
in shadirs.
parent 80c85d82
...@@ -17,6 +17,7 @@ import ConfigParser ...@@ -17,6 +17,7 @@ import ConfigParser
import hashlib import hashlib
import httplib import httplib
import json import json
import logging
import os import os
import shutil import shutil
import sys import sys
...@@ -33,6 +34,8 @@ TIMEOUT = 60 ...@@ -33,6 +34,8 @@ TIMEOUT = 60
# Same here. We just wait longer that, after having uploaded the file, the server digests it. It can take time. # Same here. We just wait longer that, after having uploaded the file, the server digests it. It can take time.
UPLOAD_TIMEOUT = 60 * 60 UPLOAD_TIMEOUT = 60 * 60
logger = logging.getLogger('networkcache')
logger.setLevel(logging.INFO)
class NetworkcacheClient(object): class NetworkcacheClient(object):
...@@ -222,26 +225,33 @@ class NetworkcacheClient(object): ...@@ -222,26 +225,33 @@ class NetworkcacheClient(object):
''' '''
return self._request('cache', sha512sum) return self._request('cache', sha512sum)
def select(self, key): def select(self, key, wanted_metadata_dict={}, required_key_list=frozenset()):
''' Download a file from shacache by selecting the entry in shadir '''Return an iterator over shadir entries that match given criteria
Raise DirectoryNotFound if no trustable file is found.
''' '''
for information_json, signature in self.select_generic(key, required_key_test = frozenset(required_key_list).issubset
self.signature_certificate_list):
break
else:
raise DirectoryNotFound('Could not find a trustable entry.')
try:
information_dict = json.loads(information_json)
except Exception:
raise DirectoryNotFound('Failed to parse json-in-json response:\n%s'
% traceback.format_exc())
try: try:
sha512 = information_dict['sha512'] data_list = self.select_generic(key, self.signature_certificate_list)
except Exception: except (urllib2.HTTPError, DirectoryNotFound), e:
raise DirectoryNotFound('No sha512 in directory response (%r):\n%s' logger.info(str(e))
% (information_dict, traceback.format_exc())) return
return self.download(sha512) for information_json, signature in data_list:
try:
information_dict = json.loads(information_json)
except Exception:
logger.info('Failed to parse json-in-json response', exc_info=1)
continue
try:
len(information_dict['sha512'])
except StandardError:
logger.info('Bad or missing sha512 in directory response (%r)',
information_dict)
continue
if required_key_test(information_dict):
for k, v in wanted_metadata_dict.iteritems():
if information_dict.get(k) != v:
break
else:
yield information_dict
def select_generic(self, key, filter=True): def select_generic(self, key, filter=True):
''' Select trustable entries from shadir. ''' Select trustable entries from shadir.
......
...@@ -4,6 +4,8 @@ import errno ...@@ -4,6 +4,8 @@ import errno
import hashlib import hashlib
import httplib import httplib
import json import json
import logging
import logging.handlers
import os import os
import urllib2 import urllib2
import random import random
...@@ -20,6 +22,8 @@ import slapos.signature ...@@ -20,6 +22,8 @@ import slapos.signature
import sys import sys
from cStringIO import StringIO from cStringIO import StringIO
logging.basicConfig()
class NCHandler(BaseHTTPServer.BaseHTTPRequestHandler): class NCHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def __init__(self, request, address, server): def __init__(self, request, address, server):
...@@ -173,6 +177,8 @@ class OnlineMixin: ...@@ -173,6 +177,8 @@ class OnlineMixin:
self.test_string = str(random.random()) self.test_string = str(random.random())
self.test_data = StringIO(self.test_string) self.test_data = StringIO(self.test_string)
self.test_shasum = hashlib.sha512(self.test_string).hexdigest() self.test_shasum = hashlib.sha512(self.test_string).hexdigest()
self.handler = logging.handlers.BufferingHandler(float('inf'))
slapos.libnetworkcache.logger.addHandler(self.handler)
def tearDown(self): def tearDown(self):
if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ: if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ:
...@@ -184,13 +190,22 @@ class OnlineMixin: ...@@ -184,13 +190,22 @@ class OnlineMixin:
if self.thread is not None: if self.thread is not None:
self.thread.join() self.thread.join()
shutil.rmtree(self.tree) shutil.rmtree(self.tree)
slapos.libnetworkcache.logger.removeHandler(self.handler)
del self.handler
def assertLog(self, msg=None):
try:
self.assertTrue(self.handler.buffer.pop(0).message.startswith(msg))
except IndexError:
self.assertEqual(msg, None)
def assertDirectoryNotFound(self, msg, func, *args, **kw): def select(self, nc, key, *args):
try: try:
func(*args, **kw) return nc.download(nc.select(key).next()['sha512'])
self.fail("DirectoryNotFound not raised") except StopIteration:
except slapos.libnetworkcache.DirectoryNotFound, e: for msg in args:
self.assertTrue(str(e).startswith(msg)) self.assertLog(msg)
self.assertLog()
key = """-----BEGIN RSA PRIVATE KEY----- key = """-----BEGIN RSA PRIVATE KEY-----
MIICXQIBAAKBgQDDrOO87nSiDcXOf+xGc4Iqcdjfwd0RTOxEkO9z8mPZVg2bTPwt MIICXQIBAAKBgQDDrOO87nSiDcXOf+xGc4Iqcdjfwd0RTOxEkO9z8mPZVg2bTPwt
...@@ -341,7 +356,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -341,7 +356,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
urlmd5 = str(random.random()) urlmd5 = str(random.random())
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file') nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
result = nc.select(key) result = self.select(nc, key)
self.assertEqual(result.read(), self.test_string) self.assertEqual(result.read(), self.test_string)
def test_upload_shadir_select_not_exists(self): def test_upload_shadir_select_not_exists(self):
...@@ -350,10 +365,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -350,10 +365,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
urlmd5 = str(random.random()) urlmd5 = str(random.random())
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file') nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
try: self.select(nc, 'key_another_key' + str(random.random()),
nc.select('key_another_key' + str(random.random())) 'HTTP Error 404: Not Found')
except urllib2.HTTPError, error:
self.assertEqual(error.code, httplib.NOT_FOUND)
def test_upload_shadir_no_filename(self): def test_upload_shadir_no_filename(self):
"""Check scenario with shadir used, but not filename passed""" """Check scenario with shadir used, but not filename passed"""
...@@ -385,6 +398,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -385,6 +398,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir) nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
try: try:
nc.download(self.test_shasum) nc.download(self.test_shasum)
self.fail("HTTPError not raised")
except urllib2.HTTPError, error: except urllib2.HTTPError, error:
self.assertEqual(error.code, httplib.NOT_FOUND) self.assertEqual(error.code, httplib.NOT_FOUND)
...@@ -398,7 +412,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -398,7 +412,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
signed_nc = slapos.libnetworkcache.NetworkcacheClient( signed_nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir, key_file.name, [self.certificate]) self.shacache, self.shadir, key_file.name, [self.certificate])
signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name) signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
result = signed_nc.select(key) result = self.select(signed_nc, key)
self.assertEqual(result.read(), self.test_string) self.assertEqual(result.read(), self.test_string)
def test_select_signed_content_several_certificates(self): def test_select_signed_content_several_certificates(self):
...@@ -412,7 +426,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -412,7 +426,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
self.shacache, self.shadir, key_file.name, self.shacache, self.shadir, key_file.name,
(self.alternate_certificate, self.certificate)) (self.alternate_certificate, self.certificate))
signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name) signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
result = signed_nc.select(key) result = self.select(signed_nc, key)
self.assertEqual(result.read(), self.test_string) self.assertEqual(result.read(), self.test_string)
def test_select_signed_content_multiple(self): def test_select_signed_content_multiple(self):
...@@ -426,7 +440,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -426,7 +440,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
self.shacache, self.shadir, key_file.name, [self.certificate]) self.shacache, self.shadir, key_file.name, [self.certificate])
signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name) signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name) signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
result = signed_nc.select(key) result = self.select(signed_nc, key)
self.assertEqual(result.read(), self.test_string) self.assertEqual(result.read(), self.test_string)
def test_select_no_entries(self): def test_select_no_entries(self):
...@@ -440,8 +454,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -440,8 +454,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
f = os.path.join(self.tree, 'shadir', key) f = os.path.join(self.tree, 'shadir', key)
# now remove the entry from shacache # now remove the entry from shacache
open(f, 'w').write(json.dumps([])) open(f, 'w').write(json.dumps([]))
self.assertDirectoryNotFound('Could not find a trustable entry.', self.assertEqual(self.select(nc, key), None)
nc.select, key)
def test_select_no_json_response(self): def test_select_no_json_response(self):
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
...@@ -454,8 +467,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -454,8 +467,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
with open(os.path.join(self.tree, 'shadir', key), 'w') as f: with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache # now remove the entry from shacache
f.write('This is not a json.') f.write('This is not a json.')
self.assertDirectoryNotFound('Failed to parse json response', self.select(nc, key, 'Failed to parse json response')
nc.select, key)
def test_select_json_no_in_json_response(self): def test_select_json_no_in_json_response(self):
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
...@@ -468,8 +480,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -468,8 +480,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
with open(os.path.join(self.tree, 'shadir', key), 'w') as f: with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache # now remove the entry from shacache
f.write(json.dumps([['This is not a json.', 'signature']])) f.write(json.dumps([['This is not a json.', 'signature']]))
self.assertDirectoryNotFound('Failed to parse json-in-json response', self.select(nc, key, 'Failed to parse json-in-json response')
nc.select, key)
def test_select_json_in_json_no_dict(self): def test_select_json_in_json_no_dict(self):
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
...@@ -482,8 +493,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -482,8 +493,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
with open(os.path.join(self.tree, 'shadir', key), 'w') as f: with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache # now remove the entry from shacache
f.write(json.dumps([[json.dumps('This is a string'), 'signature']])) f.write(json.dumps([[json.dumps('This is a string'), 'signature']]))
self.assertDirectoryNotFound('No sha512 in directory response', self.select(nc, key, 'Bad or missing sha512 in directory response')
nc.select, key)
def test_select_signed_content_server_hacked(self): def test_select_signed_content_server_hacked(self):
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
...@@ -509,8 +519,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -509,8 +519,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
# ...but as he has no access to key, no way to sign data.. # ...but as he has no access to key, no way to sign data..
# hacked_json[0][1] is still a good key # hacked_json[0][1] is still a good key
open(f, 'w').write(json.dumps(hacked_json)) open(f, 'w').write(json.dumps(hacked_json))
self.assertDirectoryNotFound('Could not find a trustable entry.', self.assertEqual(self.select(signed_nc, key), None)
signed_nc.select, key)
def test_DirectoryNotFound_non_trustable_entry(self): def test_DirectoryNotFound_non_trustable_entry(self):
key_file = tempfile.NamedTemporaryFile() key_file = tempfile.NamedTemporaryFile()
...@@ -526,18 +535,17 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -526,18 +535,17 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
self.shacache, self.shadir, signature_certificate_list=[ self.shacache, self.shadir, signature_certificate_list=[
self.certificate]) self.certificate])
# when no signature is used, all works ok # when no signature is used, all works ok
selected = nc.select(key).read() selected = self.select(nc, key).read()
self.assertEqual(selected, self.test_string) self.assertEqual(selected, self.test_string)
# but when signature is used, networkcache will complain # but when signature is used, networkcache will complain
self.assertDirectoryNotFound('Could not find a trustable entry.', self.assertEqual(self.select(signed_nc, key), None)
signed_nc.select, key)
# of course if proper key will be used to sign the content uploaded # of course if proper key will be used to sign the content uploaded
# into shacache all will work # into shacache all will work
upload_nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, upload_nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache,
self.shadir, signature_private_key_file=key_file.name) self.shadir, signature_private_key_file=key_file.name)
upload_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name) upload_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
selected = signed_nc.select(key).read() selected = self.select(signed_nc, key).read()
self.assertEqual(selected, self.test_string) self.assertEqual(selected, self.test_string)
......
...@@ -15,17 +15,11 @@ ...@@ -15,17 +15,11 @@
# BBB: Deprecated. This file is ugly and must disappear. # BBB: Deprecated. This file is ugly and must disappear.
# DO NOT EXTEND IT. Add methods to NetworkcacheClient class instead. # DO NOT EXTEND IT. Add methods to NetworkcacheClient class instead.
import json
import logging
import os import os
import shutil import shutil
import urllib2 import urllib2
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \ from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound DirectoryNotFound, logger
logging.basicConfig()
logger = logging.getLogger('networkcachehelper')
logger.setLevel(logging.INFO)
def __upload_network_cached(dir_url, cache_url, def __upload_network_cached(dir_url, cache_url,
file_descriptor, directory_key, file_descriptor, directory_key,
...@@ -57,16 +51,12 @@ def __upload_network_cached(dir_url, cache_url, ...@@ -57,16 +51,12 @@ def __upload_network_cached(dir_url, cache_url,
shadir_cert_file = None shadir_cert_file = None
if not shadir_key_file: if not shadir_key_file:
shadir_key_file = None shadir_key_file = None
try: nc = NetworkcacheClient(cache_url, dir_url,
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file, signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file, shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file, shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file, shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file) shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try: try:
return nc.upload(file_descriptor, directory_key, **metadata_dict) return nc.upload(file_descriptor, directory_key, **metadata_dict)
...@@ -110,76 +100,29 @@ def helper_download_network_cached(dir_url, cache_url, ...@@ -110,76 +100,29 @@ def helper_download_network_cached(dir_url, cache_url,
return (file_descriptor, metadata) if succeeded, False otherwise. return (file_descriptor, metadata) if succeeded, False otherwise.
""" """
if not(dir_url and cache_url): if dir_url and cache_url:
return False
if len(signature_certificate_list) == 0:
# convert [] into None in order to call nc nicely
signature_certificate_list = None
try:
nc = NetworkcacheClient(cache_url, dir_url, nc = NetworkcacheClient(cache_url, dir_url,
signature_certificate_list=signature_certificate_list) signature_certificate_list=signature_certificate_list)
except TypeError: logger.info('Downloading %s...', directory_key)
logger.warning('Incompatible version of networkcache, not using it.') result = nc.select(directory_key, wanted_metadata_dict, required_key_list)
return False if strategy:
entry = None
logger.info('Trying to download %s from network cache...' % directory_key) result = list(result)
try: if result:
file_descriptor = None entry = strategy(result)
json_entry_list = nc.select_generic(directory_key) if not entry: # XXX: this should be the choice of 'strategy' function
# For each entry shadir sent, chooses only the entry matching all
# wanted metadata, and having all wanted keys
matching_entry_list = []
for entry, _ in json_entry_list:
try:
tags = json.loads(entry)
match = True
for metadata_key, metadata_value in wanted_metadata_dict.items():
if tags.get(metadata_key) != metadata_value:
# Something doesn't match: not a good entry
match = False
break
# Now checks if all required keys are present
# i.e we want that entry contains a list key with not null
# corresponding value
for required_key in required_key_list:
if not tags.get(required_key):
match = False
break
if not match:
continue
# Everything match. Add it to list of matching entries
matching_entry_list.append(tags)
except Exception:
pass
if matching_entry_list:
# If a strategy is defined, call it to determine best entry
if strategy:
best_entry = strategy(matching_entry_list)
if not best_entry:
logger.info("Can't find best entry matching strategy, selecting " logger.info("Can't find best entry matching strategy, selecting "
"random one between acceptable ones.") "random one between acceptable ones.")
best_entry = matching_entry_list[0] entry = result[0]
else:
best_entry = matching_entry_list[0]
# download best entry
file_descriptor = nc.download(best_entry.get('sha512'))
return file_descriptor, tags
else: else:
logger.info('No matching entry to download from network cache: %s'\ entry = next(result, None)
% directory_key) if entry:
return False try:
return nc.download(entry['sha512']), entry
except (IOError, DirectoryNotFound), e: except urllib2.HTTPError, e:
if isinstance(e, urllib2.HTTPError) and e.code == 404: logger.warning('Failed to download %s: %s', directory_key, e)
logger.info('%s does not exist in network cache.' % directory_key)
else: else:
logger.warning('Failed to download from network cache %s: %s' % ( logger.info('No matching entry to download %s', directory_key)
directory_key, str(e)))
return False
def helper_download_network_cached_to_file(dir_url, cache_url, def helper_download_network_cached_to_file(dir_url, cache_url,
signature_certificate_list, signature_certificate_list,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment