Provide a simple way to use network cache safely

To simplify things, DirectoryNotFound & UploadError are merged in
NetworkcacheException.

slapos/libnetworkcache.py

@@ -93,6 +93,21 @@ class NetworkcacheClient(object):
       crypto.load_certificate(crypto.FILETYPE_PEM, certificate)
       for certificate in signature_certificate_list or ()]
 
+  # NetworkcacheClient context manager catches all exceptions and logs them
+  # with INFO severity. This provides a easy way to use a networkcache safely
+  # since most of the time, failures are not fatal.
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, t, v, tb):
+    if isinstance(v, Exception):
+      if isinstance(v, NetworkcacheException):
+        logger.info(*v.args)
+      else:
+        logger.info("ignored unhandled exception", exc_info=(t, v, tb))
+      return True
+
   def _request(self, where, name=None, data=None, headers=None):
     if data is None:
       method = 'GET'
@@ -189,9 +204,9 @@ class NetworkcacheClient(object):
           'Content-Type': 'application/octet-stream'})
         data = result.read()
         if result.status != 201 or data != sha512sum:
-          raise UploadError('Failed to upload the file to SHACACHE Server.'
-                            'Response code: %s. Response data: %s'
-                            % (result.status, data))
+          raise NetworkcacheException(
+            'Failed to upload data to SHACACHE Server.'
+            ' Response code: %s. Response data: %s' % (result.status, data))
     finally:
       f is None or f.close()
 
@@ -199,8 +214,8 @@ class NetworkcacheClient(object):
       kw['sha512'] = sha512sum # always update sha512sum
       file_name = kw.pop('file', file_name)
       if file_name is None or urlmd5 is None:
-        raise ValueError('file_name and urlmd5 are required'
-                         ' for non-generic upload')
+        raise NetworkcacheException(
+          'file_name and urlmd5 are required for non-generic upload')
       if valid_until is not None:
         kw['valid-until'] = valid_until
       if architecture is not None:
@@ -217,9 +232,9 @@ class NetworkcacheClient(object):
     result = self._request('dir', key, json.dumps(data), {
       'Content-Type': 'application/json'})
     if result.status != 201:
-      raise UploadError('Failed to upload data to SHADIR Server.'
-                        'Response code: %s. Response data: %s'
-                        % (status, result.read()))
+      raise NetworkcacheException('Failed to upload data to SHADIR Server.'
+                                  ' Response code: %s. Response data: %s'
+                                  % (status, result.read()))
 
   def download(self, sha512sum):
     ''' Download the file.
@@ -231,16 +246,13 @@ class NetworkcacheClient(object):
     '''Return an iterator over shadir entries that match given criteria
     '''
     required_key_test = frozenset(required_key_list).issubset
-    try:
-      data_list = self.select_generic(key, self.signature_certificate_list)
-    except (urllib2.HTTPError, DirectoryNotFound), e:
-      logger.info(str(e))
-      return
+    data_list = self.select_generic(key, self.signature_certificate_list)
     for information_json, signature in data_list:
       try:
         information_dict = json.loads(information_json)
       except Exception:
-        logger.info('Failed to parse json-in-json response', exc_info=1)
+        logger.info('Failed to parse json-in-json response (%r)',
+                    information_json)
         continue
       try:
         len(information_dict['sha512'])
@@ -262,8 +274,7 @@ class NetworkcacheClient(object):
     try:
       data_list = json.loads(data)
     except Exception:
-      raise DirectoryNotFound('Failed to parse json response:\n%s'
-                              % traceback.format_exc())
+      raise NetworkcacheException('Failed to parse json response (%r)' % data)
     if filter:
       return (data for data in data_list
                    if self._verifySignatureInCertificateList(*data))
@@ -292,12 +303,10 @@ class NetworkcacheClient(object):
     return False
 
 
-class DirectoryNotFound(Exception):
+class NetworkcacheException(Exception):
   pass
 
-
-class UploadError(Exception):
-  pass
+DirectoryNotFound = UploadError = NetworkcacheException # BBB
 
 
 def _newArgumentParser():

slapos/libnetworkcachetests.py

@@ -193,6 +193,9 @@ class OnlineMixin:
     slapos.libnetworkcache.logger.removeHandler(self.handler)
     del self.handler
 
+  def assertRaised(self, exc_type):
+    self.assertEqual(exc_type, self.handler.buffer.pop(0).exc_info[0])
+
   def assertLog(self, msg=None):
     try:
       self.assertTrue(self.handler.buffer.pop(0).message.startswith(msg))
@@ -365,15 +368,16 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     urlmd5 = str(random.random())
     key = 'somekey' + str(random.random())
     nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
-    self.select(nc, 'key_another_key' + str(random.random()),
-                'HTTP Error 404: Not Found')
+    with nc:
+      nc.select('key_another_key' + str(random.random())).next()
+    self.assertRaised(urllib2.HTTPError)
 
   def test_upload_shadir_no_filename(self):
     """Check scenario with shadir used, but not filename passed"""
-    nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
-    urlmd5 = str(random.random())
-    self.assertRaises(ValueError, nc.upload, self.test_data, 'somekey',
-      urlmd5)
+    with slapos.libnetworkcache.NetworkcacheClient(self.shacache,
+                                                   self.shadir) as nc:
+      nc.upload(self.test_data, 'somekey', str(random.random()))
+    self.assertLog('file_name and urlmd5 are required for non-generic upload')
 
   def test_upload_twice_same(self):
     nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
@@ -467,7 +471,9 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
       # now remove the entry from shacache
       f.write('This is not a json.')
-    self.select(nc, key, 'Failed to parse json response')
+    with nc:
+      nc.select(key).next()
+    self.assertLog('Failed to parse json response')
 
   def test_select_json_no_in_json_response(self):
     key = 'somekey' + str(random.random())
@@ -602,7 +608,7 @@ class OnlineTestPOST200(OnlineMixin, unittest.TestCase):
   def test_upload_wrong_return_code(self):
     """Check reaction on HTTP return code different then 201"""
     nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
-    self.assertRaises(slapos.libnetworkcache.UploadError, nc.upload,
+    self.assertRaises(slapos.libnetworkcache.NetworkcacheException, nc.upload,
       self.test_data)
 
 
@@ -612,7 +618,7 @@ class OnlineTestWrongChecksum(OnlineMixin, unittest.TestCase):
   def test_upload_wrong_return_sha(self):
     """Check reaction in case of wrong sha returned"""
     nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
-    self.assertRaises(slapos.libnetworkcache.UploadError, nc.upload,
+    self.assertRaises(slapos.libnetworkcache.NetworkcacheException, nc.upload,
       self.test_data)
 
 

slapos/networkcachehelper.py

@@ -17,9 +17,7 @@
 
 import os
 import shutil
-import urllib2
-from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
-    DirectoryNotFound, logger
+from slapos.libnetworkcache import NetworkcacheClient, logger
 
 def __upload_network_cached(dir_url, cache_url,
     file_descriptor, directory_key,
@@ -41,28 +39,14 @@ def __upload_network_cached(dir_url, cache_url,
   metadata_dict.setdefault('urlmd5', 'notused')
 
   # convert '' into None in order to call nc nicely
-  if not signature_private_key_file:
-    signature_private_key_file = None
-  if not shacache_cert_file:
-    shacache_cert_file = None
-  if not shacache_key_file:
-    shacache_key_file = None
-  if not shadir_cert_file:
-    shadir_cert_file = None
-  if not shadir_key_file:
-    shadir_key_file = None
-  nc = NetworkcacheClient(cache_url, dir_url,
-      signature_private_key_file=signature_private_key_file,
-      shacache_cert_file=shacache_cert_file,
-      shacache_key_file=shacache_key_file,
-      shadir_cert_file=shadir_cert_file,
-      shadir_key_file=shadir_key_file)
-
-  try:
+  with NetworkcacheClient(cache_url, dir_url,
+        signature_private_key_file=signature_private_key_file or None,
+        shacache_cert_file=shacache_cert_file or None,
+        shacache_key_file=shacache_key_file or None,
+        shadir_cert_file=shadir_cert_file or None,
+        shadir_key_file=shadir_key_file or None,
+      ) as nc:
     return nc.upload(file_descriptor, directory_key, **metadata_dict)
-  except (IOError, UploadError), e:
-    logger.info('Failed to upload file. %s' % str(e))
-    return False
 
 # BBB: slapos.buildout (1.6.0-dev-SlapOS-011) imports it without using it
 helper_upload_network_cached_from_file = None
@@ -100,9 +84,10 @@ def helper_download_network_cached(dir_url, cache_url,
 
   return (file_descriptor, metadata) if succeeded, False otherwise.
   """
-  if dir_url and cache_url:
-    nc = NetworkcacheClient(cache_url, dir_url,
-        signature_certificate_list=signature_certificate_list)
+  if not (dir_url and cache_url):
+    return
+  with NetworkcacheClient(cache_url, dir_url,
+      signature_certificate_list=signature_certificate_list) as nc:
     logger.info('Downloading %s...', directory_key)
     result = nc.select(directory_key, wanted_metadata_dict, required_key_list)
     if strategy:
@@ -117,10 +102,7 @@ def helper_download_network_cached(dir_url, cache_url,
     else:
       entry = next(result, None)
     if entry:
-      try:
-        return nc.download(entry['sha512']), entry
-      except urllib2.HTTPError, e:
-        logger.warning('Failed to download %s: %s', directory_key, e)
+      return nc.download(entry['sha512']), entry
     else:
       logger.info('No matching entry to download %s', directory_key)