Skip to content

S
slapos.buildout
Commit d257344a authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Make nc bindings immune for slapos.libnetworkcache errors. 

According to specification networkcache usage in buildout shall be fully
optional. The missing part was to make bindings immune on slapos.libnetworkcache
API changes or its internal errors.

Squashed commit of the following:

commit 79064a17
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 22 13:12:54 2011 +0200

    Proof that bindings to nc are immutable.

commit b9795d85
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 22 11:35:38 2011 +0200

    Immune buildout on any networkcache issues.

    As fallback shall be always possible make binding really immune to any problems
    in networkcache including name errors, api changes, syntax errors, etc.

commit ff3a32b9
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Thu Sep 22 11:22:10 2011 +0200

    Show how networkcache is installed.

commit eeaccfc2
Author: Łukasz Nowak <luke@nexedi.com>
Date:   Fri Sep 2 11:14:49 2011 +0200

    M2Crypto is not needed anymore.
parent 151f013b
Hide whitespace changes
Inline Side-by-side
Showing with 199 additions and 12 deletions
src/zc/buildout/buildout.py
View file @ d257344a
...@@ -51,11 +51,18 @@ import zc.buildout.easy_install ...@@ -51,11 +51,18 @@ import zc.buildout.easy_install
try: try:
import slapos.libnetworkcache try:
except ImportError: import slapos.libnetworkcache
LIBNETWORKCACHE_ENABLED = False except ImportError:
else: LIBNETWORKCACHE_ENABLED = False
LIBNETWORKCACHE_ENABLED = True else:
LIBNETWORKCACHE_ENABLED = True
except:
import traceback
print 'There was problem while trying to import slapos.libnetworkcache:'\
'\n%s' % traceback.format_exc()
LIBNETWORKCACHE_ENABLED = False
print 'Networkcache forced to be disabled.'
realpath = zc.buildout.easy_install.realpath realpath = zc.buildout.easy_install.realpath
......
src/zc/buildout/networkcache.py
View file @ d257344a
...@@ -19,14 +19,21 @@ import posixpath ...@@ -19,14 +19,21 @@ import posixpath
import re import re
import shutil import shutil
import urlparse import urlparse
import traceback
try: try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \ try:
DirectoryNotFound from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
except ImportError: DirectoryNotFound
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
except:
print 'There was problem while trying to import slapos.libnetworkcache:'\
'\n%s' % traceback.format_exc()
LIBNETWORKCACHE_ENABLED = False LIBNETWORKCACHE_ENABLED = False
else: print 'Networkcache forced to be disabled.'
LIBNETWORKCACHE_ENABLED = True
_md5_re = re.compile(r'md5=([a-f0-9]+)') _md5_re = re.compile(r'md5=([a-f0-9]+)')
...@@ -37,7 +44,23 @@ def _get_md5_from_url(url): ...@@ -37,7 +44,23 @@ def _get_md5_from_url(url):
return match.group(1) return match.group(1)
return None return None
def fallback_call(function):
"""Decorator which disallow to have any problem while calling method"""
def wrapper(self, *args, **kwd):
"""
Log the call, and the result of the call
"""
try:
return function(self, *args, **kwd)
except: # indeed, *any* exception is swallowed
print 'There was problem while calling method %r:\n%s' % (
function.__name__, traceback.format_exc())
return False
wrapper.__doc__ = function.__doc__
return wrapper
@fallback_call
def get_directory_key(url): def get_directory_key(url):
"""Returns directory hash based on url. """Returns directory hash based on url.
...@@ -51,6 +74,7 @@ def get_directory_key(url): ...@@ -51,6 +74,7 @@ def get_directory_key(url):
return 'slapos-buildout-%s' % urlmd5 return 'slapos-buildout-%s' % urlmd5
@fallback_call
def download_network_cached(dir_url, cache_url, path, url, logger, def download_network_cached(dir_url, cache_url, path, url, logger,
signature_certificate_list, md5sum=None): signature_certificate_list, md5sum=None):
"""Downloads from a network cache provider """Downloads from a network cache provider
...@@ -104,6 +128,7 @@ def download_network_cached(dir_url, cache_url, path, url, logger, ...@@ -104,6 +128,7 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
return True return True
@fallback_call
def upload_network_cached(dir_url, cache_url, external_url, path, logger, def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file, shacache_cert_file, shacache_key_file, signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file): shadir_cert_file, shadir_key_file):
...@@ -158,6 +183,7 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger, ...@@ -158,6 +183,7 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
return True return True
@fallback_call
def get_filename_from_url(url): def get_filename_from_url(url):
"""Inspired how pip get filename from url. """Inspired how pip get filename from url.
""" """
......
src/zc/buildout/networkcache.txt
View file @ d257344a
...@@ -19,7 +19,6 @@ So lets activate networkcache in buildout: ...@@ -19,7 +19,6 @@ So lets activate networkcache in buildout:
... parts = networkcache ... parts = networkcache
... find-links = ... find-links =
... http://pypi.python.org/pypi/slapos.libnetworkcache ... http://pypi.python.org/pypi/slapos.libnetworkcache
... http://chandlerproject.org/bin/view/Projects/MeTooCrypto
... # Do not register this buildout run ... # Do not register this buildout run
... installed = ... installed =
... ...
...@@ -29,7 +28,10 @@ So lets activate networkcache in buildout: ...@@ -29,7 +28,10 @@ So lets activate networkcache in buildout:
... slapos.libnetworkcache ... slapos.libnetworkcache
... zc.buildout ... zc.buildout
... ''') ... ''')
>>> ignored = system(buildout) >>> print system(buildout)
Installing networkcache.
Getting distribution for 'slapos.libnetworkcache'.
Got slapos.libnetworkcache ...
Lets check that networkcache is correclty installed: Lets check that networkcache is correclty installed:
>>> write(sample_buildout, 'buildout.cfg', >>> write(sample_buildout, 'buildout.cfg',
...@@ -834,3 +836,155 @@ parse the original url and get the file name: ...@@ -834,3 +836,155 @@ parse the original url and get the file name:
... path=globals().get('tmp_dir') + '/tmp_file', ... path=globals().get('tmp_dir') + '/tmp_file',
... nc_server_path=sample_buildout) ... nc_server_path=sample_buildout)
'id=700c7d5382b01f94e7141' 'id=700c7d5382b01f94e7141'
Buildout bindings to networkcache are immune on slapos.libnetworkcache problems.
Lets prepare networkcache with errors:
>>> mkdir(sample_buildout, 'damagednc')
>>> mkdir(sample_buildout, 'damagednc', 'slapos')
>>> write(sample_buildout, 'damagednc', 'slapos', '__init__.py',
... """
... # See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
... try:
... __import__('pkg_resources').declare_namespace(__name__)
... except ImportError:
... from pkgutil import extend_path
... __path__ = extend_path(__path__, __name__)
... """)
>>> write(sample_buildout, 'damagednc', 'slapos', 'libnetworkcache.py',
... """
... += die during import
... """)
>>> write(sample_buildout, 'damagednc', 'setup.py',
... """
... from setuptools import setup, find_packages
...
... setup(
... name = "slapos.libnetworkcache",
... namespace_packages=['slapos'],
... packages=find_packages(),
... install_requires=['setuptools']
... )
... """)
>>> write(sample_buildout, 'damagednc', 'README.txt', " ")
And install it in buildout:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... parts = networkcache
... develop = damagednc
... # Do not register this buildout run
... installed =
...
... [networkcache]
... recipe = zc.recipe.egg
... eggs =
... slapos.libnetworkcache
... zc.buildout
... ''')
>>> print system(buildout)
Develop: '/sample-buildout/damagednc'
Installing networkcache.
Generated script '/sample-buildout/bin/buildout'.
So lets use such damaged networkcache:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
...
... networkcache-section = networkcache
... develop = download damagednc
... parts = download
...
... [download]
... recipe = download
... url = %(remote_server_url)shello.txt
...
... [networkcache]
... download-cache-url = %(nc_url)sshacache
... download-dir-url = %(nc_url)sshadir
... ''' % globals())
>>> print system(buildout)
There was problem while trying to import slapos.libnetworkcache:
Traceback (most recent call last):
...
SyntaxError: invalid syntax
<BLANKLINE>
Networkcache forced to be disabled.
There was problem while trying to import slapos.libnetworkcache:
Traceback (most recent call last):
...
SyntaxError: invalid syntax
<BLANKLINE>
Networkcache forced to be disabled.
Develop: '/sample-buildout/download'
Develop: '/sample-buildout/damagednc'
Unused options for buildout: 'networkcache-section'.
Installing download.
Downloading http://localhost/hello.txt
download: Downloaded http://localhost/hello.txt
If networkcache internally has issues, or its API got changed, buildout will
still work:
>>> write(sample_buildout, 'damagednc', 'slapos', 'libnetworkcache.py',
... """
... class NetworkcacheClient(object):
... def parseUrl(*args, **kwargs):
... raise Die
... def __init__(*args, **kwargs):
... raise Die
... def upload(*args, **kwargs):
... raise Die
... def download(*args, **kwargs):
... raise Die
... def select(*args, **kwargs):
... raise Die
... class UploadError: pass
... class DirectoryNotFound: pass
... """)
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
...
... networkcache-section = networkcache
... develop = download damagednc
... parts = download
...
... [download]
... recipe = download
... url = %(remote_server_url)shello.txt
...
... [networkcache]
... download-cache-url = %(nc_url)sshacache
... download-dir-url = %(nc_url)sshadir
... upload-cache-url = %(nc_url)sshacache
... upload-dir-url = %(nc_url)sshadir
... ''' % globals())
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Develop: '/sample-buildout/download'
Develop: '/sample-buildout/damagednc'
Updating download.
Downloading http://localhost/hello.txt
There was problem while calling method 'download_network_cached':
Traceback (most recent call last):
...
raise Die
NameError: global name 'Die' is not defined
<BLANKLINE>
Uploading http://localhost/hello.txt into network cache.
There was problem while calling method 'upload_network_cached':
Traceback (most recent call last):
...
raise Die
NameError: global name 'Die' is not defined
<BLANKLINE>
download: Downloaded http://localhost:20863/hello.txt
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7