Commit 1aa316f4 authored by Łukasz Nowak's avatar Łukasz Nowak

Merge branch 'bug_778748'

parents d7fa2df1 b129d3e7
...@@ -355,6 +355,10 @@ class Buildout(UserDict.DictMixin): ...@@ -355,6 +355,10 @@ class Buildout(UserDict.DictMixin):
self.upload_dir_url = None self.upload_dir_url = None
self.signature_private_key_file = None self.signature_private_key_file = None
self.signature_certificate_list = None self.signature_certificate_list = None
self.shacache_cert_file = None
self.shacache_key_file = None
self.shadir_cert_file = None
self.shadir_key_file = None
else: else:
# support networkcache # support networkcache
networkcache_section_name = options.get('networkcache-section') networkcache_section_name = options.get('networkcache-section')
...@@ -372,6 +376,14 @@ class Buildout(UserDict.DictMixin): ...@@ -372,6 +376,14 @@ class Buildout(UserDict.DictMixin):
'signature-certificate-list', '') 'signature-certificate-list', '')
self.signature_private_key_file = networkcache_section.get( self.signature_private_key_file = networkcache_section.get(
'signature-private-key-file', '') 'signature-private-key-file', '')
self.shacache_cert_file = networkcache_section.get(
'shacache-cert-file', '')
self.shacache_key_file = networkcache_section.get(
'shacache-key-file', '')
self.shadir_cert_file = networkcache_section.get(
'shadir-cert-file', '')
self.shadir_key_file = networkcache_section.get(
'shadir-key-file', '')
# parse signature list # parse signature list
cert_marker = '-----BEGIN CERTIFICATE-----' cert_marker = '-----BEGIN CERTIFICATE-----'
...@@ -391,6 +403,10 @@ class Buildout(UserDict.DictMixin): ...@@ -391,6 +403,10 @@ class Buildout(UserDict.DictMixin):
options['__networkcache__upload-dir-url'] = self.upload_dir_url options['__networkcache__upload-dir-url'] = self.upload_dir_url
options['__networkcache__signature-certificate-list'] = signature_certificate_list options['__networkcache__signature-certificate-list'] = signature_certificate_list
options['__networkcache__signature-private-key-file'] = self.signature_private_key_file options['__networkcache__signature-private-key-file'] = self.signature_private_key_file
options['__networkcache__shacache-cert-file'] = self.shacache_cert_file
options['__networkcache__shacache-key-file'] = self.shacache_key_file
options['__networkcache__shadir-cert-file'] = self.shadir_cert_file
options['__networkcache__shadir-key-file'] = self.shadir_key_file
else: else:
self._logger.debug('Networkcache functionality not enabled. ' self._logger.debug('Networkcache functionality not enabled. '
...@@ -401,6 +417,10 @@ class Buildout(UserDict.DictMixin): ...@@ -401,6 +417,10 @@ class Buildout(UserDict.DictMixin):
self.upload_dir_url = None self.upload_dir_url = None
self.signature_private_key_file = None self.signature_private_key_file = None
self.signature_certificate_list = None self.signature_certificate_list = None
self.shacache_cert_file = None
self.shacache_key_file = None
self.shadir_cert_file = None
self.shadir_key_file = None
def _buildout_path(self, name): def _buildout_path(self, name):
if '${' in name: if '${' in name:
...@@ -442,6 +462,10 @@ class Buildout(UserDict.DictMixin): ...@@ -442,6 +462,10 @@ class Buildout(UserDict.DictMixin):
upload_dir_url=self.upload_dir_url, upload_dir_url=self.upload_dir_url,
signature_private_key_file=self.signature_private_key_file, signature_private_key_file=self.signature_private_key_file,
signature_certificate_list=self.signature_certificate_list, signature_certificate_list=self.signature_certificate_list,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file,
) )
# Now copy buildout and setuptools eggs, and record destination eggs: # Now copy buildout and setuptools eggs, and record destination eggs:
...@@ -954,6 +978,10 @@ class Buildout(UserDict.DictMixin): ...@@ -954,6 +978,10 @@ class Buildout(UserDict.DictMixin):
upload_dir_url=self.upload_dir_url, upload_dir_url=self.upload_dir_url,
signature_certificate_list=self.signature_certificate_list, signature_certificate_list=self.signature_certificate_list,
signature_private_key_file=self.signature_private_key_file, signature_private_key_file=self.signature_private_key_file,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file,
) )
upgraded = [] upgraded = []
...@@ -1193,6 +1221,14 @@ def _install_and_load(spec, group, entry, buildout): ...@@ -1193,6 +1221,14 @@ def _install_and_load(spec, group, entry, buildout):
'__networkcache__signature-private-key-file'), '__networkcache__signature-private-key-file'),
signature_certificate_list=buildout_options.get( signature_certificate_list=buildout_options.get(
'__networkcache__signature-certificate-list'), '__networkcache__signature-certificate-list'),
shacache_cert_file=buildout_options.get(
'__networkcache__shacache-cert-file'),
shacache_key_file=buildout_options.get(
'__networkcache__shacache-key-file'),
shadir_cert_file=buildout_options.get(
'__networkcache__shadir-cert-file'),
shadir_key_file=buildout_options.get(
'__networkcache__shadir-key-file'),
) )
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
......
...@@ -86,6 +86,14 @@ class Download(object): ...@@ -86,6 +86,14 @@ class Download(object):
if q.strip()] if q.strip()]
self.signature_private_key_file = options.get( self.signature_private_key_file = options.get(
'__networkcache__signature-private-key-file') '__networkcache__signature-private-key-file')
self.shacache_cert_file = options.get(
'__networkcache__shacache-cert-file')
self.shacache_key_file = options.get(
'__networkcache__shacache-key-file')
self.shadir_cert_file = options.get(
'__networkcache__shadir-cert-file')
self.shadir_key_file = options.get(
'__networkcache__shadir-key-file')
@property @property
def download_cache(self): def download_cache(self):
...@@ -206,7 +214,11 @@ class Download(object): ...@@ -206,7 +214,11 @@ class Download(object):
if self.upload_cache_url and self.upload_dir_url: if self.upload_cache_url and self.upload_dir_url:
upload_network_cached(self.upload_dir_url, upload_network_cached(self.upload_dir_url,
self.upload_cache_url, url, tmp_path, self.logger, self.upload_cache_url, url, tmp_path, self.logger,
self.signature_private_key_file) self.signature_private_key_file,
self.shacache_cert_file,
self.shacache_key_file,
self.shadir_cert_file,
self.shadir_key_file)
finally: finally:
os.close(handle) os.close(handle)
except: except:
......
...@@ -346,7 +346,11 @@ class Installer: ...@@ -346,7 +346,11 @@ class Installer:
upload_dir_url=None, upload_dir_url=None,
upload_cache_url=None, upload_cache_url=None,
signature_certificate_list=None, signature_certificate_list=None,
signature_private_key_file=None signature_private_key_file=None,
shacache_cert_file=None,
shacache_key_file=None,
shadir_cert_file=None,
shadir_key_file=None,
): ):
self._dest = dest self._dest = dest
self._allow_hosts = allow_hosts self._allow_hosts = allow_hosts
...@@ -421,6 +425,10 @@ class Installer: ...@@ -421,6 +425,10 @@ class Installer:
self._upload_cache_url = upload_cache_url self._upload_cache_url = upload_cache_url
self._signature_certificate_list = signature_certificate_list self._signature_certificate_list = signature_certificate_list
self._signature_private_key_file = signature_private_key_file self._signature_private_key_file = signature_private_key_file
self._shacache_cert_file = shacache_cert_file
self._shacache_key_file = shacache_key_file
self._shadir_cert_file = shadir_cert_file
self._shadir_key_file = shadir_key_file
_allowed_eggs_from_site_packages_regex = None _allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name): def allow_site_package_egg(self, name):
...@@ -730,7 +738,9 @@ class Installer: ...@@ -730,7 +738,9 @@ class Installer:
if self._upload_cache_url and self._upload_dir_url: if self._upload_cache_url and self._upload_dir_url:
upload_network_cached(self._upload_dir_url, upload_network_cached(self._upload_dir_url,
self._upload_cache_url, dist.location, new_location, logger, self._upload_cache_url, dist.location, new_location, logger,
self._signature_private_key_file) self._signature_private_key_file,
self._shacache_cert_file, self._shacache_key_file,
self._shadir_cert_file, self._shadir_key_file)
if (download_cache if (download_cache
and (realpath(new_location) == realpath(dist.location)) and (realpath(new_location) == realpath(dist.location))
...@@ -1111,7 +1121,9 @@ def install(specs, dest, ...@@ -1111,7 +1121,9 @@ def install(specs, dest,
include_site_packages=None, allowed_eggs_from_site_packages=None, include_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None, download_dir_url=None, download_cache_url=None, prefer_final=None, download_dir_url=None, download_cache_url=None,
upload_dir_url=None, upload_cache_url=None, upload_dir_url=None, upload_cache_url=None,
signature_certificate_list=None, signature_private_key_file=None): signature_certificate_list=None, signature_private_key_file=None,
shacache_cert_file=None, shacache_key_file=None,
shadir_cert_file=None, shadir_key_file=None):
installer = Installer( installer = Installer(
dest, links, index, executable, always_unzip, path, newest, dest, links, index, executable, always_unzip, path, newest,
versions, use_dependency_links, allow_hosts=allow_hosts, versions, use_dependency_links, allow_hosts=allow_hosts,
...@@ -1121,7 +1133,11 @@ def install(specs, dest, ...@@ -1121,7 +1133,11 @@ def install(specs, dest,
download_cache_url=download_cache_url, upload_dir_url=upload_dir_url, download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
upload_cache_url=upload_cache_url, upload_cache_url=upload_cache_url,
signature_certificate_list=signature_certificate_list, signature_certificate_list=signature_certificate_list,
signature_private_key_file=signature_private_key_file) signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
return installer.install(specs, working_set) return installer.install(specs, working_set)
......
...@@ -105,7 +105,8 @@ def download_network_cached(dir_url, cache_url, path, url, logger, ...@@ -105,7 +105,8 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
def upload_network_cached(dir_url, cache_url, external_url, path, logger, def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file): signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server""" """Upload file to a network cache server"""
if not LIBNETWORKCACHE_ENABLED: if not LIBNETWORKCACHE_ENABLED:
return False return False
...@@ -122,12 +123,24 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger, ...@@ -122,12 +123,24 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
urlmd5=hashlib.md5(external_url).hexdigest()) urlmd5=hashlib.md5(external_url).hexdigest())
f = open(path, 'r') f = open(path, 'r')
if not signature_private_key_file:
# convert '' into None in order to call nc nicely # convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None signature_private_key_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try: try:
nc = NetworkcacheClient(cache_url, dir_url, nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file) signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError: except TypeError:
logger.warning('Incompatible version of networkcache, not using it.') logger.warning('Incompatible version of networkcache, not using it.')
return False return False
......
...@@ -693,6 +693,97 @@ Of course eggs are also supported by networkcache: ...@@ -693,6 +693,97 @@ Of course eggs are also supported by networkcache:
Uploading http://localhost/demoneeded-1.2c1.zip into network cache. Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
Got demoneeded 1.2c1. Got demoneeded 1.2c1.
It is possible to pass SSL certificates and key to be used by authentication.
In order to do it NetworkcacheClient.upload method have to be patched to show
that proper keys are used.
>>> mkdir(sample_buildout, 'ncpatch')
>>> write(sample_buildout, 'ncpatch', 'ncpatch.py',
... """
... def patched(self, *args, **kwargs):
... print 'shacache_cert_file', self.shacache_cert_file
... print 'shacache_key_file', self.shacache_key_file
... print 'shadir_cert_file', self.shadir_cert_file
... print 'shadir_key_file', self.shadir_key_file
... return False
...
... def ext(buildout):
... print 'Patching slapos.libnetworkcache'
... import slapos.libnetworkcache
... slapos.libnetworkcache.NetworkcacheClient.upload = patched
... """)
>>> write(sample_buildout, 'ncpatch', 'setup.py',
... """
... from setuptools import setup
...
... setup(
... name = "ncpatch",
... entry_points = {'zc.buildout.extension': ['ncpatch = ncpatch:ext']},
... )
... """)
>>> write(sample_buildout, 'ncpatch', 'README.txt', " ")
Lets activiate this extension:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = ncpatch
... parts =
... """)
>>> print system(buildout)
Develop: '/sample-buildout/ncpatch'
Uninstalling eggs.
Lets clean networkcache:
>>> rmdir(nc_data)
>>> mkdir(nc_data)
And see what happens:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
...
... networkcache-section = networkcache
... find-links = %(link_server)s
... develop = ncpatch download
... extensions = ncpatch
... parts = download
...
... [download]
... recipe = download
... url = %(remote_server_url)shello.txt
...
... [networkcache]
... download-cache-url = %(nc_url)sshacache
... download-dir-url = %(nc_url)sshadir
... upload-cache-url = %(nc_url)sshacache
... upload-dir-url = %(nc_url)sshadir
... shacache-cert-file = /path/to/shacache/cert/file
... shacache-key-file = /path/to/shacache/key/file
... shadir-cert-file = /path/to/shadir/cert/file
... shadir-key-file = /path/to/shadir/key/file
... ''' % globals())
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Patching slapos.libnetworkcache
Develop: '/sample-buildout/ncpatch'
Develop: '/sample-buildout/download'
Installing download.
Downloading http://localhost/hello.txt
Downloading hello.txt from network cache.
Failed to download from network cache hello.txt:...404...
Uploading http://localhost/hello.txt into network cache.
shacache_cert_file /path/to/shacache/cert/file
shacache_key_file /path/to/shacache/key/file
shadir_cert_file /path/to/shadir/cert/file
shadir_key_file /path/to/shadir/key/file
download: Downloaded http://localhost/hello.txt
Lets clean networkcache: Lets clean networkcache:
>>> rmdir(nc_data) >>> rmdir(nc_data)
......
...@@ -94,7 +94,12 @@ class Eggs(object): ...@@ -94,7 +94,12 @@ class Eggs(object):
('__networkcache__upload-cache-url', 'upload_cache_url'), ('__networkcache__upload-cache-url', 'upload_cache_url'),
('__networkcache__upload-dir-url', 'upload_dir_url'), ('__networkcache__upload-dir-url', 'upload_dir_url'),
('__networkcache__signature-private-key-file', 'signature_private_key_file'), ('__networkcache__signature-private-key-file', 'signature_private_key_file'),
('__networkcache__signature-certificate-list', 'signature_certificate_list')): ('__networkcache__signature-certificate-list', 'signature_certificate_list'),
('__networkcache__shacache-cert-file', 'shacache_cert_file'),
('__networkcache__shacache-key-file', 'shacache_key_file'),
('__networkcache__shadir-cert-file', 'shadir_cert_file'),
('__networkcache__shadir-key-file', 'shadir_key_file'),
):
if option_key in b_options: if option_key in b_options:
if option_key == '__networkcache__signature-certificate-list': if option_key == '__networkcache__signature-certificate-list':
cert_marker = '-----BEGIN CERTIFICATE-----' cert_marker = '-----BEGIN CERTIFICATE-----'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment