Commit 43060927 authored by Łukasz Nowak's avatar Łukasz Nowak

Merge branch 'bug_778748'

parents 332f6462 7d49bc97
*pyc
*pyo
.installed.cfg
bin/
develop-eggs/
dist/
eggs/
parts/
src/zc.buildout.egg-info/
z3c.recipe.scripts_/src/z3c.recipe.scripts.egg-info/
zc.recipe.egg_/src/zc.recipe.egg.egg-info/
[buildout]
extends = buildout.cfg
#develop += path_to_slapos.libnetworkcache
parts += cached
[cached]
recipe = z3c.recipe.scripts
eggs =
slapos.libnetworkcache
zc.buildout
scripts =
buildout=buildout_cached
[test]
eggs +=
slapos.libnetworkcache
...@@ -82,7 +82,7 @@ setup( ...@@ -82,7 +82,7 @@ setup(
packages = ['zc', 'zc.buildout'], packages = ['zc', 'zc.buildout'],
package_dir = {'': 'src'}, package_dir = {'': 'src'},
namespace_packages = ['zc'], namespace_packages = ['zc'],
install_requires = ['setuptools'], install_requires = 'setuptools',
include_package_data = True, include_package_data = True,
entry_points = entry_points, entry_points = entry_points,
extras_require = dict(test=['zope.testing']), extras_require = dict(test=['zope.testing']),
......
...@@ -40,6 +40,12 @@ import zc.buildout ...@@ -40,6 +40,12 @@ import zc.buildout
import zc.buildout.download import zc.buildout.download
import zc.buildout.easy_install import zc.buildout.easy_install
try:
import slapos.libnetworkcache
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
realpath = zc.buildout.easy_install.realpath realpath = zc.buildout.easy_install.realpath
...@@ -330,6 +336,45 @@ class Buildout(UserDict.DictMixin): ...@@ -330,6 +336,45 @@ class Buildout(UserDict.DictMixin):
os.chdir(options['directory']) os.chdir(options['directory'])
if not LIBNETWORKCACHE_ENABLED:
self._logger.warning('Networkache disabled, no '
'slapos.libnetworkache available.')
self.download_cache_url = None
self.download_dir_url = None
self.upload_cache_url = None
self.upload_dir_url = None
else:
# support networkcache
networkcache_section_name = options.get('networkcache-section')
if networkcache_section_name:
networkcache_section = self[networkcache_section_name]
self.download_cache_url = networkcache_section[
'download-cache-url']
self.download_dir_url = networkcache_section[
'download-dir-url']
self.upload_cache_url = networkcache_section.get(
'upload-cache-url', '')
self.upload_dir_url = networkcache_section.get(
'upload-dir-url', '')
self._logger.info('Networkcache enabled.')
self._logger.info('Networkcache download cache: %r, directory '
'%r' % (self.download_cache_url, self.download_dir_url))
self._logger.info('Networkcache upload cache: %r, directory %r'%
(self.upload_cache_url, self.upload_dir_url))
# expose __networkcache__* in options for easier integration
options['__networkcache__download-cache-url'] = self.download_cache_url
options['__networkcache__download-dir-url'] = self.download_dir_url
options['__networkcache__upload-cache-url'] = self.upload_cache_url
options['__networkcache__upload-dir-url'] = self.upload_dir_url
else:
self._logger.warning('Networkcache functionality not enabled.'
'In order to activate use networkcache-section.')
self.download_cache_url = None
self.download_dir_url = None
self.upload_cache_url = None
self.upload_dir_url = None
def _buildout_path(self, name): def _buildout_path(self, name):
if '${' in name: if '${' in name:
return name return name
...@@ -345,6 +390,8 @@ class Buildout(UserDict.DictMixin): ...@@ -345,6 +390,8 @@ class Buildout(UserDict.DictMixin):
# Get a base working set for our distributions that corresponds to the # Get a base working set for our distributions that corresponds to the
# stated desires in the configuration. # stated desires in the configuration.
distributions = ['setuptools', 'zc.buildout'] distributions = ['setuptools', 'zc.buildout']
if LIBNETWORKCACHE_ENABLED:
distributions.append('slapos.libnetworkcache')
if options.get('offline') == 'true': if options.get('offline') == 'true':
ws = zc.buildout.easy_install.working_set( ws = zc.buildout.easy_install.working_set(
distributions, options['executable'], distributions, options['executable'],
...@@ -362,13 +409,18 @@ class Buildout(UserDict.DictMixin): ...@@ -362,13 +409,18 @@ class Buildout(UserDict.DictMixin):
newest=self.newest, newest=self.newest,
allow_hosts=self._allow_hosts, allow_hosts=self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases, prefer_final=not self.accept_buildout_test_releases,
sha_cache=options.get('sha-cache'), download_cache_url=self.download_cache_url,
sha_dir=options.get('sha-dir'), download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url
) )
# Now copy buildout and setuptools eggs, and record destination eggs: # Now copy buildout and setuptools eggs, and record destination eggs:
entries = [] entries = []
for name in 'setuptools', 'zc.buildout': copy_list = ['setuptools', 'zc.buildout']
if LIBNETWORKCACHE_ENABLED:
copy_list.append('slapos.libnetworkcache')
for name in copy_list:
r = pkg_resources.Requirement.parse(name) r = pkg_resources.Requirement.parse(name)
dist = ws.find(r) dist = ws.find(r)
if dist.precedence == pkg_resources.DEVELOP_DIST: if dist.precedence == pkg_resources.DEVELOP_DIST:
...@@ -862,8 +914,10 @@ class Buildout(UserDict.DictMixin): ...@@ -862,8 +914,10 @@ class Buildout(UserDict.DictMixin):
path = [options['develop-eggs-directory']], path = [options['develop-eggs-directory']],
allow_hosts = self._allow_hosts, allow_hosts = self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases, prefer_final=not self.accept_buildout_test_releases,
sha_cache=options.get('sha-cache'), download_cache_url=self.download_cache_url,
sha_dir=options.get('sha-cache'), download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url
) )
upgraded = [] upgraded = []
...@@ -1092,8 +1146,14 @@ def _install_and_load(spec, group, entry, buildout): ...@@ -1092,8 +1146,14 @@ def _install_and_load(spec, group, entry, buildout):
newest=buildout.newest, newest=buildout.newest,
allow_hosts=buildout._allow_hosts, allow_hosts=buildout._allow_hosts,
prefer_final=not buildout.accept_buildout_test_releases, prefer_final=not buildout.accept_buildout_test_releases,
sha_dir=buildout_options.get('sha-dir'), download_cache_url=buildout_options.get(
sha_cache=buildout_options.get('sha-cache')) '__networkcache__download-cache-url'),
download_dir_url=buildout_options.get(
'__networkcache__download-dir-url'),
upload_cache_url=buildout_options.get(
'__networkcache__upload-cache-url'),
upload_dir_url=buildout_options.get('__networkcache__upload-dir-url')
)
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
return pkg_resources.load_entry_point( return pkg_resources.load_entry_point(
...@@ -1454,22 +1514,6 @@ def _open(base, filename, seen, dl_options, override, downloaded): ...@@ -1454,22 +1514,6 @@ def _open(base, filename, seen, dl_options, override, downloaded):
filename) filename)
result[section] = options result[section] = options
# find and expose _profile_base_location_ and _profile_location_
for section, value in result.iteritems():
_profile_base_location_ = None
_profile_location_ = None
for k,v in value.iteritems():
if '${:_profile_base_location_}' in v:
_profile_base_location_ = base
if '${:_profile_location_}' in v:
_profile_location_ = filename
if _profile_base_location_ is not None and _profile_location_ is not None:
break
if _profile_base_location_ is not None:
value['_profile_base_location_'] = _profile_base_location_
if _profile_location_ is not None:
value['_profile_location_'] = _profile_location_
result = _annotate(result, filename) result = _annotate(result, filename)
if root_config_file and 'buildout' in result: if root_config_file and 'buildout' in result:
......
...@@ -71,8 +71,12 @@ class Download(object): ...@@ -71,8 +71,12 @@ class Download(object):
self.fallback = fallback self.fallback = fallback
self.hash_name = hash_name self.hash_name = hash_name
self.logger = logger or logging.getLogger('zc.buildout') self.logger = logger or logging.getLogger('zc.buildout')
self.sha_dir = options.get('sha-dir') self.download_dir_url = options.get('__networkcache__download-dir-url')
self.sha_cache = options.get('sha-cache') self.download_cache_url = options.get(
'__networkcache__download-cache-url')
self.upload_dir_url = options.get('__networkcache__upload-dir-url')
self.upload_cache_url = options.get(
'__networkcache__upload-cache-url')
@property @property
def download_cache(self): def download_cache(self):
...@@ -181,7 +185,7 @@ class Download(object): ...@@ -181,7 +185,7 @@ class Download(object):
try: try:
try: try:
if not download_network_cached(self.sha_dir, self.sha_cache, if not download_network_cached(self.download_dir_url, self.download_cache_url,
tmp_path, url, self.logger, md5sum): tmp_path, url, self.logger, md5sum):
# Download from original url # Download from original url
...@@ -190,8 +194,9 @@ class Download(object): ...@@ -190,8 +194,9 @@ class Download(object):
raise ChecksumError( raise ChecksumError(
'MD5 checksum mismatch downloading %r' % url) 'MD5 checksum mismatch downloading %r' % url)
# Upload the file to networkcached. # Upload the file to networkcached.
upload_network_cached(self.sha_cache, self.sha_dir, url, if self.upload_cache_url and self.upload_dir_url:
tmp_path, self.logger) upload_network_cached(self.upload_dir_url,
self.upload_cache_url, url, tmp_path, self.logger)
finally: finally:
os.close(handle) os.close(handle)
except: except:
......
...@@ -341,8 +341,10 @@ class Installer: ...@@ -341,8 +341,10 @@ class Installer:
include_site_packages=None, include_site_packages=None,
allowed_eggs_from_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None, prefer_final=None,
sha_dir=None, download_dir_url=None,
sha_cache=None download_cache_url=None,
upload_dir_url=None,
upload_cache_url=None
): ):
self._dest = dest self._dest = dest
self._allow_hosts = allow_hosts self._allow_hosts = allow_hosts
...@@ -411,8 +413,10 @@ class Installer: ...@@ -411,8 +413,10 @@ class Installer:
if versions is not None: if versions is not None:
self._versions = versions self._versions = versions
self._sha_dir = sha_dir self._download_dir_url = download_dir_url
self._sha_cache = sha_cache self._download_cache_url = download_cache_url
self._upload_dir_url = upload_dir_url
self._upload_cache_url = upload_cache_url
_allowed_eggs_from_site_packages_regex = None _allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name): def allow_site_package_egg(self, name):
...@@ -715,11 +719,12 @@ class Installer: ...@@ -715,11 +719,12 @@ class Installer:
filename = get_filename_from_url(dist.location) filename = get_filename_from_url(dist.location)
new_location = os.path.join(tmp, filename) new_location = os.path.join(tmp, filename)
if not download_network_cached(self._sha_dir, self._sha_cache, if not download_network_cached(self._download_dir_url, self._download_cache_url,
new_location, dist.location, logger): new_location, dist.location, logger):
new_location = self._index.download(dist.location, tmp) new_location = self._index.download(dist.location, tmp)
upload_network_cached(self._sha_cache, self._sha_dir, if self._upload_cache_url and self._upload_dir_url:
dist.location, new_location, logger) upload_network_cached(self._upload_dir_url,
self._upload_cache_url, dist.location, new_location, logger)
if (download_cache if (download_cache
and (realpath(new_location) == realpath(dist.location)) and (realpath(new_location) == realpath(dist.location))
...@@ -1098,13 +1103,16 @@ def install(specs, dest, ...@@ -1098,13 +1103,16 @@ def install(specs, dest,
path=None, working_set=None, newest=True, versions=None, path=None, working_set=None, newest=True, versions=None,
use_dependency_links=None, allow_hosts=('*',), use_dependency_links=None, allow_hosts=('*',),
include_site_packages=None, allowed_eggs_from_site_packages=None, include_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None, sha_dir=None, sha_cache=None): prefer_final=None, download_dir_url=None, download_cache_url=None,
upload_dir_url=None, upload_cache_url=None):
installer = Installer( installer = Installer(
dest, links, index, executable, always_unzip, path, newest, dest, links, index, executable, always_unzip, path, newest,
versions, use_dependency_links, allow_hosts=allow_hosts, versions, use_dependency_links, allow_hosts=allow_hosts,
include_site_packages=include_site_packages, include_site_packages=include_site_packages,
allowed_eggs_from_site_packages=allowed_eggs_from_site_packages, allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
prefer_final=prefer_final, sha_dir=sha_dir, sha_cache=sha_cache) prefer_final=prefer_final, download_dir_url=download_dir_url,
download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
upload_cache_url=upload_cache_url)
return installer.install(specs, working_set) return installer.install(specs, working_set)
......
...@@ -17,9 +17,14 @@ import hashlib ...@@ -17,9 +17,14 @@ import hashlib
import os import os
import posixpath import posixpath
import re import re
import urllib
import urlparse
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
_md5_re = re.compile(r'md5=([a-f0-9]+)') _md5_re = re.compile(r'md5=([a-f0-9]+)')
...@@ -32,7 +37,7 @@ def _get_md5_from_url(url): ...@@ -32,7 +37,7 @@ def _get_md5_from_url(url):
def get_directory_key(url): def get_directory_key(url):
"""It must retorn the directory key based on the URL. """Returns directory hash based on url.
Basically check if the url belongs to pypi: Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5 - if yes, the directory key will be pypi-buildout-urlmd5
...@@ -44,23 +49,18 @@ def get_directory_key(url): ...@@ -44,23 +49,18 @@ def get_directory_key(url):
return 'slapos-buildout-%s' % urlmd5 return 'slapos-buildout-%s' % urlmd5
def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None): def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None):
"""Download from a network cache provider """Downloads from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore If something fail (providor be offline, or hash_string fail), we ignore
network cached files. network cached files.
return True if download succeeded. return True if download succeeded.
""" """
try: if not LIBNETWORKCACHE_ENABLED:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
except:
return False return False
if not(dir_url and cache_url):
if sha_cache in (None, '',):
# Not able to use network cache
return False return False
if md5sum is None: if md5sum is None:
md5sum = _get_md5_from_url(url) md5sum = _get_md5_from_url(url)
...@@ -71,7 +71,7 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None): ...@@ -71,7 +71,7 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
logger.info('Downloading %s from network cache.' % file_name) logger.info('Downloading %s from network cache.' % file_name)
try: try:
nc = NetworkcacheClient(shacache=sha_cache, shadir=sha_dir) nc = NetworkcacheClient(shacache=cache_url, shadir=dir_url)
file_content = nc.select(directory_key) file_content = nc.select(directory_key)
f = open(path, 'w+b') f = open(path, 'w+b')
...@@ -86,25 +86,22 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None): ...@@ -86,25 +86,22 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
return False return False
except (IOError, DirectoryNotFound), e: except (IOError, DirectoryNotFound), e:
logger.info('Fail to download from network cache %s: %s' % \ logger.info('Failed to download from network cache %s: %s' % \
(file_name, str(e))) (file_name, str(e)))
return False return False
return True return True
def upload_network_cached(sha_cache, sha_dir, external_url, path, logger): def upload_network_cached(dir_url, cache_url, external_url, path, logger):
"""Upload file to a shacache server""" """Upload file to a network cache server"""
try: if not LIBNETWORKCACHE_ENABLED:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
except:
return False return False
if sha_cache in [None, ''] or sha_dir in [None, '']: if not (dir_url and cache_url):
logger.debug(
'Upload cache ignored, shacache or shadir was not provided!')
return False return False
logger.info('Uploading %s into network cache.' % external_url)
file_name = external_url.split('/')[-1].split('#')[0].split('?')[0] file_name = external_url.split('/')[-1].split('#')[0].split('?')[0]
directory_key = get_directory_key(external_url) directory_key = get_directory_key(external_url)
kw = dict(file=file_name, kw = dict(file=file_name,
...@@ -112,9 +109,9 @@ def upload_network_cached(sha_cache, sha_dir, external_url, path, logger): ...@@ -112,9 +109,9 @@ def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
f = open(path, 'r') f = open(path, 'r')
try: try:
nc = NetworkcacheClient(shacache=sha_cache, nc = NetworkcacheClient(shacache=cache_url,
shadir=sha_dir) shadir=dir_url)
data = nc.upload(f, directory_key, **kw) return nc.upload(f, directory_key, **kw)
except (IOError, UploadError), e: except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % \ logger.info('Fail to upload file. %s' % \
(str(e))) (str(e)))
......
Using buildout with network-cache Using buildout with cache over network
================================= ======================================
NOTE: You need to install the networkcached into your system to run this Note: We are in process of preparing docs. This is only showcase.
test, otherwise it will not be possible to get proper results.
The buildout itself must be able to cache everything which is downloadble
into an external and universal cache server.
Such feature is not enabled by default, but you can enable defining the
option called ```network-cache```.
>>> write(sample_buildout, 'buildout.cfg', >>> write(sample_buildout, 'buildout.cfg',
... ''' ... '''
... [buildout] ... [buildout]
... network-cache = http://127.0.0.1:5001/ ...
... networkcache-section = networkcache
... find-links = %(link_server)s ... find-links = %(link_server)s
... parts = eggs ... parts = eggs
... ...
... [eggs] ... [eggs]
... recipe = zc.recipe.egg:eggs ... recipe = zc.recipe.egg:eggs
... eggs = demo ==0.2 ... eggs = demo ==0.2
... ''' % globals())
>>> print system(buildout)
Installing eggs.
Getting distribution for 'demo==0.2'.
The url is not cached yet: http://localhost/demo-0.2-py2.6.egg
Got demo 0.2.
Getting distribution for 'demoneeded'.
The url is not cached yet: http://localhost/demoneeded-1.2c1.zip
Got demoneeded 1.2c1.
... ...
... [networkcache]
... download-cache-url = http://127.0.0.1:1/
... download-dir-url = http://127.0.0.1:1/
... upload-cache-url = http://127.0.0.1:1/
... upload-dir-url = http://127.0.0.1:1/
... ''' % globals())
>>> from glob import glob >>> from glob import glob
>>> from os.path import join >>> from os.path import join
...@@ -38,8 +28,20 @@ option called ```network-cache```. ...@@ -38,8 +28,20 @@ option called ```network-cache```.
... remove(sample_buildout, 'eggs', egg) ... remove(sample_buildout, 'eggs', egg)
>>> print system(buildout) >>> print system(buildout)
Updating eggs. Networkcache enabled.
Networkcache download cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
Networkcache upload cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
Installing eggs.
Getting distribution for 'demo==0.2'.
Downloading demo-0.2-py2.7.egg from network cache.
Failed to download from network cache demo-0.2-py2.7.egg: [Errno 111] Connection refused
Uploading http://localhost/demo-0.2-py2.7.egg into network cache.
Fail to upload file. [Errno 111] Connection refused
Got demo 0.2.
Getting distribution for 'demoneeded'. Getting distribution for 'demoneeded'.
Downloading from network cache http://127.0.0.1:5001/105e4a9ba3f9c7f45b46cc8c47ac2d2ca6440ae564538090c6d390a8959dd03e Downloading demoneeded-1.2c1.zip from network cache.
Failed to download from network cache demoneeded-1.2c1.zip: [Errno 111] Connection refused
Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
Fail to upload file. [Errno 111] Connection refused
Got demoneeded 1.2c1. Got demoneeded 1.2c1.
... <BLANKLINE>
...@@ -88,10 +88,13 @@ class Eggs(object): ...@@ -88,10 +88,13 @@ class Eggs(object):
kw = {} kw = {}
if 'unzip' in options: if 'unzip' in options:
kw['always_unzip'] = options.query_bool('unzip', None) kw['always_unzip'] = options.query_bool('unzip', None)
if 'sha-cache' in b_options: for option_key, kw_key in (
kw['sha_cache'] = b_options.get('sha-cache') ('__networkcache__download-cache-url', 'download_cache_url'),
if 'sha-dir' in b_options: ('__networkcache__download-dir-url', 'download_dir_url'),
kw['sha_dir'] = b_options.get('sha-dir') ('__networkcache__upload-cache-url', 'upload_cache_url'),
('__networkcache__upload-dir-url', 'upload_dir_url')):
if option_key in b_options:
kw[kw_key] = b_options[option_key]
ws = zc.buildout.easy_install.install( ws = zc.buildout.easy_install.install(
distributions, options['eggs-directory'], distributions, options['eggs-directory'],
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment