Commit 43060927 authored by Łukasz Nowak's avatar Łukasz Nowak

Merge branch 'bug_778748'

parents 332f6462 7d49bc97
*pyc
*pyo
.installed.cfg
bin/
develop-eggs/
dist/
eggs/
parts/
src/zc.buildout.egg-info/
z3c.recipe.scripts_/src/z3c.recipe.scripts.egg-info/
zc.recipe.egg_/src/zc.recipe.egg.egg-info/
[buildout]
extends = buildout.cfg
#develop += path_to_slapos.libnetworkcache
parts += cached
[cached]
recipe = z3c.recipe.scripts
eggs =
slapos.libnetworkcache
zc.buildout
scripts =
buildout=buildout_cached
[test]
eggs +=
slapos.libnetworkcache
......@@ -82,7 +82,7 @@ setup(
packages = ['zc', 'zc.buildout'],
package_dir = {'': 'src'},
namespace_packages = ['zc'],
install_requires = ['setuptools'],
install_requires = 'setuptools',
include_package_data = True,
entry_points = entry_points,
extras_require = dict(test=['zope.testing']),
......
......@@ -40,6 +40,12 @@ import zc.buildout
import zc.buildout.download
import zc.buildout.easy_install
try:
import slapos.libnetworkcache
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
realpath = zc.buildout.easy_install.realpath
......@@ -330,6 +336,45 @@ class Buildout(UserDict.DictMixin):
os.chdir(options['directory'])
if not LIBNETWORKCACHE_ENABLED:
self._logger.warning('Networkache disabled, no '
'slapos.libnetworkache available.')
self.download_cache_url = None
self.download_dir_url = None
self.upload_cache_url = None
self.upload_dir_url = None
else:
# support networkcache
networkcache_section_name = options.get('networkcache-section')
if networkcache_section_name:
networkcache_section = self[networkcache_section_name]
self.download_cache_url = networkcache_section[
'download-cache-url']
self.download_dir_url = networkcache_section[
'download-dir-url']
self.upload_cache_url = networkcache_section.get(
'upload-cache-url', '')
self.upload_dir_url = networkcache_section.get(
'upload-dir-url', '')
self._logger.info('Networkcache enabled.')
self._logger.info('Networkcache download cache: %r, directory '
'%r' % (self.download_cache_url, self.download_dir_url))
self._logger.info('Networkcache upload cache: %r, directory %r'%
(self.upload_cache_url, self.upload_dir_url))
# expose __networkcache__* in options for easier integration
options['__networkcache__download-cache-url'] = self.download_cache_url
options['__networkcache__download-dir-url'] = self.download_dir_url
options['__networkcache__upload-cache-url'] = self.upload_cache_url
options['__networkcache__upload-dir-url'] = self.upload_dir_url
else:
self._logger.warning('Networkcache functionality not enabled.'
'In order to activate use networkcache-section.')
self.download_cache_url = None
self.download_dir_url = None
self.upload_cache_url = None
self.upload_dir_url = None
def _buildout_path(self, name):
if '${' in name:
return name
......@@ -345,6 +390,8 @@ class Buildout(UserDict.DictMixin):
# Get a base working set for our distributions that corresponds to the
# stated desires in the configuration.
distributions = ['setuptools', 'zc.buildout']
if LIBNETWORKCACHE_ENABLED:
distributions.append('slapos.libnetworkcache')
if options.get('offline') == 'true':
ws = zc.buildout.easy_install.working_set(
distributions, options['executable'],
......@@ -362,13 +409,18 @@ class Buildout(UserDict.DictMixin):
newest=self.newest,
allow_hosts=self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases,
sha_cache=options.get('sha-cache'),
sha_dir=options.get('sha-dir'),
download_cache_url=self.download_cache_url,
download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url
)
# Now copy buildout and setuptools eggs, and record destination eggs:
entries = []
for name in 'setuptools', 'zc.buildout':
copy_list = ['setuptools', 'zc.buildout']
if LIBNETWORKCACHE_ENABLED:
copy_list.append('slapos.libnetworkcache')
for name in copy_list:
r = pkg_resources.Requirement.parse(name)
dist = ws.find(r)
if dist.precedence == pkg_resources.DEVELOP_DIST:
......@@ -862,8 +914,10 @@ class Buildout(UserDict.DictMixin):
path = [options['develop-eggs-directory']],
allow_hosts = self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases,
sha_cache=options.get('sha-cache'),
sha_dir=options.get('sha-cache'),
download_cache_url=self.download_cache_url,
download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url
)
upgraded = []
......@@ -1092,8 +1146,14 @@ def _install_and_load(spec, group, entry, buildout):
newest=buildout.newest,
allow_hosts=buildout._allow_hosts,
prefer_final=not buildout.accept_buildout_test_releases,
sha_dir=buildout_options.get('sha-dir'),
sha_cache=buildout_options.get('sha-cache'))
download_cache_url=buildout_options.get(
'__networkcache__download-cache-url'),
download_dir_url=buildout_options.get(
'__networkcache__download-dir-url'),
upload_cache_url=buildout_options.get(
'__networkcache__upload-cache-url'),
upload_dir_url=buildout_options.get('__networkcache__upload-dir-url')
)
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
return pkg_resources.load_entry_point(
......@@ -1454,22 +1514,6 @@ def _open(base, filename, seen, dl_options, override, downloaded):
filename)
result[section] = options
# find and expose _profile_base_location_ and _profile_location_
for section, value in result.iteritems():
_profile_base_location_ = None
_profile_location_ = None
for k,v in value.iteritems():
if '${:_profile_base_location_}' in v:
_profile_base_location_ = base
if '${:_profile_location_}' in v:
_profile_location_ = filename
if _profile_base_location_ is not None and _profile_location_ is not None:
break
if _profile_base_location_ is not None:
value['_profile_base_location_'] = _profile_base_location_
if _profile_location_ is not None:
value['_profile_location_'] = _profile_location_
result = _annotate(result, filename)
if root_config_file and 'buildout' in result:
......
......@@ -71,8 +71,12 @@ class Download(object):
self.fallback = fallback
self.hash_name = hash_name
self.logger = logger or logging.getLogger('zc.buildout')
self.sha_dir = options.get('sha-dir')
self.sha_cache = options.get('sha-cache')
self.download_dir_url = options.get('__networkcache__download-dir-url')
self.download_cache_url = options.get(
'__networkcache__download-cache-url')
self.upload_dir_url = options.get('__networkcache__upload-dir-url')
self.upload_cache_url = options.get(
'__networkcache__upload-cache-url')
@property
def download_cache(self):
......@@ -181,7 +185,7 @@ class Download(object):
try:
try:
if not download_network_cached(self.sha_dir, self.sha_cache,
if not download_network_cached(self.download_dir_url, self.download_cache_url,
tmp_path, url, self.logger, md5sum):
# Download from original url
......@@ -190,8 +194,9 @@ class Download(object):
raise ChecksumError(
'MD5 checksum mismatch downloading %r' % url)
# Upload the file to networkcached.
upload_network_cached(self.sha_cache, self.sha_dir, url,
tmp_path, self.logger)
if self.upload_cache_url and self.upload_dir_url:
upload_network_cached(self.upload_dir_url,
self.upload_cache_url, url, tmp_path, self.logger)
finally:
os.close(handle)
except:
......
......@@ -341,8 +341,10 @@ class Installer:
include_site_packages=None,
allowed_eggs_from_site_packages=None,
prefer_final=None,
sha_dir=None,
sha_cache=None
download_dir_url=None,
download_cache_url=None,
upload_dir_url=None,
upload_cache_url=None
):
self._dest = dest
self._allow_hosts = allow_hosts
......@@ -411,8 +413,10 @@ class Installer:
if versions is not None:
self._versions = versions
self._sha_dir = sha_dir
self._sha_cache = sha_cache
self._download_dir_url = download_dir_url
self._download_cache_url = download_cache_url
self._upload_dir_url = upload_dir_url
self._upload_cache_url = upload_cache_url
_allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name):
......@@ -715,11 +719,12 @@ class Installer:
filename = get_filename_from_url(dist.location)
new_location = os.path.join(tmp, filename)
if not download_network_cached(self._sha_dir, self._sha_cache,
if not download_network_cached(self._download_dir_url, self._download_cache_url,
new_location, dist.location, logger):
new_location = self._index.download(dist.location, tmp)
upload_network_cached(self._sha_cache, self._sha_dir,
dist.location, new_location, logger)
if self._upload_cache_url and self._upload_dir_url:
upload_network_cached(self._upload_dir_url,
self._upload_cache_url, dist.location, new_location, logger)
if (download_cache
and (realpath(new_location) == realpath(dist.location))
......@@ -1098,13 +1103,16 @@ def install(specs, dest,
path=None, working_set=None, newest=True, versions=None,
use_dependency_links=None, allow_hosts=('*',),
include_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None, sha_dir=None, sha_cache=None):
prefer_final=None, download_dir_url=None, download_cache_url=None,
upload_dir_url=None, upload_cache_url=None):
installer = Installer(
dest, links, index, executable, always_unzip, path, newest,
versions, use_dependency_links, allow_hosts=allow_hosts,
include_site_packages=include_site_packages,
allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
prefer_final=prefer_final, sha_dir=sha_dir, sha_cache=sha_cache)
prefer_final=prefer_final, download_dir_url=download_dir_url,
download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
upload_cache_url=upload_cache_url)
return installer.install(specs, working_set)
......
......@@ -17,9 +17,14 @@ import hashlib
import os
import posixpath
import re
import urllib
import urlparse
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
_md5_re = re.compile(r'md5=([a-f0-9]+)')
......@@ -32,7 +37,7 @@ def _get_md5_from_url(url):
def get_directory_key(url):
"""It must retorn the directory key based on the URL.
"""Returns directory hash based on url.
Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5
......@@ -44,23 +49,18 @@ def get_directory_key(url):
return 'slapos-buildout-%s' % urlmd5
def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
"""Download from a network cache provider
def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None):
"""Downloads from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached files.
return True if download succeeded.
"""
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
except:
if not LIBNETWORKCACHE_ENABLED:
return False
if sha_cache in (None, '',):
# Not able to use network cache
if not(dir_url and cache_url):
return False
if md5sum is None:
md5sum = _get_md5_from_url(url)
......@@ -71,7 +71,7 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
logger.info('Downloading %s from network cache.' % file_name)
try:
nc = NetworkcacheClient(shacache=sha_cache, shadir=sha_dir)
nc = NetworkcacheClient(shacache=cache_url, shadir=dir_url)
file_content = nc.select(directory_key)
f = open(path, 'w+b')
......@@ -86,25 +86,22 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
return False
except (IOError, DirectoryNotFound), e:
logger.info('Fail to download from network cache %s: %s' % \
logger.info('Failed to download from network cache %s: %s' % \
(file_name, str(e)))
return False
return True
def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
"""Upload file to a shacache server"""
try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound
except:
def upload_network_cached(dir_url, cache_url, external_url, path, logger):
"""Upload file to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if sha_cache in [None, ''] or sha_dir in [None, '']:
logger.debug(
'Upload cache ignored, shacache or shadir was not provided!')
if not (dir_url and cache_url):
return False
logger.info('Uploading %s into network cache.' % external_url)
file_name = external_url.split('/')[-1].split('#')[0].split('?')[0]
directory_key = get_directory_key(external_url)
kw = dict(file=file_name,
......@@ -112,9 +109,9 @@ def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
f = open(path, 'r')
try:
nc = NetworkcacheClient(shacache=sha_cache,
shadir=sha_dir)
data = nc.upload(f, directory_key, **kw)
nc = NetworkcacheClient(shacache=cache_url,
shadir=dir_url)
return nc.upload(f, directory_key, **kw)
except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % \
(str(e)))
......
Using buildout with network-cache
=================================
Using buildout with cache over network
======================================
NOTE: You need to install the networkcached into your system to run this
test, otherwise it will not be possible to get proper results.
The buildout itself must be able to cache everything which is downloadble
into an external and universal cache server.
Such feature is not enabled by default, but you can enable defining the
option called ```network-cache```.
Note: We are in process of preparing docs. This is only showcase.
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... network-cache = http://127.0.0.1:5001/
...
... networkcache-section = networkcache
... find-links = %(link_server)s
... parts = eggs
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = demo ==0.2
... ''' % globals())
>>> print system(buildout)
Installing eggs.
Getting distribution for 'demo==0.2'.
The url is not cached yet: http://localhost/demo-0.2-py2.6.egg
Got demo 0.2.
Getting distribution for 'demoneeded'.
The url is not cached yet: http://localhost/demoneeded-1.2c1.zip
Got demoneeded 1.2c1.
...
... [networkcache]
... download-cache-url = http://127.0.0.1:1/
... download-dir-url = http://127.0.0.1:1/
... upload-cache-url = http://127.0.0.1:1/
... upload-dir-url = http://127.0.0.1:1/
... ''' % globals())
>>> from glob import glob
>>> from os.path import join
......@@ -38,8 +28,20 @@ option called ```network-cache```.
... remove(sample_buildout, 'eggs', egg)
>>> print system(buildout)
Updating eggs.
Networkcache enabled.
Networkcache download cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
Networkcache upload cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
Installing eggs.
Getting distribution for 'demo==0.2'.
Downloading demo-0.2-py2.7.egg from network cache.
Failed to download from network cache demo-0.2-py2.7.egg: [Errno 111] Connection refused
Uploading http://localhost/demo-0.2-py2.7.egg into network cache.
Fail to upload file. [Errno 111] Connection refused
Got demo 0.2.
Getting distribution for 'demoneeded'.
Downloading from network cache http://127.0.0.1:5001/105e4a9ba3f9c7f45b46cc8c47ac2d2ca6440ae564538090c6d390a8959dd03e
Downloading demoneeded-1.2c1.zip from network cache.
Failed to download from network cache demoneeded-1.2c1.zip: [Errno 111] Connection refused
Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
Fail to upload file. [Errno 111] Connection refused
Got demoneeded 1.2c1.
...
<BLANKLINE>
......@@ -88,10 +88,13 @@ class Eggs(object):
kw = {}
if 'unzip' in options:
kw['always_unzip'] = options.query_bool('unzip', None)
if 'sha-cache' in b_options:
kw['sha_cache'] = b_options.get('sha-cache')
if 'sha-dir' in b_options:
kw['sha_dir'] = b_options.get('sha-dir')
for option_key, kw_key in (
('__networkcache__download-cache-url', 'download_cache_url'),
('__networkcache__download-dir-url', 'download_dir_url'),
('__networkcache__upload-cache-url', 'upload_cache_url'),
('__networkcache__upload-dir-url', 'upload_dir_url')):
if option_key in b_options:
kw[kw_key] = b_options[option_key]
ws = zc.buildout.easy_install.install(
distributions, options['eggs-directory'],
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment