Cache pypi index, extensions and recipes using buildout internals.

parent b0bcdf03
...@@ -1193,7 +1193,17 @@ class Buildout(UserDict.DictMixin): ...@@ -1193,7 +1193,17 @@ class Buildout(UserDict.DictMixin):
links = self['buildout'].get('find-links', '').split(), links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'), index = self['buildout'].get('index'),
newest=self.newest, allow_hosts=self._allow_hosts, newest=self.newest, allow_hosts=self._allow_hosts,
prefer_final=not self.accept_buildout_test_releases) prefer_final=not self.accept_buildout_test_releases,
download_cache_url=self.download_cache_url,
download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url,
signature_private_key_file=self.signature_private_key_file,
signature_certificate_list=self.signature_certificate_list,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file,)
# Clear cache because extensions might now let us read pages we # Clear cache because extensions might now let us read pages we
# couldn't read before. # couldn't read before.
......
This diff is collapsed.
...@@ -12,12 +12,12 @@ ...@@ -12,12 +12,12 @@
# #
############################################################################## ##############################################################################
#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
import hashlib import hashlib
import os import os
import posixpath import posixpath
import re import re
import shutil
import urllib2 import urllib2
import urlparse import urlparse
import traceback import traceback
...@@ -26,6 +26,10 @@ try: ...@@ -26,6 +26,10 @@ try:
try: try:
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \ from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound DirectoryNotFound
from slapos.networkcachehelper import \
helper_download_network_cached, \
helper_upload_network_cached_from_file, \
helper_download_network_cached_to_file
except ImportError: except ImportError:
LIBNETWORKCACHE_ENABLED = False LIBNETWORKCACHE_ENABLED = False
else: else:
...@@ -68,12 +72,19 @@ def get_directory_key(url): ...@@ -68,12 +72,19 @@ def get_directory_key(url):
Basically check if the url belongs to pypi: Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5 - if yes, the directory key will be pypi-buildout-urlmd5
- if not, the directory key will be slapos-buildout-urlmd5 - if not, the directory key will be slapos-buildout-urlmd5
# XXX why is that?
""" """
urlmd5 = hashlib.md5(url).hexdigest() urlmd5 = hashlib.md5(url).hexdigest()
if 'pypi' in url: if 'pypi' in url:
return 'pypi-buildout-%s' % urlmd5 return 'pypi-buildout-%s' % urlmd5
return 'slapos-buildout-%s' % urlmd5 return 'slapos-buildout-%s' % urlmd5
@fallback_call
def get_index_directory_key(url, requirement):
"""Returns directory hash based on egg requirement.
"""
return 'pypi-index-%s-%s' % (hashlib.md5(url).hexdigest(), requirement)
@fallback_call @fallback_call
def download_network_cached(dir_url, cache_url, path, url, logger, def download_network_cached(dir_url, cache_url, path, url, logger,
...@@ -88,41 +99,60 @@ def download_network_cached(dir_url, cache_url, path, url, logger, ...@@ -88,41 +99,60 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
if not LIBNETWORKCACHE_ENABLED: if not LIBNETWORKCACHE_ENABLED:
return False return False
if not(dir_url and cache_url):
return False
if md5sum is None: if md5sum is None:
md5sum = _get_md5_from_url(url) md5sum = _get_md5_from_url(url)
directory_key = get_directory_key(url) directory_key = get_directory_key(url)
url = os.path.basename(url) url = os.path.basename(url)
if len(signature_certificate_list) == 0:
# convert [] into None in order to call nc nicely
signature_certificate_list = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_certificate_list=signature_certificate_list)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
logger.debug('Trying to download %s from network cache...' % url) logger.debug('Trying to download %s from network cache...' % url)
try:
file_descriptor = nc.select(directory_key)
f = open(path, 'w+b') if helper_download_network_cached_to_file(
try: dir_url=dir_url,
shutil.copyfileobj(file_descriptor, f) cache_url=cache_url,
finally: signature_certificate_list=signature_certificate_list,
f.close() directory_key=directory_key,
file_descriptor.close() path=path):
logger.info('Downloaded %s from network cache.' % url) logger.info('Downloaded %s from network cache.' % url)
if not check_md5sum(path, md5sum): if not check_md5sum(path, md5sum):
logger.info('MD5 checksum mismatch downloading %s' % url) logger.info('MD5 checksum mismatch downloading %s' % url)
return False return False
return True
return False
@fallback_call
def download_index_network_cached(dir_url, cache_url, url, requirement, logger,
signature_certificate_list):
"""
XXX description
Downloads pypi index from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore
network cached index.
return index if succeeded, False otherwise.
"""
if not LIBNETWORKCACHE_ENABLED:
return False
directory_key = get_index_directory_key(url, requirement)
wanted_metadata_dict = {
'urlmd5':hashlib.md5(url).hexdigest(),
'requirement':requirement
}
required_key_list = ['base']
result = helper_download_network_cached(dir_url, cache_url,
signature_certificate_list,
directory_key, wanted_metadata_dict, required_key_list)
if result:
file_descriptor, metadata = result
try:
content = file_descriptor.read()
logger.info('Downloaded %s from network cache.' % url)
return content, metadata['base']
except (IOError, DirectoryNotFound), e: except (IOError, DirectoryNotFound), e:
if isinstance(e, urllib2.HTTPError) and e.code == 404: if isinstance(e, urllib2.HTTPError) and e.code == 404:
logger.debug('%s does not exist in network cache.' % url) logger.debug('%s does not exist in network cache.' % url)
...@@ -130,14 +160,13 @@ def download_network_cached(dir_url, cache_url, path, url, logger, ...@@ -130,14 +160,13 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
logger.debug('Failed to download from network cache %s: %s' % \ logger.debug('Failed to download from network cache %s: %s' % \
(url, str(e))) (url, str(e)))
return False return False
return True
@fallback_call @fallback_call
def upload_network_cached(dir_url, cache_url, external_url, path, logger, def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file, shacache_cert_file, shacache_key_file, signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file): shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server""" """Upload file to a network cache server"""
# XXX use helper and FACTOR code
if not LIBNETWORKCACHE_ENABLED: if not LIBNETWORKCACHE_ENABLED:
return False return False
...@@ -187,6 +216,64 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger, ...@@ -187,6 +216,64 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
return True return True
@fallback_call
def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file):
# XXX use helper and FACTOR code
"""Upload content of a web page to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
if not (dir_url and cache_url):
return False
logger.info('Uploading %s content into network cache.' % external_url)
directory_key = get_index_directory_key(external_url, requirement)
kw = dict(file="file",
base=base,
urlmd5=hashlib.md5(external_url).hexdigest(),
requirement=requirement)
import tempfile
f = tempfile.TemporaryFile()
f.write(content)
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
try:
return nc.upload_generic(f, directory_key, **kw)
except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % \
(str(e)))
return False
finally:
f.close()
return True
@fallback_call @fallback_call
def get_filename_from_url(url): def get_filename_from_url(url):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment