Add support for networkcache

parent 4bb995d6
...@@ -24,6 +24,7 @@ setup(name=name, ...@@ -24,6 +24,7 @@ setup(name=name,
install_requires=[ install_requires=[
'setuptools', # namespaces 'setuptools', # namespaces
'zc.buildout', # plays with buildout 'zc.buildout', # plays with buildout
'slapos.libnetworkcache>=0.13.1', # Uses helper new in this version
], ],
zip_safe=True, zip_safe=True,
entry_points={ entry_points={
......
...@@ -25,35 +25,117 @@ ...@@ -25,35 +25,117 @@
# #
############################################################################## ##############################################################################
import hashlib
import os import os
import shutil
from subprocess import check_call import time
import traceback
from zc.buildout import UserError
from subprocess import check_call, CalledProcessError
try:
try:
from slapos.networkcachehelper import \
helper_upload_network_cached_from_directory, \
helper_download_network_cached_to_directory
except ImportError:
LIBNETWORKCACHE_ENABLED = False
else:
LIBNETWORKCACHE_ENABLED = True
except:
print 'There was problem while trying to import slapos.libnetworkcache:'\
'\n%s' % traceback.format_exc()
LIBNETWORKCACHE_ENABLED = False
print 'Networkcache forced to be disabled.'
GIT_DEFAULT_REMOTE_NAME = 'origin' GIT_DEFAULT_REMOTE_NAME = 'origin'
GIT_DEFAULT_BRANCH_NAME = 'master' GIT_DEFAULT_BRANCH_NAME = 'master'
TRUE_VALUES = ['y', 'yes', '1', 'true'] TRUE_VALUES = ['y', 'yes', '1', 'true']
def upload_network_cached(path, name, revision, networkcache_options):
"""
Creates uploads repository to cache.
"""
try:
print 'Uploading git repository to cache...'
metadata_dict = {
'revision':revision,
# XXX: we set date from client side. It can be potentially dangerous
# as it can be badly configured.
'timestamp':time.time(),
}
helper_upload_network_cached_from_directory(
path=path,
directory_key='git-buildout-%s' % hashlib.md5(name).hexdigest(),
metadata_dict=metadata_dict,
# Then we give a lot of not interesting things
dir_url=networkcache_options.get('upload-dir-url'),
cache_url=networkcache_options.get('upload-cache-url'),
signature_private_key_file=networkcache_options.get(
'signature-private-key-file'),
shacache_cert_file=networkcache_options.get('shacache-cert-file'),
shacache_key_file=networkcache_options.get('shacache-key-file'),
shadir_cert_file=networkcache_options.get('shadir-cert-file'),
shadir_key_file=networkcache_options.get('shadir-key-file'),
)
print 'Uploaded git repository to cache.'
except Exception:
print 'Unable to upload to cache:\n%s.' % traceback.format_exc()
def download_network_cached(path, name, revision, networkcache_options):
"""
Download a tar of the repository from cache, and untar it.
"""
def strategy(entry_list):
"""
Get the latest entry.
"""
timestamp = 0
best_entry = None
for entry in entry_list:
if entry['timestamp'] > timestamp:
best_entry = entry
return best_entry
return helper_download_network_cached_to_directory(
path=path,
directory_key='git-buildout-%s' % hashlib.md5(name).hexdigest(),
wanted_metadata_dict={'revision':revision},
required_key_list=['timestamp'],
strategy=strategy,
# Then we give a lot of not interesting things
dir_url=networkcache_options.get('download-dir-url'),
cache_url=networkcache_options.get('download-cache-url'),
signature_certificate_list=\
networkcache_options.get('signature-certificate-list'),
)
class Recipe(object): class Recipe(object):
"""Clone a git repository.""" """Clone a git repository."""
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
self.url = options.get('url') self.repository = options.get('repository')
self.branch = options.get('branch', GIT_DEFAULT_BRANCH_NAME) self.branch = options.get('branch', GIT_DEFAULT_BRANCH_NAME)
self.revision = options.get('revision') self.revision = options.get('revision')
self.git_command = options.get('git-command', 'git') self.git_command = options.get('git-executable', 'git')
self.name = name
self.location = options.get('location', self.location = options.get('location',
os.path.join(buildout['buildout']['parts-directory'], name)) os.path.join(buildout['buildout']['parts-directory'], name))
if options.get('develop') in TRUE_VALUES: if options.get('develop') in TRUE_VALUES:
self.develop = True self.develop = True
if not self.url: # XXX clean
raise ValueError('url parameter is mandatory.') self.networkcache = buildout.get('networkcache')
# Check if input is correct
if not self.repository:
raise UserError('repository parameter is mandatory.')
if self.revision and self.branch != GIT_DEFAULT_BRANCH_NAME: if self.revision and self.branch != GIT_DEFAULT_BRANCH_NAME:
# revision and branch options are incompatible # revision and branch options are incompatible
raise ValueError('revision and branch parameters are set but are ' raise UserError('revision and branch (other than master) parameters '
'incompatible. Please specify only one of them.') 'are set but are incompatible. Please specify only one of them.')
def gitReset(self, revision=None): def gitReset(self, revision=None):
"""Operates git reset on the repository.""" """Operates git reset on the repository."""
...@@ -68,17 +150,37 @@ class Recipe(object): ...@@ -68,17 +150,37 @@ class Recipe(object):
Do a git clone. Do a git clone.
If branch is specified, checkout to it. If branch is specified, checkout to it.
If revision is specified, reset to it. If revision is specified, reset to it.
If something fails, try to download from cache.
Else, if possible, try to upload to cache.
""" """
if not os.path.exists(self.location): # If directory already exist: delete it.
if os.path.exists(self.location):
print 'destination directory already exists. Deleting it.'
shutil.rmtree(self.location)
git_clone_command = [self.git_command, 'clone', git_clone_command = [self.git_command, 'clone',
self.url, self.repository,
self.location] self.location]
if self.branch: if self.branch:
git_clone_command.extend(['--branch', self.branch]) git_clone_command.extend(['--branch', self.branch])
check_call(git_clone_command)
try:
check_call(git_clone_command)
if not os.path.exists(self.location):
raise UserError("Unknown error while cloning repository.")
if self.revision: if self.revision:
self.gitReset(self.revision) self.gitReset(self.revision)
upload_network_cached(self.location, self.name, self.revision,
self.networkcache)
except CalledProcessError:
print ("Unable to download from git repository. Trying from network "
"cache...")
if not download_network_cached(self.location, self.name, self.revision,
self.networkcache):
if os.path.exists(self.location):
shutil.rmtree(self.location)
raise UserError('Impossible to clone repository and impossible to '
'download from cache.')
return [self.location] return [self.location]
...@@ -87,7 +189,7 @@ class Recipe(object): ...@@ -87,7 +189,7 @@ class Recipe(object):
""" """
Do a git fetch. Do a git fetch.
If user doesn't develop, reset to remote revision (or branch if revision is If user doesn't develop, reset to remote revision (or branch if revision is
not specified.) not specified).
""" """
check_call([self.git_command, 'fetch', '--all'], cwd=self.location) check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment