Commit abba1a33 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Remove NetworkCache class in favor of standard NetworkcacheClient class

WARNING: this fixes a very old bug that download-binary-dir-url was used
in place of download-dir-url. This may break existing instance of
slapcache.
parent 9a8c10e5
...@@ -40,110 +40,6 @@ from random import choice ...@@ -40,110 +40,6 @@ from random import choice
from string import ascii_lowercase from string import ascii_lowercase
from slapos.libnetworkcache import NetworkcacheClient from slapos.libnetworkcache import NetworkcacheClient
from slapos.networkcachehelper import helper_download_network_cached
class NetworkCache:
def __init__(self, configuration_path):
if not os.path.exists(configuration_path):
raise ValueError("You need configuration file")
self.configuration = configuration_path
self._load()
def _load(self):
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.configuration)
network_cache_info_dict = dict(network_cache_info.items('networkcache'))
def get_(name):
return network_cache_info_dict.get(name)
self.download_binary_cache_url = get_('download-binary-cache-url')
self.download_cache_url = get_('download-cache-url')
self.download_binary_dir_url = get_('download-binary-dir-url')
self.signature_certificate_list = get_('signature-certificate-list')
# Not mandatory
self.dir_url = get_('upload-dir-url')
self.cache_url = get_('upload-cache-url')
key_file_key = 'signature-private-key-file'
self.signature_private_key_file = get_(key_file_key)
key_file_old_key = 'signature_private_key_file'
if key_file_old_key in network_cache_info_dict:
raise ValueError(
'%s is not supported anymore, use %s' % (key_file_old_key, key_file_key))
self.shacache_ca_file = get_('shacache-ca-file')
self.shacache_cert_file = get_('shacache-cert-file')
self.shacache_key_file = get_('shacache-key-file')
self.shadir_cert_file = get_('shadir-cert-file')
self.shadir_key_file = get_('shadir-key-file')
self.shadir_ca_file = get_('shadir-ca-file')
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def upload(self, path, metadata_dict, is_sha256file=False):
"""
Upload an existing file, using a file_descriptor.
"""
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
file_descriptor = open(path, 'rb')
if not (self.dir_url and self.cache_url):
raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
# backward compatibility
metadata_dict.setdefault('file', 'notused')
metadata_dict.setdefault('urlmd5', 'notused')
# convert '' into None in order to call nc nicely
with NetworkcacheClient(self.cache_url, self.dir_url,
signature_private_key_file=self.signature_private_key_file or None,
signature_certificate_list=self.signature_certificate_list or None,
shacache_ca_file=self.shacache_ca_file or None,
shacache_cert_file=self.shacache_cert_file or None,
shacache_key_file=self.shacache_key_file or None,
shadir_cert_file=self.shadir_cert_file or None,
shadir_key_file=self.shadir_key_file or None,
shadir_ca_file=self.shadir_ca_file or None,
) as nc:
return nc.upload(file_descriptor, key, **metadata_dict)
def download(self, path, wanted_metadata_dict={},
required_key_list=[], strategy=None, is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
result = helper_download_network_cached(
self.download_binary_dir_url,
self.download_binary_cache_url,
self.signature_certificate_list,
key, wanted_metadata_dict,
required_key_list, strategy)
if result:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
file_descriptor, metadata_dict = result
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
# XXX method should check MD5.
return metadata_dict
finally:
f.close()
file_descriptor.close()
return False
def strategy(entry_list): def strategy(entry_list):
...@@ -155,13 +51,20 @@ def strategy(entry_list): ...@@ -155,13 +51,20 @@ def strategy(entry_list):
best_entry = entry best_entry = entry
timestamp = entry['timestamp'] timestamp = entry['timestamp']
return best_entry return best_entry
class Signature: class Signature:
def __init__(self, config, logger=None): def __init__(self, config, logger=None):
self.config = config self.config = config
self.logger = logger self.logger = logger
self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r'))
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.config.slapos_configuration)
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def log(self, message, *args): def log(self, message, *args):
if self.logger is not None: if self.logger is not None:
...@@ -179,32 +82,66 @@ class Signature: ...@@ -179,32 +82,66 @@ class Signature:
return base return base
def save_file_hash(self, path, destination): def save_file_hash(self, path, destination):
base = self.get_file_hash(path) base = self.get_file_hash(path)
with open(destination, "wb") as f: with open(destination, "wb") as f:
f.write(base) f.write(base)
def _download_once(self, path, wanted_metadata_dict={},
required_key_list=[], is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
self.log('Downloading %s...', key)
result = self.shacache.select(key, wanted_metadata_dict, required_key_list)
entry = None
result = list(result)
if result:
entry = strategy(result)
if not entry: # XXX: this should be the choice of 'strategy' function
self.log("Can't find best entry matching strategy, selecting "
"random one between acceptable ones.")
entry = result[0]
if not entry:
self.log('No entry matching key %s', key)
else:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
f = open(path, 'w+b')
fd_download = self.shacache.download(entry['sha512'])
try:
shutil.copyfileobj(fd_download, f)
# XXX method should check MD5.
return entry
finally:
f.close()
fd_download.close()
return False
def _download(self, path): def _download(self, path):
""" """
Download a tar of the repository from cache, and untar it. Download a tar of the repository from cache, and untar it.
""" """
shacache = NetworkCache(self.config.slapos_configuration) try:
download_metadata_dict = self._download_once(path=path,
if shacache.signature_certificate_list is None: required_key_list=['timestamp'])
raise ValueError("You need at least one valid signature for download") except:
return False
download_metadata_dict = shacache.download(path=path,
required_key_list=['timestamp'], strategy=strategy)
if download_metadata_dict: if download_metadata_dict:
self.log('File downloaded in %s', path) self.log('File downloaded in %s', path)
current_sha256 = self.get_file_hash(path) current_sha256 = self.get_file_hash(path)
with tempfile.NamedTemporaryFile() as f_256: with tempfile.NamedTemporaryFile() as f_256:
sha256path = f_256.name sha256path = f_256.name
if shacache.download(path=sha256path, required_key_list=['timestamp'], try:
strategy=strategy, is_sha256file=True): sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True)
self.log('sha 256 downloaded in %s', sha256path) self.log('sha 256 downloaded in %s', sha256path)
expected_sha256 = f_256.read() except:
sha256sum_present = False
if sha256sum_present:
expected_sha256 = f_256.read()
if current_sha256 == expected_sha256: if current_sha256 == expected_sha256:
return True return True
else: else:
...@@ -230,29 +167,32 @@ class Signature: ...@@ -230,29 +167,32 @@ class Signature:
""" """
Creates uploads repository to cache. Creates uploads repository to cache.
""" """
shacache = NetworkCache(self.config.slapos_configuration)
sha256path = path + ".sha256" sha256path = path + ".sha256"
self.save_file_hash(path, sha256path) self.save_file_hash(path, sha256path)
metadata_dict = { metadata_dict = {
# XXX: we set date from client side. It can be potentially dangerous # XXX: we set date from client side. It can be potentially dangerous
# as it can be badly configured. # as it can be badly configured.
'timestamp': time.time(), 'timestamp': time.time(),
'token': ''.join([choice(ascii_lowercase) for _ in range(128)]) 'token': ''.join([choice(ascii_lowercase) for _ in range(128)]),
# backward compatibility
'file': 'notused',
'urlmd5': 'notused'
} }
try: try:
sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict) sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict)
if sha512sum: if sha512sum:
self.log( self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', path, 'Uploaded %s to cache (using %s key) with SHA512 %s.', path,
shacache.key, sha512sum) self.key, sha512sum)
sha512sum_path = shacache.upload( sha512sum_path = self.shacache.upload(
path=sha256path, metadata_dict=metadata_dict, is_sha256file=True) open(sha256path, 'rb'),
self.key + "-sha256-content",
**metadata_dict)
if sha512sum_path: if sha512sum_path:
self.log( self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path, 'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path,
shacache.key, sha512sum_path) self.key, sha512sum_path)
else: else:
self.log('Fail to upload sha256file file to cache.') self.log('Fail to upload sha256file file to cache.')
else: else:
...@@ -263,7 +203,7 @@ class Signature: ...@@ -263,7 +203,7 @@ class Signature:
def upload(self): def upload(self):
self._upload(self.config.file) self._upload(self.config.file)
# Class containing all parameters needed for configuration # Class containing all parameters needed for configuration
class Config: class Config:
def __init__(self, option_dict=None): def __init__(self, option_dict=None):
......
...@@ -5,6 +5,7 @@ key = %(key)s ...@@ -5,6 +5,7 @@ key = %(key)s
download-binary-cache-url = http://shacache.nxdcdn.com download-binary-cache-url = http://shacache.nxdcdn.com
download-cache-url = https://shacache.nxdcdn.com download-cache-url = https://shacache.nxdcdn.com
download-binary-dir-url = http://shadir.nxdcdn.com download-binary-dir-url = http://shadir.nxdcdn.com
download-dir-url = http://shadir.nxdcdn.com
# Standard Signature handled by SlapOS Administrators # Standard Signature handled by SlapOS Administrators
signature-certificate-list = signature-certificate-list =
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment