Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
slapos.core
Commits
9e88abe9
Commit
9e88abe9
authored
Feb 08, 2012
by
Yingjie Xu
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'binary_cache'
parents
77717fd8
6b18ffec
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
259 additions
and
4 deletions
+259
-4
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+54
-4
slapos/grid/networkcache.py
slapos/grid/networkcache.py
+170
-0
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+35
-0
No files found.
slapos/grid/SlapObject.py
View file @
9e88abe9
...
...
@@ -38,6 +38,8 @@ import utils
from
svcbackend
import
getSupervisorRPC
from
exception
import
BuildoutFailedError
,
WrongPermissionError
,
\
PathDoesNotExistError
from
networkcache
import
download_network_cached
,
upload_network_cached
import
tarfile
REQUIRED_COMPUTER_PARTITION_PERMISSION
=
'0750'
...
...
@@ -45,27 +47,75 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = '0750'
class
Software
(
object
):
"""This class is responsible of installing a software release"""
def
__init__
(
self
,
url
,
software_root
,
console
,
buildout
,
signature_private_key_file
=
None
,
upload_cache_url
=
None
,
upload_dir_url
=
None
,
shacache_cert_file
=
None
,
shacache_key_file
=
None
,
shadir_cert_file
=
None
,
shadir_key_file
=
None
):
signature_private_key_file
=
None
,
signature_certificate_list
=
None
,
upload_cache_url
=
None
,
upload_dir_url
=
None
,
shacache_cert_file
=
None
,
shacache_key_file
=
None
,
shadir_cert_file
=
None
,
shadir_key_file
=
None
,
download_binary_cache_url
=
None
,
upload_binary_cache_url
=
None
,
download_binary_dir_url
=
None
,
upload_binary_dir_url
=
None
):
"""Initialisation of class parameters
"""
self
.
url
=
url
self
.
software_root
=
software_root
self
.
software_url_hash
=
utils
.
getSoftwareUrlHash
(
self
.
url
)
self
.
software_path
=
os
.
path
.
join
(
self
.
software_root
,
utils
.
getSoftwareUrlHash
(
self
.
url
)
)
self
.
software_url_hash
)
self
.
buildout
=
buildout
self
.
logger
=
logging
.
getLogger
(
'BuildoutManager'
)
self
.
console
=
console
self
.
signature_private_key_file
=
signature_private_key_file
self
.
signature_certificate_list
=
signature_certificate_list
self
.
upload_cache_url
=
upload_cache_url
self
.
upload_dir_url
=
upload_dir_url
self
.
shacache_cert_file
=
shacache_cert_file
self
.
shacache_key_file
=
shacache_key_file
self
.
shadir_cert_file
=
shadir_cert_file
self
.
shadir_key_file
=
shadir_key_file
self
.
download_binary_cache_url
=
download_binary_cache_url
self
.
upload_binary_cache_url
=
upload_binary_cache_url
self
.
download_binary_dir_url
=
download_binary_dir_url
self
.
upload_binary_dir_url
=
upload_binary_dir_url
def
install
(
self
):
""" Fetches binary cache if possible.
Installs from buildout otherwise.
"""
tarname
=
self
.
software_url_hash
cache_dir
=
tempfile
.
mkdtemp
()
tarpath
=
os
.
path
.
join
(
cache_dir
,
tarname
)
if
(
not
os
.
path
.
exists
(
self
.
software_path
))
\
and
download_network_cached
(
self
.
download_binary_cache_url
,
self
.
download_binary_dir_url
,
self
.
url
,
self
.
software_root
,
self
.
software_url_hash
,
tarpath
,
self
.
logger
,
self
.
signature_certificate_list
):
tar
=
tarfile
.
open
(
tarpath
)
try
:
tar
.
extractall
(
path
=
self
.
software_root
)
finally
:
tar
.
close
()
else
:
self
.
_install_from_buildout
()
tar
=
tarfile
.
open
(
tarpath
,
"w:gz"
)
try
:
tar
.
add
(
self
.
software_path
,
arcname
=
self
.
software_url_hash
)
finally
:
tar
.
close
()
upload_network_cached
(
self
.
software_root
,
self
.
url
,
self
.
software_url_hash
,
self
.
upload_binary_cache_url
,
self
.
upload_binary_dir_url
,
tarpath
,
self
.
logger
,
self
.
signature_private_key_file
,
self
.
shacache_cert_file
,
self
.
shacache_key_file
,
self
.
shadir_cert_file
,
self
.
shadir_key_file
)
shutil
.
rmtree
(
cache_dir
)
def
_install_from_buildout
(
self
):
""" Fetches buildout configuration from the server, run buildout with
it. If it fails, we notify the server.
"""
...
...
slapos/grid/networkcache.py
0 → 100644
View file @
9e88abe9
##############################################################################
#
# Copyright (c) 2010 ViFiB SARL and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import
hashlib
import
os
import
posixpath
import
re
import
shutil
import
urlparse
import
traceback
import
utils
import
json
import
platform
try
:
try
:
from
slapos.libnetworkcache
import
NetworkcacheClient
,
UploadError
,
\
DirectoryNotFound
except
ImportError
:
LIBNETWORKCACHE_ENABLED
=
False
else
:
LIBNETWORKCACHE_ENABLED
=
True
except
:
print
'There was problem while trying to import slapos.libnetworkcache:'
\
'
\
n
%s'
%
traceback
.
format_exc
()
LIBNETWORKCACHE_ENABLED
=
False
print
'Networkcache forced to be disabled.'
def
fallback_call
(
function
):
"""Decorator which disallow to have any problem while calling method"""
def
wrapper
(
self
,
*
args
,
**
kwd
):
"""
Log the call, and the result of the call
"""
try
:
return
function
(
self
,
*
args
,
**
kwd
)
except
:
# indeed, *any* exception is swallowed
print
'There was problem while calling method %r:
\
n
%s'
%
(
function
.
__name__
,
traceback
.
format_exc
())
return
False
wrapper
.
__doc__
=
function
.
__doc__
return
wrapper
@
fallback_call
def
download_network_cached
(
cache_url
,
dir_url
,
software_url
,
software_root
,
key
,
path
,
logger
,
signature_certificate_list
):
"""Downloads from a network cache provider
return True if download succeeded.
"""
if
not
LIBNETWORKCACHE_ENABLED
:
return
False
if
not
(
cache_url
and
dir_url
and
software_url
and
software_root
):
return
False
# In order to call nc nicely.
if
len
(
signature_certificate_list
)
==
0
:
signature_certificate_list
=
None
try
:
nc
=
NetworkcacheClient
(
cache_url
,
dir_url
,
signature_certificate_list
=
signature_certificate_list
)
except
TypeError
:
logger
.
warning
(
'Incompatible version of networkcache, not using it.'
)
return
False
logger
.
info
(
'Downloading %s binary from network cache.'
%
software_url
)
try
:
file_descriptor
=
None
json_entry_list
=
nc
.
select_generic
(
key
)
for
entry
in
json_entry_list
:
json_information
,
_
=
entry
try
:
tags
=
json
.
loads
(
json_information
)
if
tags
.
get
(
'machine'
)
!=
platform
.
machine
():
continue
if
tags
.
get
(
'os'
)
!=
str
(
platform
.
linux_distribution
()):
continue
if
tags
.
get
(
'software_url'
)
!=
software_url
:
continue
if
tags
.
get
(
'software_root'
)
!=
software_root
:
continue
sha512
=
tags
.
get
(
'sha512'
)
file_descriptor
=
nc
.
download
(
sha512
)
break
except
Exception
:
continue
if
file_descriptor
is
not
None
:
f
=
open
(
path
,
'w+b'
)
try
:
shutil
.
copyfileobj
(
file_descriptor
,
f
)
finally
:
f
.
close
()
file_descriptor
.
close
()
return
True
except
(
IOError
,
DirectoryNotFound
),
e
:
logger
.
info
(
'Failed to download from network cache %s: %s'
%
\
(
software_url
,
str
(
e
)))
return
False
@
fallback_call
def
upload_network_cached
(
software_root
,
software_url
,
cached_key
,
cache_url
,
dir_url
,
path
,
logger
,
signature_private_key_file
,
shacache_cert_file
,
shacache_key_file
,
shadir_cert_file
,
shadir_key_file
):
"""Upload file to a network cache server"""
if
not
LIBNETWORKCACHE_ENABLED
:
return
False
if
not
(
software_root
and
software_url
and
cached_key
\
and
cache_url
and
dir_url
):
return
False
logger
.
info
(
'Uploading %s binary into network cache.'
%
software_url
)
# YXU: "file" and "urlmd5" should be removed when server side is ready
kw
=
dict
(
file
=
"file"
,
urlmd5
=
"urlmd5"
,
software_url
=
software_url
,
software_root
=
software_root
,
machine
=
platform
.
machine
(),
os
=
str
(
platform
.
linux_distribution
())
)
f
=
open
(
path
,
'r'
)
# convert '' into None in order to call nc nicely
if
not
signature_private_key_file
:
signature_private_key_file
=
None
if
not
shacache_cert_file
:
shacache_cert_file
=
None
if
not
shacache_key_file
:
shacache_key_file
=
None
if
not
shadir_cert_file
:
shadir_cert_file
=
None
if
not
shadir_key_file
:
shadir_key_file
=
None
try
:
nc
=
NetworkcacheClient
(
cache_url
,
dir_url
,
signature_private_key_file
=
signature_private_key_file
,
shacache_cert_file
=
shacache_cert_file
,
shacache_key_file
=
shacache_key_file
,
shadir_cert_file
=
shadir_cert_file
,
shadir_key_file
=
shadir_key_file
)
except
TypeError
:
logger
.
warning
(
'Incompatible version of networkcache, not using it.'
)
return
False
try
:
return
nc
.
upload_generic
(
f
,
cached_key
,
**
kw
)
except
(
IOError
,
UploadError
),
e
:
logger
.
info
(
'Fail to upload file. %s'
%
(
str
(
e
)))
return
False
finally
:
f
.
close
()
return
True
slapos/grid/slapgrid.py
View file @
9e88abe9
...
...
@@ -204,6 +204,17 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
if
not
option_dict
.
get
(
'supervisord_socket'
):
option_dict
[
'supervisord_socket'
]
=
\
os
.
path
.
join
(
option_dict
[
'instance_root'
],
'supervisord.socket'
)
signature_certificate_list_string
=
\
option_dict
.
get
(
'signature-certificate-list'
,
None
)
if
signature_certificate_list_string
is
not
None
:
cert_marker
=
"-----BEGIN CERTIFICATE-----"
signature_certificate_list
=
[
cert_marker
+
'
\
n
'
+
q
.
strip
()
\
for
q
in
signature_certificate_list_string
.
split
(
cert_marker
)
\
if
q
.
strip
()]
else
:
signature_certificate_list
=
None
# Returning new Slapgrid instance and options
return
([
Slapgrid
(
software_root
=
option_dict
[
'software_root'
],
instance_root
=
option_dict
[
'instance_root'
],
...
...
@@ -218,7 +229,16 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
master_ca_file
=
master_ca_file
,
certificate_repository_path
=
certificate_repository_path
,
signature_private_key_file
=
signature_private_key_file
,
signature_certificate_list
=
signature_certificate_list
,
download_binary_cache_url
=
\
option_dict
.
get
(
'download-binary-cache-url'
,
None
),
upload_binary_cache_url
=
\
option_dict
.
get
(
'upload-binary-cache-url'
,
None
),
upload_cache_url
=
option_dict
.
get
(
'upload-cache-url'
,
None
),
download_binary_dir_url
=
\
option_dict
.
get
(
'download-binary-dir-url'
,
None
),
upload_binary_dir_url
=
\
option_dict
.
get
(
'upload-binary-dir-url'
,
None
),
upload_dir_url
=
option_dict
.
get
(
'upload-dir-url'
,
None
),
console
=
option_dict
[
'console'
],
buildout
=
option_dict
.
get
(
'buildout'
),
...
...
@@ -299,7 +319,12 @@ class Slapgrid(object):
key_file
=
None
,
cert_file
=
None
,
signature_private_key_file
=
None
,
signature_certificate_list
=
None
,
download_binary_cache_url
=
None
,
upload_binary_cache_url
=
None
,
upload_cache_url
=
None
,
download_binary_dir_url
=
None
,
upload_binary_dir_url
=
None
,
upload_dir_url
=
None
,
master_ca_file
=
None
,
certificate_repository_path
=
None
,
...
...
@@ -323,7 +348,12 @@ class Slapgrid(object):
self
.
master_ca_file
=
master_ca_file
self
.
certificate_repository_path
=
certificate_repository_path
self
.
signature_private_key_file
=
signature_private_key_file
self
.
signature_certificate_list
=
signature_certificate_list
self
.
download_binary_cache_url
=
download_binary_cache_url
self
.
upload_binary_cache_url
=
upload_binary_cache_url
self
.
upload_cache_url
=
upload_cache_url
self
.
download_binary_dir_url
=
download_binary_dir_url
self
.
upload_binary_dir_url
=
upload_binary_dir_url
self
.
upload_dir_url
=
upload_dir_url
self
.
shacache_cert_file
=
shacache_cert_file
self
.
shacache_key_file
=
shacache_key_file
...
...
@@ -403,7 +433,12 @@ class Slapgrid(object):
Software
(
url
=
software_release_uri
,
software_root
=
self
.
software_root
,
console
=
self
.
console
,
buildout
=
self
.
buildout
,
signature_private_key_file
=
self
.
signature_private_key_file
,
signature_certificate_list
=
self
.
signature_certificate_list
,
download_binary_cache_url
=
self
.
download_binary_cache_url
,
upload_binary_cache_url
=
self
.
upload_binary_cache_url
,
upload_cache_url
=
self
.
upload_cache_url
,
download_binary_dir_url
=
self
.
download_binary_dir_url
,
upload_binary_dir_url
=
self
.
upload_binary_dir_url
,
upload_dir_url
=
self
.
upload_dir_url
,
shacache_cert_file
=
self
.
shacache_cert_file
,
shacache_key_file
=
self
.
shacache_key_file
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment