Skip to content

S
slapos.core
Commit 20e9ad51 authored by Marco Mariani's avatar Marco Mariani
Browse files
Options

whitespace/indent cleanup

parent 6c003ece
Hide whitespace changes
Inline Side-by-side
Showing with 84 additions and 87 deletions
slapos/grid/SlapObject.py
View file @ 20e9ad51
......@@ -41,6 +41,7 @@ import textwrap
import xmlrpclib
from supervisor import xmlrpc
from slapos.grid.utils import (md5digest, getCleanEnvironment, bootstrapBuildout,
launchBuildout, SlapPopen, dropPrivileges, updateFile)
from slapos.slap.slap import NotFoundError
......@@ -57,13 +58,14 @@ class Software(object):
"""This class is responsible for installing a software release"""
def __init__(self, url, software_root, buildout,
signature_private_key_file=None, signature_certificate_list=None,
upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None,
shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None,
download_from_binary_cache_url_blacklist = [],
upload_to_binary_cache_url_blacklist = []):
signature_private_key_file=None, signature_certificate_list=None,
upload_cache_url=None, upload_dir_url=None, shacache_cert_file=None,
shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None,
download_from_binary_cache_url_blacklist=[],
upload_to_binary_cache_url_blacklist=[]):
# XXX mutable defaults
"""Initialisation of class parameters
"""
self.url = url
......@@ -125,7 +127,7 @@ class Software(object):
if self.url.startswith(url):
blacklisted = True
self.logger.info("Can't upload to binary cache: "
"Software Release URL is blacklisted.")
"Software Release URL is blacklisted.")
break
if not blacklisted:
self.uploadSoftwareRelease(tarpath)
......@@ -137,8 +139,7 @@ class Software(object):
it. If it fails, we notify the server.
"""
root_stat_info = os.stat(self.software_root)
os.environ = getCleanEnvironment(pwd.getpwuid(root_stat_info.st_uid
).pw_dir)
os.environ = getCleanEnvironment(pwd.getpwuid(root_stat_info.st_uid).pw_dir)
if not os.path.isdir(self.software_path):
os.mkdir(self.software_path)
extends_cache = tempfile.mkdtemp()
......@@ -146,31 +147,31 @@ class Software(object):
# In case when running as root copy ownership, to simplify logic
for path in [self.software_path, extends_cache]:
path_stat_info = os.stat(path)
if root_stat_info.st_uid != path_stat_info.st_uid or\
root_stat_info.st_gid != path_stat_info.st_gid:
os.chown(path, root_stat_info.st_uid,
root_stat_info.st_gid)
if (root_stat_info.st_uid != path_stat_info.st_uid or
root_stat_info.st_gid != path_stat_info.st_gid):
os.chown(path, root_stat_info.st_uid,
root_stat_info.st_gid)
try:
buildout_parameter_list = [
'buildout:extends-cache=%s' % extends_cache,
'buildout:directory=%s' % self.software_path,]
'buildout:extends-cache=%s' % extends_cache,
'buildout:directory=%s' % self.software_path
]
if self.signature_private_key_file or \
self.upload_cache_url or \
self.upload_dir_url is not None:
if (self.signature_private_key_file or
self.upload_cache_url or
self.upload_dir_url is not None):
buildout_parameter_list.append('buildout:networkcache-section=networkcache')
for buildout_option, value in (
('%ssignature-private-key-file=%s', self.signature_private_key_file),
('%supload-cache-url=%s', self.upload_cache_url),
('%supload-dir-url=%s', self.upload_dir_url),
('%sshacache-cert-file=%s', self.shacache_cert_file),
('%sshacache-key-file=%s', self.shacache_key_file),
('%sshadir-cert-file=%s', self.shadir_cert_file),
('%sshadir-key-file=%s', self.shadir_key_file),
):
for buildout_option, value in [
('%ssignature-private-key-file=%s', self.signature_private_key_file),
('%supload-cache-url=%s', self.upload_cache_url),
('%supload-dir-url=%s', self.upload_dir_url),
('%sshacache-cert-file=%s', self.shacache_cert_file),
('%sshacache-key-file=%s', self.shacache_key_file),
('%sshadir-cert-file=%s', self.shadir_cert_file),
('%sshadir-key-file=%s', self.shadir_key_file)
]:
if value:
buildout_parameter_list.append( \
buildout_option % ('networkcache:', value))
buildout_parameter_list.append(buildout_option % ('networkcache:', value))
buildout_cfg = os.path.join(self.software_path, 'buildout.cfg')
self.createProfileIfMissing(buildout_cfg, self.url)
......@@ -285,14 +286,13 @@ class Partition(object):
self._updateCertificate()
def _updateCertificate(self):
if not os.path.exists(self.key_file) or \
not os.path.exists(self.cert_file):
if not os.path.exists(self.key_file) or not os.path.exists(self.cert_file):
self.logger.info('Certificate and key not found, downloading to %r and '
'%r' % (self.cert_file, self.key_file))
try:
partition_certificate = self.computer_partition.getCertificate()
except NotFoundError:
raise NotFoundError('Partition %s is not known from SlapOS Master.' % \
raise NotFoundError('Partition %s is not known from SlapOS Master.' %
self.partition_id)
open(self.key_file, 'w').write(partition_certificate['key'])
open(self.cert_file, 'w').write(partition_certificate['certificate'])
......@@ -308,7 +308,7 @@ class Partition(object):
return (uid, gid)
def addServiceToGroup(self, partition_id,
runner_list, path, extension = ''):
runner_list, path, extension=''):
uid, gid = self.getUserGroupId()
program_partition_template = pkg_resources.resource_stream(__name__,
'templates/program_partition_supervisord.conf.in').read()
......@@ -340,7 +340,7 @@ class Partition(object):
""" Creates configuration file from template in software_path, then
installs the software partition with the help of buildout
"""
self.logger.info("Installing Computer Partition %s..." \
self.logger.info("Installing Computer Partition %s..."
% self.computer_partition.getId())
# Checks existence and permissions of Partition directory
# Note : Partitions have to be created and configured before running slapgrid
......@@ -359,7 +359,7 @@ class Partition(object):
(self.instance_path, permission,
REQUIRED_COMPUTER_PARTITION_PERMISSION))
os.environ = getCleanEnvironment(pwd.getpwuid(
instance_stat_info.st_uid).pw_dir)
instance_stat_info.st_uid).pw_dir)
# Generates buildout part from template
template_location = os.path.join(self.software_path, 'instance.cfg')
# Backward compatibility: "instance.cfg" file was named "template.cfg".
......@@ -433,8 +433,7 @@ class Partition(object):
if not os.path.exists(buildout_binary):
# use own buildout generation
bootstrapBuildout(self.instance_path, self.buildout,
['buildout:bin-directory=%s'% os.path.join(self.instance_path,
'sbin')])
['buildout:bin-directory=%s'% os.path.join(self.instance_path, 'sbin')])
buildout_binary = os.path.join(self.instance_path, 'sbin', 'buildout')
# Launches buildout
launchBuildout(self.instance_path, buildout_binary)
......@@ -465,12 +464,12 @@ class Partition(object):
instance_id=partition_id,
program_list=','.join(['_'.join([partition_id, runner])
for runner in runner_list+service_list]))
# Same method to add to service and run
# Same method to add to service and run
self.addServiceToGroup(partition_id, runner_list,self.run_path)
self.addServiceToGroup(partition_id, service_list,self.service_path,
extension=getWatchdogID())
updateFile(self.supervisord_partition_configuration_path,
self.partition_supervisor_configuration)
self.partition_supervisor_configuration)
self.updateSupervisor()
def start(self):
......@@ -483,7 +482,7 @@ class Partition(object):
supervisor.startProcessGroup(partition_id, False)
except xmlrpclib.Fault, e:
if e.faultString.startswith('BAD_NAME:'):
self.logger.info("Nothing to start on %s..." % \
self.logger.info("Nothing to start on %s..." %
self.computer_partition.getId())
else:
self.logger.info("Requested start of %s..." % self.computer_partition.getId())
......@@ -503,7 +502,7 @@ class Partition(object):
def destroy(self):
"""Destroys the partition and makes it available for subsequent use."
"""
self.logger.info("Destroying Computer Partition %s..." \
self.logger.info("Destroying Computer Partition %s..."
% self.computer_partition.getId())
# Launches "destroy" binary if exists
destroy_executable_location = os.path.join(self.instance_path, 'sbin',
......@@ -512,11 +511,11 @@ class Partition(object):
uid, gid = self.getUserGroupId()
self.logger.debug('Invoking %r' % destroy_executable_location)
process_handler = SlapPopen([destroy_executable_location],
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=self.instance_path,
env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=self.instance_path,
env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to destroy Computer Partition in %r.' % \
self.instance_path
......@@ -589,4 +588,3 @@ class Partition(object):
supervisor.addProcessGroup(gname)
self.logger.info('Updated %r' % gname)
self.logger.debug('Supervisord updated')
slapos/grid/slapgrid.py
View file @ 20e9ad51
......@@ -155,14 +155,12 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
parser.add_argument("--maximal_delay",
help="Deprecated. Will only work from configuration file in the future.")
# Parses arguments
if not argument_tuple:
# No arguments given to entry point : we parse sys.argv.
argument_option_instance = parser.parse_args()
else:
argument_option_instance = \
parser.parse_args(list(argument_tuple))
argument_option_instance = parser.parse_args(list(argument_tuple))
# Parses arguments from config file, if needed, then merge previous arguments
option_dict = {}
configuration_file = argument_option_instance.configuration_file[0]
......@@ -243,8 +241,7 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
certificate_repository_path = option_dict.get('certificate_repository_path')
if certificate_repository_path is not None:
if not os.path.isdir(certificate_repository_path):
parser.error('Directory %r does not exist' %
certificate_repository_path)
parser.error('Directory %r does not exist' % certificate_repository_path)
# Supervisord configuration location
if not option_dict.get('supervisord_configuration_path'):
......@@ -259,9 +256,11 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
option_dict.get('signature-certificate-list', None)
if signature_certificate_list_string is not None:
cert_marker = "-----BEGIN CERTIFICATE-----"
signature_certificate_list = [cert_marker + '\n' + q.strip() \
for q in signature_certificate_list_string.split(cert_marker) \
if q.strip()]
signature_certificate_list = [
cert_marker + '\n' + q.strip()
for q in signature_certificate_list_string.split(cert_marker)
if q.strip()
]
else:
signature_certificate_list = None
......@@ -640,13 +639,12 @@ class Slapgrid(object):
def _launchSupervisord(self):
launchSupervisord(self.supervisord_socket,
self.supervisord_configuration_path,
logger=self.logger)
self.supervisord_configuration_path,
logger=self.logger)
def _checkPromises(self, computer_partition):
self.logger.info("Checking promises...")
instance_path = os.path.join(self.instance_root,
computer_partition.getId())
instance_path = os.path.join(self.instance_root, computer_partition.getId())
uid, gid = None, None
stat_info = os.stat(instance_path)
......@@ -675,9 +673,10 @@ class Slapgrid(object):
stdin=subprocess.PIPE)
process_handler = subprocess.Popen(command,
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=cwd,
env=None if sys.platform == 'cygwin' else {}, **kw)
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=cwd,
env=None if sys.platform == 'cygwin' else {},
**kw)
process_handler.stdin.flush()
process_handler.stdin.close()
process_handler.stdin = None
......@@ -739,8 +738,7 @@ class Slapgrid(object):
# Try to process it anyway, it may need to be deleted.
software_url = None
try:
software_path = os.path.join(self.software_root,
md5digest(software_url))
software_path = os.path.join(self.software_root, md5digest(software_url))
except TypeError:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
......@@ -1107,14 +1105,14 @@ class Slapgrid(object):
script_list_to_run = os.listdir(report_path)
else:
script_list_to_run = []
#We now generate the pseudorandom name for the xml file
# and we add it in the invocation_list
f = tempfile.NamedTemporaryFile()
name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
path_to_slapreport = os.path.join(instance_path, 'var', 'xml_report',
name_xml)
failed_script_list = []
for script in script_list_to_run:
invocation_list = []
......@@ -1124,7 +1122,7 @@ class Slapgrid(object):
#f = tempfile.NamedTemporaryFile()
#name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
#path_to_slapreport = os.path.join(instance_path, 'var', name_xml)
invocation_list.append(path_to_slapreport)
#Dropping privileges
uid, gid = None, None
......@@ -1134,9 +1132,10 @@ class Slapgrid(object):
gid = stat_info.st_gid
kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=os.path.join(instance_path, 'etc', 'report'),
env=None, **kw)
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=os.path.join(instance_path, 'etc', 'report'),
env=None,
**kw)
if process_handler.returncode is None:
process_handler.kill()
if process_handler.returncode != 0:
......@@ -1169,15 +1168,15 @@ class Slapgrid(object):
filename_list = []
#logger.debug('name List %s' % filename_list)
usage = ''
for filename in filename_list:
file_path = os.path.join(dir_reports, filename)
if os.path.exists(file_path):
usage_file = open(file_path, 'r')
usage = usage_file.read()
usage_file.close()
#We check the validity of xml content of each reports
if not self.validateXML(usage, partition_consumption_model):
logger.info('WARNING: The XML file %s generated by slapreport is '
......@@ -1207,7 +1206,7 @@ class Slapgrid(object):
logger.info(issue)
for computer_partition_usage in computer_partition_usage_list:
logger.info('computer_partition_usage_list : %s - %s' % \
logger.info('computer_partition_usage_list: %s - %s' % \
(computer_partition_usage.usage, computer_partition_usage.getId()))
#If there is, at least, one report
......@@ -1216,7 +1215,7 @@ class Slapgrid(object):
#We generate the final XML report with asXML method
computer_consumption = self.asXML(computer_partition_usage_list)
logger.info('Final xml report : %s' % computer_consumption)
logger.info('Final xml report: %s' % computer_consumption)
#We test the XML report before sending it
if self.validateXML(computer_consumption, computer_consumption_model):
......@@ -1240,8 +1239,7 @@ class Slapgrid(object):
computer_partition_id = computer_partition.getId()
try:
software_url = computer_partition.getSoftwareRelease().getURI()
software_path = os.path.join(self.software_root,
md5digest(software_url))
software_path = os.path.join(self.software_root, md5digest(software_url))
except (NotFoundError, TypeError):
software_url = None
software_path = None
......@@ -1292,7 +1290,7 @@ class Slapgrid(object):
computer_partition.getId())
except ServerError as server_error:
logger.debug('Ignored server error while trying to inform about '
'destroying Computer Partition %r. Error is :\n%r' %
'destroying Computer Partition %r. Error is:\n%r' %
(computer_partition.getId(), server_error.args[0]))
logger.info("Finished usage reports.")
......
slapos/grid/utils.py
View file @ 20e9ad51
......@@ -209,12 +209,12 @@ def dropPrivileges(uid, gid):
uid, gid, group_list)
new_uid, new_gid, new_group_list = os.getuid(), os.getgid(), os.getgroups()
if not (new_uid == uid and new_gid == gid and set(new_group_list) == group_list):
raise OSError('%s new_uid = %r and new_gid = %r and ' \
'new_group_list = %r which is fatal.'
% (message_pre,
new_uid,
new_gid,
new_group_list))
raise OSError('%s new_uid = %r and new_gid = %r and '
'new_group_list = %r which is fatal.'
% (message_pre,
new_uid,
new_gid,
new_group_list))
# assert that it is not possible to go back to running one
try:
......@@ -276,8 +276,9 @@ def bootstrapBuildout(path, buildout=None,
path))
kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process_handler = SlapPopen(invocation_list,
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=path, **kw)
preexec_fn=lambda: dropPrivileges(uid, gid),
cwd=path,
**kw)
if process_handler.returncode is None or process_handler.returncode != 0:
message = 'Failed to run buildout profile in directory %r' % (path)
logger.error(message)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7