Skip to content

S
slapos.core
Commit 65622b9a authored by Marco Mariani's avatar Marco Mariani
Browse files
Options

make pep8 happy

parent c1058afd
Hide whitespace changes
Inline Side-by-side
Showing with 78 additions and 89 deletions
slapos/grid/SlapObject.py
View file @ 65622b9a
......@@ -43,7 +43,7 @@ from supervisor import xmlrpc
from slapos.grid.utils import (md5digest, getCleanEnvironment,
SlapPopen, dropPrivileges, updateFile)
from slapos.grid import utils # for mocked methods
from slapos.grid import utils # for mocked methods
from slapos.slap.slap import NotFoundError
from slapos.grid.svcbackend import getSupervisorRPC
from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
......@@ -166,8 +166,8 @@ class Software(object):
]
if (self.signature_private_key_file or
self.upload_cache_url or
self.upload_dir_url):
self.upload_cache_url or
self.upload_dir_url):
buildout_parameter_list.append('buildout:networkcache-section=networkcache')
for buildout_option, value in [
('%ssignature-private-key-file=%s', self.signature_private_key_file),
......@@ -306,10 +306,7 @@ class Partition(object):
uid, gid = self.getUserGroupId()
for name, path in [
('certificate', self.cert_file),
('key', self.key_file),
]:
for name, path in [('certificate', self.cert_file), ('key', self.key_file)]:
new_content = partition_certificate[name]
old_content = None
if os.path.exists(path):
......@@ -325,7 +322,6 @@ class Partition(object):
fout.write(new_content)
os.chown(path, uid, gid)
def getUserGroupId(self):
"""Returns tuple of (uid, gid) of partition"""
stat_info = os.stat(self.instance_path)
......@@ -341,17 +337,17 @@ class Partition(object):
for runner in runner_list:
self.partition_supervisor_configuration += '\n' + \
program_partition_template % {
'program_id': '_'.join([partition_id, runner]),
'program_directory': self.instance_path,
'program_command': os.path.join(path, runner),
'program_name': runner + extension,
'instance_path': self.instance_path,
'user_id': uid,
'group_id': gid,
# As supervisord has no environment to inherit, setup a minimalistic one
'HOME': pwd.getpwuid(uid).pw_dir,
'USER': pwd.getpwuid(uid).pw_name,
}
'program_id': '_'.join([partition_id, runner]),
'program_directory': self.instance_path,
'program_command': os.path.join(path, runner),
'program_name': runner + extension,
'instance_path': self.instance_path,
'user_id': uid,
'group_id': gid,
# As supervisord has no environment to inherit, setup a minimalistic one
'HOME': pwd.getpwuid(uid).pw_dir,
'USER': pwd.getpwuid(uid).pw_name,
}
def updateSymlink(self, sr_symlink, software_path):
if os.path.lexists(sr_symlink):
......@@ -391,7 +387,7 @@ class Partition(object):
if not os.path.exists(self.software_path):
# XXX What should it raise?
raise IOError('Software Release %s is not present on system.\n'
'Cannot deploy instance.' % self.software_release_url)
'Cannot deploy instance.' % self.software_release_url)
# Generate buildout instance profile from template in Software Release
template_location = os.path.join(self.software_path, 'instance.cfg')
......@@ -412,13 +408,13 @@ class Partition(object):
buildout_text = open(config_location).read()
buildout_text += '\n\n' + pkg_resources.resource_string(__name__,
'templates/buildout-tail.cfg.in') % {
'computer_id': self.computer_id,
'partition_id': self.partition_id,
'server_url': self.server_url,
'software_release_url': self.software_release_url,
'key_file': self.key_file,
'cert_file': self.cert_file,
}
'computer_id': self.computer_id,
'partition_id': self.partition_id,
'server_url': self.server_url,
'software_release_url': self.software_release_url,
'key_file': self.key_file,
'cert_file': self.cert_file,
}
open(config_location, 'w').write(buildout_text)
os.chmod(config_location, 0o640)
# Try to find the best possible buildout:
......@@ -482,7 +478,6 @@ class Partition(object):
logger=self.logger)
self.generateSupervisorConfigurationFile()
def generateSupervisorConfigurationFile(self):
"""
Generates supervisord configuration file from template.
......@@ -510,10 +505,10 @@ class Partition(object):
group_partition_template = pkg_resources.resource_stream(__name__,
'templates/group_partition_supervisord.conf.in').read()
self.partition_supervisor_configuration = group_partition_template % {
'instance_id': partition_id,
'program_list': ','.join(['_'.join([partition_id, runner])
for runner in runner_list + service_list])
}
'instance_id': partition_id,
'program_list': ','.join(['_'.join([partition_id, runner])
for runner in runner_list + service_list])
}
# Same method to add to service and run
self.addServiceToGroup(partition_id, runner_list, self.run_path)
self.addServiceToGroup(partition_id, service_list, self.service_path,
......
slapos/grid/distribution.py
View file @ 65622b9a
......@@ -62,4 +62,3 @@ def patched_linux_distribution(distname='', version='', id='',
pass
return platform.linux_distribution(distname, version, id, supported_dists, full_distribution_name)
slapos/grid/slapgrid.py
View file @ 65622b9a
......@@ -100,23 +100,23 @@ def check_missing_parameters(options):
def check_missing_files(options):
req_files = [
options.get('key_file'),
options.get('cert_file'),
options.get('master_ca_file'),
options.get('shacache-cert-file'),
options.get('shacache-key-file'),
options.get('shadir-cert-file'),
options.get('shadir-key-file'),
options.get('signature_private_key_file')
]
options.get('key_file'),
options.get('cert_file'),
options.get('master_ca_file'),
options.get('shacache-cert-file'),
options.get('shacache-key-file'),
options.get('shadir-cert-file'),
options.get('shadir-key-file'),
options.get('signature_private_key_file')
]
req_dirs = [
options.get('certificate_repository_path')
]
options.get('certificate_repository_path')
]
for f in req_files:
if f and not os.path.exists(f):
raise RuntimeError('File %r does not exist.' % f)
raise RuntimeError('File %r does not exist.' % f)
for d in req_dirs:
if d and not os.path.isdir(d):
......@@ -149,8 +149,8 @@ def merged_options(args, configp):
# Parse cache / binary cache options
# Backward compatibility about "binary-cache-url-blacklist" deprecated option
if options.get("binary-cache-url-blacklist") and not \
options.get("download-from-binary-cache-url-blacklist"):
if (options.get("binary-cache-url-blacklist") and not
options.get("download-from-binary-cache-url-blacklist")):
options["download-from-binary-cache-url-blacklist"] = \
options["binary-cache-url-blacklist"]
options["download-from-binary-cache-url-blacklist"] = [
......@@ -175,8 +175,8 @@ def random_delay(options, logger):
maximal_delay = int(options.get('maximal_delay', '0'))
if maximal_delay:
duration = random.randint(1, maximal_delay)
logger.info('Sleeping for %s seconds. To disable this feature, ' \
'check --now parameter in slapgrid help.' % duration)
logger.info('Sleeping for %s seconds. To disable this feature, '
'check --now parameter in slapgrid help.', duration)
time.sleep(duration)
......@@ -185,10 +185,10 @@ def create_slapgrid_object(options, logger):
if 'signature-certificate-list' in options:
cert_marker = '-----BEGIN CERTIFICATE-----'
signature_certificate_list = [
cert_marker + '\n' + q.strip()
for q in options['signature-certificate-list'].split(cert_marker)
if q.strip()
]
cert_marker + '\n' + q.strip()
for q in options['signature-certificate-list'].split(cert_marker)
if q.strip()
]
op = options
return Slapgrid(software_root=op['software_root'],
......@@ -199,17 +199,17 @@ def create_slapgrid_object(options, logger):
supervisord_configuration_path=op['supervisord_configuration_path'],
buildout=op.get('buildout'),
logger=logger,
force_periodicity = op.get('force_periodicity', False),
maximum_periodicity = op.get('maximum_periodicity', 86400),
force_periodicity=op.get('force_periodicity', False),
maximum_periodicity=op.get('maximum_periodicity', 86400),
key_file=op.get('key_file'),
cert_file=op.get('cert_file'),
signature_private_key_file=op.get('signature_private_key_file'),
signature_certificate_list=signature_certificate_list,
download_binary_cache_url=op.get('download-binary-cache-url'),
upload_binary_cache_url=op.get('upload-binary-cache-url'),
download_from_binary_cache_url_blacklist=\
download_from_binary_cache_url_blacklist=
op.get('download-from-binary-cache-url-blacklist', []),
upload_to_binary_cache_url_blacklist=\
upload_to_binary_cache_url_blacklist=
op.get('upload-to-binary-cache-url-blacklist', []),
upload_cache_url=op.get('upload-cache-url'),
download_binary_dir_url=op.get('download-binary-dir-url'),
......@@ -232,8 +232,8 @@ def create_slapgrid_object(options, logger):
def check_required_only_partitions(existing, required):
missing = set(required) - set(existing)
if missing:
plural = ['s', ''][len(missing)==1]
raise ValueError('Unknown partition%s: %s' % (plural, ', '.join(sorted(missing))))
plural = ['s', ''][len(missing) == 1]
raise ValueError('Unknown partition%s: %s' % (plural, ', '.join(sorted(missing))))
class Slapgrid(object):
......@@ -334,8 +334,8 @@ class Slapgrid(object):
invocation_list = [WATCHDOG_PATH]
invocation_list.append("--master-url '%s' " % self.master_url)
if self.certificate_repository_path:
invocation_list.append("--certificate-repository-path '%s'" \
% self.certificate_repository_path)
invocation_list.append("--certificate-repository-path '%s'" %
self.certificate_repository_path)
invocation_list.append("--computer-id '%s'" % self.computer_id)
return ' '.join(invocation_list)
......@@ -371,7 +371,8 @@ class Slapgrid(object):
'supervisord_pidfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')),
'supervisord_logfile_backups': '10',
'watchdog_command': self.getWatchdogLine(),
})
}
)
def getComputerPartitionList(self):
try:
......@@ -401,9 +402,9 @@ class Slapgrid(object):
signature_certificate_list=self.signature_certificate_list,
download_binary_cache_url=self.download_binary_cache_url,
upload_binary_cache_url=self.upload_binary_cache_url,
download_from_binary_cache_url_blacklist=\
download_from_binary_cache_url_blacklist=
self.download_from_binary_cache_url_blacklist,
upload_to_binary_cache_url_blacklist=\
upload_to_binary_cache_url_blacklist=
self.upload_to_binary_cache_url_blacklist,
upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url,
......@@ -415,10 +416,10 @@ class Slapgrid(object):
shadir_key_file=self.shadir_key_file)
if state == 'available':
completed_tag = os.path.join(software_path, '.completed')
if self.develop or (not os.path.exists(completed_tag) and \
len(self.software_release_filter_list) == 0) or \
url_hash in self.software_release_filter_list or \
url_hash in (md5digest(uri) for uri in self.software_release_filter_list):
if (self.develop or (not os.path.exists(completed_tag) and
len(self.software_release_filter_list) == 0) or
url_hash in self.software_release_filter_list or
url_hash in (md5digest(uri) for uri in self.software_release_filter_list)):
try:
software_release.building()
except NotFoundError:
......@@ -471,7 +472,6 @@ class Slapgrid(object):
return SLAPGRID_FAIL
return SLAPGRID_SUCCESS
def _launchSupervisord(self):
launchSupervisord(self.supervisord_socket,
self.supervisord_configuration_path,
......@@ -592,8 +592,8 @@ class Slapgrid(object):
# Check if timestamp from server is more recent than local one.
# If not: it's not worth processing this partition (nothing has
# changed).
if computer_partition_id not in self.computer_partition_filter_list and \
(not self.develop) and os.path.exists(timestamp_path):
if (computer_partition_id not in self.computer_partition_filter_list and
not self.develop and os.path.exists(timestamp_path)):
old_timestamp = open(timestamp_path).read()
last_runtime = int(os.path.getmtime(timestamp_path))
if timestamp:
......@@ -808,7 +808,6 @@ class Slapgrid(object):
return SLAPGRID_PROMISE_FAIL
return SLAPGRID_SUCCESS
def validateXML(self, to_be_validated, xsd_model):
"""Validates a given xml file"""
#We retrieve the xsd model
......@@ -819,8 +818,8 @@ class Slapgrid(object):
try:
document = etree.fromstring(to_be_validated)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as exc:
self.logger.info('Failed to parse this XML report : %s\n%s' % \
(to_be_validated, _formatXMLError(exc)))
self.logger.info('Failed to parse this XML report : %s\n%s' %
(to_be_validated, _formatXMLError(exc)))
self.logger.error(_formatXMLError(exc))
return False
......@@ -967,9 +966,8 @@ class Slapgrid(object):
computer_partition.error('\n'.join(failed_script_list), logger=self.logger)
# Whatever happens, don't stop processing other instances
except Exception:
self.logger.info('Cannot run usage script(s) for %r: %s' % (
computer_partition.getId(),
traceback.format_exc()))
self.logger.info('Cannot run usage script(s) for %r: %s' %
(computer_partition.getId(), traceback.format_exc()))
#Now we loop through the different computer partitions to report
report_usage_issue_cp_list = []
......@@ -1002,7 +1000,7 @@ class Slapgrid(object):
# report
else:
computer_partition_usage = self.slap.registerComputerPartition(
self.computer_id, computer_partition_id)
self.computer_id, computer_partition_id)
computer_partition_usage.setUsage(usage)
computer_partition_usage_list.append(computer_partition_usage)
filename_delete_list.append(filename)
......@@ -1015,14 +1013,12 @@ class Slapgrid(object):
# Whatever happens, don't stop processing other instances
except Exception:
self.logger.info('Cannot run usage script(s) for %r: %s' % (
computer_partition.getId(),
traceback.format_exc()))
self.logger.info('Cannot run usage script(s) for %r: %s' %
(computer_partition.getId(), traceback.format_exc()))
for computer_partition_usage in computer_partition_usage_list:
self.logger.info('computer_partition_usage_list: %s - %s' % (
computer_partition_usage.usage,
computer_partition_usage.getId()))
self.logger.info('computer_partition_usage_list: %s - %s' %
(computer_partition_usage.usage, computer_partition_usage.getId()))
#If there is, at least, one report
if computer_partition_usage_list != []:
......@@ -1041,8 +1037,8 @@ class Slapgrid(object):
raise ValueError('XML file generated by asXML is not valid !')
except Exception:
issue = "Cannot report usage for %r: %s" % (
computer_partition.getId(),
traceback.format_exc())
computer_partition.getId(),
traceback.format_exc())
self.logger.info(issue)
computer_partition.error(issue, logger=self.logger)
report_usage_issue_cp_list.append(computer_partition_id)
......
slapos/grid/utils.py
View file @ 65622b9a
......@@ -124,7 +124,7 @@ def getCleanEnvironment(logger, home_path='/tmp'):
env = os.environ.copy()
# Clean python related environment variables
for k in PYTHON_ENVIRONMENT_REMOVE_LIST + SYSTEM_ENVIRONMENT_REMOVE_LIST \
+ LOCALE_ENVIRONMENT_REMOVE_LIST:
+ LOCALE_ENVIRONMENT_REMOVE_LIST:
old = env.pop(k, None)
if old is not None:
removed_env.append(k)
......@@ -197,8 +197,7 @@ def dropPrivileges(uid, gid, logger):
# assert that privileges are dropped
message_pre = 'After dropping to uid = %r and gid = %r ' \
'and group_list = %s' % (
uid, gid, group_list)
'and group_list = %s' % (uid, gid, group_list)
new_uid, new_gid, new_group_list = os.getuid(), os.getgid(), os.getgroups()
if not (new_uid == uid and new_gid == gid and set(new_group_list) == group_list):
raise OSError('%s new_uid = %r and new_gid = %r and '
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7