Skip to content

S
slapos.core
Commit e1e0f54a authored by Marco Mariani's avatar Marco Mariani
Browse files
Options

Merge branch 'cliff'

parents e549f490 255f21fb
Show whitespace changes
Inline Side-by-side
Showing with 164 additions and 150 deletions
documentation/source/index.rst
View file @ e1e0f54a
......@@ -11,6 +11,7 @@ Contents:
.. toctree::
:maxdepth: 2
slapos.usage.rst
rest.rst
slap.rst
tioformat.rst
......
documentation/source/slapos.usage.rst
View file @ e1e0f54a
......@@ -3,8 +3,8 @@ SlapOS command line usage
=========================
Notes:
------
Notes
-----
* Default SlapOS Master is https://slap.vifib.com. It can be changed by altering configuration files or with the ``--master-url``
argument for commands that support it.
......@@ -161,7 +161,7 @@ This group of commands is used to control the current SlapOS Node. They are only
node, node status
~~~~~~~~~~~~~~~~~
These are aliases for ``node supervisorctl status``.
These are both aliases for ``node supervisorctl status``.
It displays the status of the node, also running the supervisor daemon if needed.
.. program-output:: python slapos help node supervisorctl status
......
slapos/cache.py
View file @ e1e0f54a
......@@ -7,6 +7,7 @@ import re
import requests
import sys
import prettytable
from slapos.grid import networkcache
from slapos.grid.distribution import patched_linux_distribution
......@@ -19,7 +20,7 @@ def looks_like_md5(s):
return re.match('[0-9a-f]{32}', s)
def do_lookup(configp, software_url):
def do_lookup(configp, software_url, logger):
cache_dir = configp.get('networkcache', 'download-binary-dir-url')
if looks_like_md5(software_url):
......@@ -28,39 +29,39 @@ def do_lookup(configp, software_url):
md5 = hashlib.md5(software_url).hexdigest()
try:
req = requests.get('%s/%s' % (cache_dir, md5))
except requests.ConnectionError:
print 'Cannot connect to cache at %s' % cache_dir
url = '%s/%s' % (cache_dir, md5)
logger.debug('Connecting to %s', url)
req = requests.get(url, timeout=5)
except (requests.Timeout, requests.ConnectionError):
logger.critical('Cannot connect to cache server at %s', url)
sys.exit(10)
if not req.ok:
if req.status_code == 404:
print 'Object not in cache: %s' % software_url
logger.critical('Object not in cache: %s', software_url)
else:
print 'Error while looking object %s: %s' % (software_url, req.reason)
logger.critical('Error while looking object %s: %s', software_url, req.reason)
sys.exit(10)
entries = req.json()
linux_distribution = patched_linux_distribution()
if not entries:
logger.info('Object found in cache, but has no binary entries.')
return
ostable = sorted(ast.literal_eval(json.loads(entry[0])['os']) for entry in entries)
pt = prettytable.PrettyTable(['distribution', 'version', 'id', 'compatible?'])
header_printed = False
ostable = []
for entry in entries:
meta = json.loads(entry[0])
os = ast.literal_eval(meta['os'])
if not header_printed:
print 'Software URL: %s' % meta['software_url']
print 'MD5: %s' % md5
print '-------------'
print 'Available for: '
print 'distribution | version | id | compatible?'
print '-----------------+--------------+----------------+-------------'
header_printed = True
ostable.append(os)
ostable.sort()
linux_distribution = patched_linux_distribution()
for os in ostable:
compatible = 'yes' if networkcache.os_matches(os, linux_distribution) else 'no'
print '%-16s | %12s | %s | %s' % (os[0], os[1], os[2].center(14), compatible)
pt.add_row([os[0], os[1], os[2], compatible])
meta = json.loads(entries[0][0])
logger.info('Software URL: %s', meta['software_url'])
logger.info('MD5: %s', md5)
for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'):
logger.info(line)
slapos/cli/cache.py
View file @ e1e0f54a
......@@ -27,4 +27,4 @@ class CacheLookupCommand(ConfigCommand):
def take_action(self, args):
configp = self.fetch_config(args)
do_lookup(configp, args.software_url)
do_lookup(configp, args.software_url, logger=self.app.log)
slapos/cli/console.py
View file @ e1e0f54a
......@@ -8,7 +8,7 @@ from slapos.client import init, do_console, ClientConfig
class ConsoleCommand(ClientConfigCommand):
"""
python console with slap library imported
open python console with slap library imported
You can play with the global "slap" object and
with the global "request" method.
......
slapos/cli/proxy_show.py
View file @ e1e0f54a
......@@ -71,7 +71,7 @@ def coalesce(*seq):
return el
def print_table(qry, tablename, skip=None):
def log_table(logger, qry, tablename, skip=None):
if skip is None:
skip = set()
......@@ -88,19 +88,19 @@ def print_table(qry, tablename, skip=None):
pt.add_row(row)
if rows:
print 'table %s:' % tablename,
if skip:
print 'skipping %s' % ', '.join(skip)
logger.info('table %s: skipping %s', tablename, ', '.join(skip))
else:
print
logger.info('table %s', tablename)
else:
print 'table %s: empty' % tablename
logger.info('table %s: empty', tablename)
return
print pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE)
for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'):
logger.info(line)
def print_params(conn):
def log_params(logger, conn):
cur = conn.cursor()
qry = cur.execute("SELECT reference, partition_reference, software_type, connection_xml FROM %s" % tbl_partition)
......@@ -109,44 +109,44 @@ def print_params(conn):
continue
xml = str(row['connection_xml'])
print '%s: %s (type %s)' % (row['reference'], row['partition_reference'], row['software_type'])
logger.info('%s: %s (type %s)', row['reference'], row['partition_reference'], row['software_type'])
instance = lxml.etree.fromstring(xml)
for parameter in list(instance):
name = parameter.get('id')
text = parameter.text
if text and name in ('ssh-key', 'ssh-public-key'):
text = text[:20] + '...' + text[-20:]
print ' %s = %s' % (name, text)
logger.info(' %s = %s', name, text)
def print_computer_table(conn):
def log_computer_table(logger, conn):
tbl_computer = 'computer' + DB_VERSION
cur = conn.cursor()
qry = cur.execute("SELECT * FROM %s" % tbl_computer)
print_table(qry, tbl_computer)
log_table(logger, qry, tbl_computer)
def print_software_table(conn):
def log_software_table(logger, conn):
tbl_software = 'software' + DB_VERSION
cur = conn.cursor()
qry = cur.execute("SELECT *, md5(url) as md5 FROM %s" % tbl_software)
print_table(qry, tbl_software)
log_table(logger, qry, tbl_software)
def print_partition_table(conn):
def log_partition_table(logger, conn):
cur = conn.cursor()
qry = cur.execute("SELECT * FROM %s WHERE slap_state<>'free'" % tbl_partition)
print_table(qry, tbl_partition, skip=['xml', 'connection_xml', 'slave_instance_list'])
log_table(logger, qry, tbl_partition, skip=['xml', 'connection_xml', 'slave_instance_list'])
def print_slave_table(conn):
def log_slave_table(logger, conn):
tbl_slave = 'slave' + DB_VERSION
cur = conn.cursor()
qry = cur.execute("SELECT * FROM %s" % tbl_slave)
print_table(qry, tbl_slave, skip=['connection_xml'])
log_table(logger, qry, tbl_slave, skip=['connection_xml'])
def print_network(conn):
def log_network(logger, conn):
tbl_partition_network = 'partition_network' + DB_VERSION
cur = conn.cursor()
addr = collections.defaultdict(list)
......@@ -162,39 +162,31 @@ def print_network(conn):
for partition_reference in sorted(addr.keys()):
addresses = addr[partition_reference]
print '%s: %s' % (partition_reference, ', '.join(addresses))
logger.info('%s: %s', partition_reference, ', '.join(addresses))
def do_show(conf):
conf.logger.debug('Using database: %s', conf.database_uri)
conn = sqlite3.connect(conf.database_uri)
conn.row_factory = sqlite3.Row
conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest())
print_all = not any([
conf.computers,
conf.software,
conf.partitions,
conf.slaves,
conf.params,
conf.network,
])
if print_all or conf.computers:
print_computer_table(conn)
print
if print_all or conf.software:
print_software_table(conn)
print
if print_all or conf.partitions:
print_partition_table(conn)
print
if print_all or conf.slaves:
print_slave_table(conn)
print
if print_all or conf.params:
print_params(conn)
print
if print_all or conf.network:
print_network(conn)
print
call_table = [
(conf.computers, log_computer_table),
(conf.software, log_software_table),
(conf.partitions, log_partition_table),
(conf.slaves, log_slave_table),
(conf.params, log_params),
(conf.network, log_network)
]
if not any(flag for flag, func in call_table):
to_call = [func for flag, func in call_table]
else:
to_call = [func for flag, func in call_table if flag]
for idx, func in enumerate(to_call):
func(conf.logger, conn)
if idx < len(to_call) - 1:
conf.logger.info(' ')
slapos/cli/supervisorctl.py
View file @ e1e0f54a
......@@ -12,7 +12,7 @@ import supervisor.supervisorctl
class SupervisorctlCommand(ConfigCommand):
"""enter into supervisor console, for process management"""
"""open supervisor console, for process management"""
log = logging.getLogger('supervisorctl')
......
slapos/cli_legacy/cache.py deleted 100644 → 0
View file @ e549f490
# -*- coding: utf-8 -*-
import argparse
import ConfigParser
from slapos.cache import do_lookup
def cache_lookup():
ap = argparse.ArgumentParser()
ap.add_argument("configuration_file", help="SlapOS configuration file")
ap.add_argument("software_url", help="Your software url or MD5 hash")
args = ap.parse_args()
configp = ConfigParser.SafeConfigParser()
configp.read(args.configuration_file)
do_lookup(configp, args.software_url)
slapos/cli_legacy/entry.py
View file @ e1e0f54a
......@@ -37,7 +37,6 @@ from slapos.cli_legacy.request import request
from slapos.cli_legacy.remove import remove
from slapos.cli_legacy.supply import supply
from slapos.cli_legacy.format import main as format
from slapos.cli_legacy.cache import cache_lookup
from slapos.cli_legacy.slapgrid import runComputerPartition as instance
from slapos.cli_legacy.slapgrid import runSoftwareRelease as software
from slapos.cli_legacy.slapgrid import runUsageReport as report
......@@ -165,8 +164,6 @@ def dispatch(command, is_node_command):
raise EntryPointNotImplementedError(command)
elif command == 'console':
call(console, config_path=USER_SLAPOS_CONFIGURATION)
elif command == 'cache-lookup':
call(cache_lookup, config_path=GLOBAL_SLAPOS_CONFIGURATION)
else:
return False
......@@ -193,7 +190,6 @@ Client subcommands usage:
slapos request <instance-name> <software-url> [--configuration arg1=value1 arg2=value2 ... argN=valueN]
slapos supply <software-url> <node-id>
slapos console
slapos cache-lookup <software-url-or-md5>
Node subcommands usage:
slapos node
slapos node register <node-id>
......
slapos/grid/SlapObject.py
View file @ e1e0f54a
......@@ -608,7 +608,7 @@ class Partition(object):
def updateSupervisor(self):
"""Forces supervisord to reload its configuration"""
# Note: This method shall wait for results from supervisord
# In future it will be not needed, as update command
# In future it will not be needed, as update command
# is going to be implemented on server side.
self.logger.debug('Updating supervisord')
supervisor = self.getSupervisorRPC()
......
slapos/grid/utils.py
View file @ e1e0f54a
......@@ -130,7 +130,7 @@ def getCleanEnvironment(logger, home_path='/tmp'):
removed_env.append(k)
changed_env['HOME'] = env['HOME'] = home_path
for k in sorted(changed_env.iterkeys()):
logger.debug('Overriden %s = %r' % (k, changed_env[k]))
logger.debug('Overridden %s = %r' % (k, changed_env[k]))
logger.debug('Removed from environment: %s' % ', '.join(sorted(removed_env)))
return env
......@@ -175,7 +175,7 @@ def dropPrivileges(uid, gid, logger):
Do tests to check if dropping was successful and that no system call is able
to re-raise dropped privileges
Does nothing in case if uid and gid are not 0
Does nothing if uid and gid are not 0
"""
# XXX-Cedric: remove format / just do a print, otherwise formatting is done
# twice
......@@ -327,7 +327,7 @@ def launchBuildout(path, buildout_binary, logger,
def updateFile(file_path, content, mode=0o600):
"""Creates an executable with "content" as content."""
"""Creates or updates a file with "content" as content."""
altered = False
if not (os.path.isfile(file_path)) or \
not (hashlib.md5(open(file_path).read()).digest() ==
......@@ -343,12 +343,12 @@ def updateFile(file_path, content, mode=0o600):
def updateExecutable(executable_path, content):
"""Creates an executable with "content" as content."""
"""Creates or updates an executable file with "content" as content."""
return updateFile(executable_path, content, 0o700)
def createPrivateDirectory(path):
"""Creates directory belonging to root with umask 077"""
"""Creates a directory belonging to root with umask 077"""
if not os.path.isdir(path):
os.mkdir(path)
os.chmod(path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
......
slapos/proxy/views.py
View file @ e1e0f54a
......@@ -59,15 +59,17 @@ def xml2dict(xml):
return result_dict
def dict2xml(dictionnary):
def dict2xml(dictionary):
instance = etree.Element('instance')
for parameter_id, parameter_value in dictionnary.iteritems():
for parameter_id, parameter_value in dictionary.iteritems():
# cast everything to string
parameter_value = str(parameter_value)
etree.SubElement(instance, "parameter",
attrib={'id': parameter_id}).text = parameter_value
return etree.tostring(instance, pretty_print=True,
xml_declaration=True, encoding='utf-8')
return etree.tostring(instance,
pretty_print=True,
xml_declaration=True,
encoding='utf-8')
def partitiondict2partition(partition):
......@@ -90,7 +92,7 @@ def partitiondict2partition(partition):
slap_partition._parameter_dict['ip_list'] = address_list
slap_partition._parameter_dict['slap_software_type'] = \
partition['software_type']
if not partition['slave_instance_list'] == None:
if partition['slave_instance_list'] is not None:
slap_partition._parameter_dict['slave_instance_list'] = \
xml_marshaller.xml_marshaller.loads(partition['slave_instance_list'])
slap_partition._connection_dict = xml2dict(partition['connection_xml'])
......@@ -124,6 +126,7 @@ def before_request():
g.db.cursor().executescript(schema)
g.db.commit()
@app.after_request
def after_request(response):
g.db.commit()
......@@ -150,8 +153,7 @@ def getFullComputerInformation():
partition))
return xml_marshaller.xml_marshaller.dumps(slap_computer)
else:
raise NotFoundError, "Only accept request for: %s" % \
app.config['computer_id']
raise NotFoundError('Only accept request for: %s' % app.config['computer_id'])
@app.route('/setComputerPartitionConnectionXml', methods=['POST'])
def setComputerPartitionConnectionXml():
......@@ -235,8 +237,7 @@ def loadComputerConfigurationFromXML():
return 'done'
else:
raise UnauthorizedError, "Only accept request for: %s" % \
app.config['computer_id']
raise UnauthorizedError('Only accept request for: %s' % app.config['computer_id'])
@app.route('/registerComputerPartition', methods=['GET'])
def registerComputerPartition():
......@@ -250,8 +251,7 @@ def registerComputerPartition():
return xml_marshaller.xml_marshaller.dumps(
partitiondict2partition(partition))
else:
raise UnauthorizedError, "Only accept request for: %s" % \
app.config['computer_id']
raise UnauthorizedError('Only accept request for: %s' % app.config['computer_id'])
@app.route('/supplySupply', methods=['POST'])
def supplySupply():
......@@ -263,8 +263,7 @@ def supplySupply():
else:
execute_db('software', 'INSERT OR REPLACE INTO %s VALUES(?)', [url])
else:
raise UnauthorizedError, "Only accept request for: %s" % \
app.config['computer_id']
raise UnauthorizedError('Only accept request for: %s' % app.config['computer_id'])
return '%r added' % url
......@@ -358,8 +357,7 @@ def request_not_shared():
address_list.append((address['reference'], address['address']))
# XXX it should be ComputerPartition, not a SoftwareInstance
return xml_marshaller.xml_marshaller.dumps(SoftwareInstance(
xml=partition['xml'],
software_instance = SoftwareInstance(xml=partition['xml'],
connection_xml=partition['connection_xml'],
slap_computer_id=app.config['computer_id'],
slap_computer_partition_id=partition['reference'],
......@@ -368,8 +366,9 @@ def request_not_shared():
slap_software_type=partition['software_type'],
slave_instance_list=partition['slave_instance_list'],
instance_guid=partition['reference'],
ip_list=address_list
))
ip_list=address_list)
return xml_marshaller.xml_marshaller.dumps(software_instance)
def request_slave():
......@@ -379,7 +378,7 @@ def request_slave():
1. slave table having information such as slave reference,
connection information to slave (given by slave master),
hosted_by and asked_by reference.
2. A dictionnary in slave_instance_list of selected slave master
2. A dictionary in slave_instance_list of selected slave master
in which are stored slave_reference, software_type, slave_title and
partition_parameter_kw stored as individual keys.
"""
......@@ -413,8 +412,7 @@ def request_slave():
partition = execute_db('partition', q, args, one=True)
if partition is None:
app.logger.warning('No partition corresponding to slave request: %s' % \
args)
app.logger.warning('No partition corresponding to slave request: %s' % args)
abort(404)
# We set slave dictionary as described in docstring
......@@ -424,13 +422,13 @@ def request_slave():
new_slave['slap_software_type'] = software_type
new_slave['slave_reference'] = slave_reference
for key in partition_parameter_kw :
if partition_parameter_kw[key] is not None :
for key in partition_parameter_kw:
if partition_parameter_kw[key] is not None:
new_slave[key] = partition_parameter_kw[key]
# Add slave to partition slave_list if not present else replace information
slave_instance_list = partition['slave_instance_list']
if slave_instance_list == None:
if slave_instance_list is None:
slave_instance_list = []
else:
slave_instance_list = xml_marshaller.xml_marshaller.loads(slave_instance_list)
......@@ -455,12 +453,12 @@ def request_slave():
# Add slave to slave table if not there
slave = execute_db('slave', 'SELECT * FROM %s WHERE reference=?',
[slave_reference], one=True)
if slave is None :
if slave is None:
execute_db('slave',
'INSERT OR IGNORE INTO %s (reference,asked_by,hosted_by) values(:reference,:asked_by,:hosted_by)',
[slave_reference,partition_id,partition['reference']])
slave = execute_db('slave','SELECT * FROM %s WHERE reference=?',
[slave_reference], one = True)
[slave_reference, partition_id, partition['reference']])
slave = execute_db('slave', 'SELECT * FROM %s WHERE reference=?',
[slave_reference], one=True)
address_list = []
for address in execute_db('partition_network',
......@@ -469,14 +467,13 @@ def request_slave():
address_list.append((address['reference'], address['address']))
# XXX it should be ComputerPartition, not a SoftwareInstance
return xml_marshaller.xml_marshaller.dumps(SoftwareInstance(
_connection_dict=xml2dict(slave['connection_xml']),
xml = instance_xml,
software_instance = SoftwareInstance(_connection_dict=xml2dict(slave['connection_xml']),
xml=instance_xml,
slap_computer_id=app.config['computer_id'],
slap_computer_partition_id=slave['hosted_by'],
slap_software_release_url=partition['software_release'],
slap_server_url='slap_server_url',
slap_software_type=partition['software_type'],
ip_list=address_list
))
ip_list=address_list)
return xml_marshaller.xml_marshaller.dumps(software_instance)
slapos/tests/distribution.py 0 → 100644
View file @ e1e0f54a
# -*- coding: utf-8 -*-
import unittest
from slapos.grid import distribution
class TestDebianize(unittest.TestCase):
def test_debian_major(self):
"""
On debian, we only care about major release.
All the other tuples are unchanged.
"""
for provided, expected in [
(('CentOS', '6.3', 'Final'), None),
(('Ubuntu', '12.04', 'precise'), None),
(('Ubuntu', '13.04', 'raring'), None),
(('Fedora', '17', 'Beefy Miracle'), None),
(('debian', '6.0.6', ''), ('debian', '6', '')),
(('debian', '7.0', ''), ('debian', '7', '')),
]:
self.assertEqual(distribution._debianize(provided), expected or provided)
class TestOSMatches(unittest.TestCase):
def test_centos(self):
self.assertFalse(distribution.os_matches(('CentOS', '6.3', 'Final'),
('Ubuntu', '13.04', 'raring')))
self.assertFalse(distribution.os_matches(('CentOS', '6.3', 'Final'),
('debian', '6.3', '')))
def test_ubuntu(self):
self.assertFalse(distribution.os_matches(('Ubuntu', '12.04', 'precise'),
('Ubuntu', '13.04', 'raring')))
self.assertTrue(distribution.os_matches(('Ubuntu', '13.04', 'raring'),
('Ubuntu', '13.04', 'raring')))
self.assertTrue(distribution.os_matches(('Ubuntu', '12.04', 'precise'),
('Ubuntu', '12.04', 'precise')))
def test_debian(self):
self.assertFalse(distribution.os_matches(('debian', '6.0.6', ''),
('debian', '7.0', '')))
self.assertTrue(distribution.os_matches(('debian', '6.0.6', ''),
('debian', '6.0.5', '')))
self.assertTrue(distribution.os_matches(('debian', '6.0.6', ''),
('debian', '6.1', '')))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7