Merge branch 'cliff'

documentation/source/index.rst

@@ -11,6 +11,7 @@ Contents:
 .. toctree::
    :maxdepth: 2
 
+   slapos.usage.rst
    rest.rst
    slap.rst
    tioformat.rst

documentation/source/slapos.usage.rst

@@ -3,8 +3,8 @@ SlapOS command line usage
 =========================
 
 
-Notes:
-------
+Notes
+-----
 
 * Default SlapOS Master is https://slap.vifib.com. It can be changed by altering configuration files or with the ``--master-url``
   argument for commands that support it.
@@ -161,7 +161,7 @@ This group of commands is used to control the current SlapOS Node. They are only
 node, node status
 ~~~~~~~~~~~~~~~~~
 
-These are aliases for ``node supervisorctl status``.
+These are both aliases for ``node supervisorctl status``.
 It displays the status of the node, also running the supervisor daemon if needed.
 
 .. program-output:: python slapos help node supervisorctl status

slapos/cache.py

@@ -7,6 +7,7 @@ import re
 import requests
 import sys
 
+import prettytable
 from slapos.grid import networkcache
 from slapos.grid.distribution import patched_linux_distribution
 
@@ -19,7 +20,7 @@ def looks_like_md5(s):
     return re.match('[0-9a-f]{32}', s)
 
 
-def do_lookup(configp, software_url):
+def do_lookup(configp, software_url, logger):
     cache_dir = configp.get('networkcache', 'download-binary-dir-url')
 
     if looks_like_md5(software_url):
@@ -28,39 +29,39 @@ def do_lookup(configp, software_url):
         md5 = hashlib.md5(software_url).hexdigest()
 
     try:
-        req = requests.get('%s/%s' % (cache_dir, md5))
-    except requests.ConnectionError:
-        print 'Cannot connect to cache at %s' % cache_dir
+        url = '%s/%s' % (cache_dir, md5)
+        logger.debug('Connecting to %s', url)
+        req = requests.get(url, timeout=5)
+    except (requests.Timeout, requests.ConnectionError):
+        logger.critical('Cannot connect to cache server at %s', url)
         sys.exit(10)
 
     if not req.ok:
         if req.status_code == 404:
-            print 'Object not in cache: %s' % software_url
+            logger.critical('Object not in cache: %s', software_url)
         else:
-            print 'Error while looking object %s: %s' % (software_url, req.reason)
+            logger.critical('Error while looking object %s: %s', software_url, req.reason)
         sys.exit(10)
 
     entries = req.json()
 
-    linux_distribution = patched_linux_distribution()
+    if not entries:
+        logger.info('Object found in cache, but has no binary entries.')
+        return
+
+    ostable = sorted(ast.literal_eval(json.loads(entry[0])['os']) for entry in entries)
+
+    pt = prettytable.PrettyTable(['distribution', 'version', 'id', 'compatible?'])
 
-    header_printed = False
-
-    ostable = []
-    for entry in entries:
-        meta = json.loads(entry[0])
-        os = ast.literal_eval(meta['os'])
-        if not header_printed:
-            print 'Software URL: %s' % meta['software_url']
-            print 'MD5:          %s' % md5
-            print '-------------'
-            print 'Available for: '
-            print 'distribution     |   version    |       id       | compatible?'
-            print '-----------------+--------------+----------------+-------------'
-            header_printed = True
-        ostable.append(os)
-    ostable.sort()
+    linux_distribution = patched_linux_distribution()
 
     for os in ostable:
         compatible = 'yes' if networkcache.os_matches(os, linux_distribution) else 'no'
-        print '%-16s | %12s | %s | %s' % (os[0], os[1], os[2].center(14), compatible)
+        pt.add_row([os[0], os[1], os[2], compatible])
+
+    meta = json.loads(entries[0][0])
+    logger.info('Software URL: %s', meta['software_url'])
+    logger.info('MD5:          %s', md5)
+
+    for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'):
+        logger.info(line)

slapos/cli/cache.py

@@ -27,4 +27,4 @@ class CacheLookupCommand(ConfigCommand):
 
     def take_action(self, args):
         configp = self.fetch_config(args)
-        do_lookup(configp, args.software_url)
+        do_lookup(configp, args.software_url, logger=self.app.log)

slapos/cli/console.py

@@ -8,7 +8,7 @@ from slapos.client import init, do_console, ClientConfig
 
 class ConsoleCommand(ClientConfigCommand):
     """
-    python console with slap library imported
+    open python console with slap library imported
 
     You can play with the global "slap" object and
     with the global "request" method.

slapos/cli/proxy_show.py

@@ -71,7 +71,7 @@ def coalesce(*seq):
     return el
 
 
-def print_table(qry, tablename, skip=None):
+def log_table(logger, qry, tablename, skip=None):
     if skip is None:
         skip = set()
 
@@ -88,19 +88,19 @@ def print_table(qry, tablename, skip=None):
         pt.add_row(row)
 
     if rows:
-        print 'table %s:' % tablename,
         if skip:
-            print 'skipping %s' % ', '.join(skip)
+            logger.info('table %s: skipping %s', tablename, ', '.join(skip))
         else:
-            print
+            logger.info('table %s', tablename)
     else:
-        print 'table %s: empty' % tablename
+        logger.info('table %s: empty', tablename)
         return
 
-    print pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE)
+    for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'):
+        logger.info(line)
 
 
-def print_params(conn):
+def log_params(logger, conn):
     cur = conn.cursor()
 
     qry = cur.execute("SELECT reference, partition_reference, software_type, connection_xml FROM %s" % tbl_partition)
@@ -109,44 +109,44 @@ def print_params(conn):
             continue
 
         xml = str(row['connection_xml'])
-        print '%s: %s (type %s)' % (row['reference'], row['partition_reference'], row['software_type'])
+        logger.info('%s: %s (type %s)', row['reference'], row['partition_reference'], row['software_type'])
         instance = lxml.etree.fromstring(xml)
         for parameter in list(instance):
             name = parameter.get('id')
             text = parameter.text
             if text and name in ('ssh-key', 'ssh-public-key'):
                 text = text[:20] + '...' + text[-20:]
-            print '    %s = %s' % (name, text)
+            logger.info('    %s = %s', name, text)
 
 
-def print_computer_table(conn):
+def log_computer_table(logger, conn):
     tbl_computer = 'computer' + DB_VERSION
     cur = conn.cursor()
     qry = cur.execute("SELECT * FROM %s" % tbl_computer)
-    print_table(qry, tbl_computer)
+    log_table(logger, qry, tbl_computer)
 
 
-def print_software_table(conn):
+def log_software_table(logger, conn):
     tbl_software = 'software' + DB_VERSION
     cur = conn.cursor()
     qry = cur.execute("SELECT *, md5(url) as md5 FROM %s" % tbl_software)
-    print_table(qry, tbl_software)
+    log_table(logger, qry, tbl_software)
 
 
-def print_partition_table(conn):
+def log_partition_table(logger, conn):
     cur = conn.cursor()
     qry = cur.execute("SELECT * FROM %s WHERE slap_state<>'free'" % tbl_partition)
-    print_table(qry, tbl_partition, skip=['xml', 'connection_xml', 'slave_instance_list'])
+    log_table(logger, qry, tbl_partition, skip=['xml', 'connection_xml', 'slave_instance_list'])
 
 
-def print_slave_table(conn):
+def log_slave_table(logger, conn):
     tbl_slave = 'slave' + DB_VERSION
     cur = conn.cursor()
     qry = cur.execute("SELECT * FROM %s" % tbl_slave)
-    print_table(qry, tbl_slave, skip=['connection_xml'])
+    log_table(logger, qry, tbl_slave, skip=['connection_xml'])
 
 
-def print_network(conn):
+def log_network(logger, conn):
     tbl_partition_network = 'partition_network' + DB_VERSION
     cur = conn.cursor()
     addr = collections.defaultdict(list)
@@ -162,39 +162,31 @@ def print_network(conn):
 
     for partition_reference in sorted(addr.keys()):
         addresses = addr[partition_reference]
-        print '%s: %s' % (partition_reference, ', '.join(addresses))
+        logger.info('%s: %s', partition_reference, ', '.join(addresses))
 
 
 def do_show(conf):
+    conf.logger.debug('Using database: %s', conf.database_uri)
     conn = sqlite3.connect(conf.database_uri)
     conn.row_factory = sqlite3.Row
 
     conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest())
 
-    print_all = not any([
-        conf.computers,
-        conf.software,
-        conf.partitions,
-        conf.slaves,
-        conf.params,
-        conf.network,
-    ])
-
-    if print_all or conf.computers:
-        print_computer_table(conn)
-        print
-    if print_all or conf.software:
-        print_software_table(conn)
-        print
-    if print_all or conf.partitions:
-        print_partition_table(conn)
-        print
-    if print_all or conf.slaves:
-        print_slave_table(conn)
-        print
-    if print_all or conf.params:
-        print_params(conn)
-        print
-    if print_all or conf.network:
-        print_network(conn)
-        print
+    call_table = [
+        (conf.computers, log_computer_table),
+        (conf.software, log_software_table),
+        (conf.partitions, log_partition_table),
+        (conf.slaves, log_slave_table),
+        (conf.params, log_params),
+        (conf.network, log_network)
+    ]
+
+    if not any(flag for flag, func in call_table):
+        to_call = [func for flag, func in call_table]
+    else:
+        to_call = [func for flag, func in call_table if flag]
+
+    for idx, func in enumerate(to_call):
+        func(conf.logger, conn)
+        if idx < len(to_call) - 1:
+            conf.logger.info(' ')

slapos/cli/supervisorctl.py

@@ -12,7 +12,7 @@ import supervisor.supervisorctl
 
 
 class SupervisorctlCommand(ConfigCommand):
-    """enter into supervisor console, for process management"""
+    """open supervisor console, for process management"""
 
     log = logging.getLogger('supervisorctl')
 

slapos/cli_legacy/cache.py

-# -*- coding: utf-8 -*-
-
-import argparse
-import ConfigParser
-
-from slapos.cache import do_lookup
-
-
-def cache_lookup():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("configuration_file", help="SlapOS configuration file")
-    ap.add_argument("software_url", help="Your software url or MD5 hash")
-    args = ap.parse_args()
-
-    configp = ConfigParser.SafeConfigParser()
-    configp.read(args.configuration_file)
-
-    do_lookup(configp, args.software_url)

slapos/cli_legacy/entry.py

@@ -37,7 +37,6 @@ from slapos.cli_legacy.request import request
 from slapos.cli_legacy.remove import remove
 from slapos.cli_legacy.supply import supply
 from slapos.cli_legacy.format import main as format
-from slapos.cli_legacy.cache import cache_lookup
 from slapos.cli_legacy.slapgrid import runComputerPartition as instance
 from slapos.cli_legacy.slapgrid import runSoftwareRelease as software
 from slapos.cli_legacy.slapgrid import runUsageReport as report
@@ -165,8 +164,6 @@ def dispatch(command, is_node_command):
     raise EntryPointNotImplementedError(command)
   elif command == 'console':
     call(console, config_path=USER_SLAPOS_CONFIGURATION)
-  elif command == 'cache-lookup':
-    call(cache_lookup, config_path=GLOBAL_SLAPOS_CONFIGURATION)
   else:
     return False
 
@@ -193,7 +190,6 @@ Client subcommands usage:
   slapos request <instance-name> <software-url> [--configuration arg1=value1 arg2=value2 ... argN=valueN]
   slapos supply <software-url> <node-id>
   slapos console
-  slapos cache-lookup <software-url-or-md5>
 Node subcommands usage:
   slapos node
   slapos node register <node-id>

slapos/grid/SlapObject.py

@@ -608,7 +608,7 @@ class Partition(object):
   def updateSupervisor(self):
     """Forces supervisord to reload its configuration"""
     # Note: This method shall wait for results from supervisord
-    #       In future it will be not needed, as update command
+    #       In future it will not be needed, as update command
     #       is going to be implemented on server side.
     self.logger.debug('Updating supervisord')
     supervisor = self.getSupervisorRPC()

slapos/grid/utils.py

@@ -130,7 +130,7 @@ def getCleanEnvironment(logger, home_path='/tmp'):
       removed_env.append(k)
   changed_env['HOME'] = env['HOME'] = home_path
   for k in sorted(changed_env.iterkeys()):
-    logger.debug('Overriden %s = %r' % (k, changed_env[k]))
+    logger.debug('Overridden %s = %r' % (k, changed_env[k]))
   logger.debug('Removed from environment: %s' % ', '.join(sorted(removed_env)))
   return env
 
@@ -175,7 +175,7 @@ def dropPrivileges(uid, gid, logger):
   Do tests to check if dropping was successful and that no system call is able
   to re-raise dropped privileges
 
-  Does nothing in case if uid and gid are not 0
+  Does nothing if uid and gid are not 0
   """
   # XXX-Cedric: remove format / just do a print, otherwise formatting is done
   # twice
@@ -327,7 +327,7 @@ def launchBuildout(path, buildout_binary, logger,
 
 
 def updateFile(file_path, content, mode=0o600):
-  """Creates an executable with "content" as content."""
+  """Creates or updates a file with "content" as content."""
   altered = False
   if not (os.path.isfile(file_path)) or \
      not (hashlib.md5(open(file_path).read()).digest() ==
@@ -343,12 +343,12 @@ def updateFile(file_path, content, mode=0o600):
 
 
 def updateExecutable(executable_path, content):
-  """Creates an executable with "content" as content."""
+  """Creates or updates an executable file with "content" as content."""
   return updateFile(executable_path, content, 0o700)
 
 
 def createPrivateDirectory(path):
-  """Creates directory belonging to root with umask 077"""
+  """Creates a directory belonging to root with umask 077"""
   if not os.path.isdir(path):
     os.mkdir(path)
   os.chmod(path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)

slapos/proxy/views.py

@@ -59,15 +59,17 @@ def xml2dict(xml):
   return result_dict
 
 
-def dict2xml(dictionnary):
+def dict2xml(dictionary):
   instance = etree.Element('instance')
-  for parameter_id, parameter_value in dictionnary.iteritems():
+  for parameter_id, parameter_value in dictionary.iteritems():
     # cast everything to string
     parameter_value = str(parameter_value)
     etree.SubElement(instance, "parameter",
                      attrib={'id': parameter_id}).text = parameter_value
-  return etree.tostring(instance, pretty_print=True,
-                                xml_declaration=True, encoding='utf-8')
+  return etree.tostring(instance,
+                        pretty_print=True,
+                        xml_declaration=True,
+                        encoding='utf-8')
 
 
 def partitiondict2partition(partition):
@@ -90,7 +92,7 @@ def partitiondict2partition(partition):
     slap_partition._parameter_dict['ip_list'] = address_list
     slap_partition._parameter_dict['slap_software_type'] = \
         partition['software_type']
-    if not partition['slave_instance_list'] == None:
+    if partition['slave_instance_list'] is not None:
       slap_partition._parameter_dict['slave_instance_list'] = \
           xml_marshaller.xml_marshaller.loads(partition['slave_instance_list'])
     slap_partition._connection_dict = xml2dict(partition['connection_xml'])
@@ -124,6 +126,7 @@ def before_request():
   g.db.cursor().executescript(schema)
   g.db.commit()
 
+
 @app.after_request
 def after_request(response):
   g.db.commit()
@@ -150,8 +153,7 @@ def getFullComputerInformation():
         partition))
     return xml_marshaller.xml_marshaller.dumps(slap_computer)
   else:
-    raise NotFoundError, "Only accept request for: %s" % \
-                             app.config['computer_id']
+    raise NotFoundError('Only accept request for: %s' % app.config['computer_id'])
 
 @app.route('/setComputerPartitionConnectionXml', methods=['POST'])
 def setComputerPartitionConnectionXml():
@@ -235,8 +237,7 @@ def loadComputerConfigurationFromXML():
 
     return 'done'
   else:
-    raise UnauthorizedError, "Only accept request for: %s" % \
-                             app.config['computer_id']
+    raise UnauthorizedError('Only accept request for: %s' % app.config['computer_id'])
 
 @app.route('/registerComputerPartition', methods=['GET'])
 def registerComputerPartition():
@@ -250,8 +251,7 @@ def registerComputerPartition():
     return xml_marshaller.xml_marshaller.dumps(
         partitiondict2partition(partition))
   else:
-    raise UnauthorizedError, "Only accept request for: %s" % \
-                             app.config['computer_id']
+    raise UnauthorizedError('Only accept request for: %s' % app.config['computer_id'])
 
 @app.route('/supplySupply', methods=['POST'])
 def supplySupply():
@@ -263,8 +263,7 @@ def supplySupply():
     else:
       execute_db('software', 'INSERT OR REPLACE INTO %s VALUES(?)', [url])
   else:
-    raise UnauthorizedError, "Only accept request for: %s" % \
-                             app.config['computer_id']
+    raise UnauthorizedError('Only accept request for: %s' % app.config['computer_id'])
   return '%r added' % url
 
 
@@ -358,18 +357,18 @@ def request_not_shared():
     address_list.append((address['reference'], address['address']))
 
   # XXX it should be ComputerPartition, not a SoftwareInstance
-  return xml_marshaller.xml_marshaller.dumps(SoftwareInstance(
-                            xml=partition['xml'],
-                            connection_xml=partition['connection_xml'],
-                            slap_computer_id=app.config['computer_id'],
-                            slap_computer_partition_id=partition['reference'],
-                            slap_software_release_url=partition['software_release'],
-                            slap_server_url='slap_server_url',
-                            slap_software_type=partition['software_type'],
-                            slave_instance_list=partition['slave_instance_list'],
-                            instance_guid=partition['reference'],
-                            ip_list=address_list
-                            ))
+  software_instance = SoftwareInstance(xml=partition['xml'],
+                                       connection_xml=partition['connection_xml'],
+                                       slap_computer_id=app.config['computer_id'],
+                                       slap_computer_partition_id=partition['reference'],
+                                       slap_software_release_url=partition['software_release'],
+                                       slap_server_url='slap_server_url',
+                                       slap_software_type=partition['software_type'],
+                                       slave_instance_list=partition['slave_instance_list'],
+                                       instance_guid=partition['reference'],
+                                       ip_list=address_list)
+
+  return xml_marshaller.xml_marshaller.dumps(software_instance)
 
 
 def request_slave():
@@ -379,7 +378,7 @@ def request_slave():
   1. slave table having information such as slave reference,
       connection information to slave (given by slave master),
       hosted_by and asked_by reference.
-  2. A dictionnary in slave_instance_list of selected slave master
+  2. A dictionary in slave_instance_list of selected slave master
       in which are stored slave_reference, software_type, slave_title and
       partition_parameter_kw stored as individual keys.
   """
@@ -413,8 +412,7 @@ def request_slave():
 
   partition = execute_db('partition', q, args, one=True)
   if partition is None:
-    app.logger.warning('No partition corresponding to slave request: %s' % \
-        args)
+    app.logger.warning('No partition corresponding to slave request: %s' % args)
     abort(404)
 
   # We set slave dictionary as described in docstring
@@ -424,13 +422,13 @@ def request_slave():
   new_slave['slap_software_type'] = software_type
   new_slave['slave_reference'] = slave_reference
 
-  for key in partition_parameter_kw :
-    if partition_parameter_kw[key] is not None :
+  for key in partition_parameter_kw:
+    if partition_parameter_kw[key] is not None:
       new_slave[key] = partition_parameter_kw[key]
 
   # Add slave to partition slave_list if not present else replace information
   slave_instance_list = partition['slave_instance_list']
-  if slave_instance_list == None:
+  if slave_instance_list is None:
     slave_instance_list = []
   else:
     slave_instance_list = xml_marshaller.xml_marshaller.loads(slave_instance_list)
@@ -455,12 +453,12 @@ def request_slave():
   # Add slave to slave table if not there
   slave = execute_db('slave', 'SELECT * FROM %s WHERE reference=?',
                      [slave_reference], one=True)
-  if slave is None :
+  if slave is None:
     execute_db('slave',
                'INSERT OR IGNORE INTO %s (reference,asked_by,hosted_by) values(:reference,:asked_by,:hosted_by)',
-               [slave_reference,partition_id,partition['reference']])
-    slave = execute_db('slave','SELECT * FROM %s WHERE reference=?',
-                     [slave_reference], one = True)
+               [slave_reference, partition_id, partition['reference']])
+    slave = execute_db('slave', 'SELECT * FROM %s WHERE reference=?',
+                       [slave_reference], one=True)
 
   address_list = []
   for address in execute_db('partition_network',
@@ -469,14 +467,13 @@ def request_slave():
     address_list.append((address['reference'], address['address']))
 
   # XXX it should be ComputerPartition, not a SoftwareInstance
-  return xml_marshaller.xml_marshaller.dumps(SoftwareInstance(
-                                _connection_dict=xml2dict(slave['connection_xml']),
-                                xml = instance_xml,
-                                slap_computer_id=app.config['computer_id'],
-                                slap_computer_partition_id=slave['hosted_by'],
-                                slap_software_release_url=partition['software_release'],
-                                slap_server_url='slap_server_url',
-                                slap_software_type=partition['software_type'],
-                                ip_list=address_list
-                                ))
-
+  software_instance = SoftwareInstance(_connection_dict=xml2dict(slave['connection_xml']),
+                                       xml=instance_xml,
+                                       slap_computer_id=app.config['computer_id'],
+                                       slap_computer_partition_id=slave['hosted_by'],
+                                       slap_software_release_url=partition['software_release'],
+                                       slap_server_url='slap_server_url',
+                                       slap_software_type=partition['software_type'],
+                                       ip_list=address_list)
+
+  return xml_marshaller.xml_marshaller.dumps(software_instance)

slapos/tests/distribution.py

+# -*- coding: utf-8 -*-
+
+import unittest
+from slapos.grid import distribution
+
+
+class TestDebianize(unittest.TestCase):
+    def test_debian_major(self):
+        """
+        On debian, we only care about major release.
+        All the other tuples are unchanged.
+        """
+        for provided, expected in [
+            (('CentOS', '6.3', 'Final'), None),
+            (('Ubuntu', '12.04', 'precise'), None),
+            (('Ubuntu', '13.04', 'raring'), None),
+            (('Fedora', '17', 'Beefy Miracle'), None),
+            (('debian', '6.0.6', ''), ('debian', '6', '')),
+            (('debian', '7.0', ''), ('debian', '7', '')),
+        ]:
+            self.assertEqual(distribution._debianize(provided), expected or provided)
+
+
+class TestOSMatches(unittest.TestCase):
+    def test_centos(self):
+        self.assertFalse(distribution.os_matches(('CentOS', '6.3', 'Final'),
+                                                 ('Ubuntu', '13.04', 'raring')))
+        self.assertFalse(distribution.os_matches(('CentOS', '6.3', 'Final'),
+                                                 ('debian', '6.3', '')))
+
+    def test_ubuntu(self):
+        self.assertFalse(distribution.os_matches(('Ubuntu', '12.04', 'precise'),
+                                                 ('Ubuntu', '13.04', 'raring')))
+        self.assertTrue(distribution.os_matches(('Ubuntu', '13.04', 'raring'),
+                                                ('Ubuntu', '13.04', 'raring')))
+        self.assertTrue(distribution.os_matches(('Ubuntu', '12.04', 'precise'),
+                                                ('Ubuntu', '12.04', 'precise')))
+
+    def test_debian(self):
+        self.assertFalse(distribution.os_matches(('debian', '6.0.6', ''),
+                                                 ('debian', '7.0', '')))
+        self.assertTrue(distribution.os_matches(('debian', '6.0.6', ''),
+                                                ('debian', '6.0.5', '')))
+        self.assertTrue(distribution.os_matches(('debian', '6.0.6', ''),
+                                                ('debian', '6.1', '')))