Merge branch 'cliff'

documentation/source/conf.py

@@ -22,7 +22,13 @@ sys.path.append(os.path.abspath('../../'))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.ifconfig', 'repoze.sphinx.autointerface']
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.ifconfig',
+    'repoze.sphinx.autointerface',
+    'sphinxcontrib.programoutput'
+    ]
 
 # Add any paths that contain templates here, relative to this directory.
 # templates_path = ['_templates']
@@ -45,9 +51,9 @@ copyright = u'2011, Vifib'
 # built documents.
 #
 # The short X.Y version.
-version = '0.21-dev'
+from slapos.version import version
 # The full version, including alpha/beta/rc tags.
-release = '0.21-dev'
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

documentation/source/slapos

+../../slapos/cli/entry.py
\ No newline at end of file

setup.py

@@ -44,6 +44,7 @@ setup(name=name,
                                    # XML
           'zope.interface', # slap library implementes interfaces
           'zc.buildout',
+          'cliff',
         ] + additional_install_requires,
       extra_requires={'docs': ('Sphinx', 'repoze.sphinx.autointerface'),},
       tests_require=[
@@ -54,21 +55,43 @@ setup(name=name,
                       # accessing templates
       entry_points={
         'console_scripts': [
-          # One entry point to control them all
-          'slapos = slapos.entry:main',
           'slapos-watchdog = slapos.grid.watchdog:main',
-          'slapproxy = slapos.proxy:main',
-          'slapproxy-query = slapos.proxy.query:main',
+          'slapproxy = slapos.cli_legacy.proxy_start:main',
+          'slapos = slapos.cli.entry:main',
           # Deprecated entry points
-          'slapconsole = slapos.client:slapconsole',
-          'slapformat = slapos.format:main',
-          'slapgrid = slapos.grid.slapgrid:run',
-          'slapgrid-sr = slapos.grid.slapgrid:runSoftwareRelease',
-          'slapgrid-cp = slapos.grid.slapgrid:runComputerPartition',
-          'slapgrid-ur = slapos.grid.slapgrid:runUsageReport',
-          'slapgrid-supervisorctl = slapos.grid.svcbackend:supervisorctl',
-          'slapgrid-supervisord = slapos.grid.svcbackend:supervisord',
-          'bang = slapos.bang:main',
+          'slapconsole = slapos.cli_legacy.console:console',
+          'slapformat = slapos.cli_legacy.format:main',
+          'slapgrid-sr = slapos.cli_legacy.slapgrid:runSoftwareRelease',
+          'slapgrid-cp = slapos.cli_legacy.slapgrid:runComputerPartition',
+          'slapgrid-ur = slapos.cli_legacy.slapgrid:runUsageReport',
+          'slapgrid-supervisorctl = slapos.cli_legacy.svcbackend:supervisorctl',
+          'slapgrid-supervisord = slapos.cli_legacy.svcbackend:supervisord',
+          'bang = slapos.cli_legacy.bang:main',
+        ],
+        'slapos.cli': [
+          # Utilities
+          'cache lookup = slapos.cli.cache:CacheLookupCommand',
+          # SlapOS Node commands
+          'node bang = slapos.cli.bang:BangCommand',
+          'node format = slapos.cli.format:FormatCommand',
+          'node register = slapos.cli.register:RegisterCommand',
+          'node supervisord = slapos.cli.supervisord:SupervisordCommand',
+          'node supervisorctl = slapos.cli.supervisorctl:SupervisorctlCommand',
+          'node status = slapos.cli.supervisorctl:SupervisorctlStatusCommand',
+          'node start = slapos.cli.supervisorctl:SupervisorctlStartCommand',
+          'node stop = slapos.cli.supervisorctl:SupervisorctlStopCommand',
+          'node restart = slapos.cli.supervisorctl:SupervisorctlRestartCommand',
+          'node tail = slapos.cli.supervisorctl:SupervisorctlTailCommand',
+          'node report = slapos.cli.slapgrid:ReportCommand',
+          'node software = slapos.cli.slapgrid:SoftwareCommand',
+          'node instance = slapos.cli.slapgrid:InstanceCommand',
+          # SlapOS client commands
+          'console = slapos.cli.console:ConsoleCommand',
+          'proxy start = slapos.cli.proxy_start:ProxyStartCommand',
+          'proxy show = slapos.cli.proxy_show:ProxyShowCommand',
+          'supply = slapos.cli.supply:SupplyCommand',
+          'remove = slapos.cli.remove:RemoveCommand',
+          'request = slapos.cli.request:RequestCommand',
         ]
       },
       test_suite="slapos.tests",

slapos/bang.py

 # -*- coding: utf-8 -*-
+# vim: set et sts=2:
 ##############################################################################
 #
 # Copyright (c) 2011, 2012 Vifib SARL and Contributors.
@@ -28,31 +29,17 @@
 ##############################################################################
 
 import slapos.slap.slap
-import argparse
-import ConfigParser
 
-def main(*args):
-  parser = argparse.ArgumentParser()
-  parser.add_argument("-m", "--message", default='', help="Message for bang.")
-  parser.add_argument("configuration_file", nargs=1, type=argparse.FileType(),
-      help="SlapOS configuration file.")
-  if len(args) == 0:
-    argument = parser.parse_args()
-  else:
-    argument = parser.parse_args(list(args))
-  configuration_file = argument.configuration_file[0]
-  message = argument.message
-  # Loads config (if config specified)
-  configuration = ConfigParser.SafeConfigParser()
-  configuration.readfp(configuration_file)
-  computer_id = configuration.get('slapos', 'computer_id')
-  master_url = configuration.get('slapos', 'master_url')
-  if configuration.has_option('slapos', 'key_file'):
-    key_file = configuration.get('slapos', 'key_file')
+
+def do_bang(configp, message):
+  computer_id = configp.get('slapos', 'computer_id')
+  master_url = configp.get('slapos', 'master_url')
+  if configp.has_option('slapos', 'key_file'):
+    key_file = configp.get('slapos', 'key_file')
   else:
     key_file = None
-  if configuration.has_option('slapos', 'cert_file'):
-    cert_file = configuration.get('slapos', 'cert_file')
+  if configp.has_option('slapos', 'cert_file'):
+    cert_file = configp.get('slapos', 'cert_file')
   else:
     cert_file = None
   slap = slapos.slap.slap()

slapos/cache.py

 # -*- coding: utf-8 -*-
 
 import ast
-import argparse
-import ConfigParser
 import hashlib
 import json
 import re
@@ -16,29 +14,20 @@ from slapos.grid.distribution import patched_linux_distribution
 def maybe_md5(s):
     return re.match('[0-9a-f]{32}', s)
 
-def cache_lookup():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("configuration_file", help="SlapOS configuration file")
-    parser.add_argument("software_url", help="Your software url or MD5 hash")
-    args = parser.parse_args()
 
-    configuration_parser = ConfigParser.SafeConfigParser()
-    configuration_parser.read(args.configuration_file)
+def do_lookup(configp, software_url):
+    cache_dir = configp.get('networkcache', 'download-binary-dir-url')
 
-    configuration_parser.items('networkcache')
-
-    cache_dir = configuration_parser.get('networkcache', 'download-binary-dir-url')
-
-    if maybe_md5(args.software_url):
-        md5 = args.software_url
+    if maybe_md5(software_url):
+        md5 = software_url
     else:
-        md5 = hashlib.md5(args.software_url).hexdigest()
+        md5 = hashlib.md5(software_url).hexdigest()
 
     try:
         response = urllib2.urlopen('%s/%s' % (cache_dir, md5))
     except urllib2.HTTPError as e:
         if e.code == 404:
-            print 'Object not in cache: %s' % args.software_url
+            print 'Object not in cache: %s' % software_url
         else:
             print 'Error during cache lookup: %s (%s)' % (e.code, e.reason)
         sys.exit(10)
@@ -67,4 +56,3 @@ def cache_lookup():
     for os in ostable:
         compatible = 'yes' if networkcache.os_matches(os, linux_distribution) else 'no'
         print '%-16s | %12s | %s | %s' % (os[0], os[1], os[2].center(14), compatible)
-

slapos/cli/bang.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ConfigCommand
+from slapos.bang import do_bang
+
+
+class BangCommand(ConfigCommand):
+    """
+    request update on all partitions
+    """
+
+    log = logging.getLogger('bang')
+
+    def get_parser(self, prog_name):
+        ap = super(BangCommand, self).get_parser(prog_name)
+        ap.add_argument('-m', '--message',
+                        help='Message for bang')
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        do_bang(configp, args.message)

slapos/cli/cache.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ConfigCommand
+from slapos.cache import do_lookup
+
+
+class CacheLookupCommand(ConfigCommand):
+    """
+    perform a query to the networkcache
+    You can provide either a complete URL to the software release,
+    or a corresponding MD5 hash value.
+
+    The command will report which OS distribution/version have a binary
+    cache of the software release, and which ones are compatible
+    with the OS you are currently running.
+    """
+
+    log = logging.getLogger('cache-lookup')
+
+    def get_parser(self, prog_name):
+        ap = super(CacheLookupCommand, self).get_parser(prog_name)
+        ap.add_argument('software_url',
+                        help='Your software url or MD5 hash')
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        do_lookup(configp, args.software_url)

slapos/cli/command.py

+# -*- coding: utf-8 -*-
+
+import argparse
+
+import cliff
+
+
+class Command(cliff.command.Command):
+    def get_parser(self, prog_name):
+        parser = argparse.ArgumentParser(
+            description=self.get_description(),
+            prog=prog_name,
+            formatter_class=argparse.RawDescriptionHelpFormatter
+        )
+
+        return parser

slapos/cli/config.py

+# -*- coding: utf-8 -*-
+
+import ConfigParser
+import os
+
+from slapos.cli.command import Command
+
+
+class ConfigError(Exception):
+    pass
+
+
+class ConfigCommand(Command):
+    "Base class for commands that require a configuration file"
+
+    log = None
+
+    default_config_var = 'SLAPOS_CONFIGURATION'
+
+    # use this if default_config_var does not exist
+    default_config_path = '/etc/opt/slapos/slapos.cfg'
+
+    def get_parser(self, prog_name):
+        ap = super(ConfigCommand, self).get_parser(prog_name)
+        ap.add_argument('--cfg', help='SlapOS configuration file - ' +
+                                      'defaults to $%s ' % self.default_config_var +
+                                      'or %s' % self.default_config_path)
+        return ap
+
+    def _get_config(self, cfg_path, required=False):
+        """
+        Returns a configuration object if file exists/readable/valid,
+        None otherwise.
+        Will raise an error instead of returning None if required is True.
+        Even if required is False, may still raise an exception from the
+        configparser if the configuration content is very broken.
+        We don't catch that exception as it will clearly show what is
+        wrong with the file.
+        """
+
+        if not os.path.exists(cfg_path):
+            if required:
+                raise ConfigError('Configuration file does not exist: %s' % cfg_path)
+            else:
+                return None
+
+        configp = ConfigParser.SafeConfigParser()
+        if configp.read(cfg_path) != [cfg_path]:
+            # bad permission, etc.
+            if required:
+                raise ConfigError('Cannot parse configuration file: %s' % cfg_path)
+            else:
+                return None
+
+        return configp
+
+    def fetch_config(self, args):
+        if args.cfg:
+            cfg_path = args.cfg
+        else:
+            cfg_path = os.environ.get(self.default_config_var, self.default_config_path)
+
+        cfg_path = os.path.expanduser(cfg_path)
+
+        self.log.debug('Loading config: %s' % cfg_path)
+
+        return self._get_config(cfg_path, required=True)
+
+
+class ClientConfigCommand(ConfigCommand):
+    default_config_var = 'SLAPOS_CLIENT_CONFIGURATION'
+    default_config_path = '~/.slapos/slapos-client.cfg'

slapos/cli/console.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ClientConfigCommand
+from slapos.client import init, do_console, ClientConfig
+
+
+class ConsoleCommand(ClientConfigCommand):
+    """
+    python console with slap library imported
+
+    You can play with the global "slap" object and
+    with the global "request" method.
+
+    examples :
+    >>> # Request instance
+    >>> request(kvm, "myuniquekvm")
+    >>> # Request software installation on owned computer
+    >>> supply(kvm, "mycomputer")
+    >>> # Fetch instance informations on already launched instance
+    >>> request(kvm, "myuniquekvm").getConnectionParameter("url")
+    """
+
+    log = logging.getLogger('console')
+
+    def get_parser(self, prog_name):
+        ap = super(ConsoleCommand, self).get_parser(prog_name)
+
+        ap.add_argument('-u', '--master_url',
+                        help='Url of SlapOS Master to use')
+
+        ap.add_argument('-k', '--key_file',
+                        help='SSL Authorisation key file')
+
+        ap.add_argument('-c', '--cert_file',
+                        help='SSL Authorisation certificate file')
+
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        conf = ClientConfig(args, configp)
+        local = init(conf)
+        do_console(local)

slapos/cli/entry.py

+# -*- coding: utf-8 -*-
+
+import logging
+import sys
+
+import cliff
+import cliff.app
+from cliff.app import LOG
+import cliff.commandmanager
+
+import slapos.version
+
+
+class SlapOSCommandManager(cliff.commandmanager.CommandManager):
+
+    def find_command(self, argv):
+        """Given an argument list, find a command and
+        return the processor and any remaining arguments.
+        """
+        # a little cheating, 'slapos node' is not documented by the help command
+        if argv == ['node']:
+            argv = ['node', 'status']
+
+        search_args = argv[:]
+        name = ''
+        while search_args:
+            if search_args[0].startswith('-'):
+                raise ValueError('Invalid command %r' % search_args[0])
+            next_val = search_args.pop(0)
+            name = '%s %s' % (name, next_val) if name else next_val
+            if name in self.commands:
+                cmd_ep = self.commands[name]
+                cmd_factory = cmd_ep.load()
+                return (cmd_factory, name, search_args)
+        else:
+            print >>sys.stderr, ('The command %r does not exist or is not yet implemented.\n'
+                                 'Please have a look at http://community.slapos.org to read documentation or forum.\n'
+                                 'Please also make sure that SlapOS Node is up to date.' % (argv,))
+            sys.exit(5)
+
+
+class SlapOSApp(cliff.app.App):
+
+    #
+    # self.options.verbose_level:
+    #    -q  -> 0 (WARNING)
+    #        -> 1 (INFO)
+    #    -v  -> 2 (DEBUG)
+    #    -vv -> 3 (...)
+    #    etc.
+    #
+
+    log = logging.getLogger('slapos')
+
+    CONSOLE_MESSAGE_FORMAT = '%(message)s'
+    LOG_FILE_MESSAGE_FORMAT = '[%(asctime)s] %(levelname)-8s %(name)s %(message)s'
+
+    def __init__(self):
+        super(SlapOSApp, self).__init__(
+            description='SlapOS client %s' % slapos.version.version,
+            version=slapos.version.version,
+            command_manager=SlapOSCommandManager('slapos.cli'),
+        )
+
+    def build_option_parser(self, *args, **kw):
+        kw.setdefault('argparse_kwargs', {})
+        kw['argparse_kwargs']['conflict_handler'] = 'resolve'
+        parser = super(SlapOSApp, self).build_option_parser(*args, **kw)
+
+        # add two aliases for --log-file (for compatibility with old commands)
+        parser.add_argument(
+            '--log-file', '--logfile', '--log_file',
+            action='store',
+            default=None,
+            help='Specify a file to log output. Only console by default.',
+        )
+
+        # always show tracebacks on errors
+        parser.set_defaults(debug=True)
+
+        return parser
+
+    def initialize_app(self, argv):
+        if self.options.verbose_level > 2:
+            self.log.debug('initialize_app')
+
+    def prepare_to_run_command(self, cmd):
+        if self.options.verbose_level > 2:
+            self.log.debug('prepare_to_run_command %s', cmd.__class__.__name__)
+
+    def clean_up(self, cmd, result, err):
+        if self.options.verbose_level > 2:
+            self.log.debug('clean_up %s', cmd.__class__.__name__)
+
+        if err:
+            self.log.debug('got an error: %s', err)
+
+    def run(self, argv):
+        # same as cliff.App.run except that it won't re-raise
+        # a logged exception
+        try:
+            self.options, remainder = self.parser.parse_known_args(argv)
+            self.configure_logging()
+            self.interactive_mode = not remainder
+            self.initialize_app(remainder)
+        except Exception as err:
+            if hasattr(self, 'options'):
+                debug = self.options.debug
+            else:
+                debug = True
+            if debug:
+                LOG.exception(err)
+                # XXX change from cliff behaviour: avoid
+                #     displaying the exception twice
+                # raise
+            else:
+                LOG.error(err)
+            return 1
+        result = 1
+        if self.interactive_mode:
+            result = self.interact()
+        else:
+            result = self.run_subcommand(remainder)
+        return result
+
+
+def main(argv=sys.argv[1:]):
+    app = SlapOSApp()
+    if not argv:
+        argv = ['-h']
+    return app.run(argv)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

slapos/cli/format.py

+# -*- coding: utf-8 -*-
+
+import logging
+import sys
+
+from slapos.cli.config import ConfigCommand
+from slapos.format import do_format, FormatConfig, tracing_monkeypatch, UsageError
+
+
+class FormatCommand(ConfigCommand):
+    """
+    create users, partitions and network configuration
+    """
+
+    log = logging.getLogger('format')
+
+    def get_parser(self, prog_name):
+        ap = super(FormatCommand, self).get_parser(prog_name)
+
+        ap.add_argument('-x', '--computer_xml',
+                        help="Path to file with computer's XML. If does not exists, will be created",
+                        default=None)
+
+        ap.add_argument('--computer_json',
+                        help="Path to a JSON version of the computer's XML (for development only).",
+                        default=None)
+
+        ap.add_argument('-i', '--input_definition_file',
+                        help="Path to file to read definition of computer instead of "
+                        "declaration. Using definition file allows to disable "
+                        "'discovery' of machine services and allows to define computer "
+                        "configuration in fully controlled manner.")
+
+        ap.add_argument('-o', '--output_definition_file',
+                        help="Path to file to write definition of computer from "
+                        "declaration.")
+
+        ap.add_argument('-n', '--dry_run',
+                        help="Don't actually do anything.",
+                        default=False,
+                        action="store_true")
+
+        ap.add_argument('--alter_user',
+                        choices=['True', 'False'],
+                        help="Shall slapformat alter user database [default: True]")
+
+        ap.add_argument('--alter_network',
+                        choices=['True', 'False'],
+                        help="Shall slapformat alter network configuration [default: True]")
+
+        ap.add_argument('--now',
+                        help="Launch slapformat without delay",
+                        default=False,
+                        action="store_true")
+
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+
+        conf = FormatConfig(logger=self.log)
+
+        conf.mergeConfig(args, configp)
+
+        if not self.app.options.log_file and conf.log_file:
+            # no log file is provided by argparser,
+            # we set up the one from config
+            file_handler = logging.FileHandler(conf.log_file)
+            formatter = logging.Formatter(self.app.LOG_FILE_MESSAGE_FORMAT)
+            file_handler.setFormatter(formatter)
+            self.log.addHandler(file_handler)
+
+        try:
+            conf.setConfig()
+        except UsageError as err:
+            sys.stderr.write(err.message + '\n')
+            sys.stderr.write("For help use --help\n")
+            sys.exit(1)
+
+        tracing_monkeypatch(conf)
+
+        try:
+            do_format(conf=conf)
+        except:
+            self.log.exception('Uncaught exception:')
+            raise

slapos/proxy/query.py → slapos/cli/proxy_show.py

 # -*- coding: utf-8 -*-
-# vim: set et sts=4:
 
 import collections
-import ConfigParser
-from optparse import OptionParser, Option
-import sys
+import hashlib
+import logging
 
 import lxml.etree
+import prettytable
 import sqlite3
 
+from slapos.cli.config import ConfigCommand
+from slapos.proxy import ProxyConfig
 from slapos.proxy.db_version import DB_VERSION
 
 
-
-class Parser(OptionParser):
+class ProxyShowCommand(ConfigCommand):
     """
-    Parse all arguments.
+    display proxy instances and parameters
     """
-    def __init__(self, usage=None, version=None):
-      """
-      Initialize all options possibles.
-      """
-      OptionParser.__init__(self, usage=usage, version=version,
-                            option_list=[
-          Option("-u", "--database-uri",
-                 type=str,
-                 help="URI for sqlite database"),
-          Option('--show-instances',
-                 help="View instance information",
-                 default=False,
-                 action="store_true"),
-          Option('--show-params',
-                 help="View published parameters",
-                 default=False,
-                 action="store_true"),
-          Option('--show-network',
-                 help="View network information",
-                 default=False,
-                 action="store_true"),
-          Option('--show-all',
-                 help="View all information",
-                 default=False,
-                 action="store_true"),
-      ])
-
-    def check_args(self):
-        """
-        Check arguments
-        """
-        (options, args) = self.parse_args()
-        if len(args) < 1:
-            self.error("Incorrect number of arguments")
-
-        return options, args[0]
-
-class Config:
-    def setConfig(self, option_dict, configuration_file_path):
-        """
-        Set options given by parameters.
-        """
-        # Set options parameters
-        for option, value in option_dict.__dict__.items():
-          setattr(self, option, value)
-
-        # Load configuration file
-        configuration_parser = ConfigParser.SafeConfigParser()
-        configuration_parser.read(configuration_file_path)
-        # Merges the arguments and configuration
-        for section in ("slapproxy", "slapos"):
-            configuration_dict = dict(configuration_parser.items(section))
-            for key in configuration_dict:
-                if not getattr(self, key, None):
-                    setattr(self, key, configuration_dict[key])
-
-        if not self.database_uri:
-            raise ValueError('database-uri is required.')
-
-
-
-tbl_computer = 'computer' + DB_VERSION
-tbl_software = 'software' + DB_VERSION
+
+    log = logging.getLogger('proxy')
+
+    def get_parser(self, prog_name):
+        ap = super(ProxyShowCommand, self).get_parser(prog_name)
+
+        ap.add_argument('-u', '--database-uri',
+                        help='URI for sqlite database')
+
+        ap.add_argument('--computers',
+                        help='view computer information',
+                        action='store_true')
+
+        ap.add_argument('--software',
+                        help='view software releases',
+                        action='store_true')
+
+        ap.add_argument('--partitions',
+                        help='view partitions',
+                        action='store_true')
+
+        ap.add_argument('--slaves',
+                        help='view slave instances',
+                        action='store_true')
+
+        ap.add_argument('--params',
+                        help='view published parameters',
+                        action='store_true')
+
+        ap.add_argument('--network',
+                        help='view network settings',
+                        action='store_true')
+
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        conf = ProxyConfig(logger=self.log)
+        conf.mergeConfig(args, configp)
+        conf.setConfig()
+        do_show(conf=conf)
+
+
 tbl_partition = 'partition' + DB_VERSION
-tbl_partition_network = 'partition_network' + DB_VERSION
-tbl_slave = 'slave' + DB_VERSION
 
-null_str = u"-"
+
+def coalesce(*seq):
+    el = None
+    for el in seq:
+        if el is not None:
+            return el
+    return el
 
 
 def print_table(qry, tablename, skip=None):
@@ -92,50 +76,33 @@ def print_table(qry, tablename, skip=None):
         skip = set()
 
     columns = [c[0] for c in qry.description if c[0] not in skip]
+
     rows = []
     for row in qry.fetchall():
-        line = {}
-        for col in columns:
-            val = row[col]
-            if val is None:
-                val = null_str
-            line[col] = val.strip()
-        rows.append(line)
-
-    max_width = {col: len(col) for col in columns}
-    for row in rows:
-        for col in columns:
-            val = row[col]
-            max_width[col] = max(max_width[col], len(val) if val else 0)
+        rows.append([coalesce(row[col], '-') for col in columns])
 
-    hdr = [col.center(max_width[col]) for col in columns]
+    pt = prettytable.PrettyTable(columns)
+    # https://code.google.com/p/prettytable/wiki/Tutorial
 
-    print
+    for row in rows:
+        pt.add_row(row)
 
     if rows:
         print 'table %s:' % tablename,
+        if skip:
+            print 'skipping %s' % ', '.join(skip)
+        else:
+            print
     else:
         print 'table %s: empty' % tablename
         return
 
-    if skip:
-        print 'skipping %s' % ', '.join(skip)
-    else:
-        print
-
-    print ' | '.join(hdr)
-    print '-+-'.join('-'*len(h) for h in hdr)
-
-    for row in rows:
-        cells = [row[col].ljust(max_width[col]) for col in columns]
-        print ' | '.join(cells)
+    print pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE)
 
 
 def print_params(conn):
     cur = conn.cursor()
 
-    print
-
     qry = cur.execute("SELECT reference, partition_reference, software_type, connection_xml FROM %s" % tbl_partition)
     for row in qry.fetchall():
         if not row['connection_xml']:
@@ -150,18 +117,19 @@ def print_params(conn):
             if text and name in ('ssh-key', 'ssh-public-key'):
                 text = text[:20] + '...' + text[-20:]
             print '    %s = %s' % (name, text)
-        print
 
 
 def print_computer_table(conn):
+    tbl_computer = 'computer' + DB_VERSION
     cur = conn.cursor()
     qry = cur.execute("SELECT * FROM %s" % tbl_computer)
     print_table(qry, tbl_computer)
 
 
 def print_software_table(conn):
+    tbl_software = 'software' + DB_VERSION
     cur = conn.cursor()
-    qry = cur.execute("SELECT * FROM %s" % tbl_software)
+    qry = cur.execute("SELECT *, md5(url) as md5 FROM %s" % tbl_software)
     print_table(qry, tbl_software)
 
 
@@ -170,21 +138,16 @@ def print_partition_table(conn):
     qry = cur.execute("SELECT * FROM %s WHERE slap_state<>'free'" % tbl_partition)
     print_table(qry, tbl_partition, skip=['xml', 'connection_xml', 'slave_instance_list'])
 
+
 def print_slave_table(conn):
+    tbl_slave = 'slave' + DB_VERSION
     cur = conn.cursor()
     qry = cur.execute("SELECT * FROM %s" % tbl_slave)
     print_table(qry, tbl_slave, skip=['connection_xml'])
 
 
-def print_tables(conn):
-    print_computer_table(conn)
-    print_software_table(conn)
-    print_partition_table(conn)
-    print_slave_table(conn)
-
-
 def print_network(conn):
-    print
+    tbl_partition_network = 'partition_network' + DB_VERSION
     cur = conn.cursor()
     addr = collections.defaultdict(list)
     qry = cur.execute("""
@@ -202,45 +165,40 @@ def print_network(conn):
         print '%s: %s' % (partition_reference, ', '.join(addresses))
 
 
-
-
-def run(config):
-    conn = sqlite3.connect(config.database_uri)
+def do_show(conf):
+    conn = sqlite3.connect(conf.database_uri)
     conn.row_factory = sqlite3.Row
 
-    fn = []
-
-    if config.show_all or config.show_instances:
-        fn.append(print_tables)
-    if config.show_all or config.show_params:
-        fn.append(print_params)
-    if config.show_all or config.show_network:
-        fn.append(print_network)
-
-    if fn:
-        for f in fn:
-            f(conn)
-    else:
-        print 'usage: %s [ --show-params | --show-network | --show-instances | --show-all ]' % sys.argv[0]
-
-
-
-def main():
-  "Run default configuration."
-  usage = "usage: %s [options] CONFIGURATION_FILE" % sys.argv[0]
-
-  try:
-    # Parse arguments
-    config = Config()
-    config.setConfig(*Parser(usage=usage).check_args())
-
-    run(config)
-    return_code = 0
-  except SystemExit, err:
-    # Catch exception raise by optparse
-    return_code = err
-
-  sys.exit(return_code)
-
+    conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest())
+
+    print_all = not any(
+            [
+                conf.computers,
+                conf.software,
+                conf.partitions,
+                conf.slaves,
+                conf.params,
+                conf.network,
+                ]
+            )
+
+    if print_all or conf.computers:
+        print_computer_table(conn)
+        print
+    if print_all or conf.software:
+        print_software_table(conn)
+        print
+    if print_all or conf.partitions:
+        print_partition_table(conn)
+        print
+    if print_all or conf.slaves:
+        print_slave_table(conn)
+        print
+    if print_all or conf.params:
+        print_params(conn)
+        print
+    if print_all or conf.network:
+        print_network(conn)
+        print
 
 

slapos/cli/proxy_start.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ConfigCommand
+from slapos.proxy import do_proxy, ProxyConfig
+
+
+class ProxyStartCommand(ConfigCommand):
+    """
+    minimalist, stand-alone SlapOS Master
+    """
+
+    log = logging.getLogger('proxy')
+
+    def get_parser(self, prog_name):
+        ap = super(ProxyStartCommand, self).get_parser(prog_name)
+
+        ap.add_argument('-u', '--database-uri',
+                        help='URI for sqlite database')
+
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+
+        conf = ProxyConfig(logger=self.log)
+
+        conf.mergeConfig(args, configp)
+
+        if not self.app.options.log_file and hasattr(conf, 'log_file'):
+            # no log file is provided by argparser,
+            # we set up the one from config
+            file_handler = logging.FileHandler(conf.log_file)
+            formatter = logging.Formatter(self.app.LOG_FILE_MESSAGE_FORMAT)
+            file_handler.setFormatter(formatter)
+            self.log.addHandler(file_handler)
+
+        conf.setConfig()
+
+        do_proxy(conf=conf)
+

slapos/cli/register.py

+# -*- coding: utf-8 -*-
+
+import logging
+import sys
+
+from slapos.cli.command import Command
+from slapos.register.register import do_register, RegisterConfig
+
+
+class RegisterCommand(Command):
+    """
+    register a node in the SlapOS cloud
+    """
+
+    log = logging.getLogger('register')
+
+    def get_parser(self, prog_name):
+        ap = super(RegisterCommand, self).get_parser(prog_name)
+
+        ap.add_argument('node_name',
+                        help='Name of the node')
+
+        ap.add_argument('--interface-name',
+                        help='Interface name to access internet',
+                        default='eth0')
+
+        ap.add_argument('--master-url',
+                        help='URL of SlapOS master',
+                        default='https://slap.vifib.com')
+
+        ap.add_argument('--master-url-web',
+                        help='URL of SlapOS Master webservice to register certificates',
+                        default='https://www.slapos.org')
+
+        ap.add_argument('--partition-number',
+                        help='Number of partition on computer',
+                        default='10',
+                        type=int)
+
+        ap.add_argument('--ipv4-local-network',
+                        help='Base of ipv4 local network',
+                        default='10.0.0.0/16')
+
+        ap.add_argument('--ipv6-interface',
+                        help='Interface name to get ipv6',
+                        default='')
+
+        ap.add_argument('--login',
+                        help='User login on SlapOS Master webservice')
+
+        ap.add_argument('--password',
+                        help='User password on SlapOs Master webservice')
+
+        ap.add_argument('-t', '--create-tap',
+                        help='Will trigger creation of one virtual "tap" interface per '
+                             'Partition and attach it to primary interface. Requires '
+                             'primary interface to be a bridge. defaults to false. '
+                             'Needed to host virtual machines.',
+                        default=False,
+                        action='store_true')
+
+        ap.add_argument('-n', '--dry-run',
+                        help='Simulate the execution steps',
+                        default=False,
+                        action='store_true')
+
+        return ap
+
+    def take_action(self, args):
+        try:
+            conf = RegisterConfig(logger=self.log)
+            conf.setConfig(args)
+            return_code = do_register(conf)
+        except SystemExit as err:
+            return_code = err
+
+        sys.exit(return_code)

slapos/cli/remove.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ClientConfigCommand
+from slapos.client import init, do_remove, ClientConfig
+
+
+class RemoveCommand(ClientConfigCommand):
+    """
+    remove a Software from a node
+    """
+
+    log = logging.getLogger('remove')
+
+    def get_parser(self, prog_name):
+        ap = super(RemoveCommand, self).get_parser(prog_name)
+
+        ap.add_argument('software_url',
+                        help='Your software url')
+
+        ap.add_argument('node',
+                        help="Target node")
+
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        conf = ClientConfig(args, configp)
+        local = init(conf)
+        do_remove(args.software_url, args.node, local)

slapos/cli/request.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ClientConfigCommand
+from slapos.client import init, do_request, ClientConfig
+
+
+def parse_option_dict(options):
+    """
+    Parse a list of option strings like foo=bar baz=qux and return a dictionary.
+    Will raise if keys are repeated.
+    """
+    ret = {}
+    for option_pair in (options or []):
+        key, value = option_pair.split('=', 1)
+        if key in ret:
+            raise ValueError("Multiple values provided for the same key '%s'" % key)
+        ret[key] = value
+    return ret
+
+
+class RequestCommand(ClientConfigCommand):
+    """request an instance and get status and parameters of instance"""
+
+    log = logging.getLogger('request')
+
+    def get_parser(self, prog_name):
+        ap = super(RequestCommand, self).get_parser(prog_name)
+
+        ap.add_argument('reference',
+                        help='Your instance reference')
+
+        ap.add_argument('software_url',
+                        help='Your software url')
+        # XXX TODO can this be an MD5 hash?
+        # XXX TODO can we do a minimal check for correctness of this argument?
+        #          the alternative is a silent failure for mistyped/obsolete/invalid URL
+
+        ap.add_argument('--node',
+                        nargs='+',
+                        help="Node request option 'option1=value1 option2=value2'")
+
+        ap.add_argument('--type',
+                        help='Software type to be requested')
+
+        ap.add_argument('--state',
+                        help='State of the requested instance')
+
+        ap.add_argument('--slave',
+                        action='store_true',
+                        help='Ask for a slave instance')
+
+        ap.add_argument('--parameters',
+                        nargs='+',
+                        help="Give your configuration 'option1=value1 option2=value2'")
+
+        return ap
+
+    def take_action(self, args):
+        args.node = parse_option_dict(args.node)
+        args.parameters = parse_option_dict(args.parameters)
+
+        configp = self.fetch_config(args)
+        conf = ClientConfig(args, configp)
+
+        local = init(conf)
+        do_request(conf, local)

slapos/cli/slapgrid.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ConfigCommand
+
+from slapos.grid.utils import setRunning, setFinished
+from slapos.grid.slapgrid import (merged_options, check_missing_parameters, check_missing_files,
+                                  random_delay, create_slapgrid_object)
+
+
+class SlapgridCommand(ConfigCommand):
+
+    log = None
+
+    method_name = NotImplemented
+    default_pidfile = NotImplemented
+
+    def get_parser(self, prog_name):
+        ap = super(SlapgridCommand, self).get_parser(prog_name)
+
+        # TODO move more options to the instance, software and report subclasses
+
+        ap.add_argument('--instance-root',
+                        help='The instance root directory location.')
+        ap.add_argument('--software-root',
+                        help='The software_root directory location.')
+        ap.add_argument('--master-url',
+                        help='The master server URL. Mandatory.')
+        ap.add_argument('--computer-id',
+                        help='The computer id defined in the server.')
+        ap.add_argument('--supervisord-socket',
+                        help='The socket supervisor will use.')
+        ap.add_argument('--supervisord-configuration-path',
+                        help='The location where supervisord configuration will be stored.')
+        ap.add_argument('--buildout', default=None,
+                        help='Location of buildout binary.')
+        ap.add_argument('--pidfile',
+                        help='The location where pidfile will be created. '
+                             'Can be provided by configuration file, or defaults '
+                             'to %s' % self.default_pidfile)
+        ap.add_argument('--key_file',
+                        help='SSL Authorisation key file.')
+        ap.add_argument('--cert_file',
+                        help='SSL Authorisation certificate file.')
+        ap.add_argument('--signature_private_key_file',
+                        help='Signature private key file.')
+        ap.add_argument('--master_ca_file',
+                        help='Root certificate of SlapOS master key.')
+        ap.add_argument('--certificate_repository_path',
+                        help='Path to directory where downloaded certificates would be stored.')
+        ap.add_argument('--maximum-periodicity', type=int, default=None,
+                        help='Periodicity at which buildout should be run in instance.')
+        ap.add_argument('--promise-timeout', type=int, default=3,
+                        help='Promise timeout in seconds.')
+        ap.add_argument('--now', action='store_true',
+                        help='Launch slapgrid without delay. Default behavior.')
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        options = merged_options(args, configp)
+
+        check_missing_parameters(options)
+        check_missing_files(options)
+
+        random_delay(options, logger=self.log)
+
+        slapgrid_object = create_slapgrid_object(options, logger=self.log)
+
+        pidfile = options.get('pidfile') or self.default_pidfile
+
+        if pidfile:
+            setRunning(logger=self.log, pidfile=pidfile)
+        try:
+            return getattr(slapgrid_object, self.method_name)()
+        finally:
+            if pidfile:
+                setFinished(pidfile)
+
+
+class SoftwareCommand(SlapgridCommand):
+    """run software installation/deletion"""
+
+    log = logging.getLogger('software')
+
+    method_name = 'processSoftwareReleaseList'
+    default_pidfile = '/opt/slapos/slapgrid-sr.pid'
+
+    def get_parser(self, prog_name):
+        ap = super(SoftwareCommand, self).get_parser(prog_name)
+
+        only = ap.add_mutually_exclusive_group()
+        only.add_argument('--all', action='store_true',
+                          help='Process all Software Releases, even if already installed.')
+        only.add_argument('--only-sr', '--only',
+                          help='Force the update of a single software release (can be full URL or MD5 hash), '
+                               'even if is already installed. This option will make all other '
+                               'sofware releases be ignored.')
+        return ap
+
+
+class InstanceCommand(SlapgridCommand):
+    """run instance deployment"""
+
+    log = logging.getLogger('instance')
+
+    method_name = 'processComputerPartitionList'
+    default_pidfile = '/opt/slapos/slapgrid-cp.pid'
+
+    def get_parser(self, prog_name):
+        ap = super(InstanceCommand, self).get_parser(prog_name)
+
+        only = ap.add_mutually_exclusive_group()
+        only.add_argument('--all', action='store_true',
+                          help='Process all Computer Partitions.')
+        only.add_argument('--only-cp', '--only',
+                          help='Update a single or a list of computer partitions '
+                               '(ie.:slappartX, slappartY), '
+                               'this option will make all other computer partitions be ignored.')
+        return ap
+
+
+class ReportCommand(SlapgridCommand):
+    """run instance reports and garbage collection"""
+
+    log = logging.getLogger('report')
+
+    method_name = 'agregateAndSendUsage'
+    default_pidfile = '/opt/slapos/slapgrid-ur.pid'

slapos/cli/supervisorctl.py

+# -*- coding: utf-8 -*-
+
+import argparse
+import logging
+import os
+
+from slapos.cli.config import ConfigCommand
+from slapos.grid.svcbackend import launchSupervisord
+
+import supervisor.supervisorctl
+
+
+class SupervisorctlCommand(ConfigCommand):
+    """enter into supervisor console, for process management"""
+
+    log = logging.getLogger('supervisorctl')
+
+    def get_parser(self, prog_name):
+        ap = super(SupervisorctlCommand, self).get_parser(prog_name)
+        ap.add_argument('supervisor_args',
+                        nargs=argparse.REMAINDER,
+                        help='parameters passed to supervisorctl')
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        instance_root = configp.get('slapos', 'instance_root')
+        configuration_file = os.path.join(instance_root, 'etc', 'supervisord.conf')
+        launchSupervisord(socket=os.path.join(instance_root, 'supervisord.socket'),
+                          configuration_file=configuration_file,
+                          logger=self.log)
+        supervisor.supervisorctl.main(args=['-c', configuration_file] + args.supervisor_args)
+
+
+class SupervisorctlAliasCommand(SupervisorctlCommand):
+    def take_action(self, args):
+        args.supervisor_args = [self.alias] + args.supervisor_args
+        super(SupervisorctlAliasCommand, self).take_action(args)
+
+
+class SupervisorctlStatusCommand(SupervisorctlAliasCommand):
+    """alias for 'node supervisorctl status'"""
+    alias = 'status'
+
+
+class SupervisorctlStartCommand(SupervisorctlAliasCommand):
+    """alias for 'node supervisorctl start'"""
+    alias = 'start'
+
+
+class SupervisorctlStopCommand(SupervisorctlAliasCommand):
+    """alias for 'node supervisorctl stop'"""
+    alias = 'stop'
+
+
+class SupervisorctlRestartCommand(SupervisorctlAliasCommand):
+    """alias for 'node supervisorctl restart'"""
+    alias = 'restart'
+
+
+class SupervisorctlTailCommand(SupervisorctlAliasCommand):
+    """alias for 'node supervisorctl tail'"""
+    alias = 'tail'

slapos/cli/supervisord.py

+# -*- coding: utf-8 -*-
+
+import logging
+import os
+
+from slapos.cli.config import ConfigCommand
+from slapos.grid.svcbackend import launchSupervisord
+
+
+class SupervisordCommand(ConfigCommand):
+    """launch, if not already running, supervisor daemon"""
+
+    log = logging.getLogger('supervisord')
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        instance_root = configp.get('slapos', 'instance_root')
+        launchSupervisord(socket=os.path.join(instance_root, 'supervisord.socket'),
+                          configuration_file=os.path.join(instance_root, 'etc', 'supervisord.conf'),
+                          logger=self.log)

slapos/cli/supply.py

+# -*- coding: utf-8 -*-
+
+import logging
+
+from slapos.cli.config import ClientConfigCommand
+from slapos.client import init, do_supply, ClientConfig
+
+
+class SupplyCommand(ClientConfigCommand):
+    """
+    supply a Software to a node
+    """
+
+    log = logging.getLogger('supply')
+
+    def get_parser(self, prog_name):
+        ap = super(SupplyCommand, self).get_parser(prog_name)
+
+        ap.add_argument('software_url',
+                        help='Your software url')
+
+        ap.add_argument('node',
+                        help="Target node")
+
+        return ap
+
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        conf = ClientConfig(args, configp)
+        local = init(conf)
+        do_supply(args.software_url, args.node, local)

slapos/cli_legacy/bang.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2011, 2012 Vifib SARL and Contributors.
+# All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+import ConfigParser
+
+from slapos.bang import do_bang
+
+
+def main(*args):
+    ap = argparse.ArgumentParser()
+    ap.add_argument('-m', '--message', default='', help='Message for bang.')
+    ap.add_argument('configuration_file', type=argparse.FileType(),
+                    help='SlapOS configuration file.')
+    if args:
+        args = ap.parse_args(list(args))
+    else:
+        args = ap.parse_args()
+    configp = ConfigParser.SafeConfigParser()
+    configp.readfp(args.configuration_file)
+    do_bang(configp, args.message)

slapos/cli_legacy/cache.py

+# -*- coding: utf-8 -*-
+
+import argparse
+import ConfigParser
+
+from slapos.cache import do_lookup
+
+
+def cache_lookup():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("configuration_file", help="SlapOS configuration file")
+    ap.add_argument("software_url", help="Your software url or MD5 hash")
+    args = ap.parse_args()
+
+    configp = ConfigParser.SafeConfigParser()
+    configp.read(args.configuration_file)
+
+    do_lookup(configp, args.software_url)

slapos/cli_legacy/console.py

+# -*- coding: utf-8 -*-
+
+import argparse
+import os
+import textwrap
+
+from slapos.client import ClientConfig, init, do_console
+from slapos.cli_legacy.util import get_config_parser
+
+
+def console():
+  description = textwrap.dedent("""\
+    slapconsole allows you interact with slap API. You can play with the global
+    "slap" object and with the global "request" method.
+
+    examples :
+    >>> # Request instance
+    >>> request(kvm, "myuniquekvm")
+    >>> # Request software installation on owned computer
+    >>> supply(kvm, "mycomputer")
+    >>> # Fetch instance informations on already launched instance
+    >>> request(kvm, "myuniquekvm").getConnectionParameter("url")""")
+
+  ap = argparse.ArgumentParser(description=description,
+                               formatter_class=argparse.RawDescriptionHelpFormatter)
+
+  ap.add_argument('-u', '--master_url',
+                  default=None,
+                  help='Url of SlapOS Master to use.')
+
+  ap.add_argument('-k', '--key_file',
+                  help="SSL Authorisation key file.")
+
+  ap.add_argument('-c', '--cert_file',
+                  help="SSL Authorisation certificate file.")
+
+  ap.add_argument('configuration_file',
+                  help='path to slapos.cfg')
+
+  args = ap.parse_args()
+
+  if not os.path.isfile(args.configuration_file):
+    ap.error("%s: Not found or not a regular file." % args.configuration_file)
+
+  configp = get_config_parser(args.configuration_file)
+  conf = ClientConfig(args, configp)
+  local = init(conf)
+  do_console(local)

slapos/entry.py → slapos/cli_legacy/entry.py

@@ -30,19 +30,20 @@ import argparse
 import ConfigParser
 import os
 import sys
-from slapos.bang import main as bang
-from slapos.client import slapconsole as console
-from slapos.client import request as request
-from slapos.client import remove as remove
-from slapos.client import supply as supply
-from slapos.format import main as format
-from slapos.cache import cache_lookup
-from slapos.grid.slapgrid import runComputerPartition as instance
-from slapos.grid.slapgrid import runSoftwareRelease as software
-from slapos.grid.slapgrid import runUsageReport as report
-from slapos.grid.svcbackend import supervisord
-from slapos.grid.svcbackend import supervisorctl
-from slapos.register.register import main as register
+
+from slapos.cli_legacy.bang import main as bang
+from slapos.cli_legacy.console import console
+from slapos.cli_legacy.request import request
+from slapos.cli_legacy.remove import remove
+from slapos.cli_legacy.supply import supply
+from slapos.cli_legacy.format import main as format
+from slapos.cli_legacy.cache import cache_lookup
+from slapos.cli_legacy.slapgrid import runComputerPartition as instance
+from slapos.cli_legacy.slapgrid import runSoftwareRelease as software
+from slapos.cli_legacy.slapgrid import runUsageReport as report
+from slapos.cli_legacy.svcbackend import supervisord
+from slapos.cli_legacy.svcbackend import supervisorctl
+from slapos.cli_legacy.register import main as register
 from slapos.version import version
 
 # Note: this whole file is a hack. We should better try dedicated library
@@ -60,6 +61,7 @@ class EntryPointNotImplementedError(NotImplementedError):
   def __init__(self, *args, **kw_args):
     NotImplementedError.__init__(self, *args, **kw_args)
 
+
 def checkSlaposCfg():
   """
   Check if a slapos configuration file was given as a argument.
@@ -71,12 +73,13 @@ def checkSlaposCfg():
   for element in sys.argv:
     if '.cfg' in element:
       if os.path.exists(element):
-        configuration = ConfigParser.SafeConfigParser()
-        configuration.read(element)
-        if configuration.has_section('slapos'):
+        configp = ConfigParser.SafeConfigParser()
+        configp.read(element)
+        if configp.has_section('slapos'):
           return True
   return False
 
+
 def checkOption(option):
   """
   Check if a given option is already in call line
@@ -88,11 +91,12 @@ def checkOption(option):
     if key in element:
       return True
   sys.argv.append(key)
-  if len(option) > 1 :
+  if len(option) > 1:
     sys.argv = sys.argv + option[1:]
   return True
 
-def call(fun, config=False, option=None):
+
+def call(fun, config_path=False, option=None):
   """
   Add missing options to sys.argv
   Add config if asked and it is missing
@@ -102,12 +106,13 @@ def call(fun, config=False, option=None):
     option = []
   for element in option:
     checkOption(element)
-  if config:
+  if config_path:
     if not checkSlaposCfg():
-      sys.argv = [sys.argv[0]] + [os.path.expanduser(config)] + sys.argv[1:]
+      sys.argv = [sys.argv[0]] + [os.path.expanduser(config_path)] + sys.argv[1:]
   fun()
   sys.exit(0)
 
+
 def dispatch(command, is_node_command):
   """ Dispatch to correct SlapOS module.
   Here we could use introspection to get rid of the big "if" statements,
@@ -124,34 +129,34 @@ def dispatch(command, is_node_command):
     if command == 'register':
       call(register)
     elif command == 'software':
-      call(software, config=GLOBAL_SLAPOS_CONFIGURATION,
+      call(software, config_path=GLOBAL_SLAPOS_CONFIGURATION,
            option=['--pidfile /opt/slapos/slapgrid-sr.pid'])
     elif command == 'instance':
-      call(instance, config=GLOBAL_SLAPOS_CONFIGURATION,
+      call(instance, config_path=GLOBAL_SLAPOS_CONFIGURATION,
            option=['--pidfile /opt/slapos/slapgrid-cp.pid'])
     elif command == 'report':
-      call(report, config=GLOBAL_SLAPOS_CONFIGURATION,
+      call(report, config_path=GLOBAL_SLAPOS_CONFIGURATION,
            option=['--pidfile /opt/slapos/slapgrid-ur.pid'])
     elif command == 'bang':
-      call(bang, config=GLOBAL_SLAPOS_CONFIGURATION)
+      call(bang, config_path=GLOBAL_SLAPOS_CONFIGURATION)
     elif command == 'format':
-      call(format, config=GLOBAL_SLAPOS_CONFIGURATION, option=['-c', '-v'])
+      call(format, config_path=GLOBAL_SLAPOS_CONFIGURATION, option=['-c', '-v'])
     elif command == 'supervisord':
-      call(supervisord, config=GLOBAL_SLAPOS_CONFIGURATION)
+      call(supervisord, config_path=GLOBAL_SLAPOS_CONFIGURATION)
     elif command == 'supervisorctl':
-      call(supervisorctl, config=GLOBAL_SLAPOS_CONFIGURATION)
+      call(supervisorctl, config_path=GLOBAL_SLAPOS_CONFIGURATION)
     elif command in ['start', 'stop', 'restart', 'status', 'tail']:
       # Again, too hackish
       sys.argv[-2:-2] = [command]
-      call(supervisorctl, config=GLOBAL_SLAPOS_CONFIGURATION)
+      call(supervisorctl, config_path=GLOBAL_SLAPOS_CONFIGURATION)
     else:
       return False
   elif command == 'request':
-    call(request, config=USER_SLAPOS_CONFIGURATION)
+    call(request, config_path=USER_SLAPOS_CONFIGURATION)
   elif command == 'supply':
-    call(supply, config=USER_SLAPOS_CONFIGURATION)
+    call(supply, config_path=USER_SLAPOS_CONFIGURATION)
   elif command == 'remove':
-    call(remove, config=USER_SLAPOS_CONFIGURATION)
+    call(remove, config_path=USER_SLAPOS_CONFIGURATION)
   elif command == 'start':
     raise EntryPointNotImplementedError(command)
   elif command == 'stop':
@@ -159,12 +164,13 @@ def dispatch(command, is_node_command):
   elif command == 'destroy':
     raise EntryPointNotImplementedError(command)
   elif command == 'console':
-    call(console, config=USER_SLAPOS_CONFIGURATION)
+    call(console, config_path=USER_SLAPOS_CONFIGURATION)
   elif command == 'cache-lookup':
-    call(cache_lookup, config=GLOBAL_SLAPOS_CONFIGURATION)
+    call(cache_lookup, config_path=GLOBAL_SLAPOS_CONFIGURATION)
   else:
     return False
 
+
 def main():
   """
   Main entry point of SlapOS Node. Used to dispatch commands to python
@@ -206,24 +212,23 @@ Node subcommands usage:
 
   # Parse arguments
   # XXX remove the "positional arguments" from help message
-  parser = argparse.ArgumentParser(usage=usage)
-  parser.add_argument('command')
-  parser.add_argument('argument_list', nargs=argparse.REMAINDER)
+  ap = argparse.ArgumentParser(usage=usage)
+  ap.add_argument('command')
+  ap.add_argument('argument_list', nargs=argparse.REMAINDER)
 
-  namespace = parser.parse_args()
+  args = ap.parse_args()
   # Set sys.argv for the sub-entry point that we will call
-  command_line = [namespace.command]
-  command_line.extend(namespace.argument_list)
+  command_line = [args.command]
+  command_line.extend(args.argument_list)
   sys.argv = command_line
 
   try:
-    if not dispatch(namespace.command, is_node):
-      parser.print_help()
+    if not dispatch(args.command, is_node):
+      ap.print_help()
       sys.exit(1)
   except EntryPointNotImplementedError, exception:
     print ('The command %s does not exist or is not yet implemented. Please '
-        'have a look at http://community.slapos.org to read documentation or '
-        'forum. Please also make sure that SlapOS Node is up to '
-        'date.' % exception)
+           'have a look at http://community.slapos.org to read documentation or '
+           'forum. Please also make sure that SlapOS Node is up to '
+           'date.' % exception)
     sys.exit(1)
-

slapos/cli_legacy/format.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
+# All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+import ConfigParser
+import logging
+import sys
+import os
+
+from slapos.format import FormatConfig, UsageError, tracing_monkeypatch, do_format
+
+
+def main(*args):
+  "Run default configuration."
+
+  ap = argparse.ArgumentParser()
+
+  ap.add_argument('-x', '--computer_xml',
+                  help="Path to file with computer's XML. If does not exists, will be created",
+                  default=None)
+
+  ap.add_argument('--computer_json',
+                  help="Path to a JSON version of the computer's XML (for development only).",
+                  default=None)
+
+  ap.add_argument('-l', '--log_file',
+                  help="The path to the log file used by the script.")
+
+  ap.add_argument('-i', '--input_definition_file',
+                  help="Path to file to read definition of computer instead of "
+                  "declaration. Using definition file allows to disable "
+                  "'discovery' of machine services and allows to define computer "
+                  "configuration in fully controlled manner.")
+
+  ap.add_argument('-o', '--output_definition_file',
+                  help="Path to file to write definition of computer from "
+                  "declaration.")
+
+  ap.add_argument('-n', '--dry_run',
+                  help="Don't actually do anything.",
+                  default=False,
+                  action="store_true")
+
+  ap.add_argument('-v', '--verbose',
+                  default=False,
+                  action="store_true",
+                  help="Verbose output.")
+
+  # the console option is actually ignored and not used anymore.
+  ap.add_argument('-c', '--console',
+                  default=False,
+                  action="store_true",
+                  help="Console output.")
+
+  ap.add_argument('--alter_user',
+                  choices=['True', 'False'],
+                  help="Shall slapformat alter user database [default: True]")
+
+  ap.add_argument('--alter_network',
+                  choices=['True', 'False'],
+                  help="Shall slapformat alter network configuration [default: True]")
+
+  ap.add_argument('--now',
+                  help="Launch slapformat without delay",
+                  default=False,
+                  action="store_true")
+
+  ap.add_argument('configuration_file',
+                  help='path to slapos.cfg')
+
+  if args:
+    args = ap.parse_args(list(args))
+  else:
+    args = ap.parse_args()
+
+  logger = logging.getLogger("slapformat")
+  logger.addHandler(logging.StreamHandler())
+
+  if args.verbose:
+    logger.setLevel(logging.DEBUG)
+    logger.debug("Verbose mode enabled.")
+  else:
+    logger.setLevel(logging.INFO)
+
+  conf = FormatConfig(logger=logger)
+
+  configp = ConfigParser.SafeConfigParser()
+  if configp.read(args.configuration_file) != [args.configuration_file]:
+    raise UsageError('Cannot find or parse configuration file: %s' % args.configuration_file)
+
+  conf.mergeConfig(args, configp)
+
+  if conf.log_file:
+    if not os.path.isdir(os.path.dirname(conf.log_file)):
+      # fallback to console only if directory for logs does not exists and
+      # continue to run
+      raise ValueError('Please create directory %r to store %r log file' % (
+        os.path.dirname(conf.log_file), conf.log_file))
+    else:
+      file_handler = logging.FileHandler(conf.log_file)
+      file_handler.setFormatter(logging.Formatter("%(asctime)s - "
+        "%(name)s - %(levelname)s - %(message)s"))
+      conf.logger.addHandler(file_handler)
+      conf.logger.info('Configured logging to file %r' % conf.log_file)
+
+  try:
+    conf.setConfig()
+  except UsageError as exc:
+    sys.stderr.write(exc.message + '\n')
+    sys.stderr.write("For help use --help\n")
+    sys.exit(1)
+
+  tracing_monkeypatch(conf)
+
+  try:
+    do_format(conf=conf)
+  except:
+    conf.logger.exception('Uncaught exception:')
+    raise

slapos/cli_legacy/proxy_start.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
+
+import argparse
+import ConfigParser
+import logging
+import os
+import sys
+
+from slapos.proxy import ProxyConfig, do_proxy
+
+class UsageError(Exception):
+  pass
+
+
+def main():
+  ap = argparse.ArgumentParser()
+
+  ap.add_argument('-l', '--log_file',
+                  help='The path to the log file used by the script.')
+
+  ap.add_argument('-v', '--verbose',
+                  action='store_true',
+                  help='Verbose output.')
+
+  # XXX not used anymore, deprecated
+  ap.add_argument('-c', '--console',
+                  action='store_true',
+                  help='Console output.')
+
+  ap.add_argument('-u', '--database-uri',
+                  help='URI for sqlite database')
+
+  ap.add_argument('configuration_file',
+                  help='path to slapos.cfg')
+
+  args = ap.parse_args()
+
+  logger = logging.getLogger('slapproxy')
+  logger.addHandler(logging.StreamHandler())
+
+  if args.verbose:
+    logger.setLevel(logging.DEBUG)
+  else:
+    logger.setLevel(logging.INFO)
+
+  conf = ProxyConfig(logger=logger)
+
+  configp = ConfigParser.SafeConfigParser()
+  if configp.read(args.configuration_file) != [args.configuration_file]:
+    raise UsageError('Cannot find or parse configuration file: %s' % args.configuration_file)
+
+  conf.mergeConfig(args, configp)
+
+  if conf.log_file:
+    if not os.path.isdir(os.path.dirname(conf.log_file)):
+      raise ValueError('Please create directory %r to store %r log file' % (
+        os.path.dirname(conf.log_file), conf.log_file))
+    file_handler = logging.FileHandler(conf.log_file)
+    file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+    logger.addHandler(file_handler)
+    logger.info('Configured logging to file %r' % conf.log_file)
+
+  conf.setConfig()
+
+  try:
+    do_proxy(conf=conf)
+    return_code = 0
+  except SystemExit as err:
+    return_code = err
+
+  sys.exit(return_code)

slapos/cli_legacy/register.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
+##############################################################################
+#
+# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+import logging
+import sys
+
+from slapos.register.register import do_register, RegisterConfig
+
+
+def main():
+  ap = argparse.ArgumentParser()
+
+  ap.add_argument('node_name',
+                  help='Name of the node')
+
+  ap.add_argument('--interface-name',
+                  help='Interface name to access internet',
+                  default='eth0')
+
+  ap.add_argument('--master-url',
+                  help='URL of SlapOS master',
+                  default='https://slap.vifib.com')
+
+  ap.add_argument('--master-url-web',
+                  help='URL of SlapOS Master webservice to register certificates',
+                  default='https://www.slapos.org')
+
+  ap.add_argument('--partition-number',
+                  help='Number of partition on computer',
+                  default='10',
+                  type=int)
+
+  ap.add_argument('--ipv4-local-network',
+                  help='Base of ipv4 local network',
+                  default='10.0.0.0/16')
+
+  ap.add_argument('--ipv6-interface',
+                  help='Interface name to get ipv6',
+                  default='')
+
+  ap.add_argument('--login',
+                  help='User login on SlapOS Master webservice')
+
+  ap.add_argument('--password',
+                  help='User password on SlapOs Master webservice')
+
+  ap.add_argument('-t', '--create-tap',
+                  help='Will trigger creation of one virtual "tap" interface per '
+                       'Partition and attach it to primary interface. Requires '
+                       'primary interface to be a bridge. defaults to false. '
+                       'Needed to host virtual machines.',
+                  default=False,
+                  action='store_true')
+
+  ap.add_argument('-n', '--dry-run',
+                  help='Simulate the execution steps',
+                  default=False,
+                  action='store_true')
+
+  args = ap.parse_args()
+
+  if args.password and not args.login:
+    ap.error('Please enter your login with your password')
+
+  logger = logging.getLogger('Register')
+  handler = logging.StreamHandler()
+  logger.setLevel(logging.DEBUG)
+  handler.setLevel(logging.INFO)
+  handler.setFormatter(logging.Formatter('%(levelname)s - %(message)s'))
+  logger.addHandler(handler)
+
+  try:
+    conf = RegisterConfig(logger=logger)
+    conf.setConfig(args)
+    return_code = do_register(conf)
+  except SystemExit as exc:
+    return_code = exc
+
+  sys.exit(return_code)

slapos/cli_legacy/remove.py

+# -*- coding: utf-8 -*-
+
+import argparse
+
+from slapos.client import ClientConfig, init, do_remove
+from slapos.cli_legacy.util import get_config_parser
+
+
+def remove():
+  ap = argparse.ArgumentParser()
+  ap.add_argument('configuration_file',
+                  help='SlapOS configuration file')
+  ap.add_argument('software_url',
+                  help='Your software url')
+  ap.add_argument('node',
+                  help='Target node')
+  args = ap.parse_args()
+
+  configp = get_config_parser(args.configuration_file)
+  conf = ClientConfig(args, configp)
+  local = init(conf)
+  do_remove(args.software_url, args.node, local)

slapos/cli_legacy/request.py

+# -*- coding: utf-8 -*-
+
+import argparse
+
+from slapos.client import ClientConfig, init, do_request
+from slapos.cli_legacy.util import get_config_parser
+
+
+def argToDict(element):
+  """
+  convert a table of string 'key=value' to dict
+  """
+  if element is not None:
+    element_dict = dict([arg.split('=') for arg in element])
+  return element_dict
+
+
+def request():
+  """Run when invoking slapos request. Request an instance."""
+  # Parse arguments and inititate needed parameters
+  # XXX-Cedric: move argument parsing to main entry point
+
+  ap = argparse.ArgumentParser()
+  ap.add_argument('configuration_file',
+                  help='SlapOS configuration file.')
+  ap.add_argument('reference',
+                  help='Your instance reference')
+  ap.add_argument('software_url',
+                  help='Your software url')
+  ap.add_argument('--node',
+                  nargs='*',
+                  help='Node request option '
+                  "'option1=value1 option2=value2'")
+  ap.add_argument('--type',
+                  type=str,
+                  help='Define software type to be requested')
+  ap.add_argument('--slave',
+                  action='store_true', default=False,
+                  help='Ask for a slave instance')
+  ap.add_argument('--configuration',
+                  nargs='*',
+                  help='Give your configuration '
+                  "'option1=value1 option2=value2'")
+
+  args = ap.parse_args()
+
+  if args.configuration:
+    args.parameters = argToDict(args.configuration)
+
+  if args.node:
+    args.node = argToDict(args.node)
+
+  configp = get_config_parser(args.configuration_file)
+  conf = ClientConfig(args, configp)
+  local = init(conf)
+  do_request(conf, local)

slapos/cli_legacy/slapgrid.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
+
+import argparse
+import ConfigParser
+import logging
+import sys
+
+from slapos.grid.utils import setRunning, setFinished
+from slapos.grid.slapgrid import (merged_options, check_missing_parameters,
+                                  check_missing_files, random_delay, create_slapgrid_object)
+
+
+def parse_arguments(*argument_tuple):
+  """Parse arguments and return options dictionary merged with the config file."""
+
+  ap = argparse.ArgumentParser()
+  ap.add_argument('--instance-root',
+                  help='The instance root directory location.')
+  ap.add_argument('--software-root',
+                  help='The software_root directory location.')
+  ap.add_argument('--master-url',
+                  help='The master server URL. Mandatory.')
+  ap.add_argument('--computer-id',
+                  help='The computer id defined in the server.')
+  ap.add_argument('--supervisord-socket',
+                  help='The socket supervisor will use.')
+  ap.add_argument('--supervisord-configuration-path',
+                  help='The location where supervisord configuration will be stored.')
+  ap.add_argument('--buildout', default=None,
+                  help='Location of buildout binary.')
+  ap.add_argument('--pidfile',
+                  help='The location where pidfile will be created.')
+  ap.add_argument('--logfile',
+                  help='The location where slapgrid logfile will be created.')
+  ap.add_argument('--key_file',
+                  help='SSL Authorisation key file.')
+  ap.add_argument('--cert_file',
+                  help='SSL Authorisation certificate file.')
+  ap.add_argument('--signature_private_key_file',
+                  help='Signature private key file.')
+  ap.add_argument('--master_ca_file',
+                  help='Root certificate of SlapOS master key.')
+  ap.add_argument('--certificate_repository_path',
+                  help='Path to directory where downloaded certificates would be stored.')
+  ap.add_argument('-v', '--verbose', action='store_true',
+                  help='Be verbose.')
+  ap.add_argument('--maximum-periodicity', type=int, default=None,
+                  help='Periodicity at which buildout should be run in instance.')
+  ap.add_argument('--promise-timeout', type=int, default=3,
+                  help='Promise timeout in seconds.')
+  ap.add_argument('--now', action='store_true',
+                  help='Launch slapgrid without delay. Default behavior.')
+  ap.add_argument('--all', action='store_true',
+                  help='Launch slapgrid to process all Software Releases '
+                       'and/or Computer Partitions.')
+  ap.add_argument('--only-sr',
+                  help='Force the update of a single software release (use url hash), '
+                       'even if is already installed. This option will make all other '
+                       'sofware releases be ignored.')
+  ap.add_argument('--only-cp',
+                  help='Update a single or a list of computer partitions '
+                       '(ie.:slappartX, slappartY), '
+                       'this option will make all other computer partitions be ignored.')
+
+  ap.add_argument('configuration_file', type=argparse.FileType(),
+                  help='SlapOS configuration file.')
+
+  # Deprecated options
+  ap.add_argument('-c', '--console', action='store_true',
+                  help="Deprecated, doesn't do anything.")
+  ap.add_argument('--develop', action='store_true',
+                  help='Deprecated, same as --all.')
+  ap.add_argument('--only_sr',
+                  help='Deprecated, same as --only-sr.')
+  ap.add_argument('--only_cp',
+                  help='Deprecated, same as --only-cp.')
+  ap.add_argument('--maximal_delay',
+                  help='Deprecated. Will only work from configuration file in the future.')
+
+  if not argument_tuple:
+    args = ap.parse_args()
+  else:
+    args = ap.parse_args(list(argument_tuple))
+
+  return args
+
+
+def setup_logger(options):
+  logger = logging.getLogger(__name__)
+
+  if options.get('logfile'):
+    handler = logging.FileHandler(options['logfile'])
+  else:
+    handler = logging.StreamHandler()
+
+  if options['verbose']:
+    handler.setLevel(logging.DEBUG)
+  else:
+    handler.setLevel(logging.INFO)
+
+  formatter = logging.Formatter(fmt='%(asctime)s %(name)-18s: '
+                                '%(levelname)-8s %(message)s',
+                                datefmt='%Y-%m-%dT%H:%M:%S')
+
+  handler.setFormatter(formatter)
+  logger.addHandler(handler)
+
+  return logger
+
+
+def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
+  """Returns a new instance of slapgrid.Slapgrid created with argument+config parameters.
+     Also returns the pidfile path, and configures logger.
+  """
+  args = parse_arguments(*argument_tuple)
+
+  configp = ConfigParser.SafeConfigParser()
+  configp.readfp(args.configuration_file)
+
+  options = merged_options(args, configp)
+
+  logger = setup_logger(options)
+
+  check_missing_parameters(options)
+  check_missing_files(options)
+
+  random_delay(options, logger=logger)
+
+  slapgrid_object = create_slapgrid_object(options, logger=logger)
+
+  return slapgrid_object, options.get('pidfile')
+
+
+def realRun(argument_tuple, method):
+  slapgrid_object, pidfile = parseArgumentTupleAndReturnSlapgridObject(*argument_tuple)
+  if pidfile:
+    setRunning(logger=slapgrid_object.logger, pidfile=pidfile)
+  try:
+    return getattr(slapgrid_object, method)()
+  finally:
+    if pidfile:
+      setFinished(pidfile)
+
+
+def runSoftwareRelease(*argument_tuple):
+  """Hook for entry point to process Software Releases"""
+  sys.exit(realRun(argument_tuple, 'processSoftwareReleaseList'))
+
+
+def runComputerPartition(*argument_tuple):
+  """Hook for entry point to process Computer Partitions"""
+  sys.exit(realRun(argument_tuple, 'processComputerPartitionList'))
+
+
+def runUsageReport(*argument_tuple):
+  """Hook for entry point to process Usage Reports"""
+  sys.exit(realRun(argument_tuple, 'agregateAndSendUsage'))

slapos/cli_legacy/supply.py

+# -*- coding: utf-8 -*-
+
+import argparse
+
+from slapos.client import ClientConfig, init, do_supply
+from slapos.cli_legacy.util import get_config_parser
+
+
+def supply():
+  """
+  Run when invoking slapos supply. Mostly argument parsing.
+  """
+  ap = argparse.ArgumentParser()
+  ap.add_argument('configuration_file',
+                  help='SlapOS configuration file')
+  ap.add_argument('software_url',
+                  help='Your software url')
+  ap.add_argument('node',
+                  help='Target node')
+  args = ap.parse_args()
+
+  configp = get_config_parser(args.configuration_file)
+  conf = ClientConfig(args, configp)
+  local = init(conf)
+  do_supply(args.software_url, args.node, local)

slapos/cli_legacy/svcbackend.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
+##############################################################################
+#
+# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
+# All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import logging
+import os
+from optparse import OptionParser
+import ConfigParser
+
+from slapos.grid.svcbackend import launchSupervisord
+
+
+def getOptionDict(*argument_tuple):
+  usage = """
+Typical usage:
+ * %prog CONFIGURATION_FILE [arguments passed to supervisor]
+
+""".strip()
+
+  parser = OptionParser(usage=usage)
+
+  # Parses arguments
+  if argument_tuple:
+    (argument_option_instance, argument_list) = parser.parse_args(list(argument_tuple))
+  else:
+    # No arguments given to entry point : we parse sys.argv.
+    (argument_option_instance, argument_list) = parser.parse_args()
+
+  if not argument_list:
+    parser.error("Configuration file is obligatory. Consult documentation by calling with -h.")
+  configuration_file = argument_list[0]
+  if not os.path.exists(configuration_file):
+    parser.error("Could not read configuration file : %s" % configuration_file)
+
+  slapgrid_configuration = ConfigParser.SafeConfigParser()
+  slapgrid_configuration.read(configuration_file)
+
+  # Merges the two dictionnaries
+  option_dict = dict(slapgrid_configuration.items("slapos"))
+  # Supervisord configuration location
+  option_dict.setdefault('supervisord_configuration_path',
+                         os.path.join(option_dict['instance_root'], 'etc', 'supervisord.conf'))
+  # Supervisord socket
+  option_dict.setdefault('supervisord_socket',
+                         os.path.join(option_dict['instance_root'], 'supervisord.socket'))
+  return option_dict, argument_list[1:]
+
+
+def supervisorctl(*argument_tuple):
+  logger = logging.getLogger('SVCBackend')
+  logger.setLevel(logging.INFO)
+  handler = logging.StreamHandler()
+  logger.addHandler(handler)
+
+  option_dict, args = getOptionDict(*argument_tuple)
+  import supervisor.supervisorctl
+  launchSupervisord(socket=option_dict['supervisord_socket'],
+                    configuration_file=option_dict['supervisord_configuration_path'],
+                    logger=logger)
+  supervisor.supervisorctl.main(args=['-c', option_dict['supervisord_configuration_path']] + args)
+
+
+def supervisord(*argument_tuple):
+  logger = logging.getLogger('SVCBackend')
+  logger.setLevel(logging.INFO)
+  handler = logging.StreamHandler()
+  logger.addHandler(handler)
+
+  option_dict, _ = getOptionDict(*argument_tuple)
+  launchSupervisord(socket=option_dict['supervisord_socket'],
+                    configuration_file=option_dict['supervisord_configuration_path'],
+                    logger=logger)

slapos/cli_legacy/util.py

+# -*- coding: utf-8 -*-
+
+import ConfigParser
+import os
+
+
+def get_config_parser(path):
+    configp = ConfigParser.SafeConfigParser()
+    path = os.path.expanduser(path)
+    if not os.path.isfile(path):
+        raise OSError('Specified configuration file %s does not exist. Exiting.' % path)
+    configp.read(path)
+    return configp

slapos/grid/svcbackend.py

@@ -28,18 +28,16 @@
 #
 ##############################################################################
 
-from supervisor import xmlrpc
 import time
-from utils import SlapPopen
-import logging
 import os
 import sys
 import xmlrpclib
-from optparse import OptionParser
-import ConfigParser
 import socket as socketlib
 import subprocess
 
+from supervisor import xmlrpc
+from slapos.grid.utils import SlapPopen
+
 
 def getSupervisorRPC(socket):
   supervisor_transport = xmlrpc.SupervisorTransport('', '',
@@ -49,20 +47,19 @@ def getSupervisorRPC(socket):
   return getattr(server_proxy, 'supervisor')
 
 
-def launchSupervisord(socket, configuration_file):
-  logger = logging.getLogger('SVCBackend')
-  supervisor = getSupervisorRPC(socket)
+def launchSupervisord(socket, configuration_file, logger):
   if os.path.exists(socket):
     trynum = 1
     while trynum < 6:
       try:
+        supervisor = getSupervisorRPC(socket)
         status = supervisor.getState()
       except xmlrpclib.Fault as e:
         if e.faultCode == 6 and e.faultString == 'SHUTDOWN_STATE':
           logger.info('Supervisor in shutdown procedure, will check again later.')
           trynum += 1
           time.sleep(2 * trynum)
-      except Exception:
+      except Exception as e:
         # In case if there is problem with connection, assume that supervisord
         # is not running and try to run it
         break
@@ -90,85 +87,34 @@ def launchSupervisord(socket, configuration_file):
                                 env={},
                                 executable=sys.executable,
                                 stdout=subprocess.PIPE,
-                                stderr=subprocess.STDOUT)
-  result = supervisord_popen.communicate()[0]
-  if supervisord_popen.returncode == 0:
-    logger.info('Supervisord command invoked with: %s' % result)
-    try:
-      default_timeout = socketlib.getdefaulttimeout()
-      current_timeout = 1
-      trynum = 1
-      while trynum < 6:
-        try:
-          socketlib.setdefaulttimeout(current_timeout)
-          status = supervisor.getState()
-          if status['statename'] == 'RUNNING' and status['statecode'] == 1:
-            return
-          logger.warning('Wrong status name %(statename)r and code '
-            '%(statecode)r, trying again' % status)
-          trynum += 1
-        except Exception:
-          current_timeout = 5 * trynum
-          trynum += 1
-        else:
-          logger.info('Supervisord started correctly in try %s.' % trynum)
-          return
-      logger.warning('Issue while checking supervisord.')
-    finally:
-      socketlib.setdefaulttimeout(default_timeout)
-
-  else:
-    log_message = 'Supervisord unknown problem: %s' % result
-    logger.warning(log_message)
-
-
-def getOptionDict(*argument_tuple):
-  usage = """
-Typical usage:
- * %prog CONFIGURATION_FILE [arguments passed to supervisor]
-
-""".strip()
-
-  parser = OptionParser(usage=usage)
-
-  # Parses arguments
-  if argument_tuple:
-    (argument_option_instance, argument_list) = parser.parse_args(list(argument_tuple))
-  else:
-    # No arguments given to entry point : we parse sys.argv.
-    (argument_option_instance, argument_list) = parser.parse_args()
+                                stderr=subprocess.STDOUT,
+                                logger=logger)
 
-  if not argument_list:
-    parser.error("Configuration file is obligatory. Consult documentation by "
-        "calling with -h.")
-  configuration_file = argument_list[0]
-  if not os.path.exists(configuration_file):
-    parser.error("Could not read configuration file : %s" % configuration_file)
-
-  slapgrid_configuration = ConfigParser.SafeConfigParser()
-  slapgrid_configuration.read(configuration_file)
-
-  # Merges the two dictionnaries
-  option_dict = dict(slapgrid_configuration.items("slapos"))
-  # Supervisord configuration location
-  option_dict.setdefault('supervisord_configuration_path',
-                         os.path.join(option_dict['instance_root'], 'etc', 'supervisord.conf'))
-  # Supervisord socket
-  option_dict.setdefault('supervisord_socket',
-                         os.path.join(option_dict['instance_root'], 'supervisord.socket'))
-  return option_dict, argument_list[1:]
-
-
-def supervisorctl(*argument_tuple):
-  option_dict, args = getOptionDict(*argument_tuple)
-  import supervisor.supervisorctl
-  launchSupervisord(option_dict['supervisord_socket'],
-      option_dict['supervisord_configuration_path'])
-  supervisor.supervisorctl.main(args=['-c',
-    option_dict['supervisord_configuration_path']] + args)
-
-def supervisord(*argument_tuple):
-  option_dict, _ = getOptionDict(*argument_tuple)
-  launchSupervisord(option_dict['supervisord_socket'],
-      option_dict['supervisord_configuration_path'])
+  result = supervisord_popen.communicate()[0]
+  if supervisord_popen.returncode:
+    logger.warning('Supervisord unknown problem: %s' % result)
+    return
 
+  try:
+    default_timeout = socketlib.getdefaulttimeout()
+    current_timeout = 1
+    trynum = 1
+    while trynum < 6:
+      try:
+        socketlib.setdefaulttimeout(current_timeout)
+        supervisor = getSupervisorRPC(socket)
+        status = supervisor.getState()
+        if status['statename'] == 'RUNNING' and status['statecode'] == 1:
+          return
+        logger.warning('Wrong status name %(statename)r and code '
+          '%(statecode)r, trying again' % status)
+        trynum += 1
+      except Exception as e:
+        current_timeout = 5 * trynum
+        trynum += 1
+      else:
+        logger.info('Supervisord started correctly in try %s.' % trynum)
+        return
+    logger.warning('Issue while checking supervisord.')
+  finally:
+    socketlib.setdefaulttimeout(default_timeout)

slapos/slap/slap.py

@@ -34,8 +34,6 @@ __all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease",
            "Supply", "OpenOrder", "NotFoundError", "Unauthorized",
            "ResourceNotReady", "ServerError"]
 
-from interface import slap as interface
-from xml_marshaller import xml_marshaller
 import httplib
 import logging
 import socket
@@ -43,9 +41,16 @@ import ssl
 import traceback
 import urllib
 import urlparse
+
 import zope.interface
+from interface import slap as interface
+from xml_marshaller import xml_marshaller
+
+fallback_logger = logging.getLogger(__name__)
+fallback_handler = logging.StreamHandler()
+fallback_logger.setLevel(logging.INFO)
+fallback_logger.addHandler(fallback_handler)
 
-log = logging.getLogger(__name__)
 
 DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
@@ -144,7 +149,7 @@ class SoftwareRelease(SlapDocument):
     else:
       return self._software_release
 
-  def error(self, error_log):
+  def error(self, error_log, logger=None):
     try:
       # Does not follow interface
       self._connection_helper.POST('/softwareReleaseError', {
@@ -152,8 +157,7 @@ class SoftwareRelease(SlapDocument):
         'computer_id' : self.getComputerId(),
         'error_log': error_log})
     except Exception:
-      exception = traceback.format_exc()
-      log.error(exception)
+      (logger or fallback_logger).error(traceback.format_exc())
 
   def available(self):
     self._connection_helper.POST('/availableSoftwareRelease', {
@@ -417,15 +421,14 @@ class ComputerPartition(SlapRequester):
       'computer_partition_id': self.getId(),
       })
 
-  def error(self, error_log):
+  def error(self, error_log, logger=None):
     try:
       self._connection_helper.POST('/softwareInstanceError', {
         'computer_id': self._computer_id,
         'computer_partition_id': self.getId(),
         'error_log': error_log})
     except Exception:
-      exception = traceback.format_exc()
-      log.error(exception)
+      (logger or fallback_logger).error(traceback.format_exc())
 
   def bang(self, message):
     self._connection_helper.POST('/softwareInstanceBang', {
@@ -660,7 +663,7 @@ class slap:
     returns SoftwareRelease class object
     """
     return SoftwareRelease(software_release=software_release,
-      connection_helper=self._connection_helper,
+      connection_helper=self._connection_helper
     )
 
   def registerComputer(self, computer_guid):

slapos/tests/slap.py

@@ -24,12 +24,15 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import xml_marshaller
+
+import httplib
+import os
 import unittest
 import urlparse
-import httplib
+
 import slapos.slap
-import os
+import xml_marshaller
+
 
 class UndefinedYetException(Exception):
   """To catch exceptions which are not yet defined"""

slapos/tests/slapproxy.py

@@ -27,6 +27,7 @@
 #
 ##############################################################################
 
+import ConfigParser
 import os
 import logging
 import shutil
@@ -86,14 +87,16 @@ database_uri = %(tempdir)s/lib/proxy.db
     """
     Set config for slapproxy and start it
     """
-    config = slapos.proxy.Config()
-    config.setConfig(*(ProxyOption(self.proxy_db),
-                       self.slapos_cfg))
+    conf = slapos.proxy.ProxyConfig(logger=logging.getLogger())
+    configp = ConfigParser.SafeConfigParser()
+    configp.read(self.slapos_cfg)
+    conf.mergeConfig(ProxyOption(self.proxy_db), configp)
+    conf.setConfig()
     views.app.config['TESTING'] = True
     views.app.config['computer_id'] = self.computer_id
     views.app.config['DATABASE_URI'] = self.proxy_db
-    views.app.config['HOST'] = config.host
-    views.app.config['port'] = config.port
+    views.app.config['HOST'] = conf.host
+    views.app.config['port'] = conf.port
     self.app = views.app.test_client()
 
   def add_free_partition (self, partition_amount):