experimental sqlalchemy model in replacement of sql schema

setup.py

@@ -46,6 +46,7 @@ setup(name=name,
           'zc.buildout',
           'cliff',
           'requests',
+          'sqlalchemy',
         ] + additional_install_requires,
       extra_requires={'docs': (
         'Sphinx',

slapos/cli/proxy_show.py

@@ -6,12 +6,75 @@ import logging
 
 import lxml.etree
 import prettytable
-import sqlite3
 
 from slapos.cli.config import ConfigCommand
 from slapos.proxy import ProxyConfig
 from slapos.proxy.db_version import DB_VERSION
 
+import sqlalchemy
+from sqlalchemy import Column, String, ForeignKey
+import sqlalchemy.orm
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+
+
+class Partition(Base):
+    __tablename__ = 'partition' + DB_VERSION
+
+    reference = Column(String(255), primary_key=True)
+    slap_state = Column(String(255), default='free')
+    software_release = Column(String(255))
+    xml = Column(String)
+    connection_xml = Column(String)
+    slave_instance_list = Column(String)
+    software_type = Column(String(255))
+    partition_reference = Column(String(255))
+    requested_by = Column(String(255))  # only used for debugging,
+                                        # slapproxy does not support proper scope
+    requested_state = Column(String(255), nullable=False, default='started')
+
+
+class Computer(Base):
+    __tablename__ = 'computer' + DB_VERSION
+
+    address = Column(String(255), primary_key=True)
+    netmask = Column(String(255), primary_key=True)
+
+
+class Software(Base):
+    __tablename__ = 'software' + DB_VERSION
+
+    url = Column(String(255), primary_key=True)
+
+    @property
+    def md5(self):
+        return hashlib.md5(self.url).hexdigest()
+
+
+class Slave(Base):
+    __tablename__ = 'slave' + DB_VERSION
+
+    reference = Column(String(255), primary_key=True)
+    connection_xml = Column(String)
+    hosted_by = Column(String(255))
+    asked_by = Column(String(255))  # only used for debugging,
+                                    # slapproxy does not support proper scope
+
+
+class PartitionNetwork(Base):
+    __tablename__ = 'partition_network' + DB_VERSION
+
+    partition_reference = Column(String(255),
+                                 ForeignKey(Partition.reference),
+                                 primary_key=True)
+    reference = Column(String(255))
+    address = Column(String(255), primary_key=True)
+    netmask = Column(String(255))
+
+    partition = sqlalchemy.orm.relationship(Partition)
+
 
 class ProxyShowCommand(ConfigCommand):
     """
@@ -60,9 +123,6 @@ class ProxyShowCommand(ConfigCommand):
         do_show(conf=conf)
 
 
-tbl_partition = 'partition' + DB_VERSION
-
-
 def coalesce(*seq):
     el = None
     for el in seq:
@@ -71,94 +131,85 @@ def coalesce(*seq):
     return el
 
 
-def log_table(logger, qry, tablename, skip=None):
-    if skip is None:
-        skip = set()
+def log_params(logger, session):
+    for partition in session.query(Partition):
+        if not partition.connection_xml:
+            continue
+
+        xml = str(partition.connection_xml)
+        logger.info('%s: %s (type %s)', partition.reference, partition.partition_reference, partition.software_type)
+        instance = lxml.etree.fromstring(xml)
+        for parameter in list(instance):
+            name = parameter.get('id')
+            text = parameter.text
+            if text and name in ('ssh-key', 'ssh-public-key'):
+                text = text[:20] + '...' + text[-20:]
+            logger.info('    %s = %s', name, text)
 
-    columns = [c[0] for c in qry.description if c[0] not in skip]
 
-    rows = []
-    for row in qry.fetchall():
-        rows.append([coalesce(row[col], '-') for col in columns])
+def log_table(logger, objects, tablename, columns=None):
+    if columns is None:
+        columns = []
 
     pt = prettytable.PrettyTable(columns)
     # https://code.google.com/p/prettytable/wiki/Tutorial
 
-    for row in rows:
-        pt.add_row(row)
+    rows = [
+        [coalesce(getattr(obj, col), '-') for col in columns]
+        for obj in objects
+    ]
 
     if rows:
-        if skip:
-            logger.info('table %s: skipping %s', tablename, ', '.join(skip))
-        else:
-            logger.info('table %s', tablename)
+        logger.info('table %s', tablename)
     else:
         logger.info('table %s: empty', tablename)
         return
 
+    for row in rows:
+        pt.add_row(row)
+
     for line in pt.get_string(border=True, padding_width=0, vrules=prettytable.NONE).split('\n'):
         logger.info(line)
 
 
-def log_params(logger, conn):
-    cur = conn.cursor()
+def log_computer_table(logger, session):
+    computers = session.query(Computer)
+    log_table(logger, computers, Computer.__tablename__,
+              columns=['address', 'netmask'])
 
-    qry = cur.execute("SELECT reference, partition_reference, software_type, connection_xml FROM %s" % tbl_partition)
-    for row in qry.fetchall():
-        if not row['connection_xml']:
-            continue
 
-        xml = str(row['connection_xml'])
-        logger.info('%s: %s (type %s)', row['reference'], row['partition_reference'], row['software_type'])
-        instance = lxml.etree.fromstring(xml)
-        for parameter in list(instance):
-            name = parameter.get('id')
-            text = parameter.text
-            if text and name in ('ssh-key', 'ssh-public-key'):
-                text = text[:20] + '...' + text[-20:]
-            logger.info('    %s = %s', name, text)
-
-
-def log_computer_table(logger, conn):
-    tbl_computer = 'computer' + DB_VERSION
-    cur = conn.cursor()
-    qry = cur.execute("SELECT * FROM %s" % tbl_computer)
-    log_table(logger, qry, tbl_computer)
+def log_software_table(logger, session):
+    software_objs = session.query(Software)
+    log_table(logger, software_objs, Software.__tablename__,
+              columns=['url', 'md5'])
 
 
-def log_software_table(logger, conn):
-    tbl_software = 'software' + DB_VERSION
-    cur = conn.cursor()
-    qry = cur.execute("SELECT *, md5(url) as md5 FROM %s" % tbl_software)
-    log_table(logger, qry, tbl_software)
+def log_partition_table(logger, session):
+    partitions = session.query(Partition).filter(Partition.slap_state != 'free')
+    log_table(logger, partitions, Partition.__tablename__,
+              columns=[
+                  'reference', 'slap_state', 'software_release',
+                  'software_type', 'partition_reference', 'requested_by',
+                  'requested_state'
+              ])
 
 
-def log_partition_table(logger, conn):
-    cur = conn.cursor()
-    qry = cur.execute("SELECT * FROM %s WHERE slap_state<>'free'" % tbl_partition)
-    log_table(logger, qry, tbl_partition, skip=['xml', 'connection_xml', 'slave_instance_list'])
+def log_slave_table(logger, session):
+    slaves = session.query(Slave)
+    log_table(logger, slaves, Slave.__tablename__,
+              columns=['reference', 'hosted_by', 'asked_by'])
 
 
-def log_slave_table(logger, conn):
-    tbl_slave = 'slave' + DB_VERSION
-    cur = conn.cursor()
-    qry = cur.execute("SELECT * FROM %s" % tbl_slave)
-    log_table(logger, qry, tbl_slave, skip=['connection_xml'])
-
+def log_network(logger, session):
+    partition_networks = [
+        pn for pn in session.query(PartitionNetwork)
+        if pn.partition.slap_state != 'free'
+    ]
 
-def log_network(logger, conn):
-    tbl_partition_network = 'partition_network' + DB_VERSION
-    cur = conn.cursor()
     addr = collections.defaultdict(list)
-    qry = cur.execute("""
-                      SELECT * FROM %s
-                       WHERE partition_reference NOT IN (
-                                                SELECT reference
-                                                  FROM %s
-                                                 WHERE slap_state='free')
-                        """ % (tbl_partition_network, tbl_partition))
-    for row in qry:
-        addr[row['partition_reference']].append(row['address'])
+
+    for pn in partition_networks:
+        addr[pn.partition_reference].append(pn.address)
 
     for partition_reference in sorted(addr.keys()):
         addresses = addr[partition_reference]
@@ -167,10 +218,10 @@ def log_network(logger, conn):
 
 def do_show(conf):
     conf.logger.debug('Using database: %s', conf.database_uri)
-    conn = sqlite3.connect(conf.database_uri)
-    conn.row_factory = sqlite3.Row
 
-    conn.create_function('md5', 1, lambda s: hashlib.md5(s).hexdigest())
+    engine = sqlalchemy.create_engine('sqlite:///%s' % conf.database_uri)
+    Session = sqlalchemy.orm.sessionmaker(bind=engine)
+    session = Session()
 
     call_table = [
         (conf.computers, log_computer_table),
@@ -187,6 +238,6 @@ def do_show(conf):
         to_call = [func for flag, func in call_table if flag]
 
     for idx, func in enumerate(to_call):
-        func(conf.logger, conn)
+        func(conf.logger, session)
         if idx < len(to_call) - 1:
             conf.logger.info(' ')