Merge branch 'kvm-cluster'

component/firewalld/buildout.cfg

@@ -81,6 +81,7 @@ environment =
   GLIB_LIBS=-L${glib:location}/lib -lglib-2.0 -lintl -lgobject-2.0
   FFI_CFLAGS=-I${libffi:location}/include
   FFI_LIBS=-L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
+  GIR_DIR=${buildout:parts-directory}/${:_buildout_section_name_}/share/gir-1.0
 
 [pygobject3]
 recipe = slapos.recipe.cmmi
@@ -144,4 +145,3 @@ extra-paths =
   ${dbus-python:python-egg}
   ${firewalld:python-egg}
   ${pygobject3:python-egg}
-

component/qemu-kvm/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-2.2.1.tar.bz2
-md5sum = 833ff4457062456d38d6567f802ffef4
+url = http://wiki.qemu-project.org/download/qemu-2.4.0.tar.bz2
+md5sum = 186ee8194140a484a455f8e3c74589f4
 configure-options =
   --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
   --enable-system
@@ -29,9 +29,9 @@ configure-options =
   --disable-curl
   --enable-kvm
   --disable-docs
+  --enable-vnc
   --enable-vnc-png
   --disable-vnc-jpeg
-  --enable-vnc-ws
   --extra-cflags="-I${gnutls:location}/include -I${libuuid:location}/include -I${zlib:location}/include -I${libpng:location}/include"
   --extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${gpg-error:location}/lib -L${gpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls"
   --disable-werror

slapos/recipe/kvm/template/kvm_run.in

@@ -49,6 +49,11 @@ httpd_port = %(httpd-port)s
 netcat_bin = '%(netcat-binary)s'.strip()
 cluster_doc_host = '%(cluster-doc-host)s'
 cluster_doc_port = %(cluster-doc-port)s
+language = '%(language)s'
+language_list = ['ar', 'da', 'de', 'de-ch', 'en-gb', 'en-us', 'es', 'et', 'fi',
+    'fo', 'fr', 'fr-be', 'fr-ca', 'fr-ch', 'hr', 'hu', 'is', 'it', 'ja', 'lt',
+    'lv', 'mk', 'nl', 'nl-be', 'no', 'pl', 'pt', 'pt-br', 'ru', 'sl', 'sv',
+    'th', 'tr']
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -246,6 +251,8 @@ if tap_network_parameter == [] and nat_network_parameter == []:
   print 'Warning : No network interface defined.'
 else:
   kvm_argument_list += nat_network_parameter + tap_network_parameter
+if language in language_list:
+  kvm_argument_list.extend(['-k', language])
 
 for disk in additional_disk_list:
   kvm_argument_list.extend([

slapos/recipe/simplehttpserver/simplehttpserver.py

@@ -65,7 +65,7 @@ class ServerHandler(SimpleHTTPRequestHandler):
     logging.info('Writing recieved content to file %s' % file_path)
     try:
       with open(file_path, method) as myfile:
-        myfile.write(content.decode('utf-8'))
+        myfile.write(content)
         logging.info('Done.')
     except IOError as e:
       logging.error('Something happened while processing \'writeFile\'. The message is %s' %

software/kvm/common.cfg

@@ -98,7 +98,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = 52e115fe1aaf232994af9037bf7c292c
+md5sum = 90309e9a1efe6d6b981a27f503c86277
 download-only = true
 on-update = true
 
@@ -106,7 +106,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 5700de1cadde0b5bede78f08e215f47a
+md5sum = 0e06ef2879454eee140fe00e167a01d5
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster-input-schema.json

@@ -122,6 +122,25 @@
       },
       "type": "object"
     },
+    "fw-restricted-access": {
+      "title": "Restrict all access to VM with firewall.",
+      "description": "When Firewall is enabled, this parameter define if only vm of this cluster and authorized sources ip should have access to cluster.",
+      "type": "string",
+      "enum": ["on", "off"],
+      "default": "off"
+    },
+    "fw-authorized-sources": {
+        "title": "List of IP/Network address authorized to cluster, if 'Restrict all access' is on.",
+        "description": "When Firewall is enabled, this contain the list of IP address to authorize for access to all VM of this cluster.",
+        "type": "array",
+        "optional": true
+    },
+    "fw-reject-sources": {
+        "title": "List of IP/Network address rejected, if 'Restrict all access' is off.",
+        "description": "When Firewall is enabled, this contain the list of IP address which should not access to all VM of this cluster.",
+        "type": "array",
+        "optional": true
+    },
     "authorized-keys": {
       "title": "Public keys for virtual machines.",
       "description": "Set the list of public keys to add in your virtual machine. The public key file will be available in the VM via url http://10.0.2.100/authorized_keys if you keep the NAT interface enabled",
@@ -198,6 +217,12 @@
               "description": "Simulate a multi node NUMA system. If mem and cpus are omitted, resources are split equally. Each numa option are separated by space: node,nodeid=4,cpus=40-49,mem=64g node,nodeid=1,cpus=10-19,mem=128g. Set this option if you know what you're doing.",
               "type": "string"
             },
+            "keyboard-layout-language": {
+              "title": "Use keyboard layout language",
+              "description": "Use keyboard layout language (for example fr for French). Can be usefull with VNC display",
+              "type": "string",
+              "enum": ["ar", "da", "de", "de-ch", "en-gb", "en-us", "es", "et", "fi", "fo", "fr", "fr-be", "fr-ca", "fr-ch", "hr", "hu", "is", "it", "ja", "lt", "lv", "mk", "nl", "nl-be", "no", "pl", "pt", "pt-br", "ru", "sl", "sv", "th", "tr"]
+            },
             "nbd-host": {
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -74,6 +74,13 @@ config-enable-monitor = True
 config-document-host = ${apache-conf:ip}
 config-document-port = ${apache-conf:port}
 config-document-path = ${hash-code:passwd}
+config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', '')) }}
+
+{% set authorized_source_list = slapparameter_dict.get('fw-authorized-sources', []) -%}
+{% set rejected_source_list = slapparameter_dict.get('fw-reject-sources', []) -%}
+sla-fw_authorized_sources = {{ authorized_source_list | join(' ') }}
+sla-fw_rejected_sources = {{ rejected_source_list | join(' ') }}
+sla-fw_restricted_access = {{ dumps(slapparameter_dict.get('fw-restricted-access', 'off')) }}
 
 return = 
   backend-url

software/kvm/instance-kvm-input-schema.json

@@ -44,6 +44,12 @@
       "description": "Simulate a multi node NUMA system. If mem and cpus are omitted, resources are split equally. Each numa option are separated by space: node,nodeid=4,cpus=40-49,mem=64g node,nodeid=1,cpus=10-19,mem=128g. Set this option if you know what you're doing.",
       "type": "string"
     },
+    "keyboard-layout-language": {
+      "title": "Use keyboard layout language",
+      "description": "Use keyboard layout language (for example fr for French). Can be usefull with VNC display",
+      "type": "string",
+      "enum": ["ar", "da", "de", "de-ch", "en-gb", "en-us", "es", "et", "fi", "fo", "fr", "fr-be", "fr-ca", "fr-ch", "hr", "hu", "is", "it", "ja", "lt", "lv", "mk", "nl", "nl-be", "no", "pl", "pt", "pt-br", "ru", "sl", "sv", "th", "tr"]
+    },
 
     "nbd-host": {
       "title": "NBD hostname",

software/kvm/instance-kvm.cfg.jinja2

@@ -186,6 +186,7 @@ cluster-doc-host =
 cluster-doc-port = 0
 {% endif -%}
 netcat-binary = {{ netcat_bin }}
+language = ${slap-parameter:keyboard-layout-language}
 
 [kvm-vnc-promise]
 recipe = slapos.cookbook:check_port_listening
@@ -567,3 +568,6 @@ authorized-key =
 
 # send some content which will be accessible to the vm through static url: http://10.0.2.100/data
 data-to-vm =
+
+# Change keyboard layout language
+keyboard-layout-language = 

software/kvm/monitor/ansibleReport.py

+#!/usr/bin/env python
+
+# Parse Ansible result log file and and generate a report
+
+import os, time
+import json
+import sqlite3
+from datetime import datetime, timedelta
+
+import sys
+
+FIELDS = ['cmd', 'command', 'start', 'end', 'delta', 'msg', 'stdout', 'stderr',
+          'response', 'status_code', 'url', 'dest']
+
+class ansibleReport(object):
+
+  def __init__(self, db_path,
+                    ansible_log_dir,
+                    name):
+    self.db_path = db_path
+    self.ansible_log_dir = ansible_log_dir
+    self.name = name
+    self.result_OK = '127.0.0.1_OK'
+    self.result_failed = '127.0.0.1_FAILED'
+    self.result_failed_ignore = '127.0.0.1_FAILED_INGORED' # tipo in ansible log upload pluging
+    self.date_format = '%Y-%m-%d %H:%M:%S'
+    self.day_format = '%Y-%m-%d'
+    
+    self._init_db()
+
+  def _init_db(self):
+    db = sqlite3.connect(self.db_path)
+    c = db.cursor()
+    c.executescript("""
+CREATE TABLE IF NOT EXISTS ansible_report (
+  name VARCHAR(40),
+  reportdate VARCHAR(15),
+  createdate VARCHAR(15),
+  status VARCHAR(20),
+  success_count INTEGER,
+  ignored_count INTEGER,
+  failed_count INTEGER,
+  ignored TEXT,
+  failed TEXT,
+  success TEXT);
+""")
+    db.commit()
+    db.close()
+
+  def connect_db(self):
+    db = sqlite3.connect(self.db_path)
+    return db
+
+  def insertEntryDb(self, table_name, data_dict):
+    db = self.connect_db()
+    columns = data_dict.keys()
+    entries = ', '.join(columns)
+    values = '?' + ', ?' * (len(columns)-1)
+    sql_string = "insert into %s(%s) values (%s)" % (
+                    table_name, entries, values)
+    tuple_data = ()
+    for key in columns:
+      tuple_data += (data_dict[key],)
+    db.execute(sql_string, tuple_data)
+    db.commit()
+    db.close()
+
+  def selectEntriesDb(self, fields=[], start_date=None, limit=0, success=None, order='DESC', where=""):
+    db = self.connect_db()
+
+    entries = ', '.join(fields) if fields else '*'
+    query = "select %s from ansible_report " % entries
+    where = " and %s" % where if where else ""
+    if not start_date:
+      start_date = datetime.utcnow().strftime(self.day_format)
+    tuple_values = (start_date,)
+    if success is not None:
+      status = 'OK' if success else 'FAILLED'
+      query += "where createdate>=? and status=? %s order by createdate %s" % (where, order)
+      tuple_values += (status,)
+    else:
+      query += "where createdate>=? %s order by createdate %s" % (where, order)
+    if limit:
+      query += " limit ?"
+      tuple_values += (limit,)
+
+    rows = db.cursor().execute(query, tuple_values)
+    #db.close()
+    if rows:
+      return [list(row) for row in rows]
+    return []
+
+  def truncateEntriesDb(self, table_name, on_field, to_value, operator='<'):
+    db = self.connect_db()
+    query = "delete from %s where %s%s?" % (table_name, on_field,
+                                            operator)
+    db.execute(query, (to_value,))
+    db.commit()
+    db.close()
+
+  def getLogString(self, res, head=False):
+    log = ""
+    if type(res) == type(dict()):
+      log = '%s, args [%s]\n' % (res['invocation']['module_name'],
+                                        res['invocation']['module_args'])
+      if head:
+        return log
+      for field in FIELDS:
+        if field in res.keys():
+        	# use default encoding, check out sys.setdefaultencoding
+          log += '\n{0}:\n{1}'.format(field, res[field])
+  
+    return log
+
+  def _read_file(self, filepath):
+    content = '[]'
+    with open(filepath, 'r') as f:
+      content = f.read()
+    return content
+
+  def saveResult(self):
+    
+    date = datetime.utcnow().strftime(self.date_format)
+
+    files_list = os.listdir(self.ansible_log_dir)
+    if not len(files_list):
+      return
+
+    to_date = (datetime.now() - timedelta(days=2)).strftime(self.date_format)
+    cmp_file = os.path.join(self.ansible_log_dir, files_list.pop())
+    modification_date = datetime.fromtimestamp(
+                            os.path.getmtime(cmp_file)
+                          ).strftime(self.date_format)
+    # Get the latest insert date
+    result = self.selectEntriesDb(['reportdate'], start_date=to_date, limit=1)
+    if len(result):
+      latest_date = result[0][0]
+      if latest_date >= modification_date:
+        return
+
+    file_map = dict(
+          success_file=os.path.join(self.ansible_log_dir, self.result_OK),
+          failed_file=os.path.join(self.ansible_log_dir, self.result_failed),
+          ignored_file=os.path.join(self.ansible_log_dir, self.result_failed_ignore))
+    
+    data = dict(name=self.name, status='FAILED',
+                reportdate=modification_date, createdate=date,
+                success_count=0, ignored_count=0,
+                failed_count=0, success="",
+                failed="", ignored="")
+    for category in ('failed', 'success', 'ignored'):
+      file_category = file_map['%s_file' % category]
+      if os.path.exists(file_category):
+        text_content = self._read_file(file_category)
+        count = len(json.loads(text_content))
+        if count > 0:
+          data['%s_count' % category] = count
+          data[category] = text_content
+    
+    if data['failed_count'] == 0:
+      data['status'] = 'OK'
+
+    self.insertEntryDb('ansible_report', data)
+
+  def getAnsibleReport(self, start_date=None, limit=0, success=None, order='DESC', category=None, head=False, only_state=True):
+    """Get one or many entries from the ansible report table.
+    """
+    where = ""
+    get_content = category is not None
+    fields = ['name', 'reportdate', 'createdate', 'status', 'success_count',
+                'ignored_count', 'failed_count']
+    if category:
+      where = " %s_count>0" % category
+      if not only_state:
+        fields.append(category)
+
+    rows = self.selectEntriesDb(fields=fields, start_date=start_date,
+                                limit=limit, success=success, order=order,
+                                where=where)
+    result_dict = {}
+    if category and not only_state:
+      last_pos = len(fields) -1
+      for i in range (0, len(rows)):
+        message = ""
+        message_list = json.loads(rows[i][last_pos])
+        for msg in message_list:
+          message += '%s\n\n' %  self.getLogString(msg, head=head)
+        rows[i][last_pos] = message
+    else:
+      return {}
+    return rows
+
+
+if __name__ == "__main__":
+  json = """{
+  "status": "OK",
+  "message": "kvm-1: OK(114) FAILED(0) IGNORED(2)",
+  "description": "Ansible playbook report in kvm-1. Execution date is: 2015-08-28 17:42:01."
+}"""
+  parameter_dict = json.loads(sys.argv[1])
+  with open(parameter_dict['status_path'], 'w') as status_file:
+    status_file.write(json)
+  
+  
\ No newline at end of file