Merge branch 'kvm-cluster' into master

component/noVNC/buildout.cfg

@@ -4,7 +4,7 @@ parts =
 
 [noVNC]
 recipe = hexagonit.recipe.download
-# Post-0.4 release from January 2013
-url = http://github.com/kanaka/noVNC/zipball/3b2acc2258d36137a37edfbe0f03a3099189c49d
-md5sum = a276be8fa193652bb5de8a271603f11f
+# version-0.5.1 release from 29 Nov 2014
+url = https://github.com/kanaka/noVNC/archive/v0.5.1.tar.gz
+md5sum = ac55b2316b2164b6e09ae3bd89c37cb6
 strip-top-level-dir = true

component/qemu-kvm/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.6.1.tar.bz2
-md5sum = 3a897d722457c5a895cd6ac79a28fda0
+url = http://wiki.qemu-project.org/download/qemu-2.2.0.tar.bz2
+md5sum = f7a5e2da22d057eb838a91da7aff43c8
 configure-options =
   --target-list=x86_64-softmmu
   --enable-system
@@ -55,9 +55,9 @@ configure-options =
 [debian-amd64-netinst.iso]
 # Download the installer of Debian 7 (Wheezy)
 recipe = hexagonit.recipe.download
-url = http://cdimage.debian.org/debian-cd/7.2.0/amd64/iso-cd/debian-7.2.0-amd64-netinst.iso 
+url = http://cdimage.debian.org/debian-cd/7.8.0/amd64/iso-cd/debian-7.8.0-amd64-netinst.iso
 filename = ${:_buildout_section_name_}
-md5sum = b86774fe4de88be6378ba3d71b8029bd 
+md5sum = a91fba5001cf0fbccb44a7ae38c63b6e 
 download-only = true
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}

slapos/recipe/kvm/__init__.py

@@ -45,7 +45,7 @@ class Recipe(GenericBaseRecipe):
 
     path_list = []
 
-    if not self.isTrueValue(self.options.get('use-tap')):
+    if self.isTrueValue(self.options.get('use-nat')):
       # XXX This could be done using Jinja.
       for port in self.options['nat-rules'].split():
         tunnel_port = int(port) + 10000

slapos/recipe/kvm/template/kvm_run.in

@@ -7,6 +7,8 @@ import os
 import socket
 import subprocess
 import urllib
+import gzip
+import shutil
 
 # XXX: give all of this through parameter, don't use this as template, but as module
 qemu_img_path = '%(qemu-img-path)s'
@@ -19,11 +21,14 @@ default_disk_image = '%(default-disk-image)s'
 disk_path = '%(disk-path)s'
 virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
 virtual_hard_drive_md5sum = '%(virtual-hard-drive-md5sum)s'.strip()
+virtual_hard_drive_gzipped = '%(virtual-hard-drive-gzipped)s'.strip()
 nat_rules = '%(nat-rules)s'.strip()
 use_tap = '%(use-tap)s'
+use_nat = '%(use-nat)s'
 tap_interface = '%(tap-interface)s'
 listen_ip = '%(ipv4)s'
 mac_address = '%(mac-address)s'
+tap_mac_address = '%(tap-mac-address)s'
 smp_count = '%(smp-count)s'
 ram_size = '%(ram-size)s'
 pid_file_path = '%(pid-file-path)s'
@@ -61,21 +66,29 @@ def getSocketStatus(host, port):
 if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
   print('Downloading virtual hard drive...')
   try:
-    urllib.urlretrieve(virtual_hard_drive_url, disk_path)
+    downloaded_disk = disk_path
+    if virtual_hard_drive_gzipped == 'True':
+      downloaded_disk = '%%s.gz' %% disk_path
+    urllib.urlretrieve(virtual_hard_drive_url, downloaded_disk)
   except:
-    os.remove(disk_path)
+    os.remove(downloaded_disk)
     raise
   md5sum = virtual_hard_drive_md5sum.strip()
   if md5sum:
     print('Checking MD5 checksum...')
-    local_md5sum = md5Checksum(disk_path)
+    local_md5sum = md5Checksum(downloaded_disk)
     if local_md5sum != md5sum:
-      os.remove(disk_path)
+      os.remove(downloaded_disk)
       raise Exception('MD5 mismatch. MD5 of local file is %%s, Specified MD5 is %%s.' %% (
           local_md5sum, md5sum))
     print('MD5sum check passed.')
   else:
     print('Warning: not checksum specified.')
+  if downloaded_disk.endswith('.gz'):
+    with open(disk_path, 'w') as disk:
+      with gzip.open(downloaded_disk, 'rb') as disk_gz:
+        shutil.copyfileobj(disk_gz, disk)
+    os.remove(downloaded_disk)
 
 # Create disk if doesn't exist
 # XXX: move to Buildout profile
@@ -87,22 +100,35 @@ if not os.path.exists(disk_path):
 
 # Generate network parameters
 # XXX: use_tap should be a boolean
+tap_network_parameter = []
+nat_network_parameter = []
+number = -1
+if use_nat == 'True':
+  number += 1
+  rules = 'user,id=lan%%s,' %% number + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip,
+                        int(port) + 10000, port) for port in nat_rules.split())
+  nat_network_parameter = ['-netdev', rules,
+          '-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, mac_address)]
 if use_tap == 'True':
-  qemu_network_parameter = 'tap,ifname=%%s,script=no,downscript=no' %% tap_interface
-else:
-  qemu_network_parameter = 'user,' + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip, int(port) + 10000, port) for port in nat_rules.split())
+  number += 1
+  tap_network_parameter = ['-netdev',
+          'tap,id=lan%%s,ifname=%%s,script=no,downscript=no' %% (number,
+            tap_interface),
+          '-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, tap_mac_address)]
 
 kvm_argument_list = [qemu_path,
-  '-enable-kvm', '-net', 'nic,macaddr=%%s' %% mac_address,
-  '-net', qemu_network_parameter,
-  '-smp', smp_count,
-  '-m', ram_size,
+  '-enable-kvm', '-smp', smp_count,
+  '-m', ram_size, '-vga', 'std',
   '-drive', 'file=%%s,if=%%s' %% (disk_path, disk_type),
   '-vnc', '%%s:1,ipv4,password' %% listen_ip,
-  '-boot', 'menu=on',
+  '-boot', 'order=cd,menu=on',
   '-qmp', 'unix:%%s,server' %% socket_path,
   '-pidfile', pid_file_path,
 ]
+if tap_network_parameter == [] and nat_network_parameter == []:
+  print 'Warning : No network interface defined.'
+else:
+  kvm_argument_list += nat_network_parameter + tap_network_parameter
 
 # Try to connect to NBD server (and second nbd if defined).
 # If not available, don't even specify it in qemu command line parameters.

software/kvm/common.cfg

@@ -85,7 +85,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 4c8f07da2217e54163c265fe6fe3d41d
+md5sum = 5fdeb07b7baaf0dfa9219f0d6ba1b140
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -93,7 +93,15 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = aae627dc6cfc9c728f61a3f12c525cf7
+md5sum = 5ff1c3c0083ad7d331e9e2da7dc601cb
+download-only = true
+on-update = true
+
+[template-kvm-cluster]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
+mode = 644
+md5sum = 9dc4b77be3d350b41baaf57d147e07e2
 download-only = true
 on-update = true
 

software/kvm/development.cfg

@@ -19,7 +19,7 @@ develop =
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = master
+branch = kvm-cluster
 git-executable = ${git:location}/bin/git
 
 [slapos.core-repository]

software/kvm/instance-kvm-cluster-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+
+  "title": "Input Parameters",
+  "properties": {
+    "frontend": {
+      "frontend-instance-guid": {
+        "title": "Frontend Instance ID",
+        "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
+        "type": "string",
+        "default": "SOFTINST-11031"
+      },
+      "frontend-software-type": {
+        "title": "Frontend Software Type",
+        "description": "Type of the frontend instance, like \"frontend\".",
+        "type": "string",
+        "default": "frontend"
+      },
+      "frontend-software-url": {
+        "title": "Frontend Software URL",
+        "description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
+        "type": "string",
+        "format": "uri",
+        "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg"
+      },
+      "type": "object"
+    },
+    "kvm-partition-dict": {
+      "description": "kvm instances definition",
+      "patternProperties": {
+        ".*": {
+            "properties": {
+              "ram-size": {
+                "title": "RAM size",
+                "description": "RAM size, in MB.",
+                "type": "integer",
+                "default": 1024,
+                "minimum": 128,
+                "multipleOf": 128,
+                "maximum": 16384
+              },
+              "disk-size": {
+                "title": "Disk size",
+                "description": "Disk size, in GB.",
+                "type": "integer",
+                "default": 10,
+                "minimum": 1,
+                "maximum": 80
+              },
+              "disk-type": {
+                "title": "Disk type",
+                "description": "Type of QEMU disk drive.",
+                "type": "string",
+                "default": "virtio",
+                "enum": ["ide", "scsi", "sd", "mtd", "floppy", "pflash", "virtio"]
+              },
+          
+              "cpu-count": {
+                "title": "CPU count",
+                "description": "Number of CPU cores.",
+                "type": "integer",
+                "minimum": 1,
+                "maximum": 8
+              },
+          
+              "nbd-host": {
+                "title": "NBD hostname",
+                "description": "hostname (or IP) of the NBD server containing the boot image.",
+                "type": "string",
+                "format": ["host-name", "ip-address", "ipv6"],
+                "default": "debian.nbd.vifib.net"
+              },
+              "nbd-port": {
+                "title": "NBD port",
+                "description": "Port of the NBD server containing the boot image.",
+                "type": "integer",
+                "default": 1024,
+                "minimum": 1,
+                "maximum": 65535
+              },
+          
+              "nbd2-host": {
+                "title": "Second NBD hostname",
+                "description": "hostname (or IP) of the second NBD server (containing drivers for example).",
+                "type": "string",
+                "format": ["host-name", "ip-address", "ipv6"]
+              },
+              "nbd2-port": {
+                "title": "Second NBD port",
+                "description": "Port of the second NBD server containing the boot image.",
+                "type": "integer",
+                "minimum": 1,
+                "maximum": 65535
+              },
+          
+              "virtual-hard-drive-url": {
+                "title": "Existing disk image URL",
+                "description": "If specified, will download an existing disk image (qcow2, raw, ...), and will use it as main virtual hard drive. Can be used to download and use an already installed and customized virtual hard drive.",
+                "format": "uri",
+                "type": "string"
+              },
+              "virtual-hard-drive-md5sum": {
+                "title": "Checksum of virtual hard drive",
+                "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
+                "type": "string"
+              },
+              "virtual-hard-drive-gzipped": {
+                "title": "Define if virtual hard drive to download is gzipped",
+                "description": "Define if virtual hard drive to download is gzipped using gzip. This help to reduce size of file to download.",
+                "type": "boolean",
+                "default": false
+              },
+          
+              "use-tap": {
+                "title": "Use QEMU TAP network interface",
+                "description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
+                "type": "boolean",
+                "default": false
+              },
+              "nat-rules": {
+                "title": "List of rules for NAT of QEMU user mode network stack.",
+                "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
+                "type": "string"
+              }
+            },
+          "type": "object"
+        }
+      },
+      "type": "object"
+    }
+  }
+}
\ No newline at end of file

software/kvm/instance-kvm-cluster.cfg.jinja2.in

+{% set publish_dict = {} -%}
+{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
+
+[request-common]
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
+
+# Request kvm instances
+{% for instance_name, kvm_parameter_dict in slapparameter_dict.get('kvm-partition-dict', {'kvm-default': {}}).items() -%}
+{% set section = 'request-' ~ instance_name -%}
+[{{ section }}]
+<= request-common
+software-type = kvm
+name = {{ instance_name }}
+sla-computer_guid = {{ dumps(kvm_parameter_dict.get('computer-guid', computer_id)) }}
+
+config-frontend-instance-name = {{ instance_name ~ ' VNC Frontend' }}
+config-frontend-software-type = {{ dumps(frontend_dict.get('software-type', 'frontend')) }}
+config-frontend-software-url = {{ dumps(frontend_dict.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg')) }}
+config-frontend-instance-guid = {{ dumps(frontend_dict.get('instance-guid', '')) }}
+config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
+config-nbd-host = {{ dumps(kvm_parameter_dict.get('nbd-host', '')) }}
+config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
+config-nbd2-host = {{ dumps(kvm_parameter_dict.get('nbd-host2', '')) }}
+config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
+config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
+config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
+config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
+
+{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', [22, 80, 443]) -%}
+config-nat-rules = {{ nat_rules_list | join(' ') }}
+config-use-nat = {{ dumps(kvm_parameter_dict.get('use-nat', True)) }}
+config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', False)) }}
+config-virtual-hard-drive-url = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-url', '')) }}
+config-virtual-hard-drive-md5sum = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-md5sum', '')) }}
+config-virtual-hard-drive-gzipped = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-gzipped', False)) }}
+return = 
+  backend-url
+  url
+{% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
+{% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
+{% endfor %}
+
+[publish]
+recipe = slapos.cookbook:publish
+{% for name, value in publish_dict.items() -%}
+{{   name }} = {{ value }}
+{% endfor %}
+
+[buildout]
+parts = publish
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
\ No newline at end of file

software/kvm/instance-kvm.cfg.jinja2

@@ -41,6 +41,10 @@ cronstamps = ${:etc}/cronstamps
 recipe = slapos.cookbook:generate.mac
 storage-path = ${directory:srv}/mac
 
+[create-tap-mac]
+recipe = slapos.cookbook:generate.mac
+storage-path = ${directory:srv}/tap_mac
+
 [gen-passwd]
 recipe = slapos.cookbook:generate.password
 storage-path = ${directory:srv}/passwd
@@ -79,17 +83,20 @@ pid-file-path = ${directory:run}/pid_file
 smp-count = ${slap-parameter:cpu-count}
 ram-size = ${slap-parameter:ram-size}
 mac-address = ${create-mac:mac-address}
+tap-mac-address = ${create-tap-mac:mac-address}
 
 # XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
 runner-path = ${directory:services}/kvm
 controller-path = ${directory:scripts}/kvm_controller
 
 use-tap = ${slap-parameter:use-tap}
+use-nat = ${slap-parameter:use-nat}
 nat-rules = ${slap-parameter:nat-rules}
 6tunnel-wrapper-path = ${directory:services}/6tunnel
 
 virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
 virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
+virtual-hard-drive-gzipped = ${slap-parameter:virtual-hard-drive-gzipped}
 
 shell-path = {{ dash_executable_location }}
 qemu-path =  {{ qemu_executable_location }}
@@ -202,7 +209,7 @@ key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
-name = VNC Frontend
+name = ${slap-parameter:frontend-instance-name}
 software-type = ${slap-parameter:frontend-software-type}
 slave = true
 config-host = ${novnc-instance:ip}
@@ -236,6 +243,7 @@ url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slav
 frontend-software-type = frontend
 frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
 frontend-instance-guid =
+frontend-instance-name = VNC Frontend
 nbd-port = 1024
 nbd-host =
 nbd2-port = 1024
@@ -248,7 +256,9 @@ disk-type = virtio
 cpu-count = 1
 
 nat-rules = 22 80 443
+use-nat = True
 use-tap = False
 
 virtual-hard-drive-url =
 virtual-hard-drive-md5sum =
+virtual-hard-drive-gzipped = False

software/kvm/instance.cfg.in

@@ -8,6 +8,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = $${:kvm}
+kvm-cluster = $${dynamic-template-kvm-cluster:rendered}
 kvm = $${dynamic-template-kvm:rendered}
 nbd = ${template-nbd:output}
 frontend = ${template-frontend:output}
@@ -30,6 +31,30 @@ url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
 
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = $${buildout:directory}/$${:filename}
+extensions = jinja2.ext.do
+mode = 0644
+extra-context =
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key ipv4 slap-configuration:ipv4
+    key ipv6 slap-configuration:ipv6
+    key slapparameter_dict slap-configuration:configuration
+    key computer_id slap-configuration:computer
+    $${:extra-context}
+
+[dynamic-template-kvm-cluster-parameters]
+
+[dynamic-template-kvm-cluster]
+<= jinja2-template-base
+template = ${template-kvm-cluster:location}/instance-kvm-cluster.cfg.jinja2.in
+filename = template-kvm-cluster.cfg
+extra-context =
+  section parameter_dict dynamic-template-kvm-cluster-parameters
+
 [dynamic-template-kvm]
 recipe = slapos.recipe.template:jinja2
 template = ${template-kvm:location}/instance-kvm.cfg.jinja2

software/kvm/software.cfg

@@ -3,41 +3,41 @@ extends = common.cfg
 
 [versions]
 PyRSS2Gen = 1.1
-apache-libcloud = 0.15.1
-async = 0.6.1
+apache-libcloud = 0.16.0
 cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 ecdsa = 0.11
-erp5.util = 0.4.41
-gitdb = 0.5.4
+erp5.util = 0.4.42
+gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.4
-smmap = 0.8.2
+slapos.recipe.template = 2.6
+slapos.toolbox = 0.46.1
+smmap = 0.9.0
 websockify = 0.6.0
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# websockify==0.6.0
-numpy = 1.8.2
-
-# Required by:
-# slapos.toolbox==0.40.4
-GitPython = 0.3.2.RC1
+# slapos.toolbox==0.46.1
+GitPython = 0.3.6
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.46.1
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.46.1
 feedparser = 5.1.3
 
+# Required by:
+# slapos.toolbox==0.46.1
+lockfile = 0.10.2
+
 # Required by:
 # websockify==0.6.0
-numpy = 1.9.0
+numpy = 1.9.1
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.46.1
 paramiko = 1.15.2