Merge branch 'kvm'

slapos/recipe/kvm/__init__.py

@@ -41,40 +41,40 @@ class Recipe(GenericBaseRecipe):
           '"virtio" value.'
       self.options['disk-type'] = 'virtio'
 
-    config = dict(
-      tap_interface=self.options['tap'],
-      vnc_ip=self.options['vnc-ip'],
-      vnc_port=self.options['vnc-port'],
-      nbd_ip=self.options['nbd-host'],
-      nbd_port=self.options['nbd-port'],
-      nbd2_ip=self.options.get('nbd2-host', ''),
-      nbd2_port=self.options.get('nbd2-port', 1024),
-      disk_path=self.options['disk-path'],
-      disk_size=self.options['disk-size'],
-      disk_type=self.options['disk-type'],
-      mac_address=self.options['mac-address'],
-      smp_count=self.options['smp-count'],
-      ram_size=self.options['ram-size'],
-      socket_path=self.options['socket-path'],
-      pid_file_path=self.options['pid-path'],
-      python_path=sys.executable,
-      shell_path=self.options['shell-path'],
-      qemu_path=self.options['qemu-path'],
-      qemu_img_path=self.options['qemu-img-path'],
-      vnc_passwd=self.options['passwd'],
-      default_disk_image=self.options['default-disk-image'],
-    )
+    self.options['python-path'] = sys.executable
+
+    path_list = []
+
+    if not self.isTrueValue(self.options.get('use-tap')):
+      # XXX This could be done using Jinja.
+      for port in self.options['nat-rules'].split():
+        ipv6_port = int(port) + 10000
+        tunnel_path = self.createExecutable(
+            '%s-%sto%s' % (self.options['6tunnel-wrapper-path'], port, ipv6_port),
+            self.substituteTemplate(
+                self.getTemplateFilename('6to4.in'),
+                {
+                    'ipv6': self.options['ipv6'],
+                    'ipv6_port': ipv6_port,
+                    'ipv4': self.options['ipv4'],
+                    'ipv4_port': port,
+                    'shell_path': self.options['shell-path'],
+                    '6tunnel_path': self.options['6tunnel-path'],
+                },
+            ),
+        )
+        path_list.append(tunnel_path)
 
-    # Runners
     runner_path = self.createExecutable(
-      self.options['runner-path'],
-      self.substituteTemplate(self.getTemplateFilename('kvm_run.in'),
-                              config))
+        self.options['runner-path'],
+        self.substituteTemplate(self.getTemplateFilename('kvm_run.in'),
+                                self.options))
+    path_list.append(runner_path)
 
     controller_path = self.createExecutable(
       self.options['controller-path'],
       self.substituteTemplate(self.getTemplateFilename('kvm_controller_run.in'),
-                              config))
+                              self.options))
 
 
-    return [runner_path, controller_path]
+    return path_list

slapos/recipe/kvm/template/6to4.in

+#!%(shell_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(6tunnel_path)s -6 -4 -d -l %(ipv6)s %(ipv6_port)s %(ipv4)s %(ipv4_port)s
+

slapos/recipe/kvm/template/kvm_controller_run.in

-#!%(python_path)s
+#!%(python-path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 
@@ -6,12 +6,15 @@
 import socket
 import time
 
+socket_path = '%(socket-path)s'
+vnc_password = '%(vnc-passwd)s'
+
 # Connect to KVM qmp socket
 so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 connected = False
 while not connected:
   try:
-    so.connect('%(socket_path)s')
+    so.connect(socket_path)
   except socket.error:
     time.sleep(1)
   else:
@@ -25,7 +28,7 @@ data = so.recv(1024)
 # Set VNC password
 so.send('{ "execute": "change", ' \
         '"arguments": { "device": "vnc", "target": "password", ' \
-        '               "arg": "%(vnc_passwd)s" } }')
+        '               "arg": "' + vnc_password + '" } }')
 data = so.recv(1024)
 
 # Finish

slapos/recipe/kvm/template/kvm_run.in

-#!%(python_path)s
+#!%(python-path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 
-# Echo client program
+import hashlib
 import os
 import socket
 import subprocess
+import urllib
 
-# XXX: give all of this through parameter, don't use this as template
-default_disk_image = '%(default_disk_image)s'
+# XXX: give all of this through parameter, don't use this as template, but as module
+qemu_img_path = '%(qemu-img-path)s'
+qemu_path = '%(qemu-path)s'
+disk_size = '%(disk-size)s'
+disk_type = '%(disk-type)s'
+socket_path = '%(socket-path)s'
+nbd_list = (('%(nbd-host)s', %(nbd-port)s), ('%(nbd2-host)s', %(nbd2-port)s))
+default_disk_image = '%(default-disk-image)s'
+disk_path = '%(disk-path)s'
+virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
+virtual_hard_drive_md5_url = '%(virtual-hard-drive-md5-url)s'.strip()
+nat_rules = '%(nat-rules)s'.strip()
+use_tap = '%(use-tap)s'
+tap_interface = '%(tap-interface)s'
+listen_ip = '%(ipv4)s'
+mac_address = '%(mac-address)s'
+smp_count = '%(smp-count)s'
+ram_size = '%(ram-size)s'
+pid_file_path = '%(pid-file-path)s'
+
+def md5Checksum(file_path):
+    with open(file_path, 'rb') as fh:
+        m = hashlib.md5()
+        while True:
+            data = fh.read(8192)
+            if not data:
+                break
+            m.update(data)
+        return m.hexdigest()
 
 def getSocketStatus(host, port):
   s = None
@@ -29,28 +57,44 @@ def getSocketStatus(host, port):
     break
   return s
 
+# Download existing hard drive if needed at first boot
+if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
+  urllib.urlretrieve(virtual_hard_drive_url, disk_path)
+  local_md5sum = md5Checksum(disk_path)
+  md5sum = urllib.urlopen(virtual_hard_drive_md5_url).read().strip()
+  if local_md5sum != md5sum:
+    os.remove(disk_path)
+    raise Exception('MD5 mismatch.')
+
 # Create disk if doesn't exist
 # XXX: move to Buildout profile
-disk_path = '%(disk_path)s'
 if not os.path.exists(disk_path):
-  subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
-      disk_path, '%(disk_size)sG'])
+  subprocess.Popen([qemu_img_path, 'create' ,'-f', 'qcow2',
+      disk_path, '%%sG' %% disk_size])
 
-kvm_argument_list = ['%(qemu_path)s',
-  '-enable-kvm', '-net', 'nic,macaddr=%(mac_address)s',
-  '-net', 'tap,ifname=%(tap_interface)s,script=no,downscript=no',
-  '-smp', '%(smp_count)s',
-  '-m', '%(ram_size)s',
-  '-drive', 'file=%(disk_path)s,if=%(disk_type)s',
-  '-vnc', '%(vnc_ip)s:1,ipv4,password',
+# Generate network parameters
+# XXX: use_tap should be a boolean
+if use_tap == 'True':
+  qemu_network_parameter = 'tap,ifname=%%s,script=no,downscript=no' %% tap_interface
+else:
+  qemu_network_parameter = 'user,' + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip, int(port) + 10000, port) for port in nat_rules.split())
+
+kvm_argument_list = [qemu_path,
+  '-enable-kvm', '-net', 'nic,macaddr=%%s' %% mac_address,
+  '-net', qemu_network_parameter,
+  '-smp', smp_count,
+  '-m', ram_size,
+  '-drive', 'file=%%s,if=%%s' %% (disk_path, disk_type),
+  '-vnc', '%%s:1,ipv4,password' %% listen_ip,
   '-boot', 'menu=on',
-  '-qmp', 'unix:%(socket_path)s,server',
-  '-pidfile', '%(pid_file_path)s',
+  '-qmp', 'unix:%%s,server' %% socket_path,
+  '-pidfile', pid_file_path,
 ]
 
-# Try to connect to NBD server (and second nbd if defined)
-for nbd_ip, nbd_port in (
-    ('%(nbd_ip)s', %(nbd_port)s), ('%(nbd2_ip)s', %(nbd2_port)s)):
+# Try to connect to NBD server (and second nbd if defined).
+# If not available, don't even specify it in qemu command line parameters.
+# Reason: if qemu starts with unavailable NBD drive, it will just crash.
+for nbd_ip, nbd_port in nbd_list:
   if nbd_ip and nbd_port:
     s = getSocketStatus(nbd_ip, nbd_port)
     if s is None:
@@ -61,11 +105,10 @@ for nbd_ip, nbd_port in (
       kvm_argument_list.extend([
           '-drive',
           'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
-
 # If no NBD is specified/available: use internal disk image
 else:
   kvm_argument_list.extend([
       '-drive', 'file=%%s,media=cdrom' %% default_disk_image
   ])
 
-os.execv('%(qemu_path)s', kvm_argument_list)
+os.execv(qemu_path, kvm_argument_list)

software/kvm/common.cfg

 [buildout]
 
 extends =
+  ../../component/6tunnel/buildout.cfg
   ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
@@ -84,7 +85,7 @@ mode = 0644
 [template-kvm]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm.cfg.in
-md5sum = f7c0e2172dac4ee70daae50f38d610ef
+#md5sum = c3c888c78bbff334135be9e8ad5885a9
 output = ${buildout:directory}/template-kvm.cfg
 mode = 0644
 

software/kvm/instance-kvm-input-schema.json

 {
-  "name": "Input Parameters",
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+
+  "title": "Input Parameters",
   "properties": {
     "ram-size": {
       "title": "RAM size",
@@ -7,7 +10,7 @@
       "type": "integer",
       "default": 1024,
       "minimum": 128,
-      "divisibleBy": 128,
+      "multipleOf": 128,
       "maximum": 16384
     },
     "disk-size": {
@@ -34,7 +37,6 @@
       "maximum": 8
     },
 
-
     "nbd-host": {
       "title": "NBD hostname",
       "description": "hostname (or IP) of the NBD server containing the boot image.",
@@ -65,6 +67,25 @@
       "maximum": 65535
     },
 
+    "virtual-hard-drive-url": {
+      "title": "Existing disk image URL",
+      "description": "If specified, will download an existing disk image (qcow2, raw, ...), and will use it as main virtual hard drive. Can be used to download and use an already installed and customized virtual hard drive.",
+      "format": "uri",
+      "type": "string",
+    },
+
+    "use-tap": {
+      "title": "Use QEMU TAP network interface",
+      "description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
+      "type": "boolean",
+      "default": false
+    },
+    "nat-rules": {
+      "title": "List of rules for NAT of QEMU user mode network stack.",
+      "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
+      "type": "string",
+    },
+
 
     "frontend-instance-guid": {
       "title": "Frontend Instance ID",

software/kvm/instance-kvm-output-schema.json

@@ -13,13 +13,6 @@
       "description": "URL used to connect to the service.",
       "type": "uri",
       "required": false
-    },
-
-    "password": {
-      "title": "Password",
-      "description": "Password used to authenticate in the service webpage.",
-      "type": "uri",
-      "required": true
     }
   }
 }

software/kvm/instance-kvm.cfg.in

@@ -45,32 +45,56 @@ recipe = slapos.cookbook:generate.password
 storage-path = $${directory:srv}/passwd
 bytes = 8
 
+
 [kvm-instance]
 # XXX-Cedric: change "KVM" recipe to simple "create wrappers". No need for this
-# Specific code
+# Specific code. It needs Jinja.
 recipe = slapos.cookbook:kvm
-vnc-ip = $${slap-network-information:local-ipv4}
+
+vnc-passwd = $${gen-passwd:passwd}
+
+ipv4 = $${slap-network-information:local-ipv4}
+ipv6 = $${slap-network-information:global-ipv6}
+vnc-ip = $${:ipv4}
+
 vnc-port = 5901
+
+# XXX-Cedric: should be named "default-cdrom-iso"
 default-disk-image = ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
 nbd-host = $${slap-parameter:nbd-host}
 nbd-port = $${slap-parameter:nbd-port}
 nbd2-host = $${slap-parameter:nbd2-host}
 nbd2-port = $${slap-parameter:nbd2-port}
-tap = $${slap-network-information:network-interface}
+
+tap-interface = $${slap-network-information:network-interface}
+
 disk-path = $${directory:srv}/virtual.qcow2
 disk-size = $${slap-parameter:disk-size}
 disk-type = $${slap-parameter:disk-type}
+
 socket-path = $${directory:var}/qmp_socket
-pid-path = $${directory:run}/pid_file
+pid-file-path = $${directory:run}/pid_file
+
 smp-count = $${slap-parameter:cpu-count}
 ram-size = $${slap-parameter:ram-size}
 mac-address = $${create-mac:mac-address}
+
+# XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
 runner-path = $${directory:services}/kvm
 controller-path = $${directory:scripts}/kvm_controller
+
+use-tap = $${slap-parameter:use-tap}
+nat-rules = $${slap-parameter:nat-rules}
+6tunnel-wrapper-path = $${directory:services}/6tunnel
+
+virtual-hard-drive-url = $${slap-parameter:virtual-hard-drive-url}
+virtual-hard-drive-md5-url = $${slap-parameter:virtual-hard-drive-md5-url}
+
 shell-path = ${dash:location}/bin/dash
 qemu-path = ${kvm:location}/bin/qemu-system-x86_64
 qemu-img-path = ${kvm:location}/bin/qemu-img
-passwd = $${gen-passwd:passwd}
+6tunnel-path = ${6tunnel:location}/bin/6tunnel
+
 
 [kvm-promise]
 recipe = slapos.cookbook:check_port_listening
@@ -188,8 +212,8 @@ sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:passwd}
-url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:passwd}
+backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:vnc-passwd}
+url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:vnc-passwd}
 
 
 [frontend-promise]
@@ -214,3 +238,9 @@ disk-size = 10
 disk-type = virtio
 
 cpu-count = 1
+
+nat-rules = 22 80 443
+use-tap = False
+
+virtual-hard-drive-url =
+virtual-hard-drive-md5-url =