Merge branch 'kvm-cluster' into master

slapos/recipe/kvm/template/kvm_run.in

@@ -32,6 +32,15 @@ tap_mac_address = '%(tap-mac-address)s'
 smp_count = '%(smp-count)s'
 ram_size = '%(ram-size)s'
 pid_file_path = '%(pid-file-path)s'
+external_disk_number = %(external-disk-number)s
+external_disk_size = '%(external-disk-size)s'
+disk_storage_dict = {}
+disk_storage_list = """%(disk-storage-list)s""".split('\n')
+for storage in disk_storage_list:
+  if storage:
+    key, val = storage.split(' ')
+    disk_storage_dict[key.strip()] = val.strip()
+map_storage_list = ['data%%r' %% (i + 1) for i in range(0, len(disk_storage_dict))]
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -98,6 +107,25 @@ if not os.path.exists(disk_path):
       disk_path, '%%sG' %% disk_size])
   print('Done.')
 
+# Check and create external disk
+additional_disk_list = []
+if disk_storage_dict:
+  if int(external_disk_number) > 0:
+    index = 0
+    while (index < len(disk_storage_dict)) and (index < external_disk_number):
+      path = disk_storage_dict[map_storage_list[index]]
+      if os.path.exists(path):
+        disk_filepath = os.path.join(path, 'kvm_virtual_disk.qcow2')
+        if not os.path.exists(disk_filepath):
+          print('Creating one additional virtual hard drive...')
+          subprocess.Popen([qemu_img_path, 'create' ,'-f', 'qcow2',
+              disk_filepath, '%%sG' %% external_disk_size])
+        additional_disk_list.append(disk_filepath)
+      else:
+        print('Data folder %%s was not used to create external disk %%r' %% (index +1))
+      index += 1
+
+
 # Generate network parameters
 # XXX: use_tap should be a boolean
 tap_network_parameter = []
@@ -130,6 +158,9 @@ if tap_network_parameter == [] and nat_network_parameter == []:
 else:
   kvm_argument_list += nat_network_parameter + tap_network_parameter
 
+for disk in additional_disk_list:
+  kvm_argument_list.extend([
+            '-drive', 'file=%%s,if=%%s' %% (disk, disk_type)])
 # Try to connect to NBD server (and second nbd if defined).
 # If not available, don't even specify it in qemu command line parameters.
 # Reason: if qemu starts with unavailable NBD drive, it will just crash.

slapos/recipe/slapconfiguration.py

@@ -32,6 +32,7 @@ import slapos.slap
 from slapos.recipe.librecipe import unwrap
 from ConfigParser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
+from slapos.util import mkdir_p
 
 class Recipe(object):
   """
@@ -61,6 +62,10 @@ class Recipe(object):
       Partition identifier.
       Example:
         ${slap-connection:partition-id}
+    storage-home
+      Path of folder configured for data storage
+      Example:
+        ${storage-configuration:storage-home}
 
   Output:
     slap-software-type
@@ -75,8 +80,20 @@ class Recipe(object):
       One of the IPv6 addresses.
     tap
       Set of TAP interfaces.
+    tap-network-information-dict
+      Dict of set of all TAP network information
+    tap-ipv4
+      ipv4 allowed for this TAP
+    tap-gateway
+      ipv4 of gateway interface of this TAP
+    tap-netmask
+      ipv4 netmask address of this TAP
+    tap-network
+      ipv4 network address of this TAP
     configuration
       Dict of all parameters.
+    storage-dict
+      Dict of partition data path when it is configured
     configuration.<key>
       One key per partition parameter.
       Partition parameter whose name cannot be represented unambiguously in
@@ -91,7 +108,8 @@ class Recipe(object):
   OPTCRE_match = RawConfigParser.OPTCRE.match
 
   def __init__(self, buildout, name, options):
-      parameter_dict = self.fetch_parameter_dict(options)
+      parameter_dict = self.fetch_parameter_dict(options,
+                                      buildout['buildout']['directory'])
 
       match = self.OPTCRE_match
       for key, value in parameter_dict.iteritems():
@@ -99,7 +117,7 @@ class Recipe(object):
               continue
           options['configuration.' + key] = value
 
-  def fetch_parameter_dict(self, options):
+  def fetch_parameter_dict(self, options, instance_root):
       slap = slapos.slap.slap()
       slap.initializeConnection(
           options['url'],
@@ -134,6 +152,14 @@ class Recipe(object):
       v6_add = ipv6_set.add
       tap_set = set()
       tap_add = tap_set.add
+      route_gw_set = set()
+      route_gw_add = route_gw_set.add
+      route_mask_set = set()
+      route_mask_add = route_mask_set.add
+      route_ipv4_set = set()
+      route_v4_add = route_ipv4_set.add
+      route_network_set = set()
+      route_net_add = route_network_set.add
       for tap, ip in parameter_dict.pop('ip_list'):
           tap_add(tap)
           if valid_ipv4(ip):
@@ -141,6 +167,21 @@ class Recipe(object):
           elif valid_ipv6(ip):
               v6_add(ip)
           # XXX: emit warning on unknown address type ?
+
+      if 'full_ip_list' in parameter_dict:
+        for item in parameter_dict.pop('full_ip_list'):
+          if len(item) == 5:
+            tap, ip, gw, netmask, network = item
+            if  tap.startswith('route_'):
+              if valid_ipv4(gw):
+                route_gw_add(gw)
+              if valid_ipv4(netmask):
+                route_mask_add(netmask)
+              if valid_ipv4(ip):
+                route_v4_add(ip)
+              if valid_ipv4(network):
+                route_net_add(network)
+
       options['ipv4'] = ipv4_set
       options['ipv6'] = ipv6_set
 
@@ -149,6 +190,35 @@ class Recipe(object):
           options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
       if ipv6_set:
           options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
+      if route_ipv4_set:
+        options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
+        options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
+                                    netmask=route_mask_set,
+                                    gateway=route_gw_set,
+                                    network=route_network_set)
+      else:
+        options['tap-network-information-dict'] = {}
+      if route_gw_set:
+        options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
+      if route_mask_set:
+        options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
+      if route_network_set:
+        options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
+
+      storage_home = options.get('storage-home')
+      storage_dict = {}
+      if storage_home and os.path.exists(storage_home) and \
+                                  os.path.isdir(storage_home):
+        for filename in os.listdir(storage_home):
+          storage_path = os.path.join(storage_home, filename, 
+                                    options['slap-computer-partition-id'])
+          if os.path.exists(storage_path) and os.path.isdir(storage_path):
+            storage_link = os.path.join(instance_root, 'DATA', filename)
+            mkdir_p(os.path.join(instance_root, 'DATA'))
+            if not os.path.lexists(storage_link):
+              os.symlink(storage_path, storage_link)
+            storage_dict[filename] = storage_link
+      options['storage-dict'] = storage_dict
 
       options['tap'] = tap_set
       return self._expandParameterDict(options, parameter_dict)

slapos/recipe/softwaretype.py

@@ -34,6 +34,7 @@ import subprocess
 import slapos.slap
 import netaddr
 import logging
+import errno
 
 import zc.buildout
 
@@ -54,6 +55,18 @@ class Recipe:
         return ip
     raise AttributeError
 
+  def _getTapIpAddressList(self, test_method):
+    """Internal helper method to fetch full ip address assigned for tap"""
+    if not 'full_ip_list' in self.parameter_dict:
+      return ()
+    for item in self.parameter_dict['full_ip_list']:
+      if len(item) == 5:
+        tap, ip, gw, mask, net = item
+        if tap.startswith('route_') and test_method(ip) and \
+                          test_method(gw) and test_method(mask):
+          return (ip, gw, mask, net)
+    return ()
+
   def getLocalIPv4Address(self):
     """Returns local IPv4 address available on partition"""
     # XXX: Lack checking for locality of address
@@ -63,6 +76,11 @@ class Recipe:
     """Returns global IPv6 address available on partition"""
     # XXX: Lack checking for globality of address
     return self._getIpAddress(netaddr.valid_ipv6)
+  
+  def getLocalTapIPv4AddressList(self):
+    """Returns global IPv6 address available for tap interface"""
+    # XXX: Lack checking for locality of address
+    return self._getTapIpAddressList(netaddr.valid_ipv4)
 
   def getNetworkInterface(self):
     """Returns the network interface available on partition"""
@@ -72,6 +90,20 @@ class Recipe:
       if name:
         return name
     raise AttributeError, "Not network interface found"
+  
+  def mkdir_p(self, path, mode=0700):
+    """
+    Creates a directory and its parents, if needed.
+    NB: If the directory already exists, it does not change its permission.
+    """
+
+    try:
+        os.makedirs(path, mode)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
 
   def install(self):
     slap = slapos.slap.slap()
@@ -81,6 +113,8 @@ class Recipe:
     server_url = slap_connection['server_url']
     key_file = slap_connection.get('key_file')
     cert_file = slap_connection.get('cert_file')
+    storage_home = self.buildout['storage-configuration'].get('storage-home')
+    instance_root = self.buildout['buildout']['directory']
     slap.initializeConnection(server_url, key_file, cert_file)
     self.computer_partition = slap.registerComputerPartition(
       computer_id,
@@ -128,6 +162,14 @@ class Recipe:
                  self.getGlobalIPv6Address())
     buildout.set('slap-network-information', 'network-interface',
                  self.getNetworkInterface())
+    tap_ip_list = self.getLocalTapIPv4AddressList()
+    tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
+    if tap_ip_list:
+      tap_ipv4, tap_gateway, tap_netmask, tap_network= tap_ip_list
+    buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
+    buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
+    buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
+    buildout.set('slap-network-information', 'tap-network', tap_network)
 
     # Copy/paste slap_connection
     buildout.add_section('slap-connection')
@@ -137,6 +179,27 @@ class Recipe:
     # XXX: Needed for lxc. Use non standard API
     buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
 
+    # setup storage directory
+    buildout.add_section('storage-configuration')
+    buildout.set('storage-configuration', 'storage-home', storage_home)
+    if storage_home and os.path.exists(storage_home) and \
+                                  os.path.isdir(storage_home):
+      # Create folder instance_root/DATA/ if not exist
+      data_home = os.path.join(instance_root, 'DATA')
+      self.mkdir_p(data_home)
+      for filename in os.listdir(storage_home):
+        storage_path = os.path.join(storage_home, filename, computer_partition_id)
+        if os.path.exists(storage_path) and os.path.isdir(storage_path):
+          storage_link = os.path.join(data_home, filename)
+          if os.path.lexists(storage_link):
+            if not os.path.islink(storage_link):
+              raise zc.buildout.UserError(
+                  'Target %r already exists but is not a link' % storage_link)
+              #os.unlink(storage_link)
+          else:
+            os.symlink(storage_path, storage_link)
+          buildout.set('storage-configuration', filename, storage_link)
+
     work_directory = os.path.abspath(self.buildout['buildout'][
       'directory'])
     buildout_filename = os.path.join(work_directory,

software/kvm/common.cfg

@@ -85,7 +85,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 5fdeb07b7baaf0dfa9219f0d6ba1b140
+md5sum = d2413a9d4978092e939418748585bbb3
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -93,7 +93,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = 681cd5a4dddceba1e756e9aa409477a6
+md5sum = 10af5beb60655add4de26f7ae60e32aa
 download-only = true
 on-update = true
 
@@ -101,7 +101,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 0d51e71a7967ead2f88e11cc797037a4
+md5sum = 214c46a9aa7605951b8a1f98572dac28
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster-input-schema.json

@@ -202,13 +202,36 @@
                 "type": "boolean",
                 "default": false
               },
+
+              "external-disk-number": {
+                "title": "Number of additional disk to create for virtual machine",
+                "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 4,
+                "default": 0
+              },
+              "external-disk-size": {
+                "title": "Number of additional disk to create for virtual machine, in Gigabytes",
+                "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+                "type": "integer",
+                "minimum": 10,
+                "maximum": 100,
+                "default": 20
+              },
           
               "use-tap": {
                 "title": "Use QEMU TAP network interface",
-                "description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
+                "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",
                 "type": "boolean",
                 "default": false
               },
+              "use-nat": {
+                "title": "Use QEMU USER Mode networking",
+                "description": "Use QEMU user-mode network stack (NAT).",
+                "type": "boolean",
+                "default": true
+              },
               "nat-rules": {
                 "title": "List of rules for NAT of QEMU user mode network stack.",
                 "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -47,6 +47,8 @@ config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', False)) }}
 config-virtual-hard-drive-url = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-url', '')) }}
 config-virtual-hard-drive-md5sum = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-md5sum', '')) }}
 config-virtual-hard-drive-gzipped = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-gzipped', False)) }}
+config-external-disk-number = {{ dumps(kvm_parameter_dict.get('external-disk-number', 0)) }}
+config-external-disk-size = {{ dumps(kvm_parameter_dict.get('external-disk-size', 20)) }}
 return = 
   backend-url
   url
@@ -110,6 +112,8 @@ recipe = slapos.cookbook:publish
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}
+{% set disk_number = len(storage_dict) -%}
+1_info = It is possible to mount up to {{ disk_number }} external disk to your virtual machine. See parameter 'external-disk-number'
 
 [buildout]
 parts = publish

software/kvm/instance-kvm-input-schema.json

@@ -78,13 +78,42 @@
       "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
       "type": "string"
     },
+    "virtual-hard-drive-gzipped": {
+      "title": "Define if virtual hard drive to download is gzipped",
+      "description": "Define if virtual hard drive to download is gzipped using gzip. This help to reduce size of file to download.",
+      "type": "boolean",
+      "default": false
+    },
+
+    "external-disk-number": {
+      "title": "Number of additional disk to create for virtual machine",
+      "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 4,
+      "default": 0
+    },
+    "external-disk-size": {
+      "title": "Number of additional disk to create for virtual machine, in Gigabytes",
+      "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+      "type": "integer",
+      "minimum": 10,
+      "maximum": 100,
+      "default": 20
+    },
 
     "use-tap": {
       "title": "Use QEMU TAP network interface",
-      "description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
+      "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",
       "type": "boolean",
       "default": false
     },
+    "use-nat": {
+      "title": "Use QEMU USER Mode networking",
+      "description": "Use QEMU user-mode network stack (NAT).",
+      "type": "boolean",
+      "default": true
+    },
     "nat-rules": {
       "title": "List of rules for NAT of QEMU user mode network stack.",
       "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",

software/kvm/instance-kvm.cfg.jinja2

@@ -103,6 +103,12 @@ qemu-path =  {{ qemu_executable_location }}
 qemu-img-path = {{ qemu_img_executable_location }}
 6tunnel-path = {{ sixtunnel_executable_location }}
 
+disk-storage-list = 
+{% for key, path in storage_dict.items() -%}
+{{ '  ' ~ key ~ ' ' ~ path }}
+{% endfor -%}
+external-disk-number = ${slap-parameter:external-disk-number}
+external-disk-size = ${slap-parameter:external-disk-size}
 
 [kvm-vnc-promise]
 recipe = slapos.cookbook:check_port_listening
@@ -230,7 +236,9 @@ recipe = slapos.cookbook:publish
 ipv6 = ${slap-network-information:global-ipv6}
 backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-instance:vnc-passwd}
 url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-instance:vnc-passwd}
+{% set iface = 'eth0' -%}
 {% if slapparameter_dict.get('use-nat', 'True') == 'True' -%}
+{%   set iface = 'eth1' -%}
 # Publish NAT port mapping status
 # XXX: hardcoded value from [slap-parameter]
 {%   set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') %}
@@ -242,6 +250,12 @@ nat-rule-url-{{port}} = [${slap-network-information:global-ipv6}]:{{external_por
 {%     endif -%}
 {%   endfor -%}
 {% endif -%}
+{% if slapparameter_dict.get('use-tap', 'False') == 'True' and tap_network_dict.has_key('ipv4') -%}
+1_info = Use these configurations below to configure interface {{ iface }} in your VM.
+2_info = ifconfig {{ iface }} ${slap-network-information:tap-ipv4} netmask ${slap-network-information:tap-netmask}
+3_info = route add -host ${slap-network-information:tap-gateway} dev {{ iface }}
+4_info = route add -net ${slap-network-information:tap-network} netmask ${slap-network-information:tap-netmask} gw ${slap-network-information:tap-gateway}
+{% endif -%}
 
 
 [slap-parameter]
@@ -268,3 +282,6 @@ use-tap = False
 virtual-hard-drive-url =
 virtual-hard-drive-md5sum =
 virtual-hard-drive-gzipped = False
+
+external-disk-number = 0
+external-disk-size = 20

software/kvm/instance.cfg.in

@@ -23,6 +23,10 @@ test = $${dynamic-template-kvm-resilient-test:rendered}
 frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
 
+# XXX - If this configuration is not generated by slapgrid, use empty values
+[storage-configuration]
+storage-home = 
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -30,6 +34,7 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+storage-home = $${storage-configuration:storage-home}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -42,6 +47,8 @@ context =
     key eggs_directory buildout:eggs-directory
     key ipv4 slap-configuration:ipv4
     key ipv6 slap-configuration:ipv6
+    key tap_network_dict slap-configuration:tap-network-information-dict
+    key storage_dict slap-configuration:storage-dict
     key slapparameter_dict slap-configuration:configuration
     key computer_id slap-configuration:computer
     $${:extra-context}
@@ -64,6 +71,8 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
+    key storage_dict slap-configuration:storage-dict
+    key tap_network_dict slap-configuration:tap-network-information-dict
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond