kvm-cluster: Share all vms hostname and ipv4 through a simple http server

A new recipe deploy a python HTTPServer which is used to share files between all vm in the cluster
the url is something like: https:IPv6:port/HASH-PATH/FILES
Access to root https:IPv6:port/ is forbiden.

setup.py

@@ -182,6 +182,7 @@ setup(name=name,
           'shellinabox = slapos.recipe.shellinabox:Recipe',
           'signalwrapper= slapos.recipe.signal_wrapper:Recipe',
           'simplelogger = slapos.recipe.simplelogger:Recipe',
+          'simplehttpserver = slapos.recipe.simplehttpserver:Recipe',
           'siptester = slapos.recipe.siptester:SipTesterRecipe',
           'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
           'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',

slapos/recipe/kvm/template/kvm_run.in

@@ -46,6 +46,8 @@ map_storage_list = []
 etc_directory = '%(etc-directory)s'.strip()
 httpd_port = %(httpd-port)s
 netcat_bin = '%(netcat-binary)s'.strip()
+cluster_doc_host = '%(cluster-doc-host)s'
+cluster_doc_port = %(cluster-doc-port)s
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -202,6 +204,9 @@ if use_nat == 'True':
   if httpd_port > 0:
     rules += ',guestfwd=tcp:10.0.2.100:80-cmd:%%s %%s %%s' %% (netcat_bin,
                                                         listen_ip, httpd_port)
+  if cluster_doc_host and cluster_doc_port > 0:
+    rules += ',guestfwd=tcp:10.0.2.101:443-cmd:%%s %%s %%s' %% (netcat_bin,
+                                           cluster_doc_host, cluster_doc_port)
   nat_network_parameter = ['-netdev', rules,
           '-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, mac_address)]
 if use_tap == 'True':

slapos/recipe/simplehttpserver/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import string, random
+import os
+
+class Recipe(GenericBaseRecipe):
+
+  def __init__(self, buildout, name, options):
+
+    base_path = options['base-path']
+    pool = string.letters + string.digits
+    hash_string = ''.join(random.choice(pool) for i in xrange(64))
+    path = os.path.join(base_path, hash_string)
+
+    if os.path.exists(base_path):
+      path_list = os.listdir(base_path)
+      if len(path_list) == 1:
+        hash_string = path_list[0]
+        path = os.path.join(base_path, hash_string)
+      elif len(path_list) > 1:
+        raise ValueError("Folder %s should contain 0 or 1 element." % base_path)
+
+    options['root-dir'] = path
+    options['path'] = hash_string
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+
+  def install(self):
+
+    if not os.path.exists(self.options['root-dir']):
+      os.mkdir( self.options['root-dir'] )
+    parameters = {
+      'host': self.options['host'],
+      'port': int(self.options['port']),
+      'cwd': self.options['base-path'],
+      'log-file': self.options['log-file'],
+      'cert-file': self.options['cert-file'],
+      'key-file': self.options['key-file']
+    }
+
+    server = self.createPythonScript(
+        self.options['wrapper'].strip(),
+        '%s.simplehttpserver.run' % __name__, parameters
+      )
+
+    return [server]

slapos/recipe/simplehttpserver/simplehttpserver.py

+from SimpleHTTPServer import SimpleHTTPRequestHandler
+from BaseHTTPServer import HTTPServer
+import ssl
+import os
+import logging
+from netaddr import valid_ipv4, valid_ipv6
+import socket
+
+class ServerHandler(SimpleHTTPRequestHandler):
+
+  def respond(self, code=200, type='text/plain'):
+    self.send_response(code)
+    self.send_header("Content-type", type)
+    self.end_headers()
+
+  def do_GET(self):
+    logging.info('%s - GET: %s \n%s' % (self.client_address[0], self.path, self.headers))
+    if not self.path or self.path == '/':
+      # no access to root path
+      self.respond(403)
+      self.wfile.write("Forbidden")
+      return
+    SimpleHTTPRequestHandler.do_GET(self)
+
+class HTTPServerV6(HTTPServer):
+  address_family = socket.AF_INET6
+  
+
+def run(args):
+  
+  # minimal web server.  serves files relative to the
+  # current directory.
+  logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                            filename=args['log-file'] ,level=logging.INFO)
+  
+  port = args['port']
+  host = args['host']
+  os.chdir(args['cwd'])
+  
+  Handler = ServerHandler
+  
+  if valid_ipv6(host):
+    server = HTTPServerV6
+  else:
+    server = HTTPServer
+  
+  httpd = server((host, port), Handler)
+  if args.has_key('cert-file') and args.has_key('key-file'):
+    httpd.socket = ssl.wrap_socket (httpd.socket, 
+                                     server_side=True,
+                                     certfile=args['cert-file'],
+                                     keyfile=args['key-file'])
+
+  logging.info("Starting simple http server at https://%s:%s" % (host, port))
+  httpd.serve_forever()

software/kvm/common.cfg

@@ -87,7 +87,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = c5fd8ed5878901233d39d006093a6b17
+md5sum = cf67212d3155767d0d0d8a6d75d2d8ad
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -95,7 +95,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = 42763900fce72f13bf23341774e5d097
+md5sum = 3e3354844b2052609e3c49eca03b607e
 download-only = true
 on-update = true
 
@@ -103,7 +103,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = c20056662fe3aa0fa6ff7ae9a950f325
+md5sum = 3523fd4d1b9038ab8dca95fe5454778d
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster.cfg.jinja2.in

 {% set publish_dict = {} -%}
+{% set part_list = [] -%}
+{% set ipv6 = (ipv6 | list)[0] -%}
 {% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
 {% set slave_frontend_dict = slapparameter_dict.get('slave-frontend', {}) -%}
 {% set slave_frontend_sr = slave_frontend_dict.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') -%}
 {% set slave_frontend_stype = slave_frontend_dict.get('software-type', 'default') -%}
 {% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
 {% set kvm_instance_dict = {} -%}
+{% set kvm_hostname_list = [] -%}
 
 [request-common]
 recipe = slapos.cookbook:request
@@ -59,6 +62,11 @@ config-external-disk-format = {{ dumps(kvm_parameter_dict.get('external-disk-for
 config-enable-http-server = {{ dumps(kvm_parameter_dict.get('enable-http-server', True)) }}
 config-httpd-port = {{ dumps(kvm_parameter_dict.get('httpd-port', 8081)) }}
 
+# Enable simple http server on ipv6 so all VMs will access it
+config-document-host = ${http-server:host}
+config-document-port = ${http-server:port}
+config-document-path = ${http-server:path}
+
 return = 
   backend-url
   url
@@ -71,6 +79,7 @@ return =
 {{ '  ' }}tap-ipv4
 
 {% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
+{% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% endif -%}
 {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
@@ -104,7 +113,7 @@ name = Frontend {{ name }}
 software-type = {{ slave_frontend_stype }}
 slave = true
 config-url = {{ url }}
-config-domain = {{ dumps(frontend_parameter_dict.get('domain', '')) }}
+config-custom_domain = {{ dumps(frontend_parameter_dict.get('domain', '')) }}
 config-enable_cache = {{ dumps(frontend_parameter_dict.get('enable-cache', False)) }}
 config-https-only = {{ dumps(frontend_parameter_dict.get('https-only', False)) }}
 {%  if frontend_parameter_dict.get('type', '') -%}
@@ -122,17 +131,58 @@ sla-instance_guid = {{ slave_frontend_iguid }}
 {%   endif -%}
 {% endfor %}
 
+# Enable simple http server on ipv6 so all VMs will access it
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+document = ${:srv}/document
+ssl = ${:etc}/ssl
+
+[http-ssl]
+recipe = plone.recipe.command
+command = "{{ openssl_executable_location }}" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
+key = ${directory:ssl}/key
+cert = ${directory:ssl}/cert
+update-command = 
+stop-on-error = true
+
+[http-server]
+recipe = slapos.cookbook:simplehttpserver
+host = {{ ipv6 }}
+port = 9002
+base-path = ${directory:document}
+wrapper = ${directory:services}/simple-http-server
+log-file = ${directory:log}/http.log
+cert-file = ${http-ssl:cert}
+key-file = ${http-ssl:key}
+
+[write-vm-hostname]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_content }}
+filename = hosts
+rendered = ${http-server:root-dir}/${:filename}
+context =
+    raw content_list {{ kvm_hostname_list | join('#') }}
+    raw sep #
+
 [publish]
 recipe = slapos.cookbook:publish
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}
-{% set disk_number = len(storage_dict) -%}
-{% if disk_number > 0 -%}
-1_info = It is possible to mount up to {{ disk_number }} external disk to your virtual machine. See parameter 'external-disk-number'
-{% endif %}
 [buildout]
-parts = publish
+parts = 
+  http-server
+  write-vm-hostname
+  publish
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
  
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}

software/kvm/instance-kvm.cfg.jinja2

@@ -16,6 +16,9 @@ parts =
   cron
 #  cron-entry-monitor
   frontend-promise
+{% if slapparameter_dict.get('document-host', '') %}
+  cluster-url-path
+{% endif -%}
 {% if slapparameter_dict.get('enable-http-server', 'False') == 'True' %}
   httpd
   httpd-promise
@@ -129,6 +132,15 @@ httpd-port = ${slap-parameter:httpd-port}
 {% else -%}
 httpd-port = 0
 {% endif -%}
+
+# Main instance document server info
+{% if slapparameter_dict.get('document-host', '') and slapparameter_dict.get('document-port', '') -%}
+cluster-doc-host = ${tunnel-cluster-url:ipv4}
+cluster-doc-port = ${tunnel-cluster-url:ipv4-port}
+{% else -%}
+cluster-doc-host = 
+cluster-doc-port = 0
+{% endif -%}
 netcat-binary = {{ netcat_bin }}
 
 [kvm-vnc-promise]
@@ -257,6 +269,8 @@ recipe = slapos.cookbook:publish
 ipv6 = ${slap-network-information:global-ipv6}
 backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-instance:vnc-passwd}
 url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-instance:vnc-passwd}
+{% set disk_number = len(storage_dict) -%}
+maximum-extra-disk-amount = {{ disk_number }}
 {% set iface = 'eth0' -%}
 {% if slapparameter_dict.get('use-nat', 'True') == 'True' -%}
 {%   set iface = 'eth1' -%}
@@ -345,6 +359,28 @@ command =
   echo "${:local-ipv4}" > ${:path-ip}
 update-command = ${:command}
 
+# To access documents of main instance (in case of kvm-cluster) through http
+[cluster-url-path]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_content }}
+filename = cluster.hash
+rendered = ${directory:public}/${:filename}
+hash-url = https://10.0.2.101:443/{{ slapparameter_dict.get('document-path', '') }}
+context =
+    key content_list :hash-url
+    raw sep #
+
+# This 6to4 tunnel help to access document url in ipv4
+[tunnel-cluster-url]
+recipe = slapos.cookbook:ipv4toipv6
+ipv6 = {{ slapparameter_dict.get('document-host', '') }}
+ipv4 = ${slap-network-information:local-ipv4}
+ipv6-port = {{ slapparameter_dict.get('document-port', '') }}
+ipv4-port = 16936
+shell-path = {{ dash_executable_location }}
+6tunnel-path = {{ sixtunnel_executable_location }}
+runner-path = ${directory:services}/6tunnel-cluster
+
 [slap-parameter]
 # Default values if not specified
 frontend-software-type = frontend

software/kvm/instance.cfg.in

@@ -55,6 +55,7 @@ context =
     key storage_dict slap-configuration:storage-dict
     key slapparameter_dict slap-configuration:configuration
     key computer_id slap-configuration:computer
+    raw openssl_executable_location ${openssl:location}/bin/openssl
     $${:extra-context}
 
 [dynamic-template-kvm-cluster-parameters]
@@ -65,6 +66,7 @@ template = ${template-kvm-cluster:location}/instance-kvm-cluster.cfg.jinja2.in
 filename = template-kvm-cluster.cfg
 extra-context =
   section parameter_dict dynamic-template-kvm-cluster-parameters
+  raw template_content ${template-content:location}/${template-content:filename}
 
 [dynamic-template-kvm]
 recipe = slapos.recipe.template:jinja2