kvm: Download virtual-hard-drive asynchronously

For sanity virtual-hard-drive-md5sum became mandatory.

virtual-hard-drive-gzipped case is covered.

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 4f71b7616b9f4a2f968e89326a237768
+md5sum = bf3f2d45dcc55b003c03959895f1c29a
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = adf05b4255269bc7d0eec479424405e4
+md5sum = 2909821e8eebfe8928a0a0cfb94c0348
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -359,7 +359,7 @@
             },
             "virtual-hard-drive-md5sum": {
               "title": "Checksum of virtual hard drive",
-              "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
+              "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
               "type": "string"
             },
             "virtual-hard-drive-gzipped": {

software/kvm/instance-kvm-input-schema.json

@@ -160,7 +160,7 @@
     },
     "virtual-hard-drive-md5sum": {
       "title": "Checksum of virtual hard drive",
-      "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
+      "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
       "type": "string"
     },
     "virtual-hard-drive-gzipped": {

software/kvm/instance-kvm.cfg.jinja2

@@ -17,6 +17,8 @@
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
 {% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
+{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
+{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
 {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
 {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
 {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
@@ -56,6 +58,11 @@ public = ${:srv}/public/
 cron-entries = ${:etc}/cron.d
 crontabs = ${:etc}/crontabs
 cronstamps = ${:etc}/cronstamps
+{%- if virtual_hard_drive_url_enabled %}
+virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
+virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
+virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
+{%- endif %}
 {%- if boot_image_url_list_enabled %}
 boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
 boot-image-url-list-var = ${:var}/boot-image-url-list
@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
 ## boot-image-url-list support END
 {% endif %} {# if boot_image_url_list_enabled #}
 
+{% if virtual_hard_drive_url_enabled %}
+## virtual-hard-drive-url support BEGIN
+[empty-file-state-base-virtual-promise]
+<= monitor-promise-base
+module = check_file_state
+name = ${:_buildout_section_name_}.py
+config-state = empty
+# It's very hard to put the username and password correctly, after schema://
+# and before the host, as it's not the way how one can use monitor provided
+# information, so just show the information in the URL
+config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
+
+[virtual-hard-drive-url-source-config]
+recipe = slapos.recipe.template:jinja2
+template = inline:
+{%- raw %}
+  {{ virtual_hard_drive_url }}
+{% endraw -%}
+{# Enforce md5sum on virtual-hard-drive-url #}
+virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
+context =
+  key virtual_hard_drive_url :virtual-hard-drive-url
+rendered = ${directory:etc}/virtual-hard-drive-url.conf
+
+[virtual-hard-drive-url-processed-config]
+# compares if the current configuration has been used by
+# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
+# information
+recipe = slapos.recipe.build
+install =
+  import os
+  import hashlib
+  if not os.path.exists(location):
+    os.mkdir(location)
+  with open('${:state-file}', 'w') as state_handler:
+    try:
+      with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
+        config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
+        processed_md5sum = processed_handler.read()
+        if config_md5sum == processed_md5sum:
+          state_handler.write('')
+        else:
+          state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
+    except Exception as e:
+      state_handler.write(str(e))
+
+update = ${:install}
+config-file = ${virtual-hard-drive-url-source-config:rendered}
+state-filename = virtual-hard-drive-url-processed-config.state
+state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
+processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
+
+[virtual-hard-drive-url-processed-config-promise]
+# promise to check if the configuration provided by the user has been already
+# processed by the virtual-hard-drive-url-download script, which runs asynchronously
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-processed-config:state-filename}
+config-filename = ${virtual-hard-drive-url-processed-config:state-file}
+
+[virtual-hard-drive-url-json-config]
+# generates json configuration from user configuration
+recipe = plone.recipe.command
+command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
+update-command = ${:command}
+rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
+error-state-filename = virtual-hard-drive-url-json-config-error.txt
+error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
+
+[virtual-hard-drive-url-config-state-promise]
+# promise to check if configuration has been parsed without errors
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-json-config:error-state-filename}
+config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
+
+[virtual-hard-drive-url-download-wrapper]
+# wrapper to execute virtual-hard-drive-url-download on each run
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
+command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
+md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
+md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
+error-state-filename = virtual-hard-drive-url-download-controller-error.text
+error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[virtual-hard-drive-url-download-md5sum-promise]
+# promise to report errors with problems with calculating md5sum of the
+# downloaded images
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
+config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
+
+[virtual-hard-drive-url-download-state-promise]
+# promise to report errors during download
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
+config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
+## virtual-hard-drive-url support END
+{% endif %} {# if virtual_hard_drive_url_enabled #}
+
 [kvm-controller-parameter-dict]
 python-path = {{ python_eggs_executable }}
 vnc-passwd = ${gen-passwd:passwd}
@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
 vnc-port = 5901
 
 default-cdrom-iso = {{ debian_amd64_netinst_location }}
+{% if virtual_hard_drive_url_enabled %}
+virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
+{% else %}
+virtual-hard-drive-url-json-config =
+{% endif %}
 {% if boot_image_url_list_enabled %}
 boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
 {% else %}
@@ -1108,6 +1220,13 @@ parts =
   cron-service
   cron-entry-logrotate
   frontend-promise
+{% if virtual_hard_drive_url_enabled %}
+  virtual-hard-drive-url-download-wrapper
+  virtual-hard-drive-url-config-state-promise
+  virtual-hard-drive-url-download-md5sum-promise
+  virtual-hard-drive-url-download-state-promise
+  virtual-hard-drive-url-processed-config-promise
+{% endif %}
 {% if boot_image_url_list_enabled %}
   boot-image-url-list-download-wrapper
   boot-image-url-list-config-state-promise

software/kvm/template/template-kvm-run.in

@@ -6,10 +6,6 @@ import hashlib
 import os
 import socket
 import subprocess
-try:
-  from urllib.request import FancyURLopener
-except ImportError:
-  from urllib import FancyURLopener
 import gzip
 import shutil
 from random import shuffle
@@ -17,8 +13,6 @@ import glob
 import re
 import json
 
-import ssl
-
 # XXX: give all of this through parameter, don't use this as template, but as module
 qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
 qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
@@ -32,9 +26,6 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
               {{ parameter_dict.get("nbd2-port") }}))
 default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
 
-virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
-virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
-virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
 nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
 use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
 use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
@@ -98,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
 
 boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
 boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
-
-if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
-  opener = FancyURLopener(context=ssl._create_unverified_context())
-else:
-  opener = FancyURLopener({})
+virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
+virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -163,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
       lf.write('%s' % external_disk_number)
   return id_list, external_disk_number
 
-# Download existing hard drive if needed at first boot
-if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '':
-  print('Downloading virtual hard drive...')
-  try:
-    downloaded_disk = disk_info_list[0]['path']
+# Use downloaded virtual-hard-drive-url
+if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
+  print('Using virtual hard drive...')
+  with open(virtual_hard_drive_url_json_config) as fh:
+    image_config = json.load(fh)
+  if image_config['error-amount'] == 0:
+    image = image_config['image-list'][0]
+    downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
+    # previous version was using disk in place, but here it would result with
+    # redownload, so copy it
     if virtual_hard_drive_gzipped == 'true':
-      downloaded_disk = '%s.gz' % disk_info_list[0]['path']
-    opener.retrieve(virtual_hard_drive_url, downloaded_disk)
-  except:
-    if os.path.exists(downloaded_disk):
-      os.remove(downloaded_disk)
+      try:
+        with open(disk_info_list[0]['path'], 'wb') as d_fh:
+          with gzip.open(downloaded_image, 'rb') as s_fh:
+            shutil.copyfileobj(s_fh, d_fh)
+      except Exception:
+        if os.path.exists(disk_info_list[0]['path']):
+          os.unlink(disk_info_list[0]['path'])
         raise
-  md5sum = virtual_hard_drive_md5sum.strip()
-  if md5sum:
-    print('Checking MD5 checksum...')
-    local_md5sum = md5Checksum(downloaded_disk)
-    if local_md5sum != md5sum:
-      os.remove(downloaded_disk)
-      raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % (
-          local_md5sum, md5sum))
-    print('MD5sum check passed.')
     else:
-    print('Warning: not checksum specified.')
-  if downloaded_disk.endswith('.gz'):
       try:
-      with open(disk_info_list[0]['path'], 'w') as disk:
-        with gzip.open(downloaded_disk, 'rb') as disk_gz:
-          shutil.copyfileobj(disk_gz, disk)
+        shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
       except Exception:
         if os.path.exists(disk_info_list[0]['path']):
-        os.remove(disk_info_list[0]['path'])
+          os.unlink(disk_info_list[0]['path'])
         raise
-    os.remove(downloaded_disk)
+  else:
+    raise ValueError('Problems with virtual-hard-drive-url download')
 
 # Create disk if doesn't exist
 # XXX: move to Buildout profile

software/kvm/test/test.py

@@ -417,10 +417,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
           "test-machine2": dict(bootstrap_machine_param_dict, **{
               # Debian 9 image
               "virtual-hard-drive-url":
-              "http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2"
-              "eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f940"
-              "71e073f9980cc61badd6d2427f592e6e8",
-              "virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99"
+              "http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
+              "57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
+              "a508930046a78cffea6e0e29d03345333cc",
+              "virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
+              "virtual-hard-drive-gzipped": True,
           })
       }
     }))}