software/kvm: Simple "Boot image" selection

Thanks to using to draft-06 schema oneOf the "Boot image" field provides a
list of images to select one to boot.

software/kvm/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 2fb085450d33e2674b0c1d4ab2055762
+md5sum = 23493c541efef97ac5fe435114910b8e
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = be228c9e39682be53aaba491f858f848
+md5sum = bdf8549a76ec61e125d51a05e611e004
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 536c3b208fec9fa29ba6223432cd3509
+md5sum = 84e40e43a74559f3b31677c2b2052360
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
@@ -87,4 +87,4 @@ md5sum = 7e4b54f8172c364bd12d28b07f8b1600
 
 [image-download-config-creator]
 _update_hash_filename_ = template/image-download-config-creator.py
-md5sum = 7f6cd75096695922fddbba1c9292cef5
+md5sum = 54261e418ab9860efe73efd514c4d47f

software/kvm/instance-kvm-cluster-input-schema.json

@@ -48,6 +48,40 @@
           "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
           "type": "string",
           "textarea": "true"
+        },
+        "boot-image-url-select": {
+          "title": "Boot image",
+          "type": "array",
+          "oneOf": [
+            {
+              "const": ["https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"],
+              "title": "Debian Buster 10.5 netinst x86_64"
+            },
+            {
+              "const": ["https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"],
+              "title": "Centos 8.2004 Minimal x86_64"
+            },
+            {
+              "const": ["https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"],
+              "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+            },
+            {
+              "const": ["https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"],
+              "title": "openSUSE Leap 15.2 NET x86_64"
+            },
+            {
+              "const": ["https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"],
+              "title": "Arch Linux 2020.09.01 x86_64"
+            },
+            {
+              "const": ["https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"],
+              "title": "Fedora Server 32-1.6 netinst x86_64"
+            },
+            {
+              "const": ["https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"],
+              "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+            }
+          ]
         }
       },
       "type": "object"

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -131,6 +131,10 @@ config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layo
 {#- play nice: if parameter was not constructed by the original request, do not send it at all #}
 config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
 {%- endif %}
+{%- if 'boot-image-url-select' in kvm_parameter_dict %}
+{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
+config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
+{%- endif %}
 config-type = cluster
 
 {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%}

software/kvm/instance-kvm-input-schema.json

 {
   "type": "object",
-  "$schema": "http://json-schema.org/draft-04/schema",
+  "$schema": "http://json-schema.org/draft-06/schema",
   "title": "Input Parameters",
   "properties": {
     "enable-device-hotplug": {
@@ -371,6 +371,54 @@
       "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
       "type": "string",
       "textarea": "true"
+    },
+    "boot-image-url-select": {
+      "title": "Boot image",
+      "type": "array",
+      "oneOf": [
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+          ],
+          "title": "Debian Buster 10.5 netinst x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
+          ],
+          "title": "Centos 8.2004 Minimal x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
+          ],
+          "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
+          ],
+          "title": "openSUSE Leap 15.2 NET x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
+          ],
+          "title": "Arch Linux 2020.09.01 x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
+          ],
+          "title": "Fedora Server 32-1.6 netinst x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
+          ],
+          "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+        }
+      ]
     }
   }
 }

software/kvm/instance-kvm.cfg.jinja2

@@ -10,6 +10,7 @@
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
 {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
+{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
 {% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
 {% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
 {% set extends_list = [] -%}
@@ -59,6 +60,11 @@ boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
 boot-image-url-list-var = ${:var}/boot-image-url-list
 boot-image-url-list-expose = ${monitor-directory:private}/boot-image-url-list
 {%- endif %}
+{%- if boot_image_url_select_enabled %}
+boot-image-url-select-repository = ${:srv}/boot-image-url-select-repository
+boot-image-url-select-var = ${:var}/boot-image-url-select
+boot-image-url-select-expose = ${monitor-directory:private}/boot-image-url-select
+{%- endif %}
 
 [create-mac]
 recipe = slapos.cookbook:generate.mac
@@ -73,9 +79,116 @@ recipe = slapos.cookbook:generate.password
 storage-path = ${directory:srv}/passwd
 bytes = 8
 
+{% if boot_image_url_select_enabled %}
+## boot-image-url-select support BEGIN
+[empty-file-state-base-select-promise]
+<= monitor-promise-base
+module = check_file_state
+name = ${:_buildout_section_name_}.py
+config-state = empty
+# It's very hard to put the username and password correctly, after schema://
+# and before the host, as it's not the way how one can use monitor provided
+# information, so just show the information in the URL
+config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
+
+[boot-image-url-select-source-config]
+# generates configuration of the image from the user parameter
+# special "magic" is used, to properly support multiline boot-image-url-select
+# but in the same time correctly generate the configuration file
+recipe = slapos.recipe.template:jinja2
+template = inline:
+{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
+{%- raw %}
+  {%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
+  {%- if boot_image_url_select == 'None' %}
+  {#- That's insane protection, is it 'None' because of type = array? #}
+  {%-   set boot_image_url_select = '' %}
+  {%- endif %}
+  {{ boot_image_url_select }}
+{% endraw -%}
+context =
+  section slap_parameter slap-parameter
+rendered = ${directory:etc}/boot-image-url-select.json
+
+[boot-image-url-select-processed-config]
+# compares if the current configuration has been used by
+# the boot-image-url-select-download, if not, exposes it as not empty file with
+# information
+recipe = slapos.recipe.build
+install =
+  import os
+  import hashlib
+  if not os.path.exists(location):
+    os.mkdir(location)
+  with open('${:state-file}', 'w') as state_handler:
+    try:
+      with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
+        config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
+        processed_md5sum = processed_handler.read()
+        if config_md5sum == processed_md5sum:
+          state_handler.write('')
+        else:
+          state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
+    except Exception as e:
+      state_handler.write(str(e))
+
+update = ${:install}
+config-file = ${boot-image-url-select-source-config:rendered}
+state-filename = boot-image-url-select-processed-config.state
+state-file = ${directory:boot-image-url-select-expose}/${:state-filename}
+processed-md5sum = ${directory:boot-image-url-select-var}/update-image-processed.md5sum
+
+[boot-image-url-select-processed-config-promise]
+# promise to check if the configuration provided by the user has been already
+# processed by the boot-image-url-select-download script, which runs asynchronously
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-processed-config:state-filename}
+config-filename = ${boot-image-url-select-processed-config:state-file}
+
+[boot-image-url-select-json-config]
+# generates json configuration from user configuration
+recipe = plone.recipe.command
+command = {{ python_executable }} {{ image_download_config_creator }} ${boot-image-url-select-source-config:rendered} ${:rendered} ${directory:boot-image-url-select-repository} ${:error-state-file}
+update-command = ${:command}
+rendered = ${directory:boot-image-url-select-var}/boot-image-url-select.json
+error-state-filename = boot-image-url-select-json-config-error.txt
+error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
+
+[boot-image-url-select-config-state-promise]
+# promise to check if configuration has been parsed without errors
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-json-config:error-state-filename}
+config-filename = ${boot-image-url-select-json-config:error-state-file}
+
+[boot-image-url-select-download-wrapper]
+# wrapper to execute boot-image-url-select-download on each run
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/boot-image-url-select-updater
+command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-select-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-select-processed-config:processed-md5sum}
+md5sum-state-filename = boot-image-url-select-download-controller-md5sum-fail.json
+md5sum-state-file = ${directory:boot-image-url-select-expose}/${:md5sum-state-filename}
+error-state-filename = boot-image-url-select-download-controller-error.text
+error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[boot-image-url-select-download-md5sum-promise]
+# promise to report errors with problems with calculating md5sum of the
+# downloaded images
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-download-wrapper:md5sum-state-filename}
+config-filename = ${boot-image-url-select-download-wrapper:md5sum-state-file}
+
+[boot-image-url-select-download-state-promise]
+# promise to report errors during download
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-download-wrapper:error-state-filename}
+config-filename = ${boot-image-url-select-download-wrapper:error-state-file}
+## boot-image-url-select support END
+{% endif %} {# if boot_image_url_select_enabled #}
+
 {% if boot_image_url_list_enabled %}
 ## boot-image-url-list support BEGIN
-[empty-file-state-base-promise]
+[empty-file-state-base-list-promise]
 <= monitor-promise-base
 module = check_file_state
 name = ${:_buildout_section_name_}.py
@@ -130,7 +243,7 @@ processed-md5sum = ${directory:boot-image-url-list-var}/update-image-processed.m
 [boot-image-url-list-processed-config-promise]
 # promise to check if the configuration provided by the user has been already
 # processed by the boot-image-url-list-download script, which runs asynchronously
-<= empty-file-state-base-promise
+<= empty-file-state-base-list-promise
 filename = ${boot-image-url-list-processed-config:state-filename}
 config-filename = ${boot-image-url-list-processed-config:state-file}
 
@@ -145,7 +258,7 @@ error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filena
 
 [boot-image-url-list-config-state-promise]
 # promise to check if configuration has been parsed without errors
-<= empty-file-state-base-promise
+<= empty-file-state-base-list-promise
 filename = ${boot-image-url-list-json-config:error-state-filename}
 config-filename = ${boot-image-url-list-json-config:error-state-file}
 
@@ -163,13 +276,13 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 [boot-image-url-list-download-md5sum-promise]
 # promise to report errors with problems with calculating md5sum of the
 # downloaded images
-<= empty-file-state-base-promise
+<= empty-file-state-base-list-promise
 filename = ${boot-image-url-list-download-wrapper:md5sum-state-filename}
 config-filename = ${boot-image-url-list-download-wrapper:md5sum-state-file}
 
 [boot-image-url-list-download-state-promise]
 # promise to report errors during download
-<= empty-file-state-base-promise
+<= empty-file-state-base-list-promise
 filename = ${boot-image-url-list-download-wrapper:error-state-filename}
 config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
 ## boot-image-url-list support END
@@ -200,6 +313,11 @@ boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
 {% else %}
 boot-image-url-list-json-config =
 {% endif %}
+{% if boot_image_url_select_enabled %}
+boot-image-url-select-json-config = ${boot-image-url-select-json-config:rendered}
+{% else %}
+boot-image-url-select-json-config =
+{% endif %}
 nbd-host = ${slap-parameter:nbd-host}
 nbd-port = ${slap-parameter:nbd-port}
 nbd2-host = ${slap-parameter:nbd2-host}
@@ -957,6 +1075,13 @@ parts =
   boot-image-url-list-download-state-promise
   boot-image-url-list-processed-config-promise
 {% endif %}
+{% if boot_image_url_select_enabled %}
+  boot-image-url-select-download-wrapper
+  boot-image-url-select-config-state-promise
+  boot-image-url-select-download-md5sum-promise
+  boot-image-url-select-download-state-promise
+  boot-image-url-select-processed-config-promise
+{% endif %}
 {% if additional_frontend %}
   frontend-additional-promise
 {% endif %}

software/kvm/template/image-download-config-creator.py

@@ -24,7 +24,18 @@ if __name__ == "__main__":
     image_number = 0
     data = fh.read()
     configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest()
-    for entry in data.decode('utf-8').split():
+    if source_configuration.endswith('.json'):
+      data = data.strip()
+      data_list = []
+      if len(data):
+        try:
+          data_list = json.loads(data)
+        except Exception:
+          error_list.append('ERR: Data is not a JSON')
+          data_list = []
+    else:
+      data_list = data.decode('utf-8').split()
+    for entry in data_list:
       split_entry = entry.split('#')
       if len(split_entry) != 2:
         error_list.append('ERR: entry %r is incorrect' % (entry,))

software/kvm/template/template-kvm-run.in

@@ -100,6 +100,7 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
 logfile = '{{ parameter_dict.get("log-file") }}'
 
 boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
+boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
 
 if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
   opener = FancyURLopener(context=ssl._create_unverified_context())
@@ -370,6 +371,18 @@ for nbd_ip, nbd_port in nbd_list:
           '-drive',
           'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
 else:
+  if boot_image_url_select_json_config:
+    # Support boot-image-url-select
+    with open(boot_image_url_select_json_config) as fh:
+      image_config = json.load(fh)
+    if image_config['error-amount'] == 0:
+      for image in sorted(image_config['image-list'], key=lambda k: k['link']):
+        link = os.path.join(image_config['destination-directory'], image['link'])
+        if os.path.exists(link) and os.path.islink(link):
+          kvm_argument_list.extend([
+            '-drive',
+            'file=%s,media=cdrom' % (link,)
+          ])
   if boot_image_url_list_json_config:
     # Support boot-image-url-list
     with open(boot_image_url_list_json_config) as fh: