Skip to content

slapos
slapos
Commit ca15bbff authored by Łukasz Nowak's avatar Łukasz Nowak Committed by Léo-Paul Géneau
Browse files
Options

software/kvm: Simple "Boot image" selection 

Thanks to using to draft-06 schema oneOf the "Boot image" field provides a
list of images to select one to boot.
parent 9b9a875e
Show whitespace changes
Inline Side-by-side
Showing with 435 additions and 46 deletions
software/kvm/buildout.hash.cfg
View file @ ca15bbff
...@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe ...@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 2fb085450d33e2674b0c1d4ab2055762 md5sum = 23493c541efef97ac5fe435114910b8e
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = be228c9e39682be53aaba491f858f848 md5sum = bdf8549a76ec61e125d51a05e611e004
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = 536c3b208fec9fa29ba6223432cd3509 md5sum = 84e40e43a74559f3b31677c2b2052360
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
...@@ -87,4 +87,4 @@ md5sum = 7e4b54f8172c364bd12d28b07f8b1600 ...@@ -87,4 +87,4 @@ md5sum = 7e4b54f8172c364bd12d28b07f8b1600
[image-download-config-creator] [image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py _update_hash_filename_ = template/image-download-config-creator.py
md5sum = 7f6cd75096695922fddbba1c9292cef5 md5sum = 54261e418ab9860efe73efd514c4d47f
software/kvm/instance-kvm-cluster-input-schema.json
View file @ ca15bbff
...@@ -48,6 +48,40 @@ ...@@ -48,6 +48,40 @@
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.", "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string", "type": "string",
"textarea": "true" "textarea": "true"
},
"boot-image-url-select": {
"title": "Boot image",
"type": "array",
"oneOf": [
{
"const": ["https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"],
"title": "Debian Buster 10.5 netinst x86_64"
},
{
"const": ["https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"],
"title": "Centos 8.2004 Minimal x86_64"
},
{
"const": ["https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"],
"title": "Ubuntu Focal 20.04.1 Live Server x86_64"
},
{
"const": ["https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"],
"title": "openSUSE Leap 15.2 NET x86_64"
},
{
"const": ["https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"],
"title": "Arch Linux 2020.09.01 x86_64"
},
{
"const": ["https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"],
"title": "Fedora Server 32-1.6 netinst x86_64"
},
{
"const": ["https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"],
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
} }
}, },
"type": "object" "type": "object"
......
software/kvm/instance-kvm-cluster.cfg.jinja2.in
View file @ ca15bbff
...@@ -131,6 +131,10 @@ config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layo ...@@ -131,6 +131,10 @@ config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layo
{#- play nice: if parameter was not constructed by the original request, do not send it at all #} {#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }} config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
{%- endif %} {%- endif %}
{%- if 'boot-image-url-select' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
{%- endif %}
config-type = cluster config-type = cluster
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%} {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%}
......
software/kvm/instance-kvm-input-schema.json
View file @ ca15bbff
{ {
"type": "object", "type": "object",
"$schema": "http://json-schema.org/draft-04/schema", "$schema": "http://json-schema.org/draft-06/schema",
"title": "Input Parameters", "title": "Input Parameters",
"properties": { "properties": {
"enable-device-hotplug": { "enable-device-hotplug": {
...@@ -371,6 +371,54 @@ ...@@ -371,6 +371,54 @@
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.", "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string", "type": "string",
"textarea": "true" "textarea": "true"
},
"boot-image-url-select": {
"title": "Boot image",
"type": "array",
"oneOf": [
{
"const": [
"https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
],
"title": "Debian Buster 10.5 netinst x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
],
"title": "Centos 8.2004 Minimal x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
],
"title": "Ubuntu Focal 20.04.1 Live Server x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
],
"title": "openSUSE Leap 15.2 NET x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
],
"title": "Arch Linux 2020.09.01 x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
],
"title": "Fedora Server 32-1.6 netinst x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
],
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
} }
} }
} }
software/kvm/instance-kvm.cfg.jinja2
View file @ ca15bbff
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%} {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %} {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %} {% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %} {% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
{% set extends_list = [] -%} {% set extends_list = [] -%}
...@@ -59,6 +60,11 @@ boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository ...@@ -59,6 +60,11 @@ boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list boot-image-url-list-var = ${:var}/boot-image-url-list
boot-image-url-list-expose = ${monitor-directory:private}/boot-image-url-list boot-image-url-list-expose = ${monitor-directory:private}/boot-image-url-list
{%- endif %} {%- endif %}
{%- if boot_image_url_select_enabled %}
boot-image-url-select-repository = ${:srv}/boot-image-url-select-repository
boot-image-url-select-var = ${:var}/boot-image-url-select
boot-image-url-select-expose = ${monitor-directory:private}/boot-image-url-select
{%- endif %}
[create-mac] [create-mac]
recipe = slapos.cookbook:generate.mac recipe = slapos.cookbook:generate.mac
...@@ -73,9 +79,116 @@ recipe = slapos.cookbook:generate.password ...@@ -73,9 +79,116 @@ recipe = slapos.cookbook:generate.password
storage-path = ${directory:srv}/passwd storage-path = ${directory:srv}/passwd
bytes = 8 bytes = 8
{% if boot_image_url_select_enabled %}
## boot-image-url-select support BEGIN
[empty-file-state-base-select-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-select-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-select
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
{%- raw %}
{%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
{%- if boot_image_url_select == 'None' %}
{#- That's insane protection, is it 'None' because of type = array? #}
{%- set boot_image_url_select = '' %}
{%- endif %}
{{ boot_image_url_select }}
{% endraw -%}
context =
section slap_parameter slap-parameter
rendered = ${directory:etc}/boot-image-url-select.json
[boot-image-url-select-processed-config]
# compares if the current configuration has been used by
# the boot-image-url-select-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${boot-image-url-select-source-config:rendered}
state-filename = boot-image-url-select-processed-config.state
state-file = ${directory:boot-image-url-select-expose}/${:state-filename}
processed-md5sum = ${directory:boot-image-url-select-var}/update-image-processed.md5sum
[boot-image-url-select-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the boot-image-url-select-download script, which runs asynchronously
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-processed-config:state-filename}
config-filename = ${boot-image-url-select-processed-config:state-file}
[boot-image-url-select-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${boot-image-url-select-source-config:rendered} ${:rendered} ${directory:boot-image-url-select-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:boot-image-url-select-var}/boot-image-url-select.json
error-state-filename = boot-image-url-select-json-config-error.txt
error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
[boot-image-url-select-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-json-config:error-state-filename}
config-filename = ${boot-image-url-select-json-config:error-state-file}
[boot-image-url-select-download-wrapper]
# wrapper to execute boot-image-url-select-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/boot-image-url-select-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-select-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-select-processed-config:processed-md5sum}
md5sum-state-filename = boot-image-url-select-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:boot-image-url-select-expose}/${:md5sum-state-filename}
error-state-filename = boot-image-url-select-download-controller-error.text
error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[boot-image-url-select-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-download-wrapper:md5sum-state-filename}
config-filename = ${boot-image-url-select-download-wrapper:md5sum-state-file}
[boot-image-url-select-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-download-wrapper:error-state-filename}
config-filename = ${boot-image-url-select-download-wrapper:error-state-file}
## boot-image-url-select support END
{% endif %} {# if boot_image_url_select_enabled #}
{% if boot_image_url_list_enabled %} {% if boot_image_url_list_enabled %}
## boot-image-url-list support BEGIN ## boot-image-url-list support BEGIN
[empty-file-state-base-promise] [empty-file-state-base-list-promise]
<= monitor-promise-base <= monitor-promise-base
module = check_file_state module = check_file_state
name = ${:_buildout_section_name_}.py name = ${:_buildout_section_name_}.py
...@@ -130,7 +243,7 @@ processed-md5sum = ${directory:boot-image-url-list-var}/update-image-processed.m ...@@ -130,7 +243,7 @@ processed-md5sum = ${directory:boot-image-url-list-var}/update-image-processed.m
[boot-image-url-list-processed-config-promise] [boot-image-url-list-processed-config-promise]
# promise to check if the configuration provided by the user has been already # promise to check if the configuration provided by the user has been already
# processed by the boot-image-url-list-download script, which runs asynchronously # processed by the boot-image-url-list-download script, which runs asynchronously
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${boot-image-url-list-processed-config:state-filename} filename = ${boot-image-url-list-processed-config:state-filename}
config-filename = ${boot-image-url-list-processed-config:state-file} config-filename = ${boot-image-url-list-processed-config:state-file}
...@@ -145,7 +258,7 @@ error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filena ...@@ -145,7 +258,7 @@ error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filena
[boot-image-url-list-config-state-promise] [boot-image-url-list-config-state-promise]
# promise to check if configuration has been parsed without errors # promise to check if configuration has been parsed without errors
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${boot-image-url-list-json-config:error-state-filename} filename = ${boot-image-url-list-json-config:error-state-filename}
config-filename = ${boot-image-url-list-json-config:error-state-file} config-filename = ${boot-image-url-list-json-config:error-state-file}
...@@ -163,13 +276,13 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg ...@@ -163,13 +276,13 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[boot-image-url-list-download-md5sum-promise] [boot-image-url-list-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the # promise to report errors with problems with calculating md5sum of the
# downloaded images # downloaded images
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${boot-image-url-list-download-wrapper:md5sum-state-filename} filename = ${boot-image-url-list-download-wrapper:md5sum-state-filename}
config-filename = ${boot-image-url-list-download-wrapper:md5sum-state-file} config-filename = ${boot-image-url-list-download-wrapper:md5sum-state-file}
[boot-image-url-list-download-state-promise] [boot-image-url-list-download-state-promise]
# promise to report errors during download # promise to report errors during download
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${boot-image-url-list-download-wrapper:error-state-filename} filename = ${boot-image-url-list-download-wrapper:error-state-filename}
config-filename = ${boot-image-url-list-download-wrapper:error-state-file} config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END ## boot-image-url-list support END
...@@ -200,6 +313,11 @@ boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered} ...@@ -200,6 +313,11 @@ boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %} {% else %}
boot-image-url-list-json-config = boot-image-url-list-json-config =
{% endif %} {% endif %}
{% if boot_image_url_select_enabled %}
boot-image-url-select-json-config = ${boot-image-url-select-json-config:rendered}
{% else %}
boot-image-url-select-json-config =
{% endif %}
nbd-host = ${slap-parameter:nbd-host} nbd-host = ${slap-parameter:nbd-host}
nbd-port = ${slap-parameter:nbd-port} nbd-port = ${slap-parameter:nbd-port}
nbd2-host = ${slap-parameter:nbd2-host} nbd2-host = ${slap-parameter:nbd2-host}
...@@ -957,6 +1075,13 @@ parts = ...@@ -957,6 +1075,13 @@ parts =
boot-image-url-list-download-state-promise boot-image-url-list-download-state-promise
boot-image-url-list-processed-config-promise boot-image-url-list-processed-config-promise
{% endif %} {% endif %}
{% if boot_image_url_select_enabled %}
boot-image-url-select-download-wrapper
boot-image-url-select-config-state-promise
boot-image-url-select-download-md5sum-promise
boot-image-url-select-download-state-promise
boot-image-url-select-processed-config-promise
{% endif %}
{% if additional_frontend %} {% if additional_frontend %}
frontend-additional-promise frontend-additional-promise
{% endif %} {% endif %}
......
software/kvm/template/image-download-config-creator.py
View file @ ca15bbff
...@@ -24,7 +24,18 @@ if __name__ == "__main__": ...@@ -24,7 +24,18 @@ if __name__ == "__main__":
image_number = 0 image_number = 0
data = fh.read() data = fh.read()
configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest() configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest()
for entry in data.decode('utf-8').split(): if source_configuration.endswith('.json'):
data = data.strip()
data_list = []
if len(data):
try:
data_list = json.loads(data)
except Exception:
error_list.append('ERR: Data is not a JSON')
data_list = []
else:
data_list = data.decode('utf-8').split()
for entry in data_list:
split_entry = entry.split('#') split_entry = entry.split('#')
if len(split_entry) != 2: if len(split_entry) != 2:
error_list.append('ERR: entry %r is incorrect' % (entry,)) error_list.append('ERR: entry %r is incorrect' % (entry,))
......
software/kvm/template/template-kvm-run.in
View file @ ca15bbff
...@@ -100,6 +100,7 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe ...@@ -100,6 +100,7 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
logfile = '{{ parameter_dict.get("log-file") }}' logfile = '{{ parameter_dict.get("log-file") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}' boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = FancyURLopener(context=ssl._create_unverified_context()) opener = FancyURLopener(context=ssl._create_unverified_context())
...@@ -370,6 +371,18 @@ for nbd_ip, nbd_port in nbd_list: ...@@ -370,6 +371,18 @@ for nbd_ip, nbd_port in nbd_list:
'-drive', '-drive',
'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)]) 'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
else: else:
if boot_image_url_select_json_config:
# Support boot-image-url-select
with open(boot_image_url_select_json_config) as fh:
image_config = json.load(fh)
if image_config['error-amount'] == 0:
for image in sorted(image_config['image-list'], key=lambda k: k['link']):
link = os.path.join(image_config['destination-directory'], image['link'])
if os.path.exists(link) and os.path.islink(link):
kvm_argument_list.extend([
'-drive',
'file=%s,media=cdrom' % (link,)
])
if boot_image_url_list_json_config: if boot_image_url_list_json_config:
# Support boot-image-url-list # Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh: with open(boot_image_url_list_json_config) as fh:
......
software/kvm/test/test.py
View file @ ca15bbff
...@@ -489,6 +489,28 @@ class TestInstanceNbdServer(InstanceTestCase): ...@@ -489,6 +489,28 @@ class TestInstanceNbdServer(InstanceTestCase):
class TestBootImageUrlList(InstanceTestCase): class TestBootImageUrlList(InstanceTestCase):
__partition_reference__ = 'biul' __partition_reference__ = 'biul'
# variations
key = 'boot-image-url-list'
test_input = "%s#%s\n%s#%s"
image_directory = 'boot-image-url-list-repository'
config_state_promise = 'boot-image-url-list-config-state-promise.py'
download_md5sum_promise = 'boot-image-url-list-download-md5sum-promise.py'
download_state_promise = 'boot-image-url-list-download-state-promise.py'
bad_value = "jsutbad"
incorrect_md5sum_value_image = "%s#"
incorrect_md5sum_value = "url#asdasd"
single_image_value = "%s#%s"
unreachable_host_value = "evennotahost#%s"
too_many_image_value = """
image1#11111111111111111111111111111111
image2#22222222222222222222222222222222
image3#33333333333333333333333333333333
image4#44444444444444444444444444444444
image5#55555555555555555555555555555555
image6#66666666666666666666666666666666
"""
@classmethod @classmethod
def getInstanceSoftwareType(cls): def getInstanceSoftwareType(cls):
return 'default' return 'default'
...@@ -501,7 +523,7 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -501,7 +523,7 @@ class TestBootImageUrlList(InstanceTestCase):
def tearDown(self): def tearDown(self):
# clean up the instance for other tests # clean up the instance for other tests
# 1st remove all images... # 1st remove all images...
self.rerequestInstance({'boot-image-url-list': ''}) self.rerequestInstance({self.key: ''})
self.slap.waitForInstance(max_retry=10) self.slap.waitForInstance(max_retry=10)
# 2nd ...move instance to "default" state # 2nd ...move instance to "default" state
self.rerequestInstance({}) self.rerequestInstance({})
...@@ -535,7 +557,7 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -535,7 +557,7 @@ class TestBootImageUrlList(InstanceTestCase):
def test(self): def test(self):
partition_parameter_kw = { partition_parameter_kw = {
'boot-image-url-list': "%s#%s\n%s#%s" % ( self.key: self.test_input % (
self.fake_image, self.fake_image_md5sum, self.fake_image2, self.fake_image, self.fake_image_md5sum, self.fake_image2,
self.fake_image2_md5sum) self.fake_image2_md5sum)
} }
...@@ -543,8 +565,7 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -543,8 +565,7 @@ class TestBootImageUrlList(InstanceTestCase):
self.slap.waitForInstance(max_retry=10) self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked # check that image is correctly downloaded and linked
image_repository = os.path.join( image_repository = os.path.join(
self.computer_partition_root_path, 'srv', self.computer_partition_root_path, 'srv', self.image_directory)
'boot-image-url-list-repository')
image = os.path.join(image_repository, self.fake_image_md5sum) image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001') image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image)) self.assertTrue(os.path.exists(image))
...@@ -597,8 +618,8 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -597,8 +618,8 @@ class TestBootImageUrlList(InstanceTestCase):
# now the image is available in the kvm, and its above default image # now the image is available in the kvm, and its above default image
self.assertEqual( self.assertEqual(
[ [
'file=/srv/boot-image-url-list-repository/image_001,media=cdrom', 'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
'file=/srv/boot-image-url-list-repository/image_002,media=cdrom', 'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,' 'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom' 'media=cdrom'
], ],
...@@ -607,7 +628,7 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -607,7 +628,7 @@ class TestBootImageUrlList(InstanceTestCase):
# cleanup of images works, also asserts that configuration changes are # cleanup of images works, also asserts that configuration changes are
# reflected # reflected
self.rerequestInstance({'boot-image-url-list': ''}) self.rerequestInstance({self.key: ''})
self.slap.waitForInstance(max_retry=2) self.slap.waitForInstance(max_retry=2)
self.assertEqual( self.assertEqual(
os.listdir(image_repository), os.listdir(image_repository),
...@@ -645,53 +666,173 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -645,53 +666,173 @@ class TestBootImageUrlList(InstanceTestCase):
def test_bad_parameter(self): def test_bad_parameter(self):
self.rerequestInstance({ self.rerequestInstance({
'boot-image-url-list': "jsutbad" self.key: self.bad_value
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('boot-image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
def test_incorrect_md5sum(self): def test_incorrect_md5sum(self):
self.rerequestInstance({ self.rerequestInstance({
'boot-image-url-list': "%s#" % (self.fake_image,) self.key: self.incorrect_md5sum_value_image % (self.fake_image,)
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('boot-image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
self.rerequestInstance({ self.rerequestInstance({
'boot-image-url-list': "url#asdasd" self.key: self.incorrect_md5sum_value
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('boot-image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
def test_not_matching_md5sum(self): def test_not_matching_md5sum(self):
self.rerequestInstance({ self.rerequestInstance({
'boot-image-url-list': "%s#%s" % ( self.key: self.single_image_value % (
self.fake_image, self.fake_image_wrong_md5sum) self.fake_image, self.fake_image_wrong_md5sum)
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('boot-image-url-list-download-md5sum-promise.py') self.assertPromiseFails(self.download_md5sum_promise)
self.assertPromiseFails('boot-image-url-list-download-state-promise.py') self.assertPromiseFails(self.download_state_promise)
def test_unreachable_host(self): def test_unreachable_host(self):
self.rerequestInstance({ self.rerequestInstance({
'boot-image-url-list': "evennotahost#%s" % ( self.key: self.unreachable_host_value % (
self.fake_image_md5sum,) self.fake_image_md5sum,)
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('boot-image-url-list-download-state-promise.py') self.assertPromiseFails(self.download_state_promise)
def test_too_many_images(self): def test_too_many_images(self):
self.rerequestInstance({ self.rerequestInstance({
'boot-image-url-list': """ self.key: self.too_many_image_value
image1#11111111111111111111111111111111
image2#22222222222222222222222222222222
image3#33333333333333333333333333333333
image4#44444444444444444444444444444444
image5#55555555555555555555555555555555
image6#66666666666666666666666666666666
"""
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('boot-image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
@skipUnlessKvm
class TestBootImageUrlSelect(TestBootImageUrlList):
__partition_reference__ = 'bius'
# variations
key = 'boot-image-url-select'
test_input = '["%s#%s", "%s#%s"]'
image_directory = 'boot-image-url-select-repository'
config_state_promise = 'boot-image-url-select-config-state-promise.py'
download_md5sum_promise = 'boot-image-url-select-download-md5sum-promise.py'
download_state_promise = 'boot-image-url-select-download-state-promise.py'
bad_value = '["jsutbad"]'
incorrect_md5sum_value_image = '["%s#"]'
incorrect_md5sum_value = '["url#asdasd"]'
single_image_value = '["%s#%s"]'
unreachable_host_value = '["evennotahost#%s"]'
too_many_image_value = """[
"image1#11111111111111111111111111111111",
"image2#22222222222222222222222222222222",
"image3#33333333333333333333333333333333",
"image4#44444444444444444444444444444444",
"image5#55555555555555555555555555555555",
"image6#66666666666666666666666666666666"
]"""
def test_not_json(self):
self.rerequestInstance({
self.key: 'notjson#notjson'
})
self.raising_waitForInstance(3)
self.assertPromiseFails(self.config_state_promise)
def test_together(self):
partition_parameter_kw = {
'boot-image-url-list': "%s#%s" % (
self.fake_image, self.fake_image_md5sum),
'boot-image-url-select': '["%s#%s"]' % (
self.fake_image, self.fake_image_md5sum)
}
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, self.fake_image_md5sum)
self.assertTrue(os.path.islink(image_link))
self.assertEqual(os.readlink(image_link), image)
def getRunningImageList():
running_image_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
software_root = '/'.join([
self.slap.software_directory,
hashlib.md5(self.getSoftwareURL()).hexdigest()])
for entry in kvm_process.cmdline():
if entry.startswith('file') and 'media=cdrom' in entry:
# do cleanups
entry = entry.replace(software_root, '')
entry = entry.replace(self.computer_partition_root_path, '')
running_image_list.append(entry)
return running_image_list
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom'],
getRunningImageList()
)
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
# now the image is available in the kvm, and its above default image
self.assertEqual(
[
'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom'
],
getRunningImageList()
)
# cleanup of images works, also asserts that configuration changes are
# reflected
self.rerequestInstance(
{'boot-image-url-list': '', 'boot-image-url-select': ''})
self.slap.waitForInstance(max_retry=2)
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory)
self.assertEqual(
os.listdir(image_repository),
[]
)
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
# again only default image is available in the running process
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom'],
getRunningImageList()
)
@skipUnlessKvm @skipUnlessKvm
...@@ -713,18 +854,22 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase): ...@@ -713,18 +854,22 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase):
"2a74b7675a8d8ca20476dba6e",) "2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6" fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
input_value = "%s#%s"
key = 'boot-image-url-list'
config_file_name = 'boot-image-url-list.conf'
@classmethod @classmethod
def getInstanceParameterDict(cls): def getInstanceParameterDict(cls):
return {'_': json.dumps({ return {'_': json.dumps({
"kvm-partition-dict": { "kvm-partition-dict": {
"KVM0": { "KVM0": {
"disable-ansible-promise": True, "disable-ansible-promise": True,
"boot-image-url-list": "%s#%s" % ( cls.key: cls.input_value % (
cls.fake_image, cls.fake_image_md5sum) cls.fake_image, cls.fake_image_md5sum)
}, },
"KVM1": { "KVM1": {
"disable-ansible-promise": True, "disable-ansible-promise": True,
"boot-image-url-list": "%s#%s" % ( cls.key: cls.input_value % (
cls.fake_image2, cls.fake_image2_md5sum) cls.fake_image2, cls.fake_image2_md5sum)
} }
} }
...@@ -735,22 +880,31 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase): ...@@ -735,22 +880,31 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase):
# we assume ordering of the cluster requests # we assume ordering of the cluster requests
KVM0_config = os.path.join( KVM0_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '1', 'etc', self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
'boot-image-url-list.conf') self.config_file_name)
KVM1_config = os.path.join( KVM1_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '2', 'etc', self.slap.instance_directory, self.__partition_reference__ + '2', 'etc',
'boot-image-url-list.conf') self.config_file_name)
with open(KVM0_config, 'r') as fh: with open(KVM0_config, 'r') as fh:
self.assertEqual( self.assertEqual(
"%s#%s" % (self.fake_image, self.fake_image_md5sum), self.input_value % (self.fake_image, self.fake_image_md5sum),
fh.read() fh.read().strip()
) )
with open(KVM1_config, 'r') as fh: with open(KVM1_config, 'r') as fh:
self.assertEqual( self.assertEqual(
"%s#%s" % (self.fake_image2, self.fake_image2_md5sum), self.input_value % (self.fake_image2, self.fake_image2_md5sum),
fh.read() fh.read().strip()
) )
@skipUnlessKvm
class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
__partition_reference__ = 'biuskc'
input_value = "[\"%s#%s\"]"
key = 'boot-image-url-select'
config_file_name = 'boot-image-url-select.json'
@skipUnlessKvm @skipUnlessKvm
class TestCpuMemMaxDynamic(InstanceTestCase): class TestCpuMemMaxDynamic(InstanceTestCase):
__partition_reference__ = 'cmm' __partition_reference__ = 'cmm'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7