Commit 4343f31d authored by Łukasz Nowak's avatar Łukasz Nowak

feature/kvm: User selectable list of boot ISOs

See merge request nexedi/slapos!807
parents ecc880d1 92130619
...@@ -18,9 +18,8 @@ KVM instance (1GB of RAM, 10GB of SSD, one core) ...@@ -18,9 +18,8 @@ KVM instance (1GB of RAM, 10GB of SSD, one core)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Note that the KVM instance will try to request a frontend slave instance in order Note that the KVM instance will try to request a frontend slave instance in order
to be accessible from IPv4. to be accessible from IPv4::
::
myawesomekvm = request( myawesomekvm = request(
software_release=kvm, software_release=kvm,
partition_reference="My awesome KVM", partition_reference="My awesome KVM",
...@@ -92,16 +91,15 @@ KVM instance parameters: ...@@ -92,16 +91,15 @@ KVM instance parameters:
- keyboard-layout-language (default: fr) - keyboard-layout-language (default: fr)
Change keyboard layout language (Change to en-us if you face some bad bihaviors) Change keyboard layout language (Change to en-us if you face some bad bihaviors)
Language list: ['ar', 'da', 'de', 'de-ch', 'en-gb', 'en-us', 'es', 'et', 'fi', Language list: ['ar', 'da', 'de', 'de-ch', 'en-gb', 'en-us', 'es', 'et', 'fi',
'fo', 'fr', 'fr-be', 'fr-ca', 'fr-ch', 'hr', 'hu', 'is', 'it', 'ja', 'lt', 'fo', 'fr', 'fr-be', 'fr-ca', 'fr-ch', 'hr', 'hu', 'is', 'it', 'ja', 'lt',
'lv', 'mk', 'nl', 'nl-be', 'no', 'pl', 'pt', 'pt-br', 'ru', 'sl', 'sv', 'lv', 'mk', 'nl', 'nl-be', 'no', 'pl', 'pt', 'pt-br', 'ru', 'sl', 'sv',
'th', 'tr'] 'th', 'tr']
Resilient KVM instance Resilient KVM instance
~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~
Like KVM instance, but backed-up (with history) in two places. Like KVM instance, but backed-up (with history) in two places::
::
kvm = 'https://lab.nexedi.com/nexedi/slapos/raw/slapos-0.188/software/kvm/software.cfg' kvm = 'https://lab.nexedi.com/nexedi/slapos/raw/slapos-0.188/software/kvm/software.cfg'
myresilientkvm = request( myresilientkvm = request(
software_release=kvm, software_release=kvm,
...@@ -119,3 +117,18 @@ files for more instance parameters (cpu-count, ram-size, disk-size, specific loc ...@@ -119,3 +117,18 @@ files for more instance parameters (cpu-count, ram-size, disk-size, specific loc
Then, if you want one of the two clones to takeover, you need to login into Then, if you want one of the two clones to takeover, you need to login into
the hosting machine, go to the partition of the clone, and invoke bin/takeover. the hosting machine, go to the partition of the clone, and invoke bin/takeover.
Technical notes
---------------
Updating boot-image-url-select
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* download the new OS installation image
* calculate it's sha512sum and store as <SHA512>
* calculate it's md5sum and store as <MD5>
* upload it to shacache
* construct download url: ``https://shacache.nxdcdn.com/<SHA512>#<MD5>``
* update the ``boot-image-url-select`` in:
* ``instance-kvm-input-schema.json``
* ``instance-kvm-cluster-input-schema.json``
...@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe ...@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 50e78a2a34efe09afab161ae1c1efd46 md5sum = 23493c541efef97ac5fe435114910b8e
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = a4788112008cd0b38a57cd28f7252fbd md5sum = bdf8549a76ec61e125d51a05e611e004
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = dd1f581f34cf5a0b627576771347c710 md5sum = 4a6f149177a453a13436f320f6841518
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
...@@ -87,4 +87,4 @@ md5sum = 7e4b54f8172c364bd12d28b07f8b1600 ...@@ -87,4 +87,4 @@ md5sum = 7e4b54f8172c364bd12d28b07f8b1600
[image-download-config-creator] [image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py _update_hash_filename_ = template/image-download-config-creator.py
md5sum = 7f6cd75096695922fddbba1c9292cef5 md5sum = 54261e418ab9860efe73efd514c4d47f
...@@ -42,12 +42,6 @@ ...@@ -42,12 +42,6 @@
"type": "string", "type": "string",
"format": "uri", "format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg" "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
},
"image-url-list": {
"title": "List of URLs images to download",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string",
"textarea": "true"
} }
}, },
"type": "object" "type": "object"
...@@ -508,6 +502,60 @@ ...@@ -508,6 +502,60 @@
"description": "If the VM of cluster doesn't run Ansible and report status to this SlapOS instances, then this allow to disable ansible promise so your instance will not fail to check ansible promise.", "description": "If the VM of cluster doesn't run Ansible and report status to this SlapOS instances, then this allow to disable ansible promise so your instance will not fail to check ansible promise.",
"type": "boolean", "type": "boolean",
"default": false "default": false
},
"boot-image-url-list": {
"title": "Boot image list",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string",
"textarea": "true"
},
"boot-image-url-select": {
"title": "Boot image",
"type": "array",
"oneOf": [
{
"const": [
"https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
],
"title": "Debian Buster 10.5 netinst x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
],
"title": "Centos 8.2004 Minimal x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
],
"title": "Ubuntu Focal 20.04.1 Live Server x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
],
"title": "openSUSE Leap 15.2 NET x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
],
"title": "Arch Linux 2020.09.01 x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
],
"title": "Fedora Server 32-1.6 netinst x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
],
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
} }
}, },
"type": "object" "type": "object"
......
...@@ -127,9 +127,13 @@ config-document-host = ${apache-conf:ip} ...@@ -127,9 +127,13 @@ config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port} config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd} config-document-path = ${hash-code:passwd}
config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }} config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
{%- if 'image-url-list' in kvm_parameter_dict %} {%- if 'boot-image-url-list' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #} {#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-image-url-list = {{ kvm_parameter_dict['image-url-list'] }} config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
{%- endif %}
{%- if 'boot-image-url-select' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
{%- endif %} {%- endif %}
config-type = cluster config-type = cluster
......
{ {
"type": "object", "type": "object",
"$schema": "http://json-schema.org/draft-04/schema", "$schema": "http://json-schema.org/draft-06/schema",
"title": "Input Parameters", "title": "Input Parameters",
"properties": { "properties": {
"enable-device-hotplug": { "enable-device-hotplug": {
...@@ -366,11 +366,59 @@ ...@@ -366,11 +366,59 @@
"format": "uri", "format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg" "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
}, },
"image-url-list": { "boot-image-url-list": {
"title": "List of URLs images to download", "title": "Boot image list",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.", "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string", "type": "string",
"textarea": "true" "textarea": "true"
},
"boot-image-url-select": {
"title": "Boot image",
"type": "array",
"oneOf": [
{
"const": [
"https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
],
"title": "Debian Buster 10.5 netinst x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
],
"title": "Centos 8.2004 Minimal x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
],
"title": "Ubuntu Focal 20.04.1 Live Server x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
],
"title": "openSUSE Leap 15.2 NET x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
],
"title": "Arch Linux 2020.09.01 x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
],
"title": "Fedora Server 32-1.6 netinst x86_64"
},
{
"const": [
"https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
],
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
} }
} }
} }
...@@ -9,7 +9,8 @@ ...@@ -9,7 +9,8 @@
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%} {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%} {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set image_url_list_enabled = 'image-url-list' in slapparameter_dict %} {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %} {% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %} {% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
{% set extends_list = [] -%} {% set extends_list = [] -%}
...@@ -54,10 +55,15 @@ public = ${:srv}/public/ ...@@ -54,10 +55,15 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps cronstamps = ${:etc}/cronstamps
{%- if image_url_list_enabled %} {%- if boot_image_url_list_enabled %}
image-repository = ${:srv}/image-repository boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
image-url-list-var = ${:var}/image-url-list boot-image-url-list-var = ${:var}/boot-image-url-list
image-url-list-expose = ${monitor-directory:private}/image-url-list boot-image-url-list-expose = ${monitor-directory:private}/boot-image-url-list
{%- endif %}
{%- if boot_image_url_select_enabled %}
boot-image-url-select-repository = ${:srv}/boot-image-url-select-repository
boot-image-url-select-var = ${:var}/boot-image-url-select
boot-image-url-select-expose = ${monitor-directory:private}/boot-image-url-select
{%- endif %} {%- endif %}
[create-mac] [create-mac]
...@@ -73,9 +79,116 @@ recipe = slapos.cookbook:generate.password ...@@ -73,9 +79,116 @@ recipe = slapos.cookbook:generate.password
storage-path = ${directory:srv}/passwd storage-path = ${directory:srv}/passwd
bytes = 8 bytes = 8
{% if image_url_list_enabled %} {% if boot_image_url_select_enabled %}
## image-url-list support BEGIN ## boot-image-url-select support BEGIN
[empty-file-state-base-promise] [empty-file-state-base-select-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-select-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-select
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
{%- raw %}
{%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
{%- if boot_image_url_select == 'None' %}
{#- That's insane protection, is it 'None' because of type = array? #}
{%- set boot_image_url_select = '' %}
{%- endif %}
{{ boot_image_url_select }}
{% endraw -%}
context =
section slap_parameter slap-parameter
rendered = ${directory:etc}/boot-image-url-select.json
[boot-image-url-select-processed-config]
# compares if the current configuration has been used by
# the boot-image-url-select-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${boot-image-url-select-source-config:rendered}
state-filename = boot-image-url-select-processed-config.state
state-file = ${directory:boot-image-url-select-expose}/${:state-filename}
processed-md5sum = ${directory:boot-image-url-select-var}/update-image-processed.md5sum
[boot-image-url-select-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the boot-image-url-select-download script, which runs asynchronously
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-processed-config:state-filename}
config-filename = ${boot-image-url-select-processed-config:state-file}
[boot-image-url-select-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${boot-image-url-select-source-config:rendered} ${:rendered} ${directory:boot-image-url-select-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:boot-image-url-select-var}/boot-image-url-select.json
error-state-filename = boot-image-url-select-json-config-error.txt
error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
[boot-image-url-select-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-json-config:error-state-filename}
config-filename = ${boot-image-url-select-json-config:error-state-file}
[boot-image-url-select-download-wrapper]
# wrapper to execute boot-image-url-select-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/boot-image-url-select-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-select-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-select-processed-config:processed-md5sum}
md5sum-state-filename = boot-image-url-select-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:boot-image-url-select-expose}/${:md5sum-state-filename}
error-state-filename = boot-image-url-select-download-controller-error.text
error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[boot-image-url-select-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-download-wrapper:md5sum-state-filename}
config-filename = ${boot-image-url-select-download-wrapper:md5sum-state-file}
[boot-image-url-select-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-select-promise
filename = ${boot-image-url-select-download-wrapper:error-state-filename}
config-filename = ${boot-image-url-select-download-wrapper:error-state-file}
## boot-image-url-select support END
{% endif %} {# if boot_image_url_select_enabled #}
{% if boot_image_url_list_enabled %}
## boot-image-url-list support BEGIN
[empty-file-state-base-list-promise]
<= monitor-promise-base <= monitor-promise-base
module = check_file_state module = check_file_state
name = ${:_buildout_section_name_}.py name = ${:_buildout_section_name_}.py
...@@ -83,25 +196,25 @@ config-state = empty ...@@ -83,25 +196,25 @@ config-state = empty
# It's very hard to put the username and password correctly, after schema:// # It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided # and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL # information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password} config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[image-url-list-source-config] [boot-image-url-list-source-config]
# generates configuration of the image from the user parameter # generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline image-url-list # special "magic" is used, to properly support multiline boot-image-url-list
# but in the same time correctly generate the configuration file # but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = inline: template = inline:
{#- Do special trick to support image-url-list being None, if key is present with value "" #} {#- Do special trick to support boot-image-url-list being None, if key is present with value "" #}
{%- raw %} {%- raw %}
{{ slap_parameter.get('image-url-list') or '' }} {{ slap_parameter.get('boot-image-url-list') or '' }}
{% endraw -%} {% endraw -%}
context = context =
section slap_parameter slap-parameter section slap_parameter slap-parameter
rendered = ${directory:etc}/image-url-list.conf rendered = ${directory:etc}/boot-image-url-list.conf
[image-url-list-processed-config] [boot-image-url-list-processed-config]
# compares if the current configuration has been used by # compares if the current configuration has been used by
# the image-url-list-download, if not, exposes it as not empty file with # the boot-image-url-list-download, if not, exposes it as not empty file with
# information # information
recipe = slapos.recipe.build recipe = slapos.recipe.build
install = install =
...@@ -122,58 +235,58 @@ install = ...@@ -122,58 +235,58 @@ install =
state_handler.write(str(e)) state_handler.write(str(e))
update = ${:install} update = ${:install}
config-file = ${image-url-list-source-config:rendered} config-file = ${boot-image-url-list-source-config:rendered}
state-filename = image-url-list-processed-config.state state-filename = boot-image-url-list-processed-config.state
state-file = ${directory:image-url-list-expose}/${:state-filename} state-file = ${directory:boot-image-url-list-expose}/${:state-filename}
processed-md5sum = ${directory:image-url-list-var}/update-image-processed.md5sum processed-md5sum = ${directory:boot-image-url-list-var}/update-image-processed.md5sum
[image-url-list-processed-config-promise] [boot-image-url-list-processed-config-promise]
# promise to check if the configuration provided by the user has been already # promise to check if the configuration provided by the user has been already
# processed by the image-url-list-download script, which runs asynchronously # processed by the boot-image-url-list-download script, which runs asynchronously
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${image-url-list-processed-config:state-filename} filename = ${boot-image-url-list-processed-config:state-filename}
config-filename = ${image-url-list-processed-config:state-file} config-filename = ${boot-image-url-list-processed-config:state-file}
[image-url-list-json-config] [boot-image-url-list-json-config]
# generates json configuration from user configuration # generates json configuration from user configuration
recipe = plone.recipe.command recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${image-url-list-source-config:rendered} ${:rendered} ${directory:image-repository} ${:error-state-file} command = {{ python_executable }} {{ image_download_config_creator }} ${boot-image-url-list-source-config:rendered} ${:rendered} ${directory:boot-image-url-list-repository} ${:error-state-file}
update-command = ${:command} update-command = ${:command}
rendered = ${directory:image-url-list-var}/image-url-list.json rendered = ${directory:boot-image-url-list-var}/boot-image-url-list.json
error-state-filename = image-url-list-json-config-error.txt error-state-filename = boot-image-url-list-json-config-error.txt
error-state-file = ${directory:image-url-list-expose}/${:error-state-filename} error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filename}
[image-url-list-config-state-promise] [boot-image-url-list-config-state-promise]
# promise to check if configuration has been parsed without errors # promise to check if configuration has been parsed without errors
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${image-url-list-json-config:error-state-filename} filename = ${boot-image-url-list-json-config:error-state-filename}
config-filename = ${image-url-list-json-config:error-state-file} config-filename = ${boot-image-url-list-json-config:error-state-file}
[image-url-list-download-wrapper] [boot-image-url-list-download-wrapper]
# wrapper to execute image-url-list-download on each run # wrapper to execute boot-image-url-list-download on each run
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/image-updater wrapper-path = ${directory:scripts}/boot-image-url-list-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${image-url-list-processed-config:processed-md5sum} command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-list-processed-config:processed-md5sum}
md5sum-state-filename = image-download-controller-md5sum-fail.json md5sum-state-filename = boot-image-url-list-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:image-url-list-expose}/${:md5sum-state-filename} md5sum-state-file = ${directory:boot-image-url-list-expose}/${:md5sum-state-filename}
error-state-filename = image-download-controller-error.text error-state-filename = boot-image-url-list-download-controller-error.text
error-state-file = ${directory:image-url-list-expose}/${:error-state-filename} error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[image-url-list-download-md5sum-promise] [boot-image-url-list-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the # promise to report errors with problems with calculating md5sum of the
# downloaded images # downloaded images
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${image-url-list-download-wrapper:md5sum-state-filename} filename = ${boot-image-url-list-download-wrapper:md5sum-state-filename}
config-filename = ${image-url-list-download-wrapper:md5sum-state-file} config-filename = ${boot-image-url-list-download-wrapper:md5sum-state-file}
[image-url-list-download-state-promise] [boot-image-url-list-download-state-promise]
# promise to report errors during download # promise to report errors during download
<= empty-file-state-base-promise <= empty-file-state-base-list-promise
filename = ${image-url-list-download-wrapper:error-state-filename} filename = ${boot-image-url-list-download-wrapper:error-state-filename}
config-filename = ${image-url-list-download-wrapper:error-state-file} config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## image-url-list support END ## boot-image-url-list support END
{% endif %} {# if image_url_list_enabled #} {% endif %} {# if boot_image_url_list_enabled #}
[kvm-controller-parameter-dict] [kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }} python-path = {{ python_eggs_executable }}
...@@ -195,10 +308,15 @@ vnc-ip = ${:ipv4} ...@@ -195,10 +308,15 @@ vnc-ip = ${:ipv4}
vnc-port = 5901 vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }} default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if image_url_list_enabled %} {% if boot_image_url_list_enabled %}
image-url-list-json-config = ${image-url-list-json-config:rendered} boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %} {% else %}
image-url-list-json-config = boot-image-url-list-json-config =
{% endif %}
{% if boot_image_url_select_enabled %}
boot-image-url-select-json-config = ${boot-image-url-select-json-config:rendered}
{% else %}
boot-image-url-select-json-config =
{% endif %} {% endif %}
nbd-host = ${slap-parameter:nbd-host} nbd-host = ${slap-parameter:nbd-host}
nbd-port = ${slap-parameter:nbd-port} nbd-port = ${slap-parameter:nbd-port}
...@@ -823,7 +941,7 @@ nbd-port = 1024 ...@@ -823,7 +941,7 @@ nbd-port = 1024
nbd-host = nbd-host =
nbd2-port = 1024 nbd2-port = 1024
nbd2-host = nbd2-host =
image-url-list = boot-image-url-list =
enable-device-hotplug = False enable-device-hotplug = False
ram-size = 1024 ram-size = 1024
...@@ -873,7 +991,7 @@ keyboard-layout-language = fr ...@@ -873,7 +991,7 @@ keyboard-layout-language = fr
{% set key_list = v.split('\n') -%} {% set key_list = v.split('\n') -%}
{{ k }} = {{ k }} =
{{ key_list | join('\n ') }} {{ key_list | join('\n ') }}
{% elif k == 'image-url-list' %} {% elif k == 'boot-image-url-list' %}
{# needs to decorate possibly multiline or maybe unsafe value #} {# needs to decorate possibly multiline or maybe unsafe value #}
{{ k }} = {{ dumps(v) }} {{ k }} = {{ dumps(v) }}
{% else -%} {% else -%}
...@@ -950,12 +1068,19 @@ parts = ...@@ -950,12 +1068,19 @@ parts =
cron-service cron-service
cron-entry-logrotate cron-entry-logrotate
frontend-promise frontend-promise
{% if image_url_list_enabled %} {% if boot_image_url_list_enabled %}
image-url-list-download-wrapper boot-image-url-list-download-wrapper
image-url-list-config-state-promise boot-image-url-list-config-state-promise
image-url-list-download-md5sum-promise boot-image-url-list-download-md5sum-promise
image-url-list-download-state-promise boot-image-url-list-download-state-promise
image-url-list-processed-config-promise boot-image-url-list-processed-config-promise
{% endif %}
{% if boot_image_url_select_enabled %}
boot-image-url-select-download-wrapper
boot-image-url-select-config-state-promise
boot-image-url-select-download-md5sum-promise
boot-image-url-select-download-state-promise
boot-image-url-select-processed-config-promise
{% endif %} {% endif %}
{% if additional_frontend %} {% if additional_frontend %}
frontend-additional-promise frontend-additional-promise
......
...@@ -24,7 +24,18 @@ if __name__ == "__main__": ...@@ -24,7 +24,18 @@ if __name__ == "__main__":
image_number = 0 image_number = 0
data = fh.read() data = fh.read()
configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest() configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest()
for entry in data.decode('utf-8').split(): if source_configuration.endswith('.json'):
data = data.strip()
data_list = []
if len(data):
try:
data_list = json.loads(data)
except Exception:
error_list.append('ERR: Data is not a JSON')
data_list = []
else:
data_list = data.decode('utf-8').split()
for entry in data_list:
split_entry = entry.split('#') split_entry = entry.split('#')
if len(split_entry) != 2: if len(split_entry) != 2:
error_list.append('ERR: entry %r is incorrect' % (entry,)) error_list.append('ERR: entry %r is incorrect' % (entry,))
......
...@@ -99,7 +99,8 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe ...@@ -99,7 +99,8 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
logfile = '{{ parameter_dict.get("log-file") }}' logfile = '{{ parameter_dict.get("log-file") }}'
image_url_list_json_config = '{{ parameter_dict.get("image-url-list-json-config") }}' boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = FancyURLopener(context=ssl._create_unverified_context()) opener = FancyURLopener(context=ssl._create_unverified_context())
...@@ -370,25 +371,41 @@ for nbd_ip, nbd_port in nbd_list: ...@@ -370,25 +371,41 @@ for nbd_ip, nbd_port in nbd_list:
'-drive', '-drive',
'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)]) 'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
else: else:
image_url_list_used = False index = 0
if image_url_list_json_config: if boot_image_url_select_json_config:
# Support image-url-list # Support boot-image-url-select
with open(image_url_list_json_config) as fh: with open(boot_image_url_select_json_config) as fh:
image_config = json.load(fh) image_config = json.load(fh)
if image_config['error-amount'] == 0: if image_config['error-amount'] == 0:
for image in sorted(image_config['image-list'], key=lambda k: k['link']): for image in sorted(image_config['image-list'], key=lambda k: k['link']):
link = os.path.join(image_config['destination-directory'], image['link']) link = os.path.join(image_config['destination-directory'], image['link'])
if os.path.exists(link) and os.path.islink(link): if os.path.exists(link) and os.path.islink(link):
image_url_list_used = True
kvm_argument_list.extend([ kvm_argument_list.extend([
'-drive', '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
'file=%s,media=cdrom' % (link,) '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
'-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
]) ])
if not image_url_list_used: index += 1
# If no NBD is specified/available not downloadable image: use internal disk image if boot_image_url_list_json_config:
kvm_argument_list.extend([ # Support boot-image-url-list
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso with open(boot_image_url_list_json_config) as fh:
]) image_config = json.load(fh)
if image_config['error-amount'] == 0:
for image in sorted(image_config['image-list'], key=lambda k: k['link']):
link = os.path.join(image_config['destination-directory'], image['link'])
if os.path.exists(link) and os.path.islink(link):
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
'-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
'-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
])
index += 1
# Always add by default the default image
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (default_cdrom_iso, index),
'-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
'-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
])
print('Starting KVM: \n %s' % ' '.join(kvm_argument_list)) print('Starting KVM: \n %s' % ' '.join(kvm_argument_list))
......
...@@ -486,8 +486,30 @@ class TestInstanceNbdServer(InstanceTestCase): ...@@ -486,8 +486,30 @@ class TestInstanceNbdServer(InstanceTestCase):
@skipUnlessKvm @skipUnlessKvm
class TestImageUrlList(InstanceTestCase): class TestBootImageUrlList(InstanceTestCase):
__partition_reference__ = 'iul' __partition_reference__ = 'biul'
# variations
key = 'boot-image-url-list'
test_input = "%s#%s\n%s#%s"
image_directory = 'boot-image-url-list-repository'
config_state_promise = 'boot-image-url-list-config-state-promise.py'
download_md5sum_promise = 'boot-image-url-list-download-md5sum-promise.py'
download_state_promise = 'boot-image-url-list-download-state-promise.py'
bad_value = "jsutbad"
incorrect_md5sum_value_image = "%s#"
incorrect_md5sum_value = "url#asdasd"
single_image_value = "%s#%s"
unreachable_host_value = "evennotahost#%s"
too_many_image_value = """
image1#11111111111111111111111111111111
image2#22222222222222222222222222222222
image3#33333333333333333333333333333333
image4#44444444444444444444444444444444
image5#55555555555555555555555555555555
image6#66666666666666666666666666666666
"""
@classmethod @classmethod
def getInstanceSoftwareType(cls): def getInstanceSoftwareType(cls):
...@@ -501,7 +523,7 @@ class TestImageUrlList(InstanceTestCase): ...@@ -501,7 +523,7 @@ class TestImageUrlList(InstanceTestCase):
def tearDown(self): def tearDown(self):
# clean up the instance for other tests # clean up the instance for other tests
# 1st remove all images... # 1st remove all images...
self.rerequestInstance({'image-url-list': ''}) self.rerequestInstance({self.key: ''})
self.slap.waitForInstance(max_retry=10) self.slap.waitForInstance(max_retry=10)
# 2nd ...move instance to "default" state # 2nd ...move instance to "default" state
self.rerequestInstance({}) self.rerequestInstance({})
...@@ -535,7 +557,7 @@ class TestImageUrlList(InstanceTestCase): ...@@ -535,7 +557,7 @@ class TestImageUrlList(InstanceTestCase):
def test(self): def test(self):
partition_parameter_kw = { partition_parameter_kw = {
'image-url-list': "%s#%s\n%s#%s" % ( self.key: self.test_input % (
self.fake_image, self.fake_image_md5sum, self.fake_image2, self.fake_image, self.fake_image_md5sum, self.fake_image2,
self.fake_image2_md5sum) self.fake_image2_md5sum)
} }
...@@ -543,7 +565,7 @@ class TestImageUrlList(InstanceTestCase): ...@@ -543,7 +565,7 @@ class TestImageUrlList(InstanceTestCase):
self.slap.waitForInstance(max_retry=10) self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked # check that image is correctly downloaded and linked
image_repository = os.path.join( image_repository = os.path.join(
self.computer_partition_root_path, 'srv', 'image-repository') self.computer_partition_root_path, 'srv', self.image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum) image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001') image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image)) self.assertTrue(os.path.exists(image))
...@@ -562,20 +584,29 @@ class TestImageUrlList(InstanceTestCase): ...@@ -562,20 +584,29 @@ class TestImageUrlList(InstanceTestCase):
self.assertTrue(os.path.islink(image2_link)) self.assertTrue(os.path.islink(image2_link))
self.assertEqual(os.readlink(image2_link), image2) self.assertEqual(os.readlink(image2_link), image2)
def getRunningImageList():
running_image_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
software_root = '/'.join([
self.slap.software_directory,
hashlib.md5(self.getSoftwareURL()).hexdigest()])
for entry in kvm_process.cmdline():
if entry.startswith('file') and 'media=cdrom' in entry:
# do cleanups
entry = entry.replace(software_root, '')
entry = entry.replace(self.computer_partition_root_path, '')
running_image_list.append(entry)
return running_image_list
# check that the image is NOT YET available in kvm # check that the image is NOT YET available in kvm
with self.slap.instance_supervisor_rpc as instance_supervisor: self.assertEqual(
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo() ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
if 'kvm-' in q['name']][0]['pid'] 'media=cdrom,if=none,id=cdrom0'],
kvm_process = psutil.Process(kvm_pid) getRunningImageList()
cmd_line = ''.join(kvm_process.cmdline()) )
self.assertNotIn(
'srv/image-repository/image_001,media=cdrom',
cmd_line
)
self.assertNotIn(
'srv/image-repository/image_002,media=cdrom',
cmd_line
)
# mimic the requirement: restart the instance by requesting it stopped and # mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it # then started started, like user have to do it
...@@ -584,30 +615,42 @@ class TestImageUrlList(InstanceTestCase): ...@@ -584,30 +615,42 @@ class TestImageUrlList(InstanceTestCase):
self.rerequestInstance(partition_parameter_kw, state='started') self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=1)
# now the image is available in the kvm # now the image is available in the kvm, and its above default image
with self.slap.instance_supervisor_rpc as instance_supervisor: self.assertEqual(
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo() [
if 'kvm-' in q['name']][0]['pid'] 'file=/srv/%s/image_001,media=cdrom,if=none,id=cdrom0' % (
kvm_process = psutil.Process(kvm_pid) self.image_directory,),
cmd_line = ''.join(kvm_process.cmdline()) 'file=/srv/%s/image_002,media=cdrom,if=none,id=cdrom1' % (
self.assertIn( self.image_directory,),
'srv/image-repository/image_001,media=cdrom', 'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
cmd_line 'media=cdrom,if=none,id=cdrom2'
) ],
self.assertIn( getRunningImageList()
'srv/image-repository/image_002,media=cdrom', )
cmd_line
)
# cleanup of images works, also asserts that configuration changes are # cleanup of images works, also asserts that configuration changes are
# reflected # reflected
self.rerequestInstance({'image-url-list': ''}) self.rerequestInstance({self.key: ''})
self.slap.waitForInstance(max_retry=2) self.slap.waitForInstance(max_retry=2)
self.assertEqual( self.assertEqual(
os.listdir(image_repository), os.listdir(image_repository),
[] []
) )
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
# again only default image is available in the running process
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom0'],
getRunningImageList()
)
def assertPromiseFails(self, promise): def assertPromiseFails(self, promise):
monitor_run_promise = os.path.join( monitor_run_promise = os.path.join(
self.computer_partition_root_path, 'software_release', 'bin', self.computer_partition_root_path, 'software_release', 'bin',
...@@ -625,58 +668,180 @@ class TestImageUrlList(InstanceTestCase): ...@@ -625,58 +668,180 @@ class TestImageUrlList(InstanceTestCase):
def test_bad_parameter(self): def test_bad_parameter(self):
self.rerequestInstance({ self.rerequestInstance({
'image-url-list': "jsutbad" self.key: self.bad_value
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
def test_incorrect_md5sum(self): def test_incorrect_md5sum(self):
self.rerequestInstance({ self.rerequestInstance({
'image-url-list': "%s#" % (self.fake_image,) self.key: self.incorrect_md5sum_value_image % (self.fake_image,)
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
self.rerequestInstance({ self.rerequestInstance({
'image-url-list': "url#asdasd" self.key: self.incorrect_md5sum_value
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
def test_not_matching_md5sum(self): def test_not_matching_md5sum(self):
self.rerequestInstance({ self.rerequestInstance({
'image-url-list': "%s#%s" % ( self.key: self.single_image_value % (
self.fake_image, self.fake_image_wrong_md5sum) self.fake_image, self.fake_image_wrong_md5sum)
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-download-md5sum-promise.py') self.assertPromiseFails(self.download_md5sum_promise)
self.assertPromiseFails('image-url-list-download-state-promise.py') self.assertPromiseFails(self.download_state_promise)
def test_unreachable_host(self): def test_unreachable_host(self):
self.rerequestInstance({ self.rerequestInstance({
'image-url-list': "evennotahost#%s" % ( self.key: self.unreachable_host_value % (
self.fake_image_md5sum,) self.fake_image_md5sum,)
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-download-state-promise.py') self.assertPromiseFails(self.download_state_promise)
def test_too_many_images(self): def test_too_many_images(self):
self.rerequestInstance({ self.rerequestInstance({
'image-url-list': """ self.key: self.too_many_image_value
image1#11111111111111111111111111111111 })
image2#22222222222222222222222222222222 self.raising_waitForInstance(3)
image3#33333333333333333333333333333333 self.assertPromiseFails(self.config_state_promise)
image4#44444444444444444444444444444444
image5#55555555555555555555555555555555
image6#66666666666666666666666666666666 @skipUnlessKvm
""" class TestBootImageUrlSelect(TestBootImageUrlList):
__partition_reference__ = 'bius'
# variations
key = 'boot-image-url-select'
test_input = '["%s#%s", "%s#%s"]'
image_directory = 'boot-image-url-select-repository'
config_state_promise = 'boot-image-url-select-config-state-promise.py'
download_md5sum_promise = 'boot-image-url-select-download-md5sum-promise.py'
download_state_promise = 'boot-image-url-select-download-state-promise.py'
bad_value = '["jsutbad"]'
incorrect_md5sum_value_image = '["%s#"]'
incorrect_md5sum_value = '["url#asdasd"]'
single_image_value = '["%s#%s"]'
unreachable_host_value = '["evennotahost#%s"]'
too_many_image_value = """[
"image1#11111111111111111111111111111111",
"image2#22222222222222222222222222222222",
"image3#33333333333333333333333333333333",
"image4#44444444444444444444444444444444",
"image5#55555555555555555555555555555555",
"image6#66666666666666666666666666666666"
]"""
def test_not_json(self):
self.rerequestInstance({
self.key: 'notjson#notjson'
}) })
self.raising_waitForInstance(3) self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py') self.assertPromiseFails(self.config_state_promise)
def test_together(self):
partition_parameter_kw = {
'boot-image-url-list': "%s#%s" % (
self.fake_image, self.fake_image_md5sum),
'boot-image-url-select': '["%s#%s"]' % (
self.fake_image, self.fake_image_md5sum)
}
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, self.fake_image_md5sum)
self.assertTrue(os.path.islink(image_link))
self.assertEqual(os.readlink(image_link), image)
def getRunningImageList():
running_image_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
software_root = '/'.join([
self.slap.software_directory,
hashlib.md5(self.getSoftwareURL()).hexdigest()])
for entry in kvm_process.cmdline():
if entry.startswith('file') and 'media=cdrom' in entry:
# do cleanups
entry = entry.replace(software_root, '')
entry = entry.replace(self.computer_partition_root_path, '')
running_image_list.append(entry)
return running_image_list
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom0'],
getRunningImageList()
)
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
# now the image is available in the kvm, and its above default image
self.assertEqual(
[
'file=/srv/boot-image-url-select-repository/image_001,media=cdrom,'
'if=none,id=cdrom0',
'file=/srv/boot-image-url-list-repository/image_001,media=cdrom,'
'if=none,id=cdrom1',
'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom2'
],
getRunningImageList()
)
# cleanup of images works, also asserts that configuration changes are
# reflected
self.rerequestInstance(
{'boot-image-url-list': '', 'boot-image-url-select': ''})
self.slap.waitForInstance(max_retry=2)
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory)
self.assertEqual(
os.listdir(image_repository),
[]
)
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
# again only default image is available in the running process
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom,if=none,id=cdrom0'],
getRunningImageList()
)
@skipUnlessKvm @skipUnlessKvm
class TestImageUrlListKvmCluster(InstanceTestCase): class TestBootImageUrlListKvmCluster(InstanceTestCase):
__partition_reference__ = 'iulkc' __partition_reference__ = 'biulkc'
@classmethod @classmethod
def getInstanceSoftwareType(cls): def getInstanceSoftwareType(cls):
...@@ -693,18 +858,22 @@ class TestImageUrlListKvmCluster(InstanceTestCase): ...@@ -693,18 +858,22 @@ class TestImageUrlListKvmCluster(InstanceTestCase):
"2a74b7675a8d8ca20476dba6e",) "2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6" fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
input_value = "%s#%s"
key = 'boot-image-url-list'
config_file_name = 'boot-image-url-list.conf'
@classmethod @classmethod
def getInstanceParameterDict(cls): def getInstanceParameterDict(cls):
return {'_': json.dumps({ return {'_': json.dumps({
"kvm-partition-dict": { "kvm-partition-dict": {
"KVM0": { "KVM0": {
"disable-ansible-promise": True, "disable-ansible-promise": True,
"image-url-list": "%s#%s" % ( cls.key: cls.input_value % (
cls.fake_image, cls.fake_image_md5sum) cls.fake_image, cls.fake_image_md5sum)
}, },
"KVM1": { "KVM1": {
"disable-ansible-promise": True, "disable-ansible-promise": True,
"image-url-list": "%s#%s" % ( cls.key: cls.input_value % (
cls.fake_image2, cls.fake_image2_md5sum) cls.fake_image2, cls.fake_image2_md5sum)
} }
} }
...@@ -715,22 +884,31 @@ class TestImageUrlListKvmCluster(InstanceTestCase): ...@@ -715,22 +884,31 @@ class TestImageUrlListKvmCluster(InstanceTestCase):
# we assume ordering of the cluster requests # we assume ordering of the cluster requests
KVM0_config = os.path.join( KVM0_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '1', 'etc', self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
'image-url-list.conf') self.config_file_name)
KVM1_config = os.path.join( KVM1_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '2', 'etc', self.slap.instance_directory, self.__partition_reference__ + '2', 'etc',
'image-url-list.conf') self.config_file_name)
with open(KVM0_config, 'r') as fh: with open(KVM0_config, 'r') as fh:
self.assertEqual( self.assertEqual(
"%s#%s" % (self.fake_image, self.fake_image_md5sum), self.input_value % (self.fake_image, self.fake_image_md5sum),
fh.read() fh.read().strip()
) )
with open(KVM1_config, 'r') as fh: with open(KVM1_config, 'r') as fh:
self.assertEqual( self.assertEqual(
"%s#%s" % (self.fake_image2, self.fake_image2_md5sum), self.input_value % (self.fake_image2, self.fake_image2_md5sum),
fh.read() fh.read().strip()
) )
@skipUnlessKvm
class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
__partition_reference__ = 'biuskc'
input_value = "[\"%s#%s\"]"
key = 'boot-image-url-select'
config_file_name = 'boot-image-url-select.json'
@skipUnlessKvm @skipUnlessKvm
class TestCpuMemMaxDynamic(InstanceTestCase): class TestCpuMemMaxDynamic(InstanceTestCase):
__partition_reference__ = 'cmm' __partition_reference__ = 'cmm'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment