Skip to content

S
slapos
Commit e562bca8 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

software/kvm: Name field "Boot image list" for readability

parent ecc880d1
Show whitespace changes
Inline Side-by-side
Showing with 104 additions and 103 deletions
software/kvm/buildout.hash.cfg
View file @ e562bca8
......@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 50e78a2a34efe09afab161ae1c1efd46
md5sum = 2fb085450d33e2674b0c1d4ab2055762
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = a4788112008cd0b38a57cd28f7252fbd
md5sum = be228c9e39682be53aaba491f858f848
[template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = dd1f581f34cf5a0b627576771347c710
md5sum = da4947e6d67849f14b7b1ece2b945f04
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
software/kvm/instance-kvm-cluster-input-schema.json
View file @ e562bca8
......@@ -43,8 +43,8 @@
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
},
"image-url-list": {
"title": "List of URLs images to download",
"boot-image-url-list": {
"title": "Boot image list",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string",
"textarea": "true"
......
software/kvm/instance-kvm-cluster.cfg.jinja2.in
View file @ e562bca8
......@@ -127,9 +127,9 @@ config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd}
config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
{%- if 'image-url-list' in kvm_parameter_dict %}
{%- if 'boot-image-url-list' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-image-url-list = {{ kvm_parameter_dict['image-url-list'] }}
config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
{%- endif %}
config-type = cluster
......
software/kvm/instance-kvm-input-schema.json
View file @ e562bca8
......@@ -366,8 +366,8 @@
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
},
"image-url-list": {
"title": "List of URLs images to download",
"boot-image-url-list": {
"title": "Boot image list",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string",
"textarea": "true"
......
software/kvm/instance-kvm.cfg.jinja2
View file @ e562bca8
......@@ -9,7 +9,7 @@
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set image_url_list_enabled = 'image-url-list' in slapparameter_dict %}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
{% set extends_list = [] -%}
......@@ -54,10 +54,10 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps
{%- if image_url_list_enabled %}
image-repository = ${:srv}/image-repository
image-url-list-var = ${:var}/image-url-list
image-url-list-expose = ${monitor-directory:private}/image-url-list
{%- if boot_image_url_list_enabled %}
boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list
boot-image-url-list-expose = ${monitor-directory:private}/boot-image-url-list
{%- endif %}
[create-mac]
......@@ -73,8 +73,8 @@ recipe = slapos.cookbook:generate.password
storage-path = ${directory:srv}/passwd
bytes = 8
{% if image_url_list_enabled %}
## image-url-list support BEGIN
{% if boot_image_url_list_enabled %}
## boot-image-url-list support BEGIN
[empty-file-state-base-promise]
<= monitor-promise-base
module = check_file_state
......@@ -83,25 +83,25 @@ config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[image-url-list-source-config]
[boot-image-url-list-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline image-url-list
# special "magic" is used, to properly support multiline boot-image-url-list
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support image-url-list being None, if key is present with value "" #}
{#- Do special trick to support boot-image-url-list being None, if key is present with value "" #}
{%- raw %}
{{ slap_parameter.get('image-url-list') or '' }}
{{ slap_parameter.get('boot-image-url-list') or '' }}
{% endraw -%}
context =
section slap_parameter slap-parameter
rendered = ${directory:etc}/image-url-list.conf
rendered = ${directory:etc}/boot-image-url-list.conf
[image-url-list-processed-config]
[boot-image-url-list-processed-config]
# compares if the current configuration has been used by
# the image-url-list-download, if not, exposes it as not empty file with
# the boot-image-url-list-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
......@@ -122,58 +122,58 @@ install =
state_handler.write(str(e))
update = ${:install}
config-file = ${image-url-list-source-config:rendered}
state-filename = image-url-list-processed-config.state
state-file = ${directory:image-url-list-expose}/${:state-filename}
processed-md5sum = ${directory:image-url-list-var}/update-image-processed.md5sum
config-file = ${boot-image-url-list-source-config:rendered}
state-filename = boot-image-url-list-processed-config.state
state-file = ${directory:boot-image-url-list-expose}/${:state-filename}
processed-md5sum = ${directory:boot-image-url-list-var}/update-image-processed.md5sum
[image-url-list-processed-config-promise]
[boot-image-url-list-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the image-url-list-download script, which runs asynchronously
# processed by the boot-image-url-list-download script, which runs asynchronously
<= empty-file-state-base-promise
filename = ${image-url-list-processed-config:state-filename}
config-filename = ${image-url-list-processed-config:state-file}
filename = ${boot-image-url-list-processed-config:state-filename}
config-filename = ${boot-image-url-list-processed-config:state-file}
[image-url-list-json-config]
[boot-image-url-list-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${image-url-list-source-config:rendered} ${:rendered} ${directory:image-repository} ${:error-state-file}
command = {{ python_executable }} {{ image_download_config_creator }} ${boot-image-url-list-source-config:rendered} ${:rendered} ${directory:boot-image-url-list-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:image-url-list-var}/image-url-list.json
error-state-filename = image-url-list-json-config-error.txt
error-state-file = ${directory:image-url-list-expose}/${:error-state-filename}
rendered = ${directory:boot-image-url-list-var}/boot-image-url-list.json
error-state-filename = boot-image-url-list-json-config-error.txt
error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filename}
[image-url-list-config-state-promise]
[boot-image-url-list-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-promise
filename = ${image-url-list-json-config:error-state-filename}
config-filename = ${image-url-list-json-config:error-state-file}
filename = ${boot-image-url-list-json-config:error-state-filename}
config-filename = ${boot-image-url-list-json-config:error-state-file}
[image-url-list-download-wrapper]
# wrapper to execute image-url-list-download on each run
[boot-image-url-list-download-wrapper]
# wrapper to execute boot-image-url-list-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/image-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${image-url-list-processed-config:processed-md5sum}
md5sum-state-filename = image-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:image-url-list-expose}/${:md5sum-state-filename}
error-state-filename = image-download-controller-error.text
error-state-file = ${directory:image-url-list-expose}/${:error-state-filename}
wrapper-path = ${directory:scripts}/boot-image-url-list-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-list-processed-config:processed-md5sum}
md5sum-state-filename = boot-image-url-list-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:boot-image-url-list-expose}/${:md5sum-state-filename}
error-state-filename = boot-image-url-list-download-controller-error.text
error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[image-url-list-download-md5sum-promise]
[boot-image-url-list-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-promise
filename = ${image-url-list-download-wrapper:md5sum-state-filename}
config-filename = ${image-url-list-download-wrapper:md5sum-state-file}
filename = ${boot-image-url-list-download-wrapper:md5sum-state-filename}
config-filename = ${boot-image-url-list-download-wrapper:md5sum-state-file}
[image-url-list-download-state-promise]
[boot-image-url-list-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-promise
filename = ${image-url-list-download-wrapper:error-state-filename}
config-filename = ${image-url-list-download-wrapper:error-state-file}
## image-url-list support END
{% endif %} {# if image_url_list_enabled #}
filename = ${boot-image-url-list-download-wrapper:error-state-filename}
config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END
{% endif %} {# if boot_image_url_list_enabled #}
[kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }}
......@@ -195,10 +195,10 @@ vnc-ip = ${:ipv4}
vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if image_url_list_enabled %}
image-url-list-json-config = ${image-url-list-json-config:rendered}
{% if boot_image_url_list_enabled %}
boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %}
image-url-list-json-config =
boot-image-url-list-json-config =
{% endif %}
nbd-host = ${slap-parameter:nbd-host}
nbd-port = ${slap-parameter:nbd-port}
......@@ -823,7 +823,7 @@ nbd-port = 1024
nbd-host =
nbd2-port = 1024
nbd2-host =
image-url-list =
boot-image-url-list =
enable-device-hotplug = False
ram-size = 1024
......@@ -873,7 +873,7 @@ keyboard-layout-language = fr
{% set key_list = v.split('\n') -%}
{{ k }} =
{{ key_list | join('\n ') }}
{% elif k == 'image-url-list' %}
{% elif k == 'boot-image-url-list' %}
{# needs to decorate possibly multiline or maybe unsafe value #}
{{ k }} = {{ dumps(v) }}
{% else -%}
......@@ -950,12 +950,12 @@ parts =
cron-service
cron-entry-logrotate
frontend-promise
{% if image_url_list_enabled %}
image-url-list-download-wrapper
image-url-list-config-state-promise
image-url-list-download-md5sum-promise
image-url-list-download-state-promise
image-url-list-processed-config-promise
{% if boot_image_url_list_enabled %}
boot-image-url-list-download-wrapper
boot-image-url-list-config-state-promise
boot-image-url-list-download-md5sum-promise
boot-image-url-list-download-state-promise
boot-image-url-list-processed-config-promise
{% endif %}
{% if additional_frontend %}
frontend-additional-promise
......
software/kvm/template/template-kvm-run.in
View file @ e562bca8
......@@ -99,7 +99,7 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
logfile = '{{ parameter_dict.get("log-file") }}'
image_url_list_json_config = '{{ parameter_dict.get("image-url-list-json-config") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = FancyURLopener(context=ssl._create_unverified_context())
......@@ -370,21 +370,21 @@ for nbd_ip, nbd_port in nbd_list:
'-drive',
'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
else:
image_url_list_used = False
if image_url_list_json_config:
# Support image-url-list
with open(image_url_list_json_config) as fh:
boot_image_url_list_used = False
if boot_image_url_list_json_config:
# Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh:
image_config = json.load(fh)
if image_config['error-amount'] == 0:
for image in sorted(image_config['image-list'], key=lambda k: k['link']):
link = os.path.join(image_config['destination-directory'], image['link'])
if os.path.exists(link) and os.path.islink(link):
image_url_list_used = True
boot_image_url_list_used = True
kvm_argument_list.extend([
'-drive',
'file=%s,media=cdrom' % (link,)
])
if not image_url_list_used:
if not boot_image_url_list_used:
# If no NBD is specified/available not downloadable image: use internal disk image
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso
......
software/kvm/test/test.py
View file @ e562bca8
......@@ -486,8 +486,8 @@ class TestInstanceNbdServer(InstanceTestCase):
@skipUnlessKvm
class TestImageUrlList(InstanceTestCase):
__partition_reference__ = 'iul'
class TestBootImageUrlList(InstanceTestCase):
__partition_reference__ = 'biul'
@classmethod
def getInstanceSoftwareType(cls):
......@@ -501,7 +501,7 @@ class TestImageUrlList(InstanceTestCase):
def tearDown(self):
# clean up the instance for other tests
# 1st remove all images...
self.rerequestInstance({'image-url-list': ''})
self.rerequestInstance({'boot-image-url-list': ''})
self.slap.waitForInstance(max_retry=10)
# 2nd ...move instance to "default" state
self.rerequestInstance({})
......@@ -535,7 +535,7 @@ class TestImageUrlList(InstanceTestCase):
def test(self):
partition_parameter_kw = {
'image-url-list': "%s#%s\n%s#%s" % (
'boot-image-url-list': "%s#%s\n%s#%s" % (
self.fake_image, self.fake_image_md5sum, self.fake_image2,
self.fake_image2_md5sum)
}
......@@ -543,7 +543,8 @@ class TestImageUrlList(InstanceTestCase):
self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', 'image-repository')
self.computer_partition_root_path, 'srv',
'boot-image-url-list-repository')
image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image))
......@@ -569,11 +570,11 @@ class TestImageUrlList(InstanceTestCase):
kvm_process = psutil.Process(kvm_pid)
cmd_line = ''.join(kvm_process.cmdline())
self.assertNotIn(
'srv/image-repository/image_001,media=cdrom',
'srv/boot-image-url-list-repository/image_001,media=cdrom',
cmd_line
)
self.assertNotIn(
'srv/image-repository/image_002,media=cdrom',
'srv/boot-image-url-list-repository/image_002,media=cdrom',
cmd_line
)
......@@ -591,17 +592,17 @@ class TestImageUrlList(InstanceTestCase):
kvm_process = psutil.Process(kvm_pid)
cmd_line = ''.join(kvm_process.cmdline())
self.assertIn(
'srv/image-repository/image_001,media=cdrom',
'srv/boot-image-url-list-repository/image_001,media=cdrom',
cmd_line
)
self.assertIn(
'srv/image-repository/image_002,media=cdrom',
'srv/boot-image-url-list-repository/image_002,media=cdrom',
cmd_line
)
# cleanup of images works, also asserts that configuration changes are
# reflected
self.rerequestInstance({'image-url-list': ''})
self.rerequestInstance({'boot-image-url-list': ''})
self.slap.waitForInstance(max_retry=2)
self.assertEqual(
os.listdir(image_repository),
......@@ -625,43 +626,43 @@ class TestImageUrlList(InstanceTestCase):
def test_bad_parameter(self):
self.rerequestInstance({
'image-url-list': "jsutbad"
'boot-image-url-list': "jsutbad"
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
self.assertPromiseFails('boot-image-url-list-config-state-promise.py')
def test_incorrect_md5sum(self):
self.rerequestInstance({
'image-url-list': "%s#" % (self.fake_image,)
'boot-image-url-list': "%s#" % (self.fake_image,)
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
self.assertPromiseFails('boot-image-url-list-config-state-promise.py')
self.rerequestInstance({
'image-url-list': "url#asdasd"
'boot-image-url-list': "url#asdasd"
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
self.assertPromiseFails('boot-image-url-list-config-state-promise.py')
def test_not_matching_md5sum(self):
self.rerequestInstance({
'image-url-list': "%s#%s" % (
'boot-image-url-list': "%s#%s" % (
self.fake_image, self.fake_image_wrong_md5sum)
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-download-md5sum-promise.py')
self.assertPromiseFails('image-url-list-download-state-promise.py')
self.assertPromiseFails('boot-image-url-list-download-md5sum-promise.py')
self.assertPromiseFails('boot-image-url-list-download-state-promise.py')
def test_unreachable_host(self):
self.rerequestInstance({
'image-url-list': "evennotahost#%s" % (
'boot-image-url-list': "evennotahost#%s" % (
self.fake_image_md5sum,)
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-download-state-promise.py')
self.assertPromiseFails('boot-image-url-list-download-state-promise.py')
def test_too_many_images(self):
self.rerequestInstance({
'image-url-list': """
'boot-image-url-list': """
image1#11111111111111111111111111111111
image2#22222222222222222222222222222222
image3#33333333333333333333333333333333
......@@ -671,12 +672,12 @@ class TestImageUrlList(InstanceTestCase):
"""
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
self.assertPromiseFails('boot-image-url-list-config-state-promise.py')
@skipUnlessKvm
class TestImageUrlListKvmCluster(InstanceTestCase):
__partition_reference__ = 'iulkc'
class TestBootImageUrlListKvmCluster(InstanceTestCase):
__partition_reference__ = 'biulkc'
@classmethod
def getInstanceSoftwareType(cls):
......@@ -699,12 +700,12 @@ class TestImageUrlListKvmCluster(InstanceTestCase):
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True,
"image-url-list": "%s#%s" % (
"boot-image-url-list": "%s#%s" % (
cls.fake_image, cls.fake_image_md5sum)
},
"KVM1": {
"disable-ansible-promise": True,
"image-url-list": "%s#%s" % (
"boot-image-url-list": "%s#%s" % (
cls.fake_image2, cls.fake_image2_md5sum)
}
}
......@@ -715,10 +716,10 @@ class TestImageUrlListKvmCluster(InstanceTestCase):
# we assume ordering of the cluster requests
KVM0_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
'image-url-list.conf')
'boot-image-url-list.conf')
KVM1_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '2', 'etc',
'image-url-list.conf')
'boot-image-url-list.conf')
with open(KVM0_config, 'r') as fh:
self.assertEqual(
"%s#%s" % (self.fake_image, self.fake_image_md5sum),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7