Skip to content

slapos
slapos
Commit 207563ef authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

kvm: Download virtual-hard-drive asynchronously 

For sanity virtual-hard-drive-md5sum became mandatory.

virtual-hard-drive-gzipped case is covered.
parent 12a5d107
Show whitespace changes
Inline Side-by-side
Showing with 156 additions and 53 deletions
software/kvm/buildout.hash.cfg
View file @ 207563ef
......@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 4f71b7616b9f4a2f968e89326a237768
md5sum = bf3f2d45dcc55b003c03959895f1c29a
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = adf05b4255269bc7d0eec479424405e4
md5sum = 2909821e8eebfe8928a0a0cfb94c0348
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
software/kvm/instance-kvm-cluster-input-schema.json
View file @ 207563ef
......@@ -359,7 +359,7 @@
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string"
},
"virtual-hard-drive-gzipped": {
......
software/kvm/instance-kvm-input-schema.json
View file @ 207563ef
......@@ -160,7 +160,7 @@
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string"
},
"virtual-hard-drive-gzipped": {
......
software/kvm/instance-kvm.cfg.jinja2
View file @ 207563ef
......@@ -17,6 +17,8 @@
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
......@@ -56,6 +58,11 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps
{%- if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
{%- endif %}
{%- if boot_image_url_list_enabled %}
boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list
......@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END
{% endif %} {# if boot_image_url_list_enabled #}
{% if virtual_hard_drive_url_enabled %}
## virtual-hard-drive-url support BEGIN
[empty-file-state-base-virtual-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[virtual-hard-drive-url-source-config]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{{ virtual_hard_drive_url }}
{% endraw -%}
{# Enforce md5sum on virtual-hard-drive-url #}
virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
context =
key virtual_hard_drive_url :virtual-hard-drive-url
rendered = ${directory:etc}/virtual-hard-drive-url.conf
[virtual-hard-drive-url-processed-config]
# compares if the current configuration has been used by
# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${virtual-hard-drive-url-source-config:rendered}
state-filename = virtual-hard-drive-url-processed-config.state
state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
[virtual-hard-drive-url-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the virtual-hard-drive-url-download script, which runs asynchronously
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-processed-config:state-filename}
config-filename = ${virtual-hard-drive-url-processed-config:state-file}
[virtual-hard-drive-url-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
error-state-filename = virtual-hard-drive-url-json-config-error.txt
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
[virtual-hard-drive-url-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-json-config:error-state-filename}
config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
[virtual-hard-drive-url-download-wrapper]
# wrapper to execute virtual-hard-drive-url-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
error-state-filename = virtual-hard-drive-url-download-controller-error.text
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[virtual-hard-drive-url-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
[virtual-hard-drive-url-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
## virtual-hard-drive-url support END
{% endif %} {# if virtual_hard_drive_url_enabled #}
[kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd}
......@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
{% else %}
virtual-hard-drive-url-json-config =
{% endif %}
{% if boot_image_url_list_enabled %}
boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %}
......@@ -1108,6 +1220,13 @@ parts =
cron-service
cron-entry-logrotate
frontend-promise
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-download-wrapper
virtual-hard-drive-url-config-state-promise
virtual-hard-drive-url-download-md5sum-promise
virtual-hard-drive-url-download-state-promise
virtual-hard-drive-url-processed-config-promise
{% endif %}
{% if boot_image_url_list_enabled %}
boot-image-url-list-download-wrapper
boot-image-url-list-config-state-promise
......
software/kvm/template/template-kvm-run.in
View file @ 207563ef
......@@ -6,10 +6,6 @@ import hashlib
import os
import socket
import subprocess
try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip
import shutil
from random import shuffle
......@@ -17,8 +13,6 @@ import glob
import re
import json
import ssl
# XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
......@@ -32,9 +26,6 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
{{ parameter_dict.get("nbd2-port") }}))
default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
......@@ -98,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
opener = FancyURLopener({})
virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
def md5Checksum(file_path):
with open(file_path, 'rb') as fh:
......@@ -163,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
lf.write('%s' % external_disk_number)
return id_list, external_disk_number
# Download existing hard drive if needed at first boot
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...')
try:
downloaded_disk = disk_info_list[0]['path']
# Use downloaded virtual-hard-drive-url
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
print('Using virtual hard drive...')
with open(virtual_hard_drive_url_json_config) as fh:
image_config = json.load(fh)
if image_config['error-amount'] == 0:
image = image_config['image-list'][0]
downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
# previous version was using disk in place, but here it would result with
# redownload, so copy it
if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_info_list[0]['path']
opener.retrieve(virtual_hard_drive_url, downloaded_disk)
except:
if os.path.exists(downloaded_disk):
os.remove(downloaded_disk)
try:
with open(disk_info_list[0]['path'], 'wb') as d_fh:
with gzip.open(downloaded_image, 'rb') as s_fh:
shutil.copyfileobj(s_fh, d_fh)
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.unlink(disk_info_list[0]['path'])
raise
md5sum = virtual_hard_drive_md5sum.strip()
if md5sum:
print('Checking MD5 checksum...')
local_md5sum = md5Checksum(downloaded_disk)
if local_md5sum != md5sum:
os.remove(downloaded_disk)
raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % (
local_md5sum, md5sum))
print('MD5sum check passed.')
else:
print('Warning: not checksum specified.')
if downloaded_disk.endswith('.gz'):
try:
with open(disk_info_list[0]['path'], 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_info_list[0]['path'])
os.unlink(disk_info_list[0]['path'])
raise
os.remove(downloaded_disk)
else:
raise ValueError('Problems with virtual-hard-drive-url download')
# Create disk if doesn't exist
# XXX: move to Buildout profile
......
software/kvm/test/test.py
View file @ 207563ef
......@@ -417,10 +417,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
"test-machine2": dict(bootstrap_machine_param_dict, **{
# Debian 9 image
"virtual-hard-drive-url":
"http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2"
"eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f940"
"71e073f9980cc61badd6d2427f592e6e8",
"virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99"
"http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
"57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
"a508930046a78cffea6e0e29d03345333cc",
"virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
"virtual-hard-drive-gzipped": True,
})
}
}))}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7