Commit cfe9f936 authored by Łukasz Nowak's avatar Łukasz Nowak

Merge branch 'feature/kvm-follow-slapos-restart'

This work results with VM being restarted on parameter change.

The VM will be stopped in case of parameter change, then the system will try
to start it back. For external resources (like downloadable images) it will be
waited until such images are correctly present before starting back the VM.

Image download is done asynchronously with image-download-controller system.
parents c8ce5d01 3670fd90
Pipeline #15414 failed with stage
...@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70 ...@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 4f71b7616b9f4a2f968e89326a237768 md5sum = bf0c01ac7493693bb57ebef00bb20fa0
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = adf05b4255269bc7d0eec479424405e4 md5sum = b8cc7c76438212e0522ebede88649393
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
......
...@@ -359,7 +359,7 @@ ...@@ -359,7 +359,7 @@
}, },
"virtual-hard-drive-md5sum": { "virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive", "title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.", "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string" "type": "string"
}, },
"virtual-hard-drive-gzipped": { "virtual-hard-drive-gzipped": {
......
...@@ -160,7 +160,7 @@ ...@@ -160,7 +160,7 @@
}, },
"virtual-hard-drive-md5sum": { "virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive", "title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.", "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string" "type": "string"
}, },
"virtual-hard-drive-gzipped": { "virtual-hard-drive-gzipped": {
......
...@@ -17,6 +17,8 @@ ...@@ -17,6 +17,8 @@
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%} {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%} {% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %} {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %} {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%} {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
...@@ -56,6 +58,11 @@ public = ${:srv}/public/ ...@@ -56,6 +58,11 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps cronstamps = ${:etc}/cronstamps
{%- if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
{%- endif %}
{%- if boot_image_url_list_enabled %} {%- if boot_image_url_list_enabled %}
boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list boot-image-url-list-var = ${:var}/boot-image-url-list
...@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file} ...@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END ## boot-image-url-list support END
{% endif %} {# if boot_image_url_list_enabled #} {% endif %} {# if boot_image_url_list_enabled #}
{% if virtual_hard_drive_url_enabled %}
## virtual-hard-drive-url support BEGIN
[empty-file-state-base-virtual-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[virtual-hard-drive-url-source-config]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{{ virtual_hard_drive_url }}
{% endraw -%}
{# Enforce md5sum on virtual-hard-drive-url #}
virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
context =
key virtual_hard_drive_url :virtual-hard-drive-url
rendered = ${directory:etc}/virtual-hard-drive-url.conf
[virtual-hard-drive-url-processed-config]
# compares if the current configuration has been used by
# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${virtual-hard-drive-url-source-config:rendered}
state-filename = virtual-hard-drive-url-processed-config.state
state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
[virtual-hard-drive-url-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the virtual-hard-drive-url-download script, which runs asynchronously
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-processed-config:state-filename}
config-filename = ${virtual-hard-drive-url-processed-config:state-file}
[virtual-hard-drive-url-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
error-state-filename = virtual-hard-drive-url-json-config-error.txt
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
[virtual-hard-drive-url-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-json-config:error-state-filename}
config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
[virtual-hard-drive-url-download-wrapper]
# wrapper to execute virtual-hard-drive-url-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
error-state-filename = virtual-hard-drive-url-download-controller-error.text
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[virtual-hard-drive-url-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
[virtual-hard-drive-url-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
## virtual-hard-drive-url support END
{% endif %} {# if virtual_hard_drive_url_enabled #}
[kvm-controller-parameter-dict] [kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }} python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd} vnc-passwd = ${gen-passwd:passwd}
...@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4} ...@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
vnc-port = 5901 vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }} default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
{% else %}
virtual-hard-drive-url-json-config =
{% endif %}
{% if boot_image_url_list_enabled %} {% if boot_image_url_list_enabled %}
boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered} boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %} {% else %}
...@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }} ...@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }}
{% endfor -%} {% endfor -%}
{% endif -%} {% endif -%}
{%- set depend_section_list = [] %}
{%- set hash_file_list = ['${kvm-run:rendered}'] %}
{%- macro generate_depend_section(section, key) %}
{%- do depend_section_list.append('${' + section + ':command}' ) %}
{%- do hash_file_list.append('${' + key + '}') %}
[{{ section }}]
recipe = plone.recipe.command
update-command = ${:command}
command = [ ! -f {{ '${' + key + '}' }} ] && touch {{ '${' + key + '}' }}
{%- endmacro %}
{#- Create depending sections, as state files appear late, so it's better to have empty file which will impact the hash anyway #}
{%- if boot_image_url_list_enabled %}
{{ generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if boot_image_url_select_enabled %}
{{ generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if virtual_hard_drive_url_enabled %}
{{ generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:md5sum-state-file') }}
{%- endif %}
[kvm-instance] [kvm-instance]
depends =
{%- for depend_section in depend_section_list %}
{{ depend_section }}
{%- endfor %}
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
socket-path = ${kvm-controller-parameter-dict:socket-path} socket-path = ${kvm-controller-parameter-dict:socket-path}
wrapper-path = ${directory:services}/kvm wrapper-path = ${directory:services}/kvm
command-line = ${kvm-run:rendered} command-line = ${kvm-run:rendered}
kvm-controller = ${kvm-controller-wrapper:wrapper-path} kvm-controller = ${kvm-controller-wrapper:wrapper-path}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
hash-files =
{%- for hash_file in hash_file_list %}
{{ hash_file }}
{%- endfor %}
[kvm-controller-wrapper] [kvm-controller-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -1108,6 +1248,13 @@ parts = ...@@ -1108,6 +1248,13 @@ parts =
cron-service cron-service
cron-entry-logrotate cron-entry-logrotate
frontend-promise frontend-promise
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-download-wrapper
virtual-hard-drive-url-config-state-promise
virtual-hard-drive-url-download-md5sum-promise
virtual-hard-drive-url-download-state-promise
virtual-hard-drive-url-processed-config-promise
{% endif %}
{% if boot_image_url_list_enabled %} {% if boot_image_url_list_enabled %}
boot-image-url-list-download-wrapper boot-image-url-list-download-wrapper
boot-image-url-list-config-state-promise boot-image-url-list-config-state-promise
......
...@@ -6,10 +6,6 @@ import hashlib ...@@ -6,10 +6,6 @@ import hashlib
import os import os
import socket import socket
import subprocess import subprocess
try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip import gzip
import shutil import shutil
from random import shuffle from random import shuffle
...@@ -17,8 +13,6 @@ import glob ...@@ -17,8 +13,6 @@ import glob
import re import re
import json import json
import ssl
# XXX: give all of this through parameter, don't use this as template, but as module # XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }} qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
qemu_path = {{ repr(parameter_dict["qemu-path"]) }} qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
...@@ -32,9 +26,6 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}', ...@@ -32,9 +26,6 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
{{ parameter_dict.get("nbd2-port") }})) {{ parameter_dict.get("nbd2-port") }}))
default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}' default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip() nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
use_tap = '{{ parameter_dict.get("use-tap") }}'.lower() use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
use_nat = '{{ parameter_dict.get("use-nat") }}'.lower() use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
...@@ -98,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}' ...@@ -98,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}' boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}' boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
opener = FancyURLopener({})
def md5Checksum(file_path): def md5Checksum(file_path):
with open(file_path, 'rb') as fh: with open(file_path, 'rb') as fh:
...@@ -163,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number): ...@@ -163,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
lf.write('%s' % external_disk_number) lf.write('%s' % external_disk_number)
return id_list, external_disk_number return id_list, external_disk_number
# Download existing hard drive if needed at first boot # Use downloaded virtual-hard-drive-url
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '': if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
print('Downloading virtual hard drive...') print('Using virtual hard drive...')
try: with open(virtual_hard_drive_url_json_config) as fh:
downloaded_disk = disk_info_list[0]['path'] image_config = json.load(fh)
if image_config['error-amount'] == 0:
image = image_config['image-list'][0]
downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
# previous version was using disk in place, but here it would result with
# redownload, so copy it
if virtual_hard_drive_gzipped == 'true': if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_info_list[0]['path'] try:
opener.retrieve(virtual_hard_drive_url, downloaded_disk) with open(disk_info_list[0]['path'], 'wb') as d_fh:
except: with gzip.open(downloaded_image, 'rb') as s_fh:
if os.path.exists(downloaded_disk): shutil.copyfileobj(s_fh, d_fh)
os.remove(downloaded_disk) except Exception:
raise if os.path.exists(disk_info_list[0]['path']):
md5sum = virtual_hard_drive_md5sum.strip() os.unlink(disk_info_list[0]['path'])
if md5sum: raise
print('Checking MD5 checksum...') else:
local_md5sum = md5Checksum(downloaded_disk) try:
if local_md5sum != md5sum: shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
os.remove(downloaded_disk) except Exception:
raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % ( if os.path.exists(disk_info_list[0]['path']):
local_md5sum, md5sum)) os.unlink(disk_info_list[0]['path'])
print('MD5sum check passed.') raise
else: else:
print('Warning: not checksum specified.') raise ValueError('virtual-hard-drive-url not ready yet')
if downloaded_disk.endswith('.gz'):
try:
with open(disk_info_list[0]['path'], 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_info_list[0]['path'])
raise
os.remove(downloaded_disk)
# Create disk if doesn't exist # Create disk if doesn't exist
# XXX: move to Buildout profile # XXX: move to Buildout profile
...@@ -378,6 +361,9 @@ else: ...@@ -378,6 +361,9 @@ else:
'-drive', '-drive',
'file=%s,media=cdrom' % (link,) 'file=%s,media=cdrom' % (link,)
]) ])
else:
raise ValueError('boot-image-url-select not ready yet')
if boot_image_url_list_json_config: if boot_image_url_list_json_config:
# Support boot-image-url-list # Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh: with open(boot_image_url_list_json_config) as fh:
...@@ -390,6 +376,8 @@ else: ...@@ -390,6 +376,8 @@ else:
'-drive', '-drive',
'file=%s,media=cdrom' % (link,) 'file=%s,media=cdrom' % (link,)
]) ])
else:
raise ValueError('boot-image-url-list not ready yet')
# Always add by default the default image # Always add by default the default image
kvm_argument_list.extend([ kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
import six.moves.http_client as httplib import six.moves.http_client as httplib
import json import json
import os import os
import glob
import hashlib import hashlib
import psutil import psutil
import requests import requests
...@@ -43,6 +44,7 @@ from six.moves import SimpleHTTPServer ...@@ -43,6 +44,7 @@ from six.moves import SimpleHTTPServer
import multiprocessing import multiprocessing
import time import time
import shutil import shutil
import sys
from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
...@@ -120,12 +122,25 @@ class KvmMixin(object): ...@@ -120,12 +122,25 @@ class KvmMixin(object):
'software_release/buildout.cfg', 'software_release/buildout.cfg',
] ]
]) ])
# find bin/kvm_raw
kvm_raw_list = glob.glob(
os.path.join(self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
self.assertEqual(1, len(kvm_raw_list)) # allow to work only with one
hash_file_list = [
kvm_raw_list[0],
'software_release/buildout.cfg',
]
kvm_hash_value = generateHashFromFiles([
os.path.join(self.computer_partition_root_path, hash_file)
for hash_file in hash_file_list
])
with self.slap.instance_supervisor_rpc as supervisor: with self.slap.instance_supervisor_rpc as supervisor:
running_process_info = '\n'.join(sorted([ running_process_info = '\n'.join(sorted([
'%(group)s:%(name)s %(statename)s' % q for q '%(group)s:%(name)s %(statename)s' % q for q
in supervisor.getAllProcessInfo() in supervisor.getAllProcessInfo()
if q['name'] != 'watchdog' and q['group'] != 'watchdog'])) if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
return running_process_info.replace(hash_value, '{hash}') return running_process_info.replace(
hash_value, '{hash}').replace(kvm_hash_value, '{kvm-hash-value}')
def raising_waitForInstance(self, max_retry): def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError): with self.assertRaises(SlapOSNodeCommandError):
...@@ -176,7 +191,7 @@ i0:6tunnel-10443-{hash}-on-watch RUNNING ...@@ -176,7 +191,7 @@ i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:bootstrap-monitor EXITED i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING i0:crond-{hash}-on-watch RUNNING
i0:kvm-{hash}-on-watch RUNNING i0:kvm-{kvm-hash-value}-on-watch RUNNING
i0:kvm_controller EXITED i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED i0:monitor-httpd-graceful EXITED
...@@ -416,10 +431,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase): ...@@ -416,10 +431,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
"test-machine2": dict(bootstrap_machine_param_dict, **{ "test-machine2": dict(bootstrap_machine_param_dict, **{
# Debian 9 image # Debian 9 image
"virtual-hard-drive-url": "virtual-hard-drive-url":
"http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2" "http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
"eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f940" "57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
"71e073f9980cc61badd6d2427f592e6e8", "a508930046a78cffea6e0e29d03345333cc",
"virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99" "virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
"virtual-hard-drive-gzipped": True,
}) })
} }
}))} }))}
...@@ -499,7 +515,7 @@ ir2:bootstrap-monitor EXITED ...@@ -499,7 +515,7 @@ ir2:bootstrap-monitor EXITED
ir2:certificate_authority-{hash}-on-watch RUNNING ir2:certificate_authority-{hash}-on-watch RUNNING
ir2:crond-{hash}-on-watch RUNNING ir2:crond-{hash}-on-watch RUNNING
ir2:equeue-on-watch RUNNING ir2:equeue-on-watch RUNNING
ir2:kvm-{hash}-on-watch RUNNING ir2:kvm-{kvm-hash-value}-on-watch RUNNING
ir2:kvm_controller EXITED ir2:kvm_controller EXITED
ir2:monitor-httpd-{hash}-on-watch RUNNING ir2:monitor-httpd-{hash}-on-watch RUNNING
ir2:monitor-httpd-graceful EXITED ir2:monitor-httpd-graceful EXITED
...@@ -1282,7 +1298,7 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin): ...@@ -1282,7 +1298,7 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
'disk-device-path': '/dev/virt0 /dev/virt1', 'disk-device-path': '/dev/virt0 /dev/virt1',
'wipe-disk-ondestroy': True 'wipe-disk-ondestroy': True
}) })
self.slap.waitForInstance(max_retry=2) self.raising_waitForInstance(3)
instance_path = os.path.join( instance_path = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference) self.slap.instance_directory, self.kvm_instance_partition_reference)
...@@ -1298,3 +1314,210 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin): ...@@ -1298,3 +1314,210 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k""" dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
) )
self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK)) self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))
@skipUnlessKvm
class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
__partition_reference__ = 'idc'
maxDiff = None
def setUp(self):
super(TestImageDownloadController, self).setUp()
self.working_directory = tempfile.mkdtemp()
self.destination_directory = os.path.join(
self.working_directory, 'destination')
os.mkdir(self.destination_directory)
self.config_json = os.path.join(
self.working_directory, 'config.json')
self.md5sum_fail_file = os.path.join(
self.working_directory, 'md5sum_fail_file')
self.error_state_file = os.path.join(
self.working_directory, 'error_state_file')
self.processed_md5sum = os.path.join(
self.working_directory, 'processed_md5sum')
self.startImageHttpServer()
self.image_download_controller = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '0',
'software_release', 'parts', 'image-download-controller',
'image-download-controller')
def tearDown(self):
self.stopImageHttpServer()
shutil.rmtree(self.working_directory)
super(InstanceTestCase, self).tearDown()
def callImageDownloadController(self, *args):
call_list = [sys.executable, self.image_download_controller] + list(args)
try:
return (0, subprocess.check_output(
call_list, stderr=subprocess.STDOUT).decode('utf-8'))
except subprocess.CalledProcessError as e:
return (e.returncode, e.output.decode('utf-8'))
def runImageDownloadControlerWithDict(self, json_dict):
with open(self.config_json, 'w') as fh:
json.dump(json_dict, fh, indent=2)
return self.callImageDownloadController(
self.config_json,
'curl', # comes from test environemnt, considered to be recent enough
self.md5sum_fail_file,
self.error_state_file,
self.processed_md5sum
)
def assertFileContent(self, path, content):
self.assertTrue(os.path.exists, path)
with open(path, 'r') as fh:
self.assertEqual(
fh.read(),
content)
def test(self):
json_dict = {
'error-amount': 0,
'config-md5sum': 'config-md5sum',
'destination-directory': self.destination_directory,
'image-list': [
{
'destination-tmp': 'tmp',
'url': self.fake_image,
'destination': 'destination',
'link': 'image_001',
'gzipped': False,
'md5sum': self.fake_image_md5sum,
}
]
}
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
INF: %(fake_image)s : Stored with checksum %(checksum)s
INF: %(fake_image)s : Symlinking %(symlink)s -> %(destination)s
""".strip() % {
'fake_image': self.fake_image,
'checksum': self.fake_image_md5sum,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(self.destination_directory, 'destination'),
})
)
self.assertFileContent(self.md5sum_fail_file, '')
self.assertFileContent(self.error_state_file, '')
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFileContent(
os.path.join(self.destination_directory, 'destination'),
'fake_image_content'
)
# Nothing happens if all is downloaded
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : already downloaded
""".strip() % {
'fake_image': self.fake_image,
'checksum': self.fake_image_md5sum,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(self.destination_directory, 'destination'),
})
)
def test_fail(self):
json_dict = {
'error-amount': 0,
'config-md5sum': 'config-md5sum',
'destination-directory': self.destination_directory,
'image-list': [
{
'destination-tmp': 'tmp',
'url': self.fake_image,
'destination': 'destination',
'link': 'image_001',
'gzipped': False,
'md5sum': self.fake_image_wrong_md5sum,
}
]
}
for try_num in range(1, 5):
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(1, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
""". strip() % {
'fake_image': self.fake_image,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(
self.destination_directory, 'destination'),
})
)
fake_image_url = '#'.join([
self.fake_image, self.fake_image_wrong_md5sum])
self.assertFileContent(
self.md5sum_fail_file, """{
"%s": %s
}""" % (fake_image_url, try_num))
self.assertFileContent(
self.error_state_file, """
ERR: %(fake_image)s : MD5 mismatch expected is %(wrong_checksum)s """
"""but got instead %(real_checksum)s""".strip() % {
'fake_image': self.fake_image,
'wrong_checksum': self.fake_image_wrong_md5sum,
'real_checksum': self.fake_image_md5sum,
})
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFalse(
os.path.exists(
os.path.join(self.destination_directory, 'destination')))
code, result = self.runImageDownloadControlerWithDict(
json_dict
)
self.assertEqual(
(code, result.strip()),
(1, """
INF: Storing errors in %(error_state_file)s
""". strip() % {
'fake_image': self.fake_image,
'error_state_file': self.error_state_file,
'symlink': os.path.join(self.destination_directory, 'image_001'),
'destination': os.path.join(
self.destination_directory, 'destination'),
})
)
fake_image_url = '#'.join([
self.fake_image, self.fake_image_wrong_md5sum])
self.assertFileContent(
self.md5sum_fail_file, """{
"%s": %s
}""" % (fake_image_url, 4))
self.assertFileContent(
self.error_state_file, """
ERR: %(fake_image)s : Checksum is incorrect after 4 tries, will not """
"""retry""".strip() % {
'fake_image': self.fake_image,
})
self.assertFileContent(self.processed_md5sum, 'config-md5sum')
self.assertFalse(
os.path.exists(os.path.join(self.destination_directory, 'tmp')))
self.assertFalse(
os.path.exists(
os.path.join(self.destination_directory, 'destination')))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment