Skip to content

slapos
slapos
Commit 5d6d8dff authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

fixup! XXX software/kvm: Drop default image and replace it with boot-image-url-select

parent e2f2d1fd
Hide whitespace changes
Inline Side-by-side
Showing with 202 additions and 101 deletions
software/kvm/buildout.hash.cfg
View file @ 5d6d8dff
......@@ -19,7 +19,7 @@ md5sum = cca7eb0f975da97491aa7803d5bd753b
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 1e51153b0498b6b0d44140f3bab8f4b0
md5sum = 34b32389f9a1d5e1a93e5d57d2ae674e
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -59,7 +59,7 @@ md5sum = 6328f99728284847b8dd1146aadeae1b
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = 75696496b7d7efbd54dce7d2aa73c6dc
md5sum = c7bba51a59eef9dd1d6d7068ade4e92d
[template-kvm-controller]
filename = template/kvm-controller-run.in
......@@ -99,4 +99,4 @@ md5sum = e9d40162ba77472775256637a2617d14
[boot-image-select-source-config]
filename = template/boot-image-select-source-config.json.in
md5sum = a80df3f3b445f8b1a575bc5e5bd0f541
md5sum = 201c9ed07d7464e9f426a3ea7309a77d
software/kvm/instance-kvm.cfg.jinja2
View file @ 5d6d8dff
......@@ -23,13 +23,15 @@
{% if 'boot-image-url-select' in slapparameter_dict %}
{% set boot_image_url_select_enabled = True %}
{% else %}
{% if boot_image_url_list_enabled or 'nbd-host' in slapparameter_dict or 'nbd2-host' in slapparameter_dict or virtual_hard_drive_url_enabled %}
{% if boot_image_url_list_enabled or 'nbd-host' in slapparameter_dict or 'nbd2-host' in slapparameter_dict or boot_image_url_list_enabled or virtual_hard_drive_url_enabled %}
{% set boot_image_url_select_enabled = False %}
{% else %}
{# Become default if no other way is used to provide the boot image #}
{% set boot_image_url_select_enabled = True %}
{% endif %}
{% endif %}
# boot_image_url_list_enabled = {{ boot_image_url_list_enabled }}
# boot_image_url_select_enabled = {{ boot_image_url_select_enabled }}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 2)) + 1)) %}
{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 4096)) + 512)) %}
......
software/kvm/template/boot-image-select-source-config.json.in
View file @ 5d6d8dff
......@@ -8,5 +8,6 @@
"Fedora Server 32 netinst x86_64" : "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713",
"FreeBSD 12.1 RELEASE bootonly x86_64" : "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23",
} -%}
["{{ IMAGE_URL_MAPPING[boot_image_url_select] }}"]
{#- Note: Above is explicitely set to fail in case if wrong key is present, as this will force users to select correct image name, especially after upgrade -#}
["{{ IMAGE_URL_MAPPING.get(boot_image_url_select, '') }}"]
{#- Note-XXX: Above is explicitely set to fail in case if wrong key is present, as this will force users to select correct image name, especially after upgrade -#}
{#- Idea dropped, as promise shall be used, and when buildout fails, no promise runs -#}
software/kvm/template/template-kvm-run.in
View file @ 5d6d8dff
......@@ -405,12 +405,13 @@ for nbd_ip, nbd_port in nbd_list:
else:
# Note: Do not get tempted to use virtio-scsi-pci, as it does not work with
# Debian installation CDs, rendering it uninstallable
if boot_image_url_select_json_config:
# Support boot-image-url-select
handle_image(boot_image_url_select_json_config, 'boot-image-url-select')
# Note: boot-image-url-list has precedence over boot-image-url-select
if boot_image_url_list_json_config:
# Support boot-image-url-list
handle_image(boot_image_url_list_json_config, 'boot-image-url-list')
if boot_image_url_select_json_config:
# Support boot-image-url-select
handle_image(boot_image_url_select_json_config, 'boot-image-url-select')
print('Starting KVM: \n %s' % ' '.join(kvm_argument_list))
os.execv(qemu_path, kvm_argument_list)
software/kvm/test/test.py
View file @ 5d6d8dff
......@@ -156,6 +156,30 @@ class KVMTestCase(InstanceTestCase):
class KvmMixin:
def getRunningImageList(
self, kvm_instance_partition,
_match_cdrom=re.compile('file=(.+),media=cdrom$').match,
_sub_iso=re.compile(r'(/debian)(-[^-/]+)(-[^/]+-netinst\.iso)$').sub,
):
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name'])['pid']
sub_shared = re.compile(r'^%s/[^/]+/[0-9a-f]{32}/'
% re.escape(self.slap.shared_directory)).sub
image_list = []
for entry in psutil.Process(kvm_pid).cmdline():
m = _match_cdrom(entry)
if m:
path = m.group(1)
image_list.append(
_sub_iso(
r'\1-${ver}\3',
sub_shared(
r'${shared}/',
path.replace(kvm_instance_partition, '${inst}')
)))
return image_list
def getConnectionParameterDictJson(self):
return json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
......@@ -172,8 +196,12 @@ class KvmMixin:
os.path.join(self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
self.assertEqual(1, len(kvm_raw_list)) # allow to work only with one
hash_file_list = [
# Note: order is important:
# * shall follow generated order in the hash_files
# * last shall be hash_existsing_files (software_release/buildout.cfg)
kvm_raw_list[0],
'software_release/buildout.cfg',
'var/boot-image-url-select/boot-image-url-select.json',
'software_release/buildout.cfg'
]
kvm_hash_value = generateHashFromFiles([
os.path.join(self.computer_partition_root_path, hash_file)
......@@ -249,6 +277,7 @@ class TestInstance(KVMTestCase, KvmMixin):
"""i0:6tunnel-10022-{hash}-on-watch RUNNING
i0:6tunnel-10080-{hash}-on-watch RUNNING
i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:boot-image-url-select-updater-{hash} EXITED
i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING
......@@ -263,6 +292,15 @@ i0:whitelist-firewall-{hash} RUNNING""",
self.getProcessInfo()
)
# assure that the default image is used
self.assertEqual(
[
'${inst}/srv/boot-image-url-select-repository/'
'326b7737c4262e8eb09cd26773f3356a'
],
self.getRunningImageList(self.computer_partition_root_path)
)
@skipUnlessKvm
class TestInstanceJson(
......@@ -878,6 +916,20 @@ class HttpHandler(http.server.SimpleHTTPRequestHandler):
class FakeImageServerMixin(KvmMixin):
@classmethod
def setUpClass(cls):
try:
cls.startImageHttpServer()
super().setUpClass()
except BaseException:
cls.stopImageHttpServer()
raise
@classmethod
def tearDownClass(cls):
super().tearDownClass()
cls.stopImageHttpServer()
@classmethod
def startImageHttpServer(cls):
cls.image_source_directory = tempfile.mkdtemp()
......@@ -976,54 +1028,13 @@ class TestBootImageUrlList(KVMTestCase, FakeImageServerMixin):
cls.fake_image2_md5sum)
}
@classmethod
def setUpClass(cls):
try:
cls.startImageHttpServer()
super().setUpClass()
except BaseException:
cls.stopImageHttpServer()
raise
@classmethod
def tearDownClass(cls):
super().tearDownClass()
cls.stopImageHttpServer()
def tearDown(self):
# clean up the instance for other tests
# 1st remove all images...
self.rerequestInstance({self.key: ''})
self.slap.waitForInstance(max_retry=10)
# 2nd ...move instance to "default" state
# move instance to "default" state
self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10)
super().tearDown()
def getRunningImageList(
self, kvm_instance_partition,
_match_cdrom=re.compile('file=(.+),media=cdrom$').match,
_sub_iso=re.compile(r'(/debian)(-[^-/]+)(-[^/]+-netinst\.iso)$').sub,
):
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name'])['pid']
sub_shared = re.compile(r'^%s/[^/]+/[0-9a-f]{32}/'
% re.escape(self.slap.shared_directory)).sub
image_list = []
for entry in psutil.Process(kvm_pid).cmdline():
m = _match_cdrom(entry)
if m:
path = m.group(1)
image_list.append(
_sub_iso(
r'\1-${ver}\3',
sub_shared(
r'${shared}/',
path.replace(kvm_instance_partition, '${inst}')
)))
return image_list
def test(self):
# check that image is correctly downloaded
kvm_instance_partition = os.path.join(
......@@ -1174,32 +1185,69 @@ class TestBootImageUrlListResilientJson(
@skipUnlessKvm
class TestBootImageUrlSelect(TestBootImageUrlList):
class TestBootImageUrlSelect(FakeImageServerMixin, KVMTestCase):
__partition_reference__ = 'bius'
kvm_instance_partition_reference = 'bius0'
# variations
key = 'boot-image-url-select'
test_input = '["%s#%s", "%s#%s"]'
empty_input = '[]'
test_input = '"%s#%s"'
empty_input = ''
image_directory = 'boot-image-url-select-repository'
config_state_promise = 'boot-image-url-select-config-state-promise.py'
download_md5sum_promise = 'boot-image-url-select-download-md5sum-promise.py'
download_state_promise = 'boot-image-url-select-download-state-promise.py'
bad_value = '["jsutbad"]'
incorrect_md5sum_value_image = '["%s#"]'
incorrect_md5sum_value = '["url#asdasd"]'
single_image_value = '["%s#%s"]'
unreachable_host_value = '["evennotahost#%s"]'
too_many_image_value = """[
"image1#11111111111111111111111111111111",
"image2#22222222222222222222222222222222",
"image3#33333333333333333333333333333333",
"image4#44444444444444444444444444444444",
"image5#55555555555555555555555555555555",
"image6#66666666666666666666666666666666"
]"""
bad_value = 'jsutbad'
incorrect_md5sum_value_image = '%s#'
incorrect_md5sum_value = '"url#asdasd"'
single_image_value = '%s#%s'
unreachable_host_value = 'evennotahost#%s'
def test(self):
# check default image
image_repository = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', 'boot-image-url-select-repository')
self.assertEqual(
['326b7737c4262e8eb09cd26773f3356a'],
os.listdir(image_repository)
)
image = os.path.join(image_repository, '326b7737c4262e8eb09cd26773f3356a')
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, '326b7737c4262e8eb09cd26773f3356a')
self.assertEqual(
[
'${inst}/srv/boot-image-url-select-repository/'
'326b7737c4262e8eb09cd26773f3356a'
],
self.getRunningImageList(self.computer_partition_root_path)
)
# switch the image
self.rerequestInstance({
'boot-image-url-select': "Debian Bullseye 11 netinst x86_64"})
self.slap.waitForInstance(max_retry=10)
image_repository = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', 'boot-image-url-select-repository')
self.assertEqual(
['b710c178eb434d79ce40ce703d30a5f0'],
os.listdir(image_repository)
)
image = os.path.join(image_repository, 'b710c178eb434d79ce40ce703d30a5f0')
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, 'b710c178eb434d79ce40ce703d30a5f0')
self.assertEqual(
[
'${inst}/srv/boot-image-url-select-repository/'
'b710c178eb434d79ce40ce703d30a5f0'
],
self.getRunningImageList(self.computer_partition_root_path)
)
def test_not_json(self):
self.rerequestInstance({
......@@ -1212,67 +1260,116 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
partition_parameter_kw = {
'boot-image-url-list': "{}#{}".format(
self.fake_image, self.fake_image_md5sum),
'boot-image-url-select': '["{}#{}"]'.format(
self.fake_image, self.fake_image_md5sum)
'boot-image-url-select': "Debian Bullseye 11 netinst x86_64"
}
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum)
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, self.fake_image_md5sum)
image_repository = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', 'boot-image-url-list-repository')
self.assertEqual(
[self.fake_image_md5sum],
os.listdir(image_repository)
)
image = os.path.join(image_repository, self.fake_image_md5sum)
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, self.fake_image_md5sum)
image_repository = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', 'boot-image-url-select-repository')
self.assertEqual(
['b710c178eb434d79ce40ce703d30a5f0'],
os.listdir(image_repository)
)
image = os.path.join(image_repository, 'b710c178eb434d79ce40ce703d30a5f0')
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, 'b710c178eb434d79ce40ce703d30a5f0')
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
self.assertEqual(
[
'${{inst}}/srv/boot-image-url-select-repository/{}'.format(
'${{inst}}/srv/boot-image-url-list-repository/{}'.format(
self.fake_image_md5sum),
'${inst}/srv/boot-image-url-select-repository/'
'b710c178eb434d79ce40ce703d30a5f0'
],
self.getRunningImageList(kvm_instance_partition)
)
# check that using only boot-image-url-list results with not having
# boot-image-url-select if nothing is provided
partition_parameter_kw = {
'boot-image-url-list': "{}#{}".format(
self.fake_image, self.fake_image_md5sum),
}
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded
image_repository = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', 'boot-image-url-list-repository')
self.assertEqual(
[self.fake_image_md5sum],
os.listdir(image_repository)
)
image = os.path.join(image_repository, self.fake_image_md5sum)
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, self.fake_image_md5sum)
image_repository = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', 'boot-image-url-select-repository')
self.assertEqual(
['b710c178eb434d79ce40ce703d30a5f0'], # XXX: No cleanup after deselecting it
os.listdir(image_repository)
)
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
self.assertEqual(
[
'${{inst}}/srv/boot-image-url-list-repository/{}'.format(
self.fake_image_md5sum),
'${shared}/debian-${ver}-amd64-netinst.iso',
],
self.getRunningImageList(kvm_instance_partition)
)
# cleanup of images works, also asserts that configuration changes are
# reflected
self.rerequestInstance(
{'boot-image-url-list': '', 'boot-image-url-select': ''})
self.slap.waitForInstance(max_retry=2)
for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join(
kvm_instance_partition, 'srv', image_directory)
self.assertEqual(
os.listdir(image_repository),
[]
)
self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10)
# cleanup of images works, also asserts that configuration changes are
# reflected
partition_parameter_kw[self.key] = ''
partition_parameter_kw['boot-image-url-list'] = ''
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=2)
self.assertEqual(
os.listdir(image_repository),
os.listdir(os.path.join(
kvm_instance_partition, 'srv', 'boot-image-url-select-repository')),
['326b7737c4262e8eb09cd26773f3356a']
)
self.assertEqual(
os.listdir(os.path.join(
kvm_instance_partition, 'srv', 'boot-image-url-list-repository')),
[]
)
# again only default image is available in the running process
self.assertEqual(
['${shared}/debian-${ver}-amd64-netinst.iso'],
[
'${inst}/srv/boot-image-url-select-repository/'
'326b7737c4262e8eb09cd26773f3356a'
],
self.getRunningImageList(kvm_instance_partition)
)
self.fail('XXX: No cleanup after deselecting it')
@skipUnlessKvm
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7