Commit 79cc1ec0 authored by Jérome Perrin's avatar Jérome Perrin

check_software fixes

 - only consider shared parts from checked software
 - support new `.buildout-shared.json` signature files
 - unit test coverage

See merge request nexedi/slapos.core!361
parents 1e74ed08 77c724a8
Pipeline #19314 passed with stage
in 0 seconds
...@@ -34,6 +34,7 @@ import warnings ...@@ -34,6 +34,7 @@ import warnings
import pkg_resources import pkg_resources
import requests import requests
from six.moves.configparser import ConfigParser
try: try:
import subprocess32 as subprocess import subprocess32 as subprocess
...@@ -209,19 +210,34 @@ def checkSoftware(slap, software_url): ...@@ -209,19 +210,34 @@ def checkSoftware(slap, software_url):
**locals())) **locals()))
return executable_link_error_list return executable_link_error_list
paths_to_check = ( software_directory = os.path.join(slap.software_directory, software_hash)
os.path.join(slap.software_directory, software_hash), paths_to_check = set((software_directory, ))
slap.shared_directory,
) # Compute the paths to check by inspecting buildout installed database
# for this software. We are looking for shared parts installed by recipes.
config_parser = ConfigParser()
config_parser.read(os.path.join(software_directory, '.installed.cfg'))
for section_name in config_parser.sections():
for option_name in 'location', '__buildout_installed__':
if config_parser.has_option(section_name, option_name):
for section_path in config_parser.get(section_name, option_name).splitlines():
if section_path and not section_path.startswith(software_directory):
paths_to_check.add(section_path)
error_list.extend( error_list.extend(
checkExecutableLink( checkExecutableLink(
paths_to_check, paths_to_check,
paths_to_check + tuple(slap._shared_part_list), tuple(paths_to_check) + tuple(slap._shared_part_list),
)) ))
# check this software is not referenced in any shared parts. # check this software is not referenced in any shared parts.
for signature_file in glob.glob(os.path.join(slap.shared_directory, '*', '*', for signature_file in glob.glob(
'.*slapos.*.signature')): os.path.join(
slap.shared_directory,
'*',
'*',
'.buildout-shared.json',
)):
with open(signature_file) as f: with open(signature_file) as f:
signature_content = f.read() signature_content = f.read()
if software_hash in signature_content: if software_hash in signature_content:
......
LDLIBS = -lz
include Makefile.conf
.PHONY: all install uninstall clean
all: check
slapos-core-test: main.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS)
install: all
install -Dp slapos-core-test $(DESTDIR)$(PREFIX)/bin/slapos-core-test
clean:
-rm -f slapos-core-test
check: slapos-core-test
./slapos-core-test
dist: main.c Makefile configure
tar --transform 's,^,slapos-core-test-0.0.0/,' -czvf dist.tar.gz $?
#!/bin/sh
echo "configured with: $*"
echo "PREFIX=$(echo "$1" | sed 's/^[^=]*=//g')" > Makefile.conf
#include "zlib.h"
#include <stdio.h>
int main(int argc, char *argv[]){
printf("%s: using zlib: %s\n", argv[0], zlibVersion());
return 0;
}
##############################################################################
#
# Copyright (c) 2021 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import absolute_import
import unittest
import mock
import os
import glob
import tempfile
import textwrap
from slapos.testing.check_software import checkSoftware
from .test_standalone import SlapOSStandaloneTestCase
class TestCheckSoftware(SlapOSStandaloneTestCase):
# BBB python2
assertRaisesRegex = getattr(
unittest.TestCase,
'assertRaisesRegex',
unittest.TestCase.assertRaisesRegexp,
)
def _get_zlib_environment(self, with_rpath=True):
"""returns an environment that will compile with slapos' zlib
"""
# find a zlib in SLAPOS_TEST_SHARED_PART_LIST
zlib_location = None
for shared_part in self.standalone._shared_part_list:
for zlib_location in glob.glob(os.path.join(shared_part, 'zlib', '*')):
break
if zlib_location:
break
assert zlib_location
return {
'CFLAGS':
'-I{zlib_location}/include'.format(**locals()),
'LDFLAGS':
'-L{zlib_location}/lib -Wl,-rpath={zlib_location}/lib'.format(
**locals()) if with_rpath else '-L{zlib_location}/lib'.format(
**locals())
}
def _install_software(self, environment, shared=True):
environment_option = ''
for k, v in environment.items():
environment_option += ' {k}={v}\n'.format(k=k, v=v)
shared_option = 'true' if shared else 'false'
test_software_archive_url = os.path.join(
os.path.dirname(__file__), 'data', 'cmmi', 'dist.tar.gz')
with tempfile.NamedTemporaryFile(
suffix="-%s.cfg" % self.id(),
mode='w',
) as f:
f.write(
textwrap.dedent('''
[buildout]
parts = cmmi
newest = false
offline = true
eggs-directory = {os.environ[SLAPOS_TEST_EGGS_DIRECTORY]}
develop-eggs-directory = {os.environ[SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY]}
[cmmi]
recipe = slapos.recipe.cmmi
url = {test_software_archive_url}
shared = {shared_option}
environment =
{environment_option}
''').format(os=os, **locals()))
f.flush()
self.standalone.supply(f.name)
self.standalone.waitForSoftware()
return f.name
def test_software_using_system_libraries(self):
software_url = self._install_software(
self._get_zlib_environment(with_rpath=False), shared=False)
with self.assertRaisesRegex(
RuntimeError,
'./bin/slapos-core-test uses system library .*libz.so.* for libz.so',
):
checkSoftware(self.standalone, software_url)
def test_shared_part_using_system_libraries(self):
software_url = self._install_software(
self._get_zlib_environment(with_rpath=False))
with self.assertRaisesRegex(
RuntimeError,
'./bin/slapos-core-test uses system library .*libz.so.* for libz.so',
):
checkSoftware(self.standalone, software_url)
def test_shared_part_referencing_software(self):
environment = self._get_zlib_environment()
environment['reference-to-part'] = '${buildout:parts-directory}'
software_url = self._install_software(environment=environment)
with self.assertRaisesRegex(
RuntimeError,
'Software hash present in signature',
):
checkSoftware(self.standalone, software_url)
def test_ok(self):
software_url = self._install_software(
environment=self._get_zlib_environment())
checkSoftware(self.standalone, software_url)
def test_software_check_isolated(self):
# if a software populated shared parts with wrong parts, this does not
# impact checking other softwares, as long as they don't use the problematic
# parts
self.test_shared_part_using_system_libraries()
self.test_ok()
...@@ -30,7 +30,6 @@ import mock ...@@ -30,7 +30,6 @@ import mock
import os import os
import tempfile import tempfile
import textwrap import textwrap
import shutil
import hashlib import hashlib
import socket import socket
import errno import errno
...@@ -47,6 +46,8 @@ from slapos.slap.standalone import StandaloneSlapOS ...@@ -47,6 +46,8 @@ from slapos.slap.standalone import StandaloneSlapOS
from slapos.slap.standalone import SlapOSNodeSoftwareError from slapos.slap.standalone import SlapOSNodeSoftwareError
from slapos.slap.standalone import PartitionForwardConfiguration from slapos.slap.standalone import PartitionForwardConfiguration
from slapos.slap.standalone import PartitionForwardAsPartitionConfiguration from slapos.slap.standalone import PartitionForwardAsPartitionConfiguration
import slapos.util
SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4'] SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4']
SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6'] SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6']
...@@ -80,7 +81,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -80,7 +81,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
def test_format(self): def test_format(self):
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
standalone = StandaloneSlapOS( standalone = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
self.addCleanup(standalone.stop) self.addCleanup(standalone.stop)
...@@ -100,7 +101,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -100,7 +101,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
def test_reformat_less_partitions(self): def test_reformat_less_partitions(self):
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
standalone = StandaloneSlapOS( standalone = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
self.addCleanup(standalone.stop) self.addCleanup(standalone.stop)
...@@ -115,7 +116,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -115,7 +116,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
def test_reformat_less_chmod_files(self): def test_reformat_less_chmod_files(self):
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
standalone = StandaloneSlapOS( standalone = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
self.addCleanup(standalone.stop) self.addCleanup(standalone.stop)
...@@ -132,7 +133,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -132,7 +133,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
def test_reformat_different_base_name(self): def test_reformat_different_base_name(self):
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
standalone = StandaloneSlapOS( standalone = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
self.addCleanup(standalone.stop) self.addCleanup(standalone.stop)
...@@ -152,7 +153,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -152,7 +153,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
def test_reformat_refuse_deleting_running_partition(self): def test_reformat_refuse_deleting_running_partition(self):
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
standalone = StandaloneSlapOS( standalone = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
self.addCleanup(standalone.stop) self.addCleanup(standalone.stop)
...@@ -163,7 +164,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -163,7 +164,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
def test_slapos_node_format(self): def test_slapos_node_format(self):
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
standalone = StandaloneSlapOS( standalone = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
self.addCleanup(standalone.stop) self.addCleanup(standalone.stop)
...@@ -180,13 +181,13 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -180,13 +181,13 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
partitions = glob.glob(glob_pattern) partitions = glob.glob(glob_pattern)
self.assertEqual(partition_count, len(partitions)) self.assertEqual(partition_count, len(partitions))
for path in partitions: for path in partitions:
shutil.rmtree(path) slapos.util.rmtree(path)
subprocess.check_call(format_command) subprocess.check_call(format_command)
self.assertEqual(partition_count, len(glob.glob(glob_pattern))) self.assertEqual(partition_count, len(glob.glob(glob_pattern)))
def test_two_instance_from_same_directory(self): def test_two_instance_from_same_directory(self):
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
standalone1 = StandaloneSlapOS( standalone1 = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
...@@ -213,7 +214,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -213,7 +214,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
def test_partition_forward(self): def test_partition_forward(self):
# type: () -> None # type: () -> None
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
partition_forward_config = [ partition_forward_config = [
PartitionForwardConfiguration( PartitionForwardConfiguration(
'https://slapos1.example.com', 'https://slapos1.example.com',
...@@ -303,9 +304,16 @@ class SlapOSStandaloneTestCase(unittest.TestCase): ...@@ -303,9 +304,16 @@ class SlapOSStandaloneTestCase(unittest.TestCase):
def setUp(self): def setUp(self):
checkPortIsFree() checkPortIsFree()
working_dir = tempfile.mkdtemp(prefix=__name__) working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir) self.addCleanup(slapos.util.rmtree, working_dir)
self.standalone = StandaloneSlapOS( self.standalone = StandaloneSlapOS(
working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT) working_dir,
SLAPOS_TEST_IPV4,
SLAPOS_TEST_PORT,
shared_part_list=[
os.path.expanduser(p) for p in os.environ.get(
'SLAPOS_TEST_SHARED_PART_LIST', '').split(os.pathsep) if p
],
)
if self._auto_stop_standalone: if self._auto_stop_standalone:
self.addCleanup(self.standalone.stop) self.addCleanup(self.standalone.stop)
self.standalone.format(1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6) self.standalone.format(1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment