Commit d9e55c3e authored by Jérome Perrin's avatar Jérome Perrin

Managed Resources for Software Release Tests

Introduce new "Managed Resources" and assorted fixes accompanying nexedi/slapos!840 

Also respect meaning of verbose (more output in the console) and debug (drop in debugger on errors and keep things around for inspection)

See merge request nexedi/slapos.core!259
parents 9654a1d2 68ea12db
......@@ -46,6 +46,7 @@ except ImportError:
subprocess # pyflakes
from .utils import getPortFromPath
from .utils import ManagedResource
from ..slap.standalone import StandaloneSlapOS
from ..slap.standalone import SlapOSNodeCommandError
......@@ -54,7 +55,8 @@ from ..grid.utils import md5digest
from ..util import mkdir_p
try:
from typing import Iterable, Tuple, Callable, Type, Dict, List, Optional
from typing import Iterable, Tuple, Callable, Type, Dict, List, Optional, TypeVar
ManagedResourceType = TypeVar("ManagedResourceType", bound=ManagedResource)
except ImportError:
pass
......@@ -127,7 +129,7 @@ def makeModuleSetUpAndTestCaseClass(
logger = logging.getLogger()
console_handler = logging.StreamHandler()
console_handler.setLevel(
logging.DEBUG if (verbose or debug) else logging.WARNING)
logging.DEBUG if verbose else logging.WARNING)
logger.addHandler(console_handler)
if debug:
......@@ -412,7 +414,7 @@ def installSoftwareUrlList(cls, software_url_list, max_retry=10, debug=False):
cls._copySnapshot(path, name)
try:
cls.logger.debug("Starting")
cls.logger.debug("Starting SlapOS")
cls.slap.start()
for software_url in software_url_list:
cls.logger.debug("Supplying %s", software_url)
......@@ -484,6 +486,11 @@ class SlapOSInstanceTestCase(unittest.TestCase):
_ipv4_address = ""
_ipv6_address = ""
_resources = {} # type: Dict[str, ManagedResource]
_instance_parameter_dict = None # type: Dict
computer_partition = None # type: ComputerPartition
computer_partition_root_path = None # type: str
# a short name of that software URL.
# eg. helloworld instead of
# https://lab.nexedi.com/nexedi/slapos/raw/software/helloworld/software.cfg
......@@ -503,6 +510,29 @@ class SlapOSInstanceTestCase(unittest.TestCase):
'etc/',
)
@classmethod
def getManagedResource(cls, resource_name, resource_class):
# type: (str, Type[ManagedResourceType]) -> ManagedResourceType
"""Get the managed resource for this name.
If resouce was not created yet, it is created and `open`. The
resource will automatically be `close` at the end of the test
class.
"""
try:
existing_resource = cls._resources[resource_name]
except KeyError:
resource = resource_class(cls, resource_name)
cls._resources[resource_name] = resource
resource.open()
return resource
else:
if not isinstance(existing_resource, resource_class):
raise ValueError(
"Resource %s is of unexpected class %s" %
(resource_name, existing_resource), )
return existing_resource
# Methods to be defined by subclasses.
@classmethod
def getSoftwareURL(cls):
......@@ -540,7 +570,7 @@ class SlapOSInstanceTestCase(unittest.TestCase):
snapshot_name = "{}.{}.setUpClass".format(cls.__module__, cls.__name__)
try:
cls.logger.debug("Starting")
cls.logger.debug("Starting setUpClass %s", cls)
cls.slap.start()
cls.logger.debug(
"Formatting to remove old partitions XXX should not be needed because we delete ..."
......@@ -692,6 +722,12 @@ class SlapOSInstanceTestCase(unittest.TestCase):
"""Destroy all instances and stop subsystem.
Catches and log all exceptions and take snapshot named `snapshot_name` + the failing step.
"""
for resource_name in list(cls._resources):
cls.logger.debug("closing resource %s", resource_name)
try:
cls._resources.pop(resource_name).close()
except:
cls.logger.exception("Error closing resource %s", resource_name)
try:
if hasattr(cls, '_instance_parameter_dict'):
cls.requestDefaultInstance(state='destroyed')
......@@ -700,8 +736,16 @@ class SlapOSInstanceTestCase(unittest.TestCase):
cls._storeSystemSnapshot(
"{}._cleanup request destroy".format(snapshot_name))
try:
# To make debug usable, we tolerate report_max_retry-1 errors and
# only debug the last.
for _ in range(3):
cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug)
if cls._debug and cls.report_max_retry:
try:
cls.slap.waitForReport(max_retry=cls.report_max_retry - 1)
except SlapOSNodeCommandError:
cls.slap.waitForReport(debug=True)
else:
cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug)
except:
cls.logger.exception("Error during actual destruction")
cls._storeSystemSnapshot(
......@@ -732,8 +776,16 @@ class SlapOSInstanceTestCase(unittest.TestCase):
"{}._cleanup leaked_partitions request destruction".format(
snapshot_name))
try:
# To make debug usable, we tolerate report_max_retry-1 errors and
# only debug the last.
for _ in range(3):
cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug)
if cls._debug and cls.report_max_retry:
try:
cls.slap.waitForReport(max_retry=cls.report_max_retry - 1)
except SlapOSNodeCommandError:
cls.slap.waitForReport(debug=True)
else:
cls.slap.waitForReport(max_retry=cls.report_max_retry, debug=cls._debug)
except:
cls.logger.exception(
"Error during leaked partitions actual destruction")
......
......@@ -30,15 +30,22 @@ import socket
import hashlib
import unittest
import os
import logging
import multiprocessing
import shutil
import subprocess
import tempfile
import sys
import json
from contextlib import closing
from six.moves import BaseHTTPServer
from six.moves import urllib_parse
try:
import typing
if typing.TYPE_CHECKING:
from PIL import Image # pylint:disable=unused-import
from .testcase import SlapOSInstanceTestCase
except ImportError:
pass
......@@ -84,8 +91,111 @@ print(json.dumps(extra_config_dict))
return json.loads(extra_config_dict_json)
class ManagedResource(object):
"""A resource that will be available for the lifetime of test.
"""
def __init__(self, slapos_instance_testcase_class, resource_name):
# type: (typing.Type[SlapOSInstanceTestCase], str) -> None
self._cls = slapos_instance_testcase_class
self._name = resource_name
def open(self):
NotImplemented
def close(self):
NotImplemented
class ManagedHTTPServer(ManagedResource):
"""Simple HTTP Server for testing.
"""
# Request Handler, needs to be defined by subclasses.
RequestHandler = None # type: typing.Type[BaseHTTPServer.BaseHTTPRequestHandler]
proto = 'http'
# hostname to listen to, default to ipv4 address of the current test
hostname = None # type: str
# port to listen to, default
port = None # type: int
@property
def url(self):
# type: () -> str
return '{self.proto}://{self.hostname}:{self.port}'.format(self=self)
@property
def netloc(self):
# type: () -> str
return urllib_parse.urlparse(self.url).netloc
def _makeServer(self):
# type: () -> BaseHTTPServer.HTTPServer
"""build the server class.
This is a method to make it possible to subclass to add https support.
"""
logger = self._cls.logger
class ErrorLoggingHTTPServer(BaseHTTPServer.HTTPServer):
def handle_error(self, request , client_addr):
# redirect errors to log
logger.info("Error processing request from %s", client_addr, exc_info=True)
logger.debug(
"starting %s (%s) on %s:%s",
self.__class__.__name__,
self._name,
self.hostname,
self.port,
)
server = ErrorLoggingHTTPServer(
(self.hostname, self.port),
self.RequestHandler,
)
return server
def open(self):
# type: () -> None
if not self.hostname:
self.hostname = self._cls._ipv4_address
if not self.port:
self.port = findFreeTCPPort(self.hostname)
server = self._makeServer()
self._process = multiprocessing.Process(
target=server.serve_forever,
name=self._name,
)
self._process.start()
def close(self):
# type: () -> None
self._process.terminate()
self._process.join()
class ManagedHTTPSServer(ManagedHTTPServer):
"""An HTTPS Server
"""
proto = 'https'
certificate_file = None # type: str
def _makeServer(self):
# type: () -> BaseHTTPServer.HTTPServer
server = super(ManagedHTTPSServer, self)._makeServer()
raise NotImplementedError("TODO")
return server
class ManagedTemporaryDirectory(ManagedResource):
path = None # type: str
def open(self):
self.path = tempfile.mkdtemp()
def close(self):
shutil.rmtree(self.path)
class CrontabMixin(object):
computer_partition_root_path = None # type: str
logger = None # type: logging.Logger
def _getCrontabCommand(self, crontab_name):
# type: (str) -> str
"""Read a crontab and return the command that is executed.
......@@ -108,10 +218,12 @@ class CrontabMixin(object):
be a relative time.
"""
crontab_command = self._getCrontabCommand(crontab_name)
subprocess.check_call(
"faketime {date} bash -o pipefail -e -c '{crontab_command}'".format(**locals()),
crontab_output = subprocess.check_output(
# XXX we unset PYTHONPATH set by `setup.py test`
"env PYTHONPATH= faketime {date} bash -o pipefail -e -c '{crontab_command}'".format(**locals()),
shell=True,
)
self.logger.debug("crontab %s output: %s", crontab_command, crontab_output)
class ImageComparisonTestCase(unittest.TestCase):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment