Commit eb753808 authored by Jérome Perrin's avatar Jérome Perrin

cli/request: support passing instance parameters from a file

Support a syntax like:

      slapos request \
        --node=computer_guid=local \
        --parameters-file=~/request.json \
        ERP5 \
        https://lab.nexedi.com/nexedi/slapos/raw/1.0.145/software/erp5/software.cfg

to request an instance with parameters in ~/request.json file.

This also teach slapos.core about the instance schema, which makes it possible to
understand the serialisation, so that we no longer need to use `_ = json.dump(params)`
to encode parameters.

This also emits a warning when requesting with parameters which do not match the schema.


See merge request nexedi/slapos.core!215
parents a244d797 451c74d1
...@@ -74,6 +74,8 @@ setup(name=name, ...@@ -74,6 +74,8 @@ setup(name=name,
'six', 'six',
'cachecontrol', 'cachecontrol',
'lockfile', 'lockfile',
'jsonschema',
'PyYAML',
'uritemplate', # used by hateoas navigator 'uritemplate', # used by hateoas navigator
'subprocess32; python_version<"3"', 'subprocess32; python_version<"3"',
'ipaddress; python_version<"3"', # used by whitelistfirewall 'ipaddress; python_version<"3"', # used by whitelistfirewall
......
...@@ -27,15 +27,49 @@ ...@@ -27,15 +27,49 @@
# #
############################################################################## ##############################################################################
import argparse
import json
import os.path
import pprint import pprint
import lxml.etree
import six
import yaml
from slapos.cli.config import ClientConfigCommand from slapos.cli.config import ClientConfigCommand
from slapos.client import init, ClientConfig, _getSoftwareReleaseFromSoftwareString from slapos.client import (ClientConfig, _getSoftwareReleaseFromSoftwareString,
init)
from slapos.slap import ResourceNotReady from slapos.slap import ResourceNotReady
from slapos.util import SoftwareReleaseSchema, SoftwareReleaseSerialisation
try:
from typing import IO, Dict
except ImportError:
pass
def getParametersFromFile(file, serialisation):
# type: (IO[str], SoftwareReleaseSerialisation) -> Dict
extension = os.path.splitext(file.name)[1]
if extension in ('.yaml', '.yml'):
params = yaml.safe_load(file)
elif extension == '.xml':
tree = lxml.etree.parse(file)
params = {e.attrib['id']: e.text for e in tree.findall('/parameter')}
# because the use case of xml files is to copy paste existing XML parameters
# as found on slapos web interface, we aren't clever regarding the
# serialisation and assume they are already correct.
serialisation = None
else:
params = json.load(file)
if serialisation == SoftwareReleaseSerialisation.JsonInXml and list(params.keys()) != ['_']:
params = {'_': json.dumps(params)}
return params
def parse_option_dict(options): def parse_option_dict(options):
# type: (str) -> Dict
""" """
Parse a list of option strings like foo=bar baz=qux and return a dictionary. Parse a list of option strings like foo=bar baz=qux and return a dictionary.
Will raise if keys are repeated. Will raise if keys are repeated.
...@@ -78,10 +112,18 @@ class RequestCommand(ClientConfigCommand): ...@@ -78,10 +112,18 @@ class RequestCommand(ClientConfigCommand):
action='store_true', action='store_true',
help='Ask for a slave instance') help='Ask for a slave instance')
ap.add_argument('--parameters', parameter_args = ap.add_mutually_exclusive_group()
nargs='+',
help="Give your configuration 'option1=value1 option2=value2'") parameter_args.add_argument(
'--parameters',
nargs='+',
help="Instance parameters, in the form 'option1=value1 option2=value2'.")
parameter_args.add_argument(
'--parameters-file',
type=argparse.FileType('r'),
help="Instance parameters, in a file.\n"
"The file will be interpreted as json, yaml or xml depending on the file extension.")
return ap return ap
def take_action(self, args): def take_action(self, args):
...@@ -95,6 +137,16 @@ class RequestCommand(ClientConfigCommand): ...@@ -95,6 +137,16 @@ class RequestCommand(ClientConfigCommand):
do_request(self.app.log, conf, local) do_request(self.app.log, conf, local)
# BBB on python3 we can use pprint.pformat
class StrPrettyPrinter(pprint.PrettyPrinter):
"""A PrettyPrinter which produces consistent output on python 2 and 3
"""
def format(self, object, context, maxlevels, level):
if six.PY2 and isinstance(object, six.text_type):
object = object.encode('utf-8')
return pprint.PrettyPrinter.format(self, object, context, maxlevels, level)
def do_request(logger, conf, local): def do_request(logger, conf, local):
logger.info('Requesting %s as instance of %s...', logger.info('Requesting %s as instance of %s...',
conf.reference, conf.software_url) conf.reference, conf.software_url)
...@@ -104,11 +156,17 @@ def do_request(logger, conf, local): ...@@ -104,11 +156,17 @@ def do_request(logger, conf, local):
if conf.software_url in local: if conf.software_url in local:
conf.software_url = local[conf.software_url] conf.software_url = local[conf.software_url]
software_schema = SoftwareReleaseSchema(conf.software_url, conf.type)
software_schema_serialisation = software_schema.getSerialisation()
parameters = conf.parameters
if conf.parameters_file:
parameters = getParametersFromFile(conf.parameters_file, software_schema_serialisation)
try: try:
partition = local['slap'].registerOpenOrder().request( partition = local['slap'].registerOpenOrder().request(
software_release=conf.software_url, software_release=conf.software_url,
partition_reference=conf.reference, partition_reference=conf.reference,
partition_parameter_kw=conf.parameters, partition_parameter_kw=parameters,
software_type=conf.type, software_type=conf.type,
filter_kw=conf.node, filter_kw=conf.node,
state=conf.state, state=conf.state,
...@@ -116,7 +174,11 @@ def do_request(logger, conf, local): ...@@ -116,7 +174,11 @@ def do_request(logger, conf, local):
) )
logger.info('Instance requested.\nState is : %s.', partition.getState()) logger.info('Instance requested.\nState is : %s.', partition.getState())
logger.info('Connection parameters of instance are:') logger.info('Connection parameters of instance are:')
logger.info(pprint.pformat(partition.getConnectionParameterDict())) connection_parameter_dict = partition.getConnectionParameterDict()
if software_schema_serialisation == SoftwareReleaseSerialisation.JsonInXml:
if '_' in connection_parameter_dict:
connection_parameter_dict = json.loads(connection_parameter_dict['_'])
logger.info(StrPrettyPrinter().pformat(connection_parameter_dict))
logger.info('You can rerun the command to get up-to-date information.') logger.info('You can rerun the command to get up-to-date information.')
except ResourceNotReady: except ResourceNotReady:
logger.warning('Instance requested. Master is provisioning it. Please rerun in a ' logger.warning('Instance requested. Master is provisioning it. Please rerun in a '
......
...@@ -40,14 +40,17 @@ import os ...@@ -40,14 +40,17 @@ import os
import logging import logging
import re import re
from functools import wraps from functools import wraps
import warnings
import json import json
import jsonschema
import six import six
from .exception import ResourceNotReady, ServerError, NotFoundError, \ from .exception import ResourceNotReady, ServerError, NotFoundError, \
ConnectionError ConnectionError
from .hateoas import SlapHateoasNavigator, ConnectionHelper from .hateoas import SlapHateoasNavigator, ConnectionHelper
from slapos.util import loads, dumps, bytes2str, unicode2str, xml2dict, dict2xml, calculate_dict_hash from slapos.util import (SoftwareReleaseSchema, bytes2str, calculate_dict_hash,
dict2xml, dumps, loads, unicode2str, xml2dict)
from xml.sax import saxutils from xml.sax import saxutils
from zope.interface import implementer from zope.interface import implementer
...@@ -87,6 +90,27 @@ class SlapRequester(SlapDocument): ...@@ -87,6 +90,27 @@ class SlapRequester(SlapDocument):
""" """
def _requestComputerPartition(self, request_dict): def _requestComputerPartition(self, request_dict):
try:
SoftwareReleaseSchema(
request_dict['software_release'],
request_dict['software_type']
).validateInstanceParameterDict(
loads(request_dict['partition_parameter_xml']))
except jsonschema.ValidationError as e:
warnings.warn(
"Request parameters do not validate against schema definition:\n{e}".format(e=e),
UserWarning,
)
except Exception as e:
# note that we intentionally catch wide exceptions, so that if anything
# is wrong with fetching the schema or the schema itself this does not
# prevent users from requesting instances.
warnings.warn(
"Error validating request parameters against schema definition:\n{e.__class__.__name__} {e}".format(e=e),
UserWarning,
)
try: try:
xml = self._connection_helper.POST('requestComputerPartition', data=request_dict) xml = self._connection_helper.POST('requestComputerPartition', data=request_dict)
except ResourceNotReady: except ResourceNotReady:
......
...@@ -119,8 +119,10 @@ class ManagedHTTPServer(ManagedResource): ...@@ -119,8 +119,10 @@ class ManagedHTTPServer(ManagedResource):
proto = 'http' proto = 'http'
# hostname to listen to, default to ipv4 address of the current test # hostname to listen to, default to ipv4 address of the current test
hostname = None # type: str hostname = None # type: str
# port to listen to, default # port to listen to, default to a free port selected with `findFreeTCPPort`
port = None # type: int port = None # type: int
# current working directory of the server process
working_directory = None # type: str
@property @property
def url(self): def url(self):
...@@ -166,10 +168,19 @@ class ManagedHTTPServer(ManagedResource): ...@@ -166,10 +168,19 @@ class ManagedHTTPServer(ManagedResource):
if not self.port: if not self.port:
self.port = findFreeTCPPort(self.hostname) self.port = findFreeTCPPort(self.hostname)
def serve_forever(server, cwd):
if cwd:
os.chdir(cwd)
server.serve_forever()
server = self._makeServer() server = self._makeServer()
self._process = multiprocessing.Process( self._process = multiprocessing.Process(
target=server.serve_forever, target=serve_forever,
name=self._name, name=self._name,
kwargs={
'server': server,
'cwd': self.working_directory,
}
) )
self._process.start() self._process.start()
# from now on, socket is used by server subprocess, we can close it # from now on, socket is used by server subprocess, we can close it
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
# #
############################################################################## ##############################################################################
import json
import logging import logging
import pprint import pprint
import unittest import unittest
...@@ -40,6 +41,7 @@ from contextlib import contextmanager ...@@ -40,6 +41,7 @@ from contextlib import contextmanager
from mock import patch, create_autospec from mock import patch, create_autospec
import mock import mock
from slapos.util import sqlite_connect, bytes2str from slapos.util import sqlite_connect, bytes2str
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
import slapos.cli.console import slapos.cli.console
import slapos.cli.entry import slapos.cli.entry
...@@ -49,10 +51,12 @@ import slapos.cli.computer_info ...@@ -49,10 +51,12 @@ import slapos.cli.computer_info
import slapos.cli.computer_list import slapos.cli.computer_list
import slapos.cli.computer_token import slapos.cli.computer_token
import slapos.cli.supervisorctl import slapos.cli.supervisorctl
import slapos.cli.request
from slapos.cli.proxy_show import do_show, StringIO from slapos.cli.proxy_show import do_show, StringIO
from slapos.cli.cache import do_lookup as cache_do_lookup from slapos.cli.cache import do_lookup as cache_do_lookup
from slapos.cli.cache_source import do_lookup as cache_source_do_lookup from slapos.cli.cache_source import do_lookup as cache_source_do_lookup
from slapos.client import ClientConfig from slapos.client import ClientConfig
from slapos.slap import SoftwareProductCollection
import slapos.grid.svcbackend import slapos.grid.svcbackend
import slapos.proxy import slapos.proxy
import slapos.slap import slapos.slap
...@@ -65,8 +69,8 @@ def raiseNotFoundError(*args, **kwargs): ...@@ -65,8 +69,8 @@ def raiseNotFoundError(*args, **kwargs):
class CliMixin(unittest.TestCase): class CliMixin(unittest.TestCase):
def setUp(self): def setUp(self):
slap = slapos.slap.slap() slap = slapos.slap.slap()
self.local = {'slap': slap}
self.logger = create_autospec(logging.Logger) self.logger = create_autospec(logging.Logger)
self.local = {'slap': slap, 'product': SoftwareProductCollection(self.logger, slap)}
self.conf = create_autospec(ClientConfig) self.conf = create_autospec(ClientConfig)
class TestCliCache(CliMixin): class TestCliCache(CliMixin):
...@@ -641,3 +645,227 @@ class TestCliComplete(CliMixin): ...@@ -641,3 +645,227 @@ class TestCliComplete(CliMixin):
with patch.object(sys, 'stdout', StringIO()) as app_stdout: with patch.object(sys, 'stdout', StringIO()) as app_stdout:
self.assertEqual(slapos.cli.entry.SlapOSApp().run(['complete', '--shell=fish']), 0) self.assertEqual(slapos.cli.entry.SlapOSApp().run(['complete', '--shell=fish']), 0)
self.assertIn('__fish_seen_subcommand_from', app_stdout.getvalue()) self.assertIn('__fish_seen_subcommand_from', app_stdout.getvalue())
class TestCliRequest(CliMixin):
def test_parse_option_dict(self):
parse_option_dict = slapos.cli.request.parse_option_dict
self.assertEqual(parse_option_dict(['foo=bar', 'a=b']), {'foo': 'bar', 'a': 'b'})
# malformed option = assignment
self.assertRaises(ValueError, parse_option_dict, ['a'])
# duplicated key
self.assertRaises(ValueError, parse_option_dict, ['a=b', 'a=c'])
# corner cases
self.assertEqual(parse_option_dict(['a=a=b']), {'a': 'a=b'})
self.assertEqual(parse_option_dict(['a=a\nb']), {'a': 'a\nb'})
self.assertEqual(parse_option_dict([]), {})
def test_request(self):
self.conf.reference = 'instance reference'
self.conf.software_url = 'software URL'
self.conf.parameters = {'key': 'value'}
self.conf.parameters_file = None
self.conf.node = {'computer_guid': 'COMP-1234'}
self.conf.type = None
self.conf.state = None
self.conf.slave = False
with patch.object(
slapos.slap.slap,
'registerOpenOrder',
return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
slapos.cli.request.do_request(self.logger, self.conf, self.local)
registerOpenOrder().request.assert_called_once_with(
software_release='software URL',
partition_reference='instance reference',
partition_parameter_kw={'key': 'value'},
software_type=None,
filter_kw={'computer_guid': 'COMP-1234'},
state=None,
shared=False,
)
self.logger.info.assert_any_call(
'Requesting %s as instance of %s...',
'instance reference',
'software URL',
)
def test_request_json_in_xml_published_parameters(self):
tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmpdir)
with open(os.path.join(tmpdir, 'software.cfg.json'), 'w') as f:
json.dump(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
DEFAULT_SOFTWARE_TYPE: {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
}, f)
self.conf.reference = 'instance reference'
self.conf.software_url = os.path.join(tmpdir, 'software.cfg')
self.conf.parameters = {'key': 'value'}
self.conf.parameters_file = None
self.conf.node = {'computer_guid': 'COMP-1234'}
self.conf.type = None
self.conf.state = None
self.conf.slave = False
cp = slapos.slap.ComputerPartition(
'computer_%s' % self.id(),
'partition_%s' % self.id())
cp._requested_state = 'started'
cp._connection_dict = {'_': json.dumps({'foo': 'bar'})}
with patch.object(
slapos.slap.slap,
'registerOpenOrder',
return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
registerOpenOrder().request.return_value = cp
slapos.cli.request.do_request(self.logger, self.conf, self.local)
registerOpenOrder().request.assert_called_once()
self.assertEqual(self.logger.info.mock_calls, [
mock.call('Requesting %s as instance of %s...', self.conf.reference,
self.conf.software_url),
mock.call('Instance requested.\nState is : %s.', 'started'),
mock.call('Connection parameters of instance are:'),
mock.call("{'foo': 'bar'}"),
mock.call('You can rerun the command to get up-to-date information.'),
])
class TestCliRequestParametersFileJson(CliMixin):
"""Request with --parameter-file, with a .json file.
"""
expected_partition_parameter_kw = {'foo': ['bar']}
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.json', mode='w', delete=False)
self.addCleanup(os.unlink, f.name)
f.write(textwrap.dedent('''\
{
"foo": ["bar"]
}
'''))
f.flush()
return f.name
def test_request_parameters_file(self):
self.conf.reference = 'instance reference'
self.conf.software_url = 'software URL'
self.conf.parameters = None
f = open(self._makeParameterFile())
self.addCleanup(f.close)
self.conf.parameters_file = f
self.conf.node = {'computer_guid': 'COMP-1234'}
self.conf.type = None
self.conf.state = None
self.conf.slave = False
with patch.object(
slapos.slap.slap,
'registerOpenOrder',
return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
slapos.cli.request.do_request(self.logger, self.conf, self.local)
registerOpenOrder().request.assert_called_once_with(
software_release='software URL',
partition_reference='instance reference',
partition_parameter_kw=self.expected_partition_parameter_kw,
software_type=None,
filter_kw={'computer_guid': 'COMP-1234'},
state=None,
shared=False,
)
self.logger.info.assert_any_call(
'Requesting %s as instance of %s...',
'instance reference',
'software URL',
)
class TestCliRequestParametersFileJsonJsonInXMLSerialisation(
TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .json file and a software using
json-in-xml for serialisation. In that case, the parameters are automatically
serialised with {'_': json.dumps(params)}
"""
expected_partition_parameter_kw = {"_": "{\"foo\": [\"bar\"]}"}
def test_request_parameters_file(self):
with mock.patch(
'slapos.cli.request.SoftwareReleaseSchema.getSerialisation',
return_value='json-in-xml'):
super(TestCliRequestParametersFileJsonJsonInXMLSerialisation,
self).test_request_parameters_file()
class TestCliRequestParametersFileJsonJsonInXMLSerialisationAlreadySerialised(
TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .json file and a software using
json-in-xml for serialisation and parameters already serialised with
{'_': json.dumps(params)}. In that case, parameters are not serialized one
more time.
"""
expected_partition_parameter_kw = {"_": "{\"foo\": [\"bar\"]}"}
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.json', mode='w', delete=False)
self.addCleanup(os.unlink, f.name)
f.write(textwrap.dedent(r'''
{"_": "{\"foo\": [\"bar\"]}"}
'''))
f.flush()
return f.name
def test_request_parameters_file(self):
with mock.patch(
'slapos.cli.request.SoftwareReleaseSchema.getSerialisation',
return_value='json-in-xml'):
super(
TestCliRequestParametersFileJsonJsonInXMLSerialisationAlreadySerialised,
self).test_request_parameters_file()
class TestCliRequestParametersFileYaml(TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .yaml file. This behaves like json.
"""
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.yaml', mode='w', delete=False)
self.addCleanup(os.unlink, f.name)
f.write(textwrap.dedent('''\
foo:
- bar
'''))
f.flush()
return f.name
class TestCliRequestParametersFileXml(TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .xml file
"""
expected_partition_parameter_kw = {'foo': 'bar'}
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.xml', mode='w', delete=False)
f.write(textwrap.dedent('''\
<?xml version="1.0" encoding="utf-8"?>
<instance>
<parameter id="foo">bar</parameter>
</instance>
'''))
f.flush()
self.addCleanup(os.unlink, f.name)
return f.name
...@@ -32,6 +32,7 @@ from six.moves.urllib import parse ...@@ -32,6 +32,7 @@ from six.moves.urllib import parse
from six import PY3 from six import PY3
import tempfile import tempfile
import logging import logging
import warnings
from collections import OrderedDict from collections import OrderedDict
import httmock import httmock
...@@ -872,6 +873,188 @@ class TestComputerPartition(SlapMixin): ...@@ -872,6 +873,188 @@ class TestComputerPartition(SlapMixin):
content_list = f.read().splitlines() content_list = f.read().splitlines()
self.assertEqual(sorted(content_list), ['myref', 'mysecondref']) self.assertEqual(sorted(content_list), ['myref', 'mysecondref'])
def test_request_validate_request_parameter(self):
def handler(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return json.dumps(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple instance parameters schema for tests",
"required": ["foo"],
"properties": {
"foo": {
"$ref": "./schemas-definitions.json#/foo"
}
},
"type": "object"
})
if url.path.endswith('/schemas-definitions.json'):
return json.dumps({"foo": {"type": "string", "const": "bar"}})
raise ValueError(404)
with httmock.HTTMock(handler):
with mock.patch.object(warnings, 'warn') as warn:
cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
cp._connection_helper = mock.Mock()
cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
cp.request(
'https://example.com/software.cfg', 'default', 'reference',
partition_parameter_kw={'foo': 'bar'})
warn.assert_not_called()
with httmock.HTTMock(handler):
with mock.patch.object(warnings, 'warn') as warn:
cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
cp._connection_helper = mock.Mock()
cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
cp.request(
'https://example.com/software.cfg', 'default', 'reference',
partition_parameter_kw={'foo': 'baz'})
if PY3:
warn.assert_called_with(
"Request parameters do not validate against schema definition:\n"
"'bar' was expected\n\n"
"Failed validating 'const' in schema['properties']['foo']:\n"
" {'const': 'bar', 'type': 'string'}\n\n"
"On instance['foo']:\n 'baz'", UserWarning
)
else: # BBB
warn.assert_called_with(
"Request parameters do not validate against schema definition:\n"
"u'bar' was expected\n\n"
"Failed validating u'const' in schema[u'properties'][u'foo']:\n"
" {u'const': u'bar', u'type': u'string'}\n\n"
"On instance[u'foo']:\n 'baz'", UserWarning
)
def test_request_validate_request_parameter_broken_software_release_schema(self):
"""Corner case tests for incorrect software release schema, these should
not prevent the request (mostly for backward compatibility)
"""
def wrong_software_cfg_schema(url, req):
if url.path.endswith('/software.cfg.json'):
return "wrong"
raise ValueError(404)
def wrong_instance_parameter_schema(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return "wrong"
raise ValueError(404)
def invalid_instance_parameter_schema(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return json.dumps(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Invalid json schema",
"required": {"wrong": True},
"properties": {
["wrong schema"]
},
"type": "object"
})
raise ValueError(404)
def broken_reference(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return json.dumps(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple instance parameters schema for tests",
"required": ["foo"],
"properties": {
"foo": {
"$ref": "broken"
}
},
"type": "object"
})
raise ValueError(404)
for handler, warning_expected in (
(broken_reference, True),
(wrong_software_cfg_schema, False),
(wrong_instance_parameter_schema, True),
(invalid_instance_parameter_schema, True),
):
with httmock.HTTMock(handler):
with mock.patch.object(warnings, 'warn') as warn:
cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
cp._connection_helper = mock.Mock()
cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
cp.request(
'https://example.com/software.cfg', 'default', 'reference',
partition_parameter_kw={'foo': 'bar'})
if warning_expected:
warn.assert_called()
else:
warn.assert_not_called()
def _test_new_computer_partition_state(self, state): def _test_new_computer_partition_state(self, state):
""" """
Helper method to automate assertions of failing states on new Computer Helper method to automate assertions of failing states on new Computer
......
...@@ -24,14 +24,26 @@ ...@@ -24,14 +24,26 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import functools
import json
import logging
import os import os
import slapos.util import shutil
from slapos.util import string_to_boolean, unicode2str
import tempfile import tempfile
import textwrap
import unittest import unittest
import shutil
from pwd import getpwnam from pwd import getpwnam
from six.moves import SimpleHTTPServer
import jsonschema
import slapos.util
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
from slapos.testing.utils import ManagedHTTPServer
from slapos.util import (SoftwareReleaseSchema, SoftwareReleaseSerialisation,
string_to_boolean, unicode2str)
class TestUtil(unittest.TestCase): class TestUtil(unittest.TestCase):
""" """
Tests methods available in the slapos.util module. Tests methods available in the slapos.util module.
...@@ -232,5 +244,221 @@ class TestUtil(unittest.TestCase): ...@@ -232,5 +244,221 @@ class TestUtil(unittest.TestCase):
self.assertRaises(Exception, slapos.util.dumps, Nasty()) self.assertRaises(Exception, slapos.util.dumps, Nasty())
class SoftwareReleaseSchemaTestXmlSerialisationMixin:
serialisation = SoftwareReleaseSerialisation.Xml
class SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin:
serialisation = SoftwareReleaseSerialisation.JsonInXml
class SoftwareReleaseSchemaTestMixin(object):
"""Mixin with test methods
"""
software_url = None # type: str
serialisation = None # type: SoftwareReleaseSerialisation
def test_software_schema(self):
schema = SoftwareReleaseSchema(self.software_url, None)
software_schema = schema.getSoftwareSchema()
self.assertEqual(software_schema['name'], 'Test Software')
self.assertEqual(len(software_schema['software-type']), 2)
def test_serialisation(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertEqual(schema.getSerialisation(), self.serialisation)
def test_instance_request_parameter_schema_default_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_parameter_schema = schema.getInstanceRequestParameterSchema()
self.assertEqual(instance_parameter_schema['description'],
"Simple instance parameters schema for tests")
def test_connection_parameter_schema(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertTrue(schema.getInstanceConnectionParameterSchemaURL())
instance_parameter_schema = schema.getInstanceConnectionParameterSchema()
self.assertEqual(instance_parameter_schema['description'],
"Simple connection parameters schema for tests")
def test_instance_request_parameter_validate_default_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_ok = {'key': 'value', 'type': 'default'}
schema.validateInstanceParameterDict(instance_ok)
if self.serialisation == SoftwareReleaseSerialisation.JsonInXml:
# already serialized values are also tolerated
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({"wrong": True})
instance_ok['key'] = False # wrong type
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict(instance_ok)
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
def test_instance_request_parameter_validate_alternate_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, 'alternate')
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_ok = {'key': 'value', 'type': 'alternate'}
schema.validateInstanceParameterDict(instance_ok)
if self.serialisation == SoftwareReleaseSerialisation.JsonInXml:
# already serialized values are also tolerated
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({"wrong": True})
instance_ok['type'] = 'wrong'
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict(instance_ok)
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
def test_instance_request_parameter_schema_alternate_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, 'alternate')
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_parameter_schema = schema.getInstanceRequestParameterSchema()
self.assertEqual(instance_parameter_schema['description'],
"Simple instance parameters schema for tests")
class SoftwareReleaseSchemaTestFileSoftwareReleaseMixin(SoftwareReleaseSchemaTestMixin):
"""Mixin with tests and software release profiles and schema in a
temporary directory.
"""
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.tmpdir)
tmpfile = functools.partial(os.path.join, self.tmpdir)
with open(tmpfile('software.cfg'), 'w') as f:
f.write(
textwrap.dedent("""\
[buildout]
"""))
with open(tmpfile('software.cfg.json'), 'w') as f:
json.dump(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": self.serialisation,
"software-type": {
DEFAULT_SOFTWARE_TYPE: {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
"alternate": {
"title": "Alternate",
"description": "Alternate type",
"request": "instance-alternate-input-schema.json",
"response": "instance-alternate-output-schema.json",
"index": 0
},
}
}, f)
for software_type in ('default', 'alternate'):
with open(
tmpfile('instance-{software_type}-input-schema.json'.format(
software_type=software_type)), 'w') as f:
json.dump(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple instance parameters schema for tests",
"required": ["key", "type"],
"properties": {
"key": {
"$ref": "./schemas-definitions.json#/key"
},
"type": {
"type": "string",
"const": software_type
}
},
"type": "object"
}, f)
with open(
tmpfile('instance-{software_type}-output-schema.json'.format(
software_type=software_type)), 'w') as f:
json.dump(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple connection parameters schema for tests",
}, f)
with open(tmpfile('schemas-definitions.json'), 'w') as f:
json.dump({"key": {"type": "string"}}, f)
self.software_url = tmpfile('software.cfg')
class SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin(SoftwareReleaseSchemaTestFileSoftwareReleaseMixin):
"""Mixin serving software release files over http.
"""
def setUp(self):
super(SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin, self).setUp()
class ProfileHTTPServer(ManagedHTTPServer):
hostname = os.environ['SLAPOS_TEST_IPV4']
working_directory = self.tmpdir
RequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
self.logger = logging.getLogger(self.id())
self.logger.propagate = False
server = ProfileHTTPServer(self, 'server')
server.open()
self.addCleanup(server.close)
self.software_url = server.url + '/software.cfg'
class TestSoftwareReleaseSchemaFileSoftwareReleaseXmlSerialisation(
SoftwareReleaseSchemaTestXmlSerialisationMixin,
SoftwareReleaseSchemaTestFileSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaFileSoftwareReleaseJsonInXmlSerialisation(
SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin,
SoftwareReleaseSchemaTestFileSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaHTTPSoftwareReleaseXmlSerialisation(
SoftwareReleaseSchemaTestXmlSerialisationMixin,
SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaHTTPSoftwareReleaseJsonInXmlSerialisation(
SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin,
SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaEdgeCases(unittest.TestCase):
def test_software_schema_file_not_exist(self):
schema = SoftwareReleaseSchema('/file/not/exist', None)
self.assertIsNone(schema.getSoftwareSchema())
def test_software_schema_wrong_URL(self):
schema = SoftwareReleaseSchema('http://slapos.invalid/software.cfg', None)
self.assertIsNone(schema.getSoftwareSchema())
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
...@@ -27,19 +27,32 @@ ...@@ -27,19 +27,32 @@
# #
############################################################################## ##############################################################################
import enum
import errno import errno
import hashlib
import json
import os import os
import shutil
import socket import socket
import sqlite3
import struct import struct
import subprocess import subprocess
import sqlite3 import warnings
from xml_marshaller.xml_marshaller import Marshaller, Unmarshaller
from lxml import etree import jsonschema
import netaddr
import requests
import six import six
from lxml import etree
from six.moves.urllib import parse from six.moves.urllib import parse
import hashlib from six.moves.urllib_parse import urljoin
import netaddr from xml_marshaller.xml_marshaller import Marshaller, Unmarshaller
import shutil
try:
from typing import Dict, Optional, IO
except ImportError:
pass
try: try:
...@@ -307,3 +320,150 @@ def rmtree(path): ...@@ -307,3 +320,150 @@ def rmtree(path):
raise e # XXX make pylint happy raise e # XXX make pylint happy
shutil.rmtree(path, onerror=chmod_retry) shutil.rmtree(path, onerror=chmod_retry)
def _readAsJson(url):
# type: (str) -> Optional[Dict]
"""Reads and parse the json file located at `url`.
`url` can also be the path of a local file.
"""
if url.startswith('file://'):
url = url[len('file://'):]
path = url if os.path.exists(url) else None
if path:
with open(path) as f:
try:
return json.load(f)
except ValueError:
return None
if url.startswith('http://') or url.startswith('https://'):
try:
r = requests.get(url)
r.raise_for_status()
return r.json()
except (requests.exceptions.RequestException, ValueError):
return None
return None
class SoftwareReleaseSerialisation(str, enum.Enum):
Xml = 'xml'
JsonInXml = 'json-in-xml'
class SoftwareReleaseSchema(object):
def __init__(self, software_url, software_type):
# type: (str, Optional[str]) -> None
self.software_url = software_url
self.software_type = software_type
def getSoftwareSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema for this software.
"""
return _readAsJson(self.software_url + '.json')
def getSoftwareTypeSchema(self):
# type: () -> Optional[Dict]
"""Returns schema for this software type.
"""
software_schema = self.getSoftwareSchema()
if software_schema is None:
return None
software_type = self.software_type
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
if software_type is None:
software_type = DEFAULT_SOFTWARE_TYPE
# XXX Some software are using "default" for default software type, because
# we are transitionning from DEFAULT_SOFTWARE_TYPE ("RootSoftwareInstance")
# to "default".
if software_type == DEFAULT_SOFTWARE_TYPE \
and software_type not in software_schema['software-type'] \
and 'default' in software_schema['software-type']:
warnings.warn(
"Software release {} does not have schema for DEFAULT_SOFTWARE_TYPE but has one for 'default'."
" Using 'default' instead.".format(self.software_url),
UserWarning,
)
software_type = 'default'
return software_schema['software-type'].get(software_type)
def getSerialisation(self):
# type: () -> Optional[SoftwareReleaseSerialisation]
"""Returns the serialisation method used for parameters.
"""
software_schema = self.getSoftwareSchema()
if software_schema is None:
return None
return SoftwareReleaseSerialisation(software_schema['serialisation'])
def getInstanceRequestParameterSchemaURL(self):
# type: () -> Optional[str]
"""Returns the URL of the schema defining instance parameters.
"""
software_type_schema = self.getSoftwareTypeSchema()
if software_type_schema is None:
return None
software_url = self.software_url
if os.path.exists(software_url):
software_url = 'file://' + software_url
return urljoin(software_url, software_type_schema['request'])
def getInstanceRequestParameterSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema defining instance parameters.
"""
instance_parameter_schema_url = self.getInstanceRequestParameterSchemaURL()
if instance_parameter_schema_url is None:
return None
schema = _readAsJson(instance_parameter_schema_url)
if schema:
# so that jsonschema knows how to resolve references
schema.setdefault('$id', instance_parameter_schema_url)
return schema
def getInstanceConnectionParameterSchemaURL(self):
# type: () -> Optional[str]
"""Returns the URL of the schema defining connection parameters published by the instance.
"""
software_type_schema = self.getSoftwareTypeSchema()
if software_type_schema is None:
return None
return urljoin(self.software_url, software_type_schema['response'])
def getInstanceConnectionParameterSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema defining connection parameters published by the instance.
"""
instance_parameter_schema_url = self.getInstanceConnectionParameterSchemaURL()
if instance_parameter_schema_url is None:
return None
schema = _readAsJson(instance_parameter_schema_url)
if schema:
# so that jsonschema knows how to resolve references
schema.setdefault('$id', instance_parameter_schema_url)
return schema
def validateInstanceParameterDict(self, parameter_dict):
# type: (Dict) -> None
"""Validate instance parameters against the software schema.
Raise jsonschema.ValidationError if parameters does not validate.
"""
schema_url = self.getInstanceRequestParameterSchemaURL()
if schema_url:
instance = parameter_dict
if self.getSerialisation() == SoftwareReleaseSerialisation.JsonInXml:
try:
instance = json.loads(parameter_dict['_'])
except KeyError:
instance = parameter_dict
instance.pop('$schema', None)
jsonschema.validate(
instance=instance,
schema=self.getInstanceRequestParameterSchema(),
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment