diff --git a/slapos/tests/test_slap.py b/slapos/tests/test_slap.py index 74ba9cb318675bb29df3d69a4371d50b2828e233..921f7ada769490c30da5c2952218fa83c922af28 100644 --- a/slapos/tests/test_slap.py +++ b/slapos/tests/test_slap.py @@ -875,94 +875,100 @@ class TestComputerPartition(SlapMixin): def test_request_validate_request_parameter(self): - def handler(url, req): - if url.path.endswith('/software.cfg.json'): - return json.dumps( - { - "name": "Test Software", - "description": "Dummy software for Test", - "serialisation": "json-in-xml", - "software-type": { - 'default': { - "title": "Default", - "description": "Default type", - "request": "instance-default-input-schema.json", - "response": "instance-default-output-schema.json", - "index": 0 - }, - } - }) - if url.path.endswith('/instance-default-input-schema.json'): - return json.dumps( - { - "$schema": "http://json-schema.org/draft-07/schema", - "description": "Simple instance parameters schema for tests", - "required": ["foo"], - "properties": { - "foo": { - "$ref": "./schemas-definitions.json#/foo" - } - }, - "additionalProperties": False, - "type": "object" - }) - if url.path.endswith('/schemas-definitions.json'): - return json.dumps({"foo": {"type": "string", "const": "bar"}}) - raise ValueError(404) - - with httmock.HTTMock(handler): - with mock.patch.object(warnings, 'warn') as warn: - cp = slapos.slap.ComputerPartition('computer_id', 'partition_id') - cp._connection_helper = mock.Mock() - cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady - cp.request( - 'https://example.com/software.cfg', 'default', 'reference', - partition_parameter_kw={'foo': 'bar'}) - warn.assert_not_called() + def _readAsJson(url, set_schema_id=False): + if url == 'https://example.com/software.cfg.json': + assert not set_schema_id + return { + "name": "Test Software", + "description": "Dummy software for Test", + "serialisation": "json-in-xml", + "software-type": { + "default": { + "title": "Default", + "description": "Default type", + "request": "instance-default-input-schema.json", + "response": "instance-default-output-schema.json", + "index": 0 + }, + } + } + if url == 'https://example.com/instance-default-input-schema.json': + assert set_schema_id + return { + "$id": url, + "$schema": "http://json-schema.org/draft-07/schema", + "description": "Simple instance parameters schema for tests", + "required": ["foo"], + "properties": { + "foo": { + "$ref": "./schemas-definitions.json#/foo", + } + }, + "additionalProperties": False, + "type": "object", + } + if url == 'https://example.com/schemas-definitions.json': + assert not set_schema_id + return { + "foo": { + "type": "string", + "const": "bar", + }, + } + assert False, "Unexpected url %s" % url + + with mock.patch( + 'slapos.util.SoftwareReleaseSchema._readAsJson', + side_effect=_readAsJson) as _readAsJson_mock, \ + mock.patch.object(warnings, 'warn') as warn: + cp = slapos.slap.ComputerPartition('computer_id', 'partition_id') + cp._connection_helper = mock.Mock() + cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady + cp.request( + 'https://example.com/software.cfg', 'default', 'reference', + partition_parameter_kw={'foo': 'bar'}) + self.assertEqual( + _readAsJson_mock.call_args_list, + [ + mock.call('https://example.com/software.cfg.json'), + mock.call('https://example.com/instance-default-input-schema.json', True), + mock.call('https://example.com/schemas-definitions.json'), + ]) + warn.assert_not_called() + + with mock.patch( + 'slapos.util.SoftwareReleaseSchema._readAsJson', + side_effect=_readAsJson), \ + mock.patch.object(warnings, 'warn') as warn: + cp = slapos.slap.ComputerPartition('computer_id', 'partition_id') + cp._connection_helper = mock.Mock() + cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady + cp.request( + 'https://example.com/software.cfg', 'default', 'reference', + partition_parameter_kw={'foo': 'baz'}) + warn.assert_called_with( + "Request parameters do not validate against schema definition:\n" + " $.foo: 'bar' was expected", + UserWarning + ) - with httmock.HTTMock(handler): - with mock.patch.object(warnings, 'warn') as warn: - cp = slapos.slap.ComputerPartition('computer_id', 'partition_id') - cp._connection_helper = mock.Mock() - cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady - cp.request( - 'https://example.com/software.cfg', 'default', 'reference', - partition_parameter_kw={'foo': 'baz'}) - if PY3: - warn.assert_called_with( - "Request parameters do not validate against schema definition:\n" - " $.foo: 'bar' was expected", - UserWarning - ) - else: # BBB - warn.assert_called_with( - "Request parameters do not validate against schema definition:\n" - " $.foo: u'bar' was expected", - UserWarning - ) + with mock.patch( + 'slapos.util.SoftwareReleaseSchema._readAsJson', + side_effect=_readAsJson), \ + mock.patch.object(warnings, 'warn') as warn: + cp = slapos.slap.ComputerPartition('computer_id', 'partition_id') + cp._connection_helper = mock.Mock() + cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady + cp.request( + 'https://example.com/software.cfg', 'default', 'reference', + partition_parameter_kw={'fooo': 'xxx'}) + warn.assert_called_with( + "Request parameters do not validate against schema definition:\n" + " $: 'foo' is a required property\n" + " $: Additional properties are not allowed ('fooo' was unexpected)", + UserWarning + ) - with httmock.HTTMock(handler): - with mock.patch.object(warnings, 'warn') as warn: - cp = slapos.slap.ComputerPartition('computer_id', 'partition_id') - cp._connection_helper = mock.Mock() - cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady - cp.request( - 'https://example.com/software.cfg', 'default', 'reference', - partition_parameter_kw={'fooo': 'xxx'}) - if PY3: - warn.assert_called_with( - "Request parameters do not validate against schema definition:\n" - " $: 'foo' is a required property\n" - " $: Additional properties are not allowed ('fooo' was unexpected)", - UserWarning - ) - else: # BBB - warn.assert_called_with( - "Request parameters do not validate against schema definition:\n" - " $: u'foo' is a required property\n" - " $: Additional properties are not allowed ('fooo' was unexpected)", - UserWarning - ) def test_request_validate_request_parameter_broken_software_release_schema(self): """Corner case tests for incorrect software release schema, these should diff --git a/slapos/util.py b/slapos/util.py index c77c9b901549a6927ae6a3d0ed758039e86c528d..009ddc0e7fd1d9dcfb19a1c16fefd2f2aad67cdb 100644 --- a/slapos/util.py +++ b/slapos/util.py @@ -43,7 +43,7 @@ import warnings import jsonschema import netaddr -import requests +import zc.buildout.download import six from lxml import etree from six.moves.urllib import parse @@ -401,37 +401,15 @@ def rmtree(path): -def _readAsJson(url, set_schema_id=False): - # type: (str) -> Optional[Dict] - """Reads and parse the json file located at `url`. - - `url` can also be the path of a local file. - """ - try: - if url.startswith('http://') or url.startswith('https://'): - r = requests.get(url, timeout=60) # we need a timeout ! - r.raise_for_status() - r = r.json() - else: - # XXX: https://discuss.python.org/t/file-uris-in-python/15600 - if url.startswith('file://'): - path = url[7:] - else: - path = url - url = 'file:' + url - with open(path) as f: - r = json.load(f) - if set_schema_id and r: - r.setdefault('$id', url) - return r - except Exception as e: - warnings.warn("Unable to load JSON %s (%s: %s)" - % (url, type(e).__name__, e)) - - class _RefResolver(jsonschema.validators.RefResolver): + @classmethod + def from_schema(cls, schema, read_as_json): + instance = super(_RefResolver, cls).from_schema(schema) + instance._read_as_json = read_as_json + return instance + def resolve_remote(self, uri): - result = _readAsJson(uri) + result = self._read_as_json(uri) if self.cache_remote: self.store[uri] = result return result @@ -484,8 +462,8 @@ class SoftwareReleaseSchemaValidationError(ValueError): class SoftwareReleaseSchema(object): - def __init__(self, software_url, software_type): - # type: (str, Optional[str]) -> None + def __init__(self, software_url, software_type, download=None): + # type: (str, Optional[str], Optional[zc.buildout.download.Download]) -> None self.software_url = software_url # XXX: Transition from OLD_DEFAULT_SOFTWARE_TYPE ("RootSoftwareInstance") # to DEFAULT_SOFTWARE_TYPE ("default") is already complete for SR schemas. @@ -493,6 +471,9 @@ class SoftwareReleaseSchema(object): if software_type == OLD_DEFAULT_SOFTWARE_TYPE: software_type = None self.software_type = software_type or DEFAULT_SOFTWARE_TYPE + if download is None: + download = zc.buildout.download.Download() + self._download = download.download def _warn(self, message, e): warnings.warn( @@ -500,6 +481,37 @@ class SoftwareReleaseSchema(object): % (message, self.software_type, self.software_url, type(e).__name__, e), stacklevel=2) + def _readAsJson(self, url, set_schema_id=False): + # type: (str, bool) -> Optional[Dict] + """Reads and parse the json file located at `url`. + + `url` can also be the path of a local file. + """ + try: + if url.startswith('http://') or url.startswith('https://'): + path, is_temp = self._download(url) + try: + with open(path) as f: + r = json.load(f) + finally: + if is_temp: + os.remove(path) + else: + # XXX: https://discuss.python.org/t/file-uris-in-python/15600 + if url.startswith('file://'): + path = url[7:] + else: + path = url + url = 'file:' + url + with open(path) as f: + r = json.load(f) + if set_schema_id and r: + r.setdefault('$id', url) + return r + except Exception as e: + warnings.warn("Unable to load JSON %s (%s: %s)" + % (url, type(e).__name__, e)) + def getSoftwareSchema(self): # type: () -> Optional[Dict] """Returns the schema for this software. @@ -507,7 +519,7 @@ class SoftwareReleaseSchema(object): try: return self._software_schema except AttributeError: - schema = self._software_schema = _readAsJson(self.software_url + '.json') + schema = self._software_schema = self._readAsJson(self.software_url + '.json') return schema def getSoftwareTypeSchema(self): @@ -556,7 +568,7 @@ class SoftwareReleaseSchema(object): return self._request_schema except AttributeError: url = self.getInstanceRequestParameterSchemaURL() - schema = None if url is None else _readAsJson(url, True) + schema = None if url is None else self._readAsJson(url, True) self._request_schema = schema return schema @@ -577,7 +589,7 @@ class SoftwareReleaseSchema(object): return self._response_schema except AttributeError: url = self.getInstanceConnectionParameterSchemaURL() - schema = None if url is None else _readAsJson(url, True) + schema = None if url is None else self._readAsJson(url, True) self._response_schema = schema return schema @@ -599,7 +611,7 @@ class SoftwareReleaseSchema(object): validator = jsonschema.validators.validator_for(schema)( schema, - resolver=_RefResolver.from_schema(schema), + resolver=_RefResolver.from_schema(schema, self._readAsJson), ) errors = list(validator.iter_errors(parameter_dict)) if errors: