util: use zc.buildout.download API to download schema

This will support buildout extension to download from gitlab API with
authentication.

The code has already been refactored a bit to be able to pass a
download instance to SoftwareReleaseSchema, for now to instanciate only
one Download but in the future this might be extended to pass some
configuration to the Download instance.

slapos/tests/test_slap.py

@@ -875,15 +875,15 @@ class TestComputerPartition(SlapMixin):
 
   def test_request_validate_request_parameter(self):
 
-    def handler(url, req):
-      if url.path.endswith('/software.cfg.json'):
-        return json.dumps(
-          {
+    def _readAsJson(url, set_schema_id=False):
+      if url == 'https://example.com/software.cfg.json':
+        assert not set_schema_id
+        return {
           "name": "Test Software",
           "description": "Dummy software for Test",
           "serialisation": "json-in-xml",
           "software-type": {
-                  'default': {
+            "default": {
               "title": "Default",
               "description": "Default type",
               "request": "instance-default-input-schema.json",
@@ -891,78 +891,84 @@ class TestComputerPartition(SlapMixin):
               "index": 0
             },
           }
-          })
-      if url.path.endswith('/instance-default-input-schema.json'):
-        return json.dumps(
-            {
+        }
+      if url == 'https://example.com/instance-default-input-schema.json':
+        assert set_schema_id
+        return {
+          "$id": url,
           "$schema": "http://json-schema.org/draft-07/schema",
           "description": "Simple instance parameters schema for tests",
           "required": ["foo"],
           "properties": {
             "foo": {
-                        "$ref": "./schemas-definitions.json#/foo"
+              "$ref": "./schemas-definitions.json#/foo",
             }
           },
           "additionalProperties": False,
-                "type": "object"
-            })
-      if url.path.endswith('/schemas-definitions.json'):
-        return json.dumps({"foo": {"type": "string", "const": "bar"}})
-      raise ValueError(404)
+          "type": "object",
+        }
+      if url == 'https://example.com/schemas-definitions.json':
+        assert not set_schema_id
+        return {
+          "foo": {
+            "type": "string",
+            "const": "bar",
+          },
+        }
+      assert False, "Unexpected url %s" % url
 
-    with httmock.HTTMock(handler):
-      with mock.patch.object(warnings, 'warn') as warn:
+    with mock.patch(
+        'slapos.util.SoftwareReleaseSchema._readAsJson',
+        side_effect=_readAsJson) as _readAsJson_mock, \
+        mock.patch.object(warnings, 'warn') as warn:
       cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
       cp._connection_helper = mock.Mock()
       cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
       cp.request(
           'https://example.com/software.cfg', 'default', 'reference',
           partition_parameter_kw={'foo': 'bar'})
+    self.assertEqual(
+      _readAsJson_mock.call_args_list,
+      [
+        mock.call('https://example.com/software.cfg.json'),
+        mock.call('https://example.com/instance-default-input-schema.json', True),
+        mock.call('https://example.com/schemas-definitions.json'),
+      ])
     warn.assert_not_called()
 
-    with httmock.HTTMock(handler):
-      with mock.patch.object(warnings, 'warn') as warn:
+    with mock.patch(
+        'slapos.util.SoftwareReleaseSchema._readAsJson',
+        side_effect=_readAsJson), \
+        mock.patch.object(warnings, 'warn') as warn:
       cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
       cp._connection_helper = mock.Mock()
       cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
       cp.request(
           'https://example.com/software.cfg', 'default', 'reference',
           partition_parameter_kw={'foo': 'baz'})
-      if PY3:
     warn.assert_called_with(
       "Request parameters do not validate against schema definition:\n"
       "  $.foo: 'bar' was expected",
       UserWarning
     )
-      else: # BBB
-        warn.assert_called_with(
-          "Request parameters do not validate against schema definition:\n"
-          "  $.foo: u'bar' was expected",
-          UserWarning
-        )
 
-    with httmock.HTTMock(handler):
-      with mock.patch.object(warnings, 'warn') as warn:
+    with mock.patch(
+        'slapos.util.SoftwareReleaseSchema._readAsJson',
+        side_effect=_readAsJson), \
+        mock.patch.object(warnings, 'warn') as warn:
       cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
       cp._connection_helper = mock.Mock()
       cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
       cp.request(
           'https://example.com/software.cfg', 'default', 'reference',
           partition_parameter_kw={'fooo': 'xxx'})
-      if PY3:
     warn.assert_called_with(
       "Request parameters do not validate against schema definition:\n"
       "  $: 'foo' is a required property\n"
       "  $: Additional properties are not allowed ('fooo' was unexpected)",
       UserWarning
     )
-      else: # BBB
-        warn.assert_called_with(
-          "Request parameters do not validate against schema definition:\n"
-          "  $: u'foo' is a required property\n"
-          "  $: Additional properties are not allowed ('fooo' was unexpected)",
-          UserWarning
-        )
+
 
   def test_request_validate_request_parameter_broken_software_release_schema(self):
     """Corner case tests for incorrect software release schema, these should

slapos/util.py

@@ -43,7 +43,7 @@ import warnings
 
 import jsonschema
 import netaddr
-import requests
+import zc.buildout.download
 import six
 from lxml import etree
 from six.moves.urllib import parse
@@ -401,37 +401,15 @@ def rmtree(path):
 
 
 
-def _readAsJson(url, set_schema_id=False):
-  # type: (str) -> Optional[Dict]
-  """Reads and parse the json file located at `url`.
-
-  `url` can also be the path of a local file.
-  """
-  try:
-    if url.startswith('http://') or url.startswith('https://'):
-      r = requests.get(url, timeout=60) # we need a timeout !
-      r.raise_for_status()
-      r = r.json()
-    else:
-      # XXX: https://discuss.python.org/t/file-uris-in-python/15600
-      if url.startswith('file://'):
-        path = url[7:]
-      else:
-        path = url
-        url = 'file:' + url
-      with open(path) as f:
-        r = json.load(f)
-    if set_schema_id and r:
-      r.setdefault('$id', url)
-    return r
-  except Exception as e:
-    warnings.warn("Unable to load JSON %s (%s: %s)"
-                  % (url, type(e).__name__, e))
-
-
 class _RefResolver(jsonschema.validators.RefResolver):
+  @classmethod
+  def from_schema(cls, schema, read_as_json):
+    instance = super(_RefResolver, cls).from_schema(schema)
+    instance._read_as_json = read_as_json
+    return instance
+
   def resolve_remote(self, uri):
-    result = _readAsJson(uri)
+    result = self._read_as_json(uri)
     if self.cache_remote:
       self.store[uri] = result
     return result
@@ -484,8 +462,8 @@ class SoftwareReleaseSchemaValidationError(ValueError):
 
 class SoftwareReleaseSchema(object):
 
-  def __init__(self, software_url, software_type):
-    # type: (str, Optional[str]) ->  None
+  def __init__(self, software_url, software_type, download=None):
+    # type: (str, Optional[str], Optional[zc.buildout.download.Download]) ->  None
     self.software_url = software_url
     # XXX: Transition from OLD_DEFAULT_SOFTWARE_TYPE ("RootSoftwareInstance")
     #      to DEFAULT_SOFTWARE_TYPE ("default") is already complete for SR schemas.
@@ -493,6 +471,9 @@ class SoftwareReleaseSchema(object):
     if software_type == OLD_DEFAULT_SOFTWARE_TYPE:
       software_type = None
     self.software_type = software_type or DEFAULT_SOFTWARE_TYPE
+    if download is None:
+      download = zc.buildout.download.Download()
+    self._download = download.download
 
   def _warn(self, message, e):
     warnings.warn(
@@ -500,6 +481,37 @@ class SoftwareReleaseSchema(object):
       % (message, self.software_type, self.software_url, type(e).__name__, e),
       stacklevel=2)
 
+  def _readAsJson(self, url, set_schema_id=False):
+    # type: (str, bool) -> Optional[Dict]
+    """Reads and parse the json file located at `url`.
+
+    `url` can also be the path of a local file.
+    """
+    try:
+      if url.startswith('http://') or url.startswith('https://'):
+        path, is_temp = self._download(url)
+        try:
+          with open(path) as f:
+            r = json.load(f)
+        finally:
+          if is_temp:
+            os.remove(path)
+      else:
+        # XXX: https://discuss.python.org/t/file-uris-in-python/15600
+        if url.startswith('file://'):
+          path = url[7:]
+        else:
+          path = url
+          url = 'file:' + url
+        with open(path) as f:
+          r = json.load(f)
+      if set_schema_id and r:
+        r.setdefault('$id', url)
+      return r
+    except Exception as e:
+      warnings.warn("Unable to load JSON %s (%s: %s)"
+                    % (url, type(e).__name__, e))
+
   def getSoftwareSchema(self):
     # type: () -> Optional[Dict]
     """Returns the schema for this software.
@@ -507,7 +519,7 @@ class SoftwareReleaseSchema(object):
     try:
       return self._software_schema
     except AttributeError:
-      schema = self._software_schema = _readAsJson(self.software_url + '.json')
+      schema = self._software_schema = self._readAsJson(self.software_url + '.json')
     return schema
 
   def getSoftwareTypeSchema(self):
@@ -556,7 +568,7 @@ class SoftwareReleaseSchema(object):
       return self._request_schema
     except AttributeError:
       url = self.getInstanceRequestParameterSchemaURL()
-      schema = None if url is None else _readAsJson(url, True)
+      schema = None if url is None else self._readAsJson(url, True)
       self._request_schema = schema
     return schema
 
@@ -577,7 +589,7 @@ class SoftwareReleaseSchema(object):
       return self._response_schema
     except AttributeError:
       url = self.getInstanceConnectionParameterSchemaURL()
-      schema = None if url is None else _readAsJson(url, True)
+      schema = None if url is None else self._readAsJson(url, True)
       self._response_schema = schema
     return schema
 
@@ -599,7 +611,7 @@ class SoftwareReleaseSchema(object):
 
       validator = jsonschema.validators.validator_for(schema)(
         schema,
-        resolver=_RefResolver.from_schema(schema),
+        resolver=_RefResolver.from_schema(schema, self._readAsJson),
       )
       errors = list(validator.iter_errors(parameter_dict))
       if errors: