erp5_web_service: Add mixin component to use RESTAPI

See merge request nexedi/erp5!1872

bt5/erp5_web_service/MixinTemplateItem/portal_components/mixin.erp5.RESTAPIClientConnectorMixin.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+import time
+import urlparse
+import ssl
+import httplib
+import json
+from Products.ERP5Type.Timeout import getTimeLeft
+from contextlib import contextmanager
+from Products.ERP5Type.XMLObject import XMLObject
+from AccessControl import ClassSecurityInfo
+from Products.ERP5Type import Permissions
+from Products.ERP5Type.Timeout import Deadline, TimeoutReachedError
+from Products.ERP5Type.UnrestrictedMethod import super_user
+from zLOG import LOG, ERROR
+
+def isJson(header_dict):
+  return header_dict.get('content-type', '').split(';', 1)[0] == 'application/json'
+
+class TimeTracker(object):
+  def __init__(self):
+    self.__stack = []
+    self.__history = []
+
+  @contextmanager
+  def __call__(self, reason):
+    stack = self.__stack
+    entry = [len(stack), reason, time.time(), None]
+    stack.append(entry)
+    self.__history.append(entry)
+    try:
+      yield
+    finally:
+      stack.pop()[3] = time.time()
+
+  def __str__(self):
+    return '\n'.join(
+      '%s%s: %.3fs' % ('  ' * depth, reason, end - begin)
+      for depth, reason, begin, end in self.__history
+      if end is not None
+    )
+
+class RESTAPIClientConnectorMixin(XMLObject):
+  security = ClassSecurityInfo()
+  security.declareObjectProtected(Permissions.AccessContentsInformation)
+  __EXPIRED_TOKEN = (0, None)
+
+  # Credential scheme:
+  # - primary credentials (client_id and client_secret) are persistent,
+  #   set by admin on the connector instance
+  # - refresh token is persistent on the connector instance, and is
+  #   expected to virtually never change once set
+  # - access token is volatile on connector instances
+  #   so as to be per-ZODB.Connection. 2-tuple:
+  #   - expiration timestamp
+  #   - access token
+  _v_access_token = __EXPIRED_TOKEN
+
+  def _clearAccessToken(self):
+    """
+    Forget current access token, so next _getAccessToken call retrieves a new one.
+    """
+    self._v_access_token = self._EXPIRED_TOKEN
+
+  def _call(self, method, path, header_dict=(), body=None):
+    """
+    body (None, string, json-serialisable objects)
+      If body is not None and not a string, it is serialised in json,
+      and the appropriate content-type is added to the headers.
+
+    Returns a 3-tuple:
+    - response header dict (header names lower-cased)
+    - response body
+      If response header content type is "application/json", the body
+      is json-decoded before being returned
+    - response status
+    """
+    header_dict = dict(header_dict)
+    if body is not None and not isinstance(body, basestring):
+      header_dict['content-type'] = 'application/json'
+      body = json.dumps(body)
+    plain_url = self.getBaseUrl().rstrip('/') + '/' + path.lstrip('/')
+    parsed_url = urlparse.urlparse(plain_url)
+    ssl_context = ssl.create_default_context(
+      cadata=self.getCaCertificatePem(),
+    )
+    ssl_context.verify_mode = ssl.CERT_REQUIRED
+    ssl_context.check_hostname = True
+    bind_address = self.getBindAddress()
+    if bind_address:
+      bind_address = (bind_address, 0)
+    time_left_before_timeout = getTimeLeft()
+    http_connection = httplib.HTTPSConnection(
+      host=parsed_url.hostname,
+      port=parsed_url.port,
+      strict=True,
+      timeout=time_left_before_timeout,
+      source_address=bind_address,
+      context=ssl_context,
+    )
+    request_start_time = time.time()
+    http_connection.request(
+      method=method,
+      url=path,
+      body=body,
+      headers=header_dict,
+    )
+
+    try:
+      http_response = http_connection.getresponse()
+      request_stop_time = time.time()
+    except ssl.SSLError as exc:
+      if 'The read operation timed out' == exc.message:
+        LOG(__name__, ERROR, "Call to %s %s raised Timeout (%ss)" %(
+          method, path, round(time_left_before_timeout, 6)
+        ), error=True)
+        raise TimeoutReachedError
+      raise
+    except Exception:
+      LOG(__name__, ERROR, "Call to %s %s raised after %ss" %(
+          method, path, round(time_left_before_timeout, 6)
+        ), error=True)
+      raise
+
+    response_body = http_response.read()
+    response_header_dict = {
+      name.lower(): value
+      for name, value in http_response.getheaders()
+    }
+    if isJson(response_header_dict):
+      response_body = json.loads(response_body)
+    return (
+      response_header_dict,
+      response_body,
+      http_response.status,
+      request_stop_time - request_start_time,
+    )
+
+  security.declarePrivate('call')
+  def call(
+    self,
+    archive_resource,
+    method,
+    path,
+    header_dict=(),
+    body=None,
+    archive_kw=None,
+    archive_document_relative_url=None,
+    archive_value_list=None,
+    timeout=None,
+  ):
+    # default timeout should be kept very low
+    # to not block an instance with default zope configuration
+    timeout = timeout if timeout is not None else self.getTimeout(1)
+    original_header_dict = header_dict
+    header_dict = dict(header_dict)
+    time_tracker = TimeTracker()
+    try:
+      with time_tracker('call'), Deadline(timeout):
+        # Limit numbers of retries, in case the authentication API succeeds
+        # but the token is not usable.
+        for _ in xrange(2):
+          with time_tracker('token'):
+            access_token = self._getAccessToken()
+            if access_token is not None:
+              header_dict['Authorization'] = 'Bearer ' + self._getAccessToken()
+          with time_tracker('_call'):
+            (
+              response_header_dict,
+              response_body,
+              response_status,
+              response_time_duration,
+            ) = self._call(
+              path=path,
+              method=method,
+              header_dict=header_dict,
+              body=body,
+            )
+          if response_status == 401:
+            self._clearAccessToken()
+          else:
+            # Success (or at least not an authentication failure), exit retry loop
+            break
+    except Exception:
+      LOG(__name__, ERROR, str(time_tracker), error=True)
+      raise
+    if archive_resource is not None:
+      archiveExchange = self._getTypeBasedMethod('archiveExchange')
+      if archiveExchange is not None:
+        with super_user():
+          archiveExchange(
+            resource_path=archive_resource,
+            raw_request=(
+              # XXX: how to avoid double request serialisation ?
+              path
+              if body is None else
+              json.dumps(body)
+            ),
+            raw_response=(
+              # XXX: how to avoid deserialisation and then re-serialisation ?
+              response_body
+              if isinstance(response_body, basestring) else
+              json.dumps(response_body)
+            ),
+            time_duration=response_time_duration,
+            archive_kw=archive_kw,
+            archive_document_relative_url=archive_document_relative_url,
+            archive_value_list=archive_value_list,
+          )
+    if response_status >= 300:
+      __traceback_info__ = { # pylint: disable=unused-variable
+        'request': {
+          'method': method,
+          'path': path,
+          # Do not put authentication headers in logs
+          'header_dict': original_header_dict,
+          'body': body,
+        },
+        'response': {
+          'header_dict': response_header_dict,
+          'body': response_body,
+          'status': response_status,
+        },
+      }
+      raise self.ClientConnectorError(
+        header_dict=response_header_dict,
+        body=response_body,
+        status=response_status,
+      )
+    return (
+      response_header_dict,
+      response_body,
+      response_status,
+    )

bt5/erp5_web_service/MixinTemplateItem/portal_components/mixin.erp5.RESTAPIClientConnectorMixin.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Mixin Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>RESTAPIClientConnectorMixin</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>mixin.erp5.RESTAPIClientConnectorMixin</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_service/TestTemplateItem/portal_components/test.erp5.testRESTAPIClientConnectorMixin.py

+# -*- coding: utf-8 -*-
+# Copyright (c) 2002-2015 Nexedi SA and Contributors. All Rights Reserved.
+from json import dumps
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+from httplib import HTTPSConnection
+from erp5.component.mixin.RESTAPIClientConnectorMixin import RESTAPIClientConnectorMixin
+from ssl import SSLError
+from Products.ERP5Type.Timeout import TimeoutReachedError
+import mock
+
+expected_output_body_dict = {
+  u'output': 'output',
+}
+input_body_dict = {
+  'input': 'input',
+}
+
+class HTTPResponse_getresponse():
+  def __init__(self, status=200):
+    self.status = status
+
+  def getheaders(self):
+    return [
+      ('content-type', 'application/json'),
+    ]
+
+  def read(self):
+    return dumps(expected_output_body_dict)
+
+class RESTAPIError(Exception):
+  __allow_access_to_unprotected_subobjects__ = {
+    'header_dict': 1,
+    'body': 1,
+    'status': 1,
+  }
+
+  def __init__(self, header_dict, body, status):
+    super(RESTAPIError, self).__init__()
+    self.header_dict = header_dict
+    self.body = body
+    self.status = status
+
+class RESTAPIClientConnector(RESTAPIClientConnectorMixin):
+  meta_type = 'REST API Client Connector'
+  security = RESTAPIClientConnectorMixin.security
+  ClientConnectorError = RESTAPIError
+
+  def _getAccessToken(self):
+    return 'access_token'
+  
+  def getTimeout(self, timeout):
+    return 5
+
+  def getBaseUrl(self):
+    return 'https://example.com/'
+  
+  def getCaCertificatePem(self):
+    return 'ca_certificate_pem'
+
+  def getBindAddress(self):
+    return 'bind_address'
+
+class TestRESTAPIClientConnector(ERP5TypeTestCase):
+
+  def afterSetUp(self):
+    self.rest_api_client_connection = RESTAPIClientConnector(id='rest_api_client_connection')
+
+  def test_api_call(self):
+    timeout = 1
+
+    with mock.patch(
+      'ssl.create_default_context',
+    ) as mock_ssl_create_default_context, mock.patch(
+      'httplib.HTTPSConnection.request',
+    ) as mock_https_connection_request, mock.patch(
+      'httplib.HTTPSConnection.getresponse',
+      return_value=HTTPResponse_getresponse()
+    ), mock.patch('httplib.HTTPSConnection', return_value=HTTPSConnection) as mock_https_connection:
+      header_dict, body_dict, status = self.rest_api_client_connection.call(
+        archive_resource=None,
+        method='POST',
+        path='/path',
+        body=input_body_dict,
+        timeout=timeout,
+      )
+
+      # Check request
+      ssl_create_default_context_argument_dict = mock_ssl_create_default_context.call_args.kwargs
+      self.assertEqual(
+        ssl_create_default_context_argument_dict['cadata'],
+        'ca_certificate_pem'
+      )
+
+      https_connection_argument_dict = mock_https_connection.call_args.kwargs
+      self.assertTrue(
+        https_connection_argument_dict['timeout'] <= timeout
+      )
+      self.assertEqual(
+        https_connection_argument_dict['host'],
+        'example.com'
+      )
+      self.assertEqual(
+        https_connection_argument_dict['source_address'],
+        ('bind_address', 0)
+      )
+
+      https_connection_request_argument_dict = mock_https_connection_request.call_args.kwargs
+      self.assertEqual(
+        https_connection_request_argument_dict['body'],
+        dumps(input_body_dict)
+      )
+      self.assertEqual(
+        https_connection_request_argument_dict['url'],
+        '/path'
+      )
+      self.assertEqual(
+        https_connection_request_argument_dict['headers']['Authorization'],
+        'Bearer access_token'
+      )
+      self.assertEqual(
+        https_connection_request_argument_dict['headers']['content-type'],
+        'application/json'
+      )
+      self.assertEqual(
+        https_connection_request_argument_dict['method'],
+        'POST'
+      )
+
+      # Check response
+      self.assertEqual(
+        header_dict['content-type'],
+        'application/json'
+      )
+      self.assertEqual(
+        body_dict,
+        expected_output_body_dict
+      )
+      self.assertEqual(
+        status,
+        200
+      )
+
+
+  def test_api_call_error(self):
+    with mock.patch(
+      'ssl.create_default_context',
+    ), mock.patch(
+      'httplib.HTTPSConnection.request',
+    ), mock.patch(
+      'httplib.HTTPSConnection.getresponse',
+      return_value=HTTPResponse_getresponse(498)
+    ):
+      with self.assertRaises(RESTAPIError) as error:
+        self.rest_api_client_connection.call(
+          archive_resource=None,
+          method='POST',
+          path='/path',
+          body=input_body_dict
+        )
+        
+        self.assertEqual(
+          error.status,
+          498
+        )
+        self.assertEqual(
+          error.header_dict['content-type'],
+          'application/json'
+        )
+        self.assertEqual(
+          error.body,
+          expected_output_body_dict
+        )
+
+  def test_api_call_timeout(self):
+    with mock.patch(
+      'ssl.create_default_context',
+    ), mock.patch(
+      'httplib.HTTPSConnection.request',
+    ), mock.patch(
+      'httplib.HTTPSConnection.getresponse',
+    ) as mock_https_connection_getresponse:
+      mock_https_connection_getresponse.side_effect = SSLError('The read operation timed out')
+      self.assertRaises(
+        TimeoutReachedError,
+        self.rest_api_client_connection.call,
+        archive_resource=None,
+        method='POST',
+        path='/path',
+        body=input_body_dict
+      )
\ No newline at end of file

bt5/erp5_web_service/TestTemplateItem/portal_components/test.erp5.testRESTAPIClientConnectorMixin.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Test Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>testRESTAPIClientConnectorMixin</string> </value>
+        </item>
+        <item>
+            <key> <string>default_source_reference</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>test.erp5.testRESTAPIClientConnectorMixin</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Test Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_service/bt/template_mixin_id_list

+mixin.erp5.RESTAPIClientConnectorMixin
\ No newline at end of file

bt5/erp5_web_service/bt/template_test_id_list

 test.erp5.testFTPConnection
+test.erp5.testRESTAPIClientConnectorMixin
 test.erp5.testWebServiceTool
\ No newline at end of file