Revert "stack/erp5: stop using caucase managed certificate for balancer"

This reverts commit 620c9332 and add a
caucase service for balancer in the balancer partition. The caucase
service from the root partition (that was not used) is remvoed.

software/erp5/README.rst

@@ -90,7 +90,6 @@ This software release assigns the following port ranges by default:
   balancer              2150-2199
   zope                  2200-*
   jupyter               8888
-  caucase               8890,8891
   ====================  ==========
 
 Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to

software/erp5/instance-erp5-input-schema.json

@@ -508,22 +508,15 @@
     },
     "caucase": {
       "description": "Caucase certificate authority parameters",
-      "allOf": [
-        {
       "properties": {
         "url": {
           "title": "Caucase URL",
-              "description": "URL of existing caucase instance to use. If empty, a new caucase instance will be deployed. If not empty, other properties in this section will be ignored.",
+          "description": "URL of existing caucase instance to use. If empty, caucase instances will be deployed inside partitions.",
           "default": "",
           "type": "string",
           "format": "uri"
         }
-          }
       },
-        {
-          "$ref": "../caucase/instance-caucase-input-schema.json"
-        }
-      ],
       "type": "object"
     },
     "test-runner": {

software/erp5/test/test/test_balancer.py

@@ -5,13 +5,17 @@ import logging
 import os
 import re
 import shutil
+import socket
 import subprocess
 import tempfile
 import time
 import urllib.parse
 from http.server import BaseHTTPRequestHandler
 from unittest import mock
+from typing import Any, Optional
 
+import idna
+import OpenSSL.SSL
 import pexpect
 import psutil
 import requests
@@ -128,6 +132,20 @@ class CaucaseService(ManagedResource):
     self._caucased_process.stdout.close()
     shutil.rmtree(self.directory)
 
+  @property
+  def ca_crt_path(self):
+    # type: () -> str
+    """Path of the CA certificate from this caucase.
+    """
+    ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
+    if not os.path.exists(ca_crt_path):
+      with open(ca_crt_path, 'w') as f:
+        f.write(
+            requests.get(urllib.parse.urljoin(
+                self.url,
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    return ca_crt_path
 
 
 class BalancerTestCase(ERP5InstanceTestCase):
@@ -143,8 +161,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
     return 'balancer'
 
   @classmethod
-  def _getInstanceParameterDict(cls):
-    # type: () -> dict
+  def _getInstanceParameterDict(cls) -> dict:
     return {
         'tcpv4-port': 8000,
         'computer-memory-percent-threshold': 100,
@@ -179,12 +196,10 @@ class BalancerTestCase(ERP5InstanceTestCase):
       }
 
   @classmethod
-  def getInstanceParameterDict(cls):
-    # type: () -> dict
+  def getInstanceParameterDict(cls) -> dict:
     return {'_': json.dumps(cls._getInstanceParameterDict())}
 
-  def setUp(self):
-    # type: () -> None
+  def setUp(self) -> None:
     self.default_balancer_url = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])['default']
 
@@ -195,8 +210,7 @@ class SlowHTTPServer(ManagedHTTPServer):
   Timeout is 2 seconds by default, and can be specified in the path of the URL
   """
   class RequestHandler(BaseHTTPRequestHandler):
-    def do_GET(self):
-      # type: () -> None
+    def do_GET(self) -> None:
       self.send_response(200)
       self.send_header("Content-Type", "text/plain")
       timeout = 2
@@ -584,6 +598,113 @@ class TestHTTP(BalancerTestCase):
       ])
 
 
+class ServerTLSMixin:
+
+  def _getServerCertificate(self, hostname, port):
+    # type: (Optional[str], Optional[int]) -> Any
+    hostname_idna = idna.encode(hostname)
+    sock = socket.socket()
+
+    sock.connect((hostname, port))
+    ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
+    ctx.check_hostname = False
+    ctx.verify_mode = OpenSSL.SSL.VERIFY_NONE
+
+    sock_ssl = OpenSSL.SSL.Connection(ctx, sock)
+    sock_ssl.set_connect_state()
+    sock_ssl.set_tlsext_host_name(hostname_idna)
+    sock_ssl.do_handshake()
+    cert = sock_ssl.get_peer_certificate()
+    crypto_cert = cert.to_cryptography()
+    sock_ssl.close()
+    sock.close()
+    return crypto_cert
+
+  def _getCaucaseCACertificatePath(self) -> str:
+    """Returns the path of the caucase certificate on file system.
+    """
+    raise NotImplementedError()
+
+  def test_certificate_validates_with_caucase_ca(self) -> None:
+    requests.get(self.default_balancer_url, verify=self._getCaucaseCACertificatePath())
+
+  def test_certificate_renewal(self) -> None:
+    balancer_parsed_url = urllib.parse.urlparse(self.default_balancer_url)
+    certificate_before_renewal = self._getServerCertificate(
+      balancer_parsed_url.hostname,
+      balancer_parsed_url.port)
+
+    # run caucase updater 90 days in the future, so that certificate is
+    # renewed.
+    caucase_updater = os.path.join(
+      self.computer_partition_root_path,
+      'etc',
+      'service',
+      'caucase-updater',
+    )
+    process = pexpect.spawnu(
+      "faketime +90days %s" % caucase_updater,
+      env=dict(os.environ, PYTHONPATH=''),
+    )
+    logger = self.logger
+    class DebugLogFile:
+      def write(self, msg):
+        logger.info("output from caucase_updater: %s", msg)
+      def flush(self):
+        pass
+    process.logfile = DebugLogFile()
+    process.expect(u"Renewing .*\nNext wake-up.*")
+    process.terminate()
+    process.wait()
+
+    # wait for server to use new certificate
+    for _ in range(30):
+      certificate_after_renewal = self._getServerCertificate(
+        balancer_parsed_url.hostname,
+        balancer_parsed_url.port)
+      if certificate_after_renewal.not_valid_before > certificate_before_renewal.not_valid_before:
+        break
+      time.sleep(.5)
+
+    self.assertGreater(
+      certificate_after_renewal.not_valid_before,
+      certificate_before_renewal.not_valid_before,
+    )
+
+    # requests are served properly after cert renewal
+    requests.get(self.default_balancer_url, verify=self._getCaucaseCACertificatePath()).raise_for_status()
+
+
+class TestServerTLSEmbeddedCaucase(BalancerTestCase, ServerTLSMixin):
+  """Check Server TLS with embedded caucase
+  """
+  __partition_reference__ = 's'
+
+  @classmethod
+  def _getInstanceParameterDict(cls) -> dict:
+    parameter_dict = super()._getInstanceParameterDict()
+    parameter_dict['ssl'] = {}
+    return parameter_dict
+
+  def _getCaucaseCACertificatePath(self) -> str:
+    """Returns the path of the caucase certificate on file system.
+    """
+    breakpoint()
+    # TODO: get CA from the running caucase and save to disk
+    return self.getManagedResource("caucase", CaucaseService).ca_crt_path
+
+
+class TestServerTLSExternalCaucase(BalancerTestCase, ServerTLSMixin):
+  """Check Server TLS with external caucase
+  """
+  __partition_reference__ = 's'
+
+  def _getCaucaseCACertificatePath(self) -> str:
+    """Returns the path of the caucase certificate on file system.
+    """
+    return self.getManagedResource("caucase", CaucaseService).ca_crt_path
+
+
 class ContentTypeHTTPServer(ManagedHTTPServer):
   """An HTTP/1.1 Server which reply with content type from path.
 

software/erp5/test/test/test_erp5.py

@@ -98,6 +98,21 @@ class TestPublishedURLIsReachableMixin:
       self.assertEqual(r.status_code, requests.codes.ok)
       self.assertIn("ERP5", r.text)
 
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    )
+    ca_cert.write(
+        requests.get(
+            urllib.parse.urlparse.urljoin(
+                self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
@@ -105,7 +120,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['family-default-v6'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
   def test_published_family_default_v4_is_reachable(self):
@@ -115,7 +130,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['family-default'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
   def test_published_frontend_default_is_reachable(self):
@@ -125,7 +140,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['url-frontend-default'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
 

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 55232eae0bcdb68a7cb2598d2ba9d60c
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = a85867f7d05c13e576b5453c49bd8997
+md5sum = a89b533746cc4a7a46c43e2b57d4a44c
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,7 +90,7 @@ md5sum = 4ab71c2cb50310b948828a07c521e0b7
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = b0751d3d12cfcc8934cb1027190f5e5e
+md5sum = 41aaa64a4df13555e645686b0f252fba
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -15,6 +15,32 @@ per partition. No more (undefined result), no less (IndexError).
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 
+
+{% if not ssl_parameter_dict['caucase-url'] -%}
+{%   if use_ipv6 -%}
+{%     set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
+{%-  else -%}
+{%     set caucase_host = (ipv4_set | list)[0] %}
+{%-  endif %}
+{%   set caucase_port = 2199 -%}
+{%   set caucase_netloc = caucase_host ~ ':' ~ caucase_port -%}
+{%   do ssl_parameter_dict.__setitem__('caucase-url', 'http://' + caucase_netloc) %}
+{{   caucase.caucased(
+       prefix='caucased',
+       buildout_bin_directory=bin_directory,
+       caucased_path='${directory:service-on-watch}/caucased',
+       backup_dir='${directory:backup-caucased}',
+       data_dir='${directory:srv}/caucased',
+       netloc=caucase_netloc,
+       tmp='${directory:tmp}',
+       service_auto_approve_count=ssl_parameter_dict.get('service-auto-approve-amount', 1),
+       user_auto_approve_count=ssl_parameter_dict.get('user-auto-approve-amount', 0),
+       key_len=ssl_parameter_dict.get('key-length', 2048),
+)}}
+{% do section('caucased') -%}
+{% do section('caucased-promise') -%}
+{% endif -%}
+
 {{ caucase.updater(
      prefix='caucase-updater',
      buildout_bin_directory=parameter_dict['bin-directory'],
@@ -30,10 +56,7 @@ recipe = slapos.recipe.template:jinja2
      template_csr_pem=ssl_parameter_dict.get('csr'),
      openssl=parameter_dict['openssl'] ~ '/bin/openssl',
 )}}
-{# XXX we don't use caucase yet.
-{% do section('caucase-updater') -%}
-{% do section('caucase-updater-promise') -%}
-#}
+
 
 {% set frontend_caucase_url_hash_list = [] -%}
 {% for frontend_caucase_url in frontend_caucase_url_list -%}

stack/erp5/instance-erp5.cfg.in

@@ -44,6 +44,7 @@
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-memory-percent-threshold', 80)) }}
+# TODO: user can pass a caucase URL
 {% set caucase_dict = slapparameter_dict.get('caucase', {}) -%}
 {% set caucase_url = caucase_dict.get('url', '') -%}