software/headless-chromium: Update chromium to 114.0.5735.340

We had to introduce a web page to bootstrap the web version of devtools
because since https://bugs.chromium.org/p/chromium/issues/detail?id=1232509
chrome debugger port no longer serve such a page via HTTP.

The URL also changed, /serve_file/@{version_hash}. pattern is no longer
used, both the devtools and the websocket endpoint are in /devtools

The test was made a bit more complete by actually making requests and
trying to connect to websocket endpoints.

Some problems were found with incognito and block-new-web-contents
options:
 - they are boolean type, but the software parameter serialisation is
 XML, which as of today does not support boolean types. This is left
 TODO for now
 - When both --incognito and --block-new-web-contents are true, the
 command line flag was --incognito--block-new-web-contents, which is
 unknown and was ignored. Some minmal changes are included to fix this.

component/depot_tools/buildout.cfg

@@ -7,5 +7,5 @@ parts =
 [depot_tools]
 recipe = slapos.recipe.build:gitclone
 repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
-revision = e023d4482012d89690f6a483e877eceb47c4501e
+revision = eb48a6ac0fa5835353ddd137ac35f44eee011716
 git-executable = ${git:location}/bin/git

component/headless-chromium/buildout.cfg

@@ -45,9 +45,9 @@ gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # called "src".
 name = src
 
-# 96.0.4664.129 version is the latest stable version in December 2021.
-# Note that we need a version compiling without python2
-version = 96.0.4664.129
+
+# 114.0.5735.340 version is the latest stable version in November 2023.
+version = 114.0.5735.340
 
 
 [headless-chromium]

software/headless-chromium/README.md

@@ -5,7 +5,7 @@ exposes an interface to connect to it remotely from another browser.
 After deployment, the instance is configured like this:
 
 ```
-   Caddy frontend
+   Rapid CDN Frontend
      |
    (HTTPS, IPv6)
      |
@@ -27,7 +27,7 @@ The following instance parameters can be configured:
 - nginx-proxy-port:       Port for Ningx proxy to listen on.
 - monitor-httpd-port:     Port for monitor.
 - incognito:              Force Incognito mode
-- window-size:            Initial windo size
+- window-size:            Initial window size
 - block-new-web-contents: Block new web contents
 
 See `instance-headless-chromium-input-schema.json` for default values.

software/headless-chromium/buildout.hash.cfg

 [template-cfg]
 filename = instance.cfg.in
-md5sum = 6315598b2c7c19f9e2d9cdf090492e2c
+md5sum = c6cdcee1e16dd4bd3bc462d286dcb999
 
 [instance-headless-chromium]
 _update_hash_filename_ = instance-headless-chromium.cfg.in
-md5sum = feaef60353c94e02d38cfec66f0eb861
+md5sum = 8a7e024569d92b0992f40ddac232cff5
 
 [template-nginx-conf]
 _update_hash_filename_ = templates/nginx.conf.in
-md5sum = 1f35f91fa7e490cd1e2194264a8a6ed8
+md5sum = 6ba793ce45bc67882ab2eea319984e3f
 
 [template-mime-types]
 _update_hash_filename_ = templates/mime_types.in
 md5sum = 4ef94a7b458d885cd79ba0b930a5727e
+
+[template-index-html]
+_update_hash_filename_ = templates/index.html
+md5sum = 9314b30f97535a4e516f4ea0c2029ab0

software/headless-chromium/instance-headless-chromium.cfg.in

@@ -9,6 +9,8 @@ log = ${:home}/log
 etc = ${:home}/etc
 ssl = ${:etc}/ssl
 service = ${:etc}/service
+srv = ${:home}/srv
+nginx-root = ${:srv}/nginx-root
 
 # Options for instance configuration. See README.md for a list of
 # options that can be configured when requesting an instance.
@@ -17,9 +19,7 @@ ipv4 = {{ partition_ipv4 }}
 ipv6 = {{ partition_ipv6 }}
 remote-debugging-port = {{ parameter_dict['remote-debugging-port'] }}
 target-url = {{ parameter_dict['target-url'] }}
-incognito = {{ parameter_dict['incognito'] }}
 window-size = {{ parameter_dict['window-size'] }}
-block-new-web-contents = {{ parameter_dict['block-new-web-contents'] }}
 remote-debugging-address = ${:ipv4}:${:remote-debugging-port}
 devtools-frontend-root = {{ parameter_list['devtools-frontend'] }}
 
@@ -34,7 +34,8 @@ nginx-htpasswd-file = ${directory:etc}/.htpasswd
 nginx-key-file = ${frontend-instance-certificate:key-file}
 nginx-cert-file = ${frontend-instance-certificate:cert-file}
 nginx-mime-types = ${directory:etc}/mime-types
-
+nginx-root = ${directory:nginx-root}
+nginx-index-html = ${:nginx-root}/index.html
 
 # Create a wrapper script in /bin/chromium for the headless shell
 # executable.
@@ -45,10 +46,11 @@ command-line =
   {{ parameter_list['chromium-wrapper'] }}
   --remote-debugging-address=${headless-chromium:ipv4}
   --remote-debugging-port=${headless-chromium:remote-debugging-port}
+  --remote-allow-origins=*
   --user-data-dir=${directory:tmp}
   --window-size="${headless-chromium:window-size}"
-  {% if parameter_dict['incognito'] %}--incognito{% endif -%}
-  {% if parameter_dict['block-new-web-contents'] %}--block-new-web-contents{% endif -%}
+  {% if parameter_dict['incognito'] %} --incognito{% endif -%}
+  {% if parameter_dict['block-new-web-contents'] %} --block-new-web-contents{% endif -%}
 {{ '\n  "${headless-chromium:target-url}"' }}
 environment =
   FONTCONFIG_FILE=${font-config:output}
@@ -74,6 +76,11 @@ recipe = slapos.recipe.template
 url = {{ parameter_list['template-mime-types'] }}
 output = ${headless-chromium:nginx-mime-types}
 
+[nginx-index-html]
+recipe = slapos.recipe.template
+url = {{ parameter_list['template-index-html'] }}
+output = ${headless-chromium:nginx-index-html}
+
 [nginx-launcher]
 recipe = slapos.cookbook:wrapper
 command-line =
@@ -195,6 +202,7 @@ parts =
   generate-passwd-file
   nginx-config
   nginx-mime-types
+  nginx-index-html
   nginx-launcher
   logrotate-entry-nginx
   remote-debugging-frontend

software/headless-chromium/instance.cfg.in

@@ -17,6 +17,7 @@ template-nginx-config = {{ template_nginx_config_target }}
 template-fonts-conf = {{ template_fonts_conf_target }}
 template-monitor = {{ template_monitor }}
 template-mime-types = {{ template_mime_types_target }}
+template-index-html = {{ template_index_html_target }}
 
 [instance-headless-chromium]
 recipe = slapos.recipe.template:jinja2

software/headless-chromium/software.cfg

@@ -27,6 +27,7 @@ context =
   key devtools_frontend headless-chromium:devtools-frontend
   key template_nginx_config_target template-nginx-conf:target
   key template_mime_types_target template-mime-types:target
+  key template_index_html_target template-index-html:target
   key template_fonts_conf_target template-fonts-conf:output
   key template_instance_headless_chromium_target instance-headless-chromium:target
   key template_monitor monitor2-template:output
@@ -43,3 +44,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 
 [template-mime-types]
 <= download-base
+
+[template-index-html]
+<= download-base

software/headless-chromium/templates/index.html

+<script>
+    fetch("/json")
+        .then(r => r.json())
+        .then(pages => window.location.replace(new URL(pages[0].devtoolsFrontendUrl, window.location)))
+</script>
\ No newline at end of file

software/headless-chromium/templates/nginx.conf.in

@@ -12,6 +12,7 @@ http {
 
   include {{ param_headless_chromium['nginx-mime-types'] }};
   default_type application/octet-stream;
+  root {{ param_headless_chromium['nginx-root'] }};
 
   server {
     listen {{ param_headless_chromium['proxy-address'] }} ssl;
@@ -30,8 +31,13 @@ http {
     uwsgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
     scgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
 
-    # All websocket connections are served from /devtools.
-    location /devtools {
+    # A minimal page to bootstrap the DevTools frontend
+    location = / {
+      try_files $uri $uri/index.html =404;
+    }
+
+    # All websocket connections are served from /devtools/page.
+    location /devtools/page {
       proxy_http_version 1.1;
       proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
       proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
@@ -39,9 +45,9 @@ http {
       proxy_set_header Connection "Upgrade";
     }
 
-    # The DevTools frontend is served from /serve_file/@{version_hash}.
-    location ~ "^\/serve_file\/@[0-9a-f]{5,40}\/(.*)" {
-      alias {{ param_headless_chromium['devtools-frontend-root'] }}/$1;
+    # Static content from DevTools frontend
+    location /devtools {
+      alias {{ param_headless_chromium['devtools-frontend-root'] }};
     }
 
     location / {
@@ -59,11 +65,11 @@ http {
       # frontend CDN URL. The tricky thing is that the frontend URL is
       # not available yet when this file is built; what we do instead is
       # use the given Host header.
-      sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$host";
+      sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$http_host";
       sub_filter_once on;
       sub_filter_types application/json;
 
-      sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$host";
+      sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$http_host";
       sub_filter_types application/json;
 
       # We want to use our own DevTools frontend rather than

software/headless-chromium/test/setup.py

@@ -44,6 +44,7 @@ setup(
         'slapos.core',
         'slapos.libnetworkcache',
         'requests',
+        'websocket-client',
     ],
     zip_safe=True,
     test_suite='test',

software/headless-chromium/test/test.py

@@ -25,8 +25,13 @@
 #
 ##############################################################################
 
+import base64
 import os
+import ssl
+import urllib.parse
+
 import requests
+import websocket
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
@@ -35,16 +40,17 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '../software.cfg')))
 
+
 class TestHeadlessChromium(SlapOSInstanceTestCase):
   def setUp(self):
-    self.connection_parameters = self.requestDefaultInstance().getConnectionParameterDict()
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
 
   def test_remote_debugging_port(self):
     # The headless browser should respond at /json with a nonempty list
     # of available pages, each of which has a webSocketDebuggerUrl and a
     # devtoolsFrontendUrl.
     url = self.connection_parameters['remote-debug-url']
-    response = requests.get('%s/json' % url)
+    response = requests.get(urllib.parse.urljoin(url, '/json'))
 
     # Check that request was successful and the response was a nonempty
     # list.
@@ -53,27 +59,72 @@ class TestHeadlessChromium(SlapOSInstanceTestCase):
 
     # Check that the first page has the correct fields.
     first_page = response.json()[0]
-    self.assertIn('webSocketDebuggerUrl', first_page)
     self.assertIn('devtoolsFrontendUrl', first_page)
+    websocket.create_connection(first_page['webSocketDebuggerUrl'], sslopt={"cert_reqs": ssl.CERT_NONE}).close()
 
   def test_devtools_frontend_ok(self):
-    # The proxy should serve the DevTools frontend from
-    # /serve_file/@{hash}/inspector.html, where {hash} is a 5-32 digit
-    # hash.
-    proxyURL = self.connection_parameters['proxy-url']
-    username = self.connection_parameters['username']
-    password = self.connection_parameters['password']
-    frontend = '/serve_file/@aaaaa/inspector.html'
-
-    response = requests.get(proxyURL + frontend, verify=False,
-                            auth=(username, password))
-    self.assertEqual(requests.codes['ok'], response.status_code)
+    param = self.computer_partition.getConnectionParameterDict()
+
+    # when accessed through RapidCDN, frontend rewrite WSS URLs with the host header but without port.
+    page, = requests.get(
+      urllib.parse.urljoin(param['proxy-url'], '/json'),
+      auth=(param['username'], param['password']),
+      headers={
+        'Host': 'hostname'
+      },
+      verify=False).json()
+    ws_debug_url = urllib.parse.urlparse(page['webSocketDebuggerUrl'])
+    self.assertEqual(
+      (ws_debug_url.scheme, ws_debug_url.netloc), ('wss', 'hostname'))
+
+    devtools_frontend_url = dict(
+      urllib.parse.parse_qsl(page['devtoolsFrontendUrl'].split('?')[1]))
+    # devtoolsFrontendUrl is a relative URL, like this:
+    # 'devtoolsFrontendUrl': '/devtools/inspector.html?wss=[::1]:9442/devtools/page/22C91CF307002BFA22DF0B4E34D2D026'
+    # and the query string argument wss must also have been rewritten:
+    self.assertTrue(
+      devtools_frontend_url['wss'].startswith('hostname/devtools/page/'))
+
+    requests.get(
+        urllib.parse.urljoin(param['proxy-url'], page['devtoolsFrontendUrl']),
+        auth=(param['username'], param['password']),
+        headers={
+          'Host': 'hostname'
+        },
+        verify=False).raise_for_status()
+
+    # when accessed directly, the :port is kept, as a consequence the debugger interface can
+    # be accessed directly from the nginx ipv6
+    page, = requests.get(
+      urllib.parse.urljoin(param['proxy-url'], '/json'),
+      auth=(param['username'], param['password']),
+      verify=False).json()
+    ws_debug_url = urllib.parse.urlparse(page['webSocketDebuggerUrl'])
+    self.assertEqual(ws_debug_url.port, 9224)
+
+    devtools_frontend_url = dict(urllib.parse.parse_qsl(page['devtoolsFrontendUrl'].split('?')[1]))
+    # devtoolsFrontendUrl is not rewritten
+    self.assertEqual(f"wss://{devtools_frontend_url['wss']}", page['webSocketDebuggerUrl'])
+
+    requests.get(
+        urllib.parse.urljoin(param['proxy-url'], page['devtoolsFrontendUrl']),
+        auth=(param['username'], param['password']),
+        verify=False).raise_for_status()
+
+    # the websocket is usable
+    websocket.create_connection(
+      page['webSocketDebuggerUrl'],
+      sslopt={"cert_reqs": ssl.CERT_NONE},
+      header={'Authorization': 'Basic ' + base64.b64encode(
+        f"{param['username']}:{param['password']}".encode()).strip().decode()}).close()
+
 
 class TestHeadlessChromiumParameters(SlapOSInstanceTestCase):
 
   instance_parameter_dict = {
     # this website echoes the get request for debugging purposes
     'target-url': 'https://httpbin.org/get?a=6&b=4',
+    # TODO: this does not work, this software uses 'xml' serialisation and only support strings
     'incognito': True,
     "block-new-web-contents": False,
     "window-size": "900,600"