add headless chromium software release

This also adds a headless-chromium component and enables the http_sub
module in the Nginx component. Some basic tests are included. See merge
request nexedi/slapos!1014 for more details and discussion.

component/headless-chromium/buildout.cfg

 [buildout]
 extends =
-  ../binutils/buildout.cfg
-  ../bison/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../gperf/buildout.cfg
-  ../ninja/buildout.cfg
-  ../freetype/buildout.cfg
-  ../cmake/buildout.cfg
-  ../git/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../cups/buildout.cfg
+# Build dependencies:
   ../coreutils/buildout.cfg
+  ../curl/buildout.cfg
   ../depot_tools/buildout.cfg
-  ../findutils/buildout.cfg
-  ../fontconfig/buildout.cfg
-  ../gettext/buildout.cfg
-  ../glib/buildout.cfg
-  ../gtk-2/buildout.cfg
-  ../libexpat/buildout.cfg
-  ../libffi/buildout.cfg
-  ../libpng/buildout.cfg
-  ../libxml2/buildout.cfg
-  ../mesa/buildout.cfg
+  ../git/buildout.cfg
+  ../gperf/buildout.cfg
+  ../pkgconfig/buildout.cfg
+# Runtime dependencies:
   ../nspr/buildout.cfg
   ../nss/buildout.cfg
-  ../pcre/buildout.cfg
-  ../sqlite3/buildout.cfg
-  ../xorg/buildout.cfg
-  ../zlib/buildout.cfg
 
 parts =
-  chromium
+  headless-chromium-wrapper
 
-[gconf]
-recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz
-md5sum = 2b16996d0e4b112856ee5c59130e822c
-configure-options = --disable-orbit --disable-static
-environment =
-  PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH
-
-[chromedriver]
-recipe = hexagonit.recipe.download
-url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
-md5sum = a72088c0a6b018ded2c0fff616da8f65
-
-[chromium-download]
+# The Chromium project recommends using their own `fetch' tool rather
+# than doing a `git clone', but this is a little more flexible and works
+# fine.
+#
+# Setting depth=1000 is a middle ground between cloning no history
+# (causes some scripts to break) and cloning the full history (takes a
+# really long time).
+#
+# Note: we should add a `depth' option to slapos.recipe.build:gitclone
+# and then migrate this part to use that recipe at some point.
+[chromium-source]
 recipe = plone.recipe.command
-# This revision is 56.0.2924.122. Because the chromedriver only support some certain
-# version. Which version 56 is in the middle of 55-57.
-revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
-path = ${buildout:parts-directory}/${:_buildout_section_name_}
 command =
-  set -e
-  PATH=${depot_tools:location}:${git:location}/bin:$PATH
-  [ -d ${:path} ] && rm -r ${:path}
-  mkdir -p ${:path}
-  cd ${:path}
-  # Do never use `fetch` with the `--no-history` option unless you find an
-  # option to fetch directly at the wanted revision. `--no-history` could
-  # reduce the download size significantly but even if it retrieves enough
-  # commits at the time you test this section, development continues upstream
-  # and at some point the next command (gclient) would break.
-  # ...
-  # This command only could work in an empty dir.
-  fetch --nohooks chromium
-  gclient sync --revision ${:revision} --with_branch_heads
+  export PATH=$PATH:${git:location}/bin
+  git clone ${:repository} ${:location} \
+    --branch ${:version} \
+    --depth 1000
+repository = https://chromium.googlesource.com/chromium/src.git
 stop-on-error = true
 
-[chromium]
+# We should place the .gclient file in the parent directory of the
+# checkout/repository itself.
+location = ${:gclient-location}/${:name}
+gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+# Theoretically the checkout can be put anywhere as long as you specify
+# "name" appropriately in the .gclient file, but in practice it seems
+# that some automated tools break. It's safest to put it in a directory
+# called "src".
+name = src
+
+# There's nothing special about version 92.0.4515.107. It just happened
+# to be the current Chromium stable version at the time of writing.
+version = 92.0.4515.107
+
+
+[headless-chromium]
 recipe = slapos.recipe.cmmi
-path = ${chromium-download:path}/src
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+path = ${chromium-source:location}
+location = ${:path}/out/headless
+
+# Configuration file for GN, the tool to build the actual compilation
+# configuration file.
+default-gclient-config =
+  solutions = [
+    {
+      "name": "${chromium-source:name}",
+      "url": "${chromium-source:repository}",
+      "managed": False,
+      "custom_deps": {},
+      "custom_vars": {},
+    },
+  ]
+
+# Configuration for a headless build.
+build-config-options =
+  import("//build/args/headless.gn")
+  is_debug = false
+  symbol_level = 0
+  blink_symbol_level = 0
+
+  # We need to unbundle the build toolchain in order to set our own
+  # LDFLAGS.
+  custom_toolchain = "//build/toolchain/linux/unbundle:default"
+  host_toolchain = "//build/toolchain/linux/unbundle:default"
+  current_os = "linux"
+  current_cpu = "x64"
+
+# Chromium bundles its own LLVM toolchain, so we might as well use it.
+llvm-toolchain = ${:path}/third_party/llvm-build/Release+Asserts/bin
+
 configure-command =
-  gclient runhooks
+# Sync build dependencies---this is a little finnicky.
+  echo '${:default-gclient-config}' \
+    > ${chromium-source:gclient-location}/.gclient
+  gclient sync --no-history
+# Generate build configuration files.
   mkdir -p ${:location}
-  echo 'use_udev = true
-  is_debug = false
-  enable_nacl = false' > ${:location}/args.gn
+  echo '${:build-config-options}' > ${:location}/args.gn
   gn gen ${:location}
 
-# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu
-make-binary = ninja -C ${:location} chrome
+# You can run the headless Chromium shell using
+# ${:binary} --remote-debugging-port=1234
+make-binary =
+  autoninja -C ${:location} headless_shell
+# By building our own version of Chromedriver, we can ensure version
+# compatibility. The build is quite cheap compared to Chromium, anyway.
+  autoninja -C ${:location} chromedriver
 environment =
-  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH
-  PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s
-  CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH
-  LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH
+  PATH=${depot_tools:location}:${gperf:location}/bin:${pkgconfig:location}/bin:${coreutils:location}/bin:${git:location}/bin:${curl:location}/bin:%(PATH)s
+  LDFLAGS="-Wl,-rpath=${nss:location}/lib,-rpath=${nspr:location}/lib"
+  CC="${:llvm-toolchain}/clang"
+  CXX="${:llvm-toolchain}/clang++"
+  AR="${:llvm-toolchain}/llvm-ar"
+  NM="${:llvm-toolchain}/llvm-nm"
+
+# Expose devtools frontend location.
+devtools-frontend = ${:location}/gen/third_party/devtools-frontend/src/front_end
+
+binary = ${:location}/headless_shell
+chromedriver = ${:location}/chromedriver
+promises =
+  ${:binary}
+  ${:chromedriver}
+
+
+# At runtime, Chromium tries to dynamically load the NSS certificate
+# database from "libnssckbi.so". But Chromium does this through NSPR,
+# which doesn't know where SlapOS installed NSS. Since we don't want to
+# modify the NSPR component from the Chromium component, we just set
+# LD_LIBRARY_PATH in a wrapper script so that NSPR knows where NSS is.
+#
+# Alternatively, we could patch crypto/nss_util.cc in the Chromium
+# source code to use an absolute path for libnssckbi.so, but this is not
+# as future-proof against new versions of Chromium.
+[headless-chromium-wrapper]
+recipe = slapos.recipe.template:jinja2
+template =
+  inline:#!/bin/sh
+  export LD_LIBRARY_PATH="{{ nss_location }}/lib:$LD_LIBRARY_PATH"
+  exec {{ chromium_binary }} "$@"
+rendered = ${buildout:bin-directory}/headless-chromium
+context =
+  key nss_location nss:location
+  key chromium_binary headless-chromium:binary

component/nginx/buildout.cfg

@@ -24,6 +24,7 @@ configure-options=
   --with-http_v2_module
   --with-http_gzip_static_module
   --with-http_realip_module
+  --with-http_sub_module
   --with-mail
   --with-mail_ssl_module
   --with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"

software/headless-chromium/README.md

+# Headless Chromium
+This software release compiles and runs a headless Chromium shell and
+exposes an interface to connect to it remotely from another browser.
+
+After deployment, the instance is configured like this:
+
+```
+   Caddy frontend
+     |
+   (HTTPS, IPv6)
+     |
+   Nginx proxy, basic authentication
+     |
+   (HTTP, IPv4)
+     |
+   Chromium shell
+```
+
+The proxy is necessary because Chromium only accepts local connections
+for remote debugging.
+
+## Parameters
+The following instance parameters can be configured:
+
+- target-url:             URL for Chromium to load on startup.
+- remote-debugging-port:  Port for Chromium to listen on.
+- nginx-proxy-port:       Port for Ningx proxy to listen on.
+- monitor-httpd-port:     Port for monitor.
+
+See `instance-headless-chromium-input-schema.json` for default values.

software/headless-chromium/buildout.hash.cfg

+[template-cfg]
+filename = instance.cfg.in
+md5sum = 5dfeeb5eca125dcaa5f9e537f941dd41
+
+[instance-headless-chromium]
+_update_hash_filename_ = instance-headless-chromium.cfg.in
+md5sum = fad685238b26ca20537c12ce7432e7e7
+
+[template-nginx-conf]
+_update_hash_filename_ = templates/nginx.conf.in
+md5sum = c4d09d2b819f624087ef4c38551dfe2f
+
+[template-mime-types]
+_update_hash_filename_ = templates/mime-types.in
+md5sum = 4ef94a7b458d885cd79ba0b930a5727e

software/headless-chromium/instance-headless-chromium-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Input Parameters",
+  "properties": {
+    "target-url": {
+      "description": "URL for Chromium to load on startup.",
+      "title": "Target URL",
+      "type": "string",
+      "default": "https://www.example.com"
+    },
+    "remote-debugging-port": {
+      "description": "Port for Chromium to listen on.",
+      "title": "Remote Debugging Port",
+      "type": "integer",
+      "default": 8081
+    },
+    "nginx-proxy-port": {
+      "description": "Port for Nginx proxy to listen on.",
+      "title": "Nginx Proxy Port",
+      "type": "integer",
+      "default": 8082
+    },
+    "monitor-httpd-port": {
+      "description": "Port for monitor frontend.",
+      "title": "Monitor Httpd Port",
+      "type": "integer",
+      "default": 8083
+    }
+  }
+}

software/headless-chromium/instance-headless-chromium-output-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Values returned by headless Chromium instantiation",
+  "properties": {
+    "frontend-url": {
+      "description": "URL to access remote debugging interface",
+      "type": "string"
+    },
+    "username": {
+      "description": "Username for remote debugging interface",
+      "type": "string"
+    },
+    "password": {
+      "description": "Password for remote debugging interface",
+      "type": "string"
+    },
+    "monitor-base-url": {
+      "description": "Base URL used by monitor",
+      "type": "string"
+    },
+    "monitor-setup-url": {
+      "description": "One-click link to setup and monitor feeds",
+      "type": "string"
+    },
+    "proxy-url": {
+      "description": "Raw IPv6 address used by Nginx proxy",
+      "type": "string"
+    },
+    "remote-debug-url": {
+      "description": "Local IPv4 address used by Chromium",
+      "type": "string"
+    }
+  }
+}

software/headless-chromium/instance-headless-chromium.cfg.in

+{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
+
+[buildout]
+parts =
+  chromium-launcher
+  generate-passwd-file
+  nginx-config
+  nginx-mime-types
+  nginx-launcher
+  logrotate-entry-nginx
+  publish-connection-information
+  frontend-ok-promise
+  frontend-secure-promise
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+extends = {{ parameter_list['template-monitor'] }}
+
+# Create necessary directories.
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+tmp = ${:home}/tmp
+log = ${:home}/log
+etc = ${:home}/etc
+ssl = ${:etc}/ssl
+service = ${:etc}/service
+
+# Options for instance configuration. See README.md for a list of
+# options that can be configured when requesting an instance.
+[headless-chromium]
+ipv4 = {{ partition_ipv4 }}
+ipv6 = {{ partition_ipv6 }}
+remote-debugging-port = {{ parameter_dict['remote-debugging-port'] }}
+url = {{ parameter_dict['target-url'] }}
+remote-debugging-address = ${:ipv4}:${:remote-debugging-port}
+devtools-frontend-root = {{ parameter_list['devtools-frontend'] }}
+
+nginx-port = {{ parameter_dict['nginx-proxy-port'] }}
+proxy-address = [${:ipv6}]:${:nginx-port}
+nginx-config-target = ${directory:etc}/nginx.conf
+nginx-pid-path = ${directory:log}/nginx.pid
+nginx-temp-path = ${directory:tmp}
+nginx-error-log = ${directory:log}/nginx-error.log
+nginx-access-log = ${directory:log}/nginx-access.log
+nginx-htpasswd-file = ${directory:etc}/.htpasswd
+nginx-key-file = ${frontend-instance-certificate:key-file}
+nginx-cert-file = ${frontend-instance-certificate:cert-file}
+nginx-mime-types = ${directory:etc}/mime-types
+
+
+# Create a launcher script in /etc/service for the headless shell
+# executable.
+[chromium-launcher]
+recipe = slapos.recipe.template:jinja2
+template =
+  inline:#!/bin/sh
+
+  export FONTCONFIG_FILE=${font-config:rendered}
+  exec {{ parameter_list['chromium-wrapper'] }} \
+    --remote-debugging-address=${headless-chromium:ipv4} \
+    --remote-debugging-port=${headless-chromium:remote-debugging-port} \
+    ${headless-chromium:url}
+rendered = ${directory:service}/chromium
+
+
+# Configure and launch the proxy server.
+[nginx-config]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-nginx-config'] }}
+rendered = ${headless-chromium:nginx-config-target}
+mode = 700
+context =
+  section param_headless_chromium headless-chromium
+
+[nginx-mime-types]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-mime-types'] }}
+rendered = ${headless-chromium:nginx-mime-types}
+
+[nginx-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = {{ parameter_list['nginx-location'] }}/sbin/nginx -c ${headless-chromium:nginx-config-target}
+wrapper-path = ${directory:service}/nginx
+
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = ${headless-chromium:nginx-error-log} ${headless-chromium:nginx-access-log}
+
+[frontend-instance-password]
+recipe = slapos.cookbook:generate.password
+username = admin
+bytes = 12
+
+[generate-passwd-file]
+recipe = plone.recipe.command
+command =
+  echo -n '${frontend-instance-password:username}:' > ${headless-chromium:nginx-htpasswd-file}
+  openssl passwd -apr1 '${frontend-instance-password:passwd}' >> ${headless-chromium:nginx-htpasswd-file}
+environment =
+  PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
+
+# Generate a self-signed TLS certificate.
+[frontend-instance-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${:key-file} ]
+  then
+    openssl req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${:common-name}" \
+      -newkey rsa:1024 -keyout ${:key-file} \
+      -out ${:cert-file}
+    openssl x509 -addtrust serverAuth \
+      -in ${:cert-file} \
+      -out ${:cert-file}
+  fi
+update-command = ${:command}
+key-file = ${directory:ssl}/${:_buildout_section_name_}.key
+cert-file = ${directory:ssl}/${:_buildout_section_name_}.cert
+common-name = ${headless-chromium:ipv6}
+environment =
+  PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
+
+
+# Generate a fonts.conf file.
+[font-config]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-fonts-conf'] }}
+rendered = ${directory:etc}/fonts.conf
+context =
+  key cachedir :cache-dir
+  key fonts :fonts
+  key includes :includes
+cache-dir =
+  ${directory:etc}/.fontconfig.cache
+fonts =
+  {{ parameter_list['liberation-fonts-location'] }}
+includes =
+  {{ parameter_list['fontconfig-location'] }}/etc/fonts/conf.d
+
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+<= monitor-publish
+remote-debug-url = http://${headless-chromium:remote-debugging-address}
+proxy-url = https://${headless-chromium:proxy-address}
+frontend-url = ${remote-debugging-frontend:connection-secure_access}
+username = ${frontend-instance-password:username}
+password = ${frontend-instance-password:passwd}
+
+# Request a frontend URL from the CDN for the remote debugging interface.
+[remote-debugging-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Headless Chromium Remote Debugging Frontend
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://${headless-chromium:proxy-address}
+config-https-only = true
+config-type = websocket
+config-websocket-path-list = /devtools
+return = domain secure_access
+
+
+# Monitoring: check that the Chromium process is alive and responding to
+# requests through the proxy.
+[monitor-instance-parameter]
+monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
+
+# Promise to make sure the remote debugging frontend returns 200 when
+# queried with the correct credentials.
+[frontend-ok-promise]
+<= monitor-promise-base
+module = check_url_available
+name = headless-chromium-frontend-ok.py
+url = ${remote-debugging-frontend:connection-secure_access}
+config-url = ${:url}
+config-username = ${frontend-instance-password:username}
+config-password = ${frontend-instance-password:passwd}
+
+# Promise to make sure that the remote debugging frontend returns 401
+# when queried with no credentials.
+[frontend-secure-promise]
+<= monitor-promise-base
+module = check_url_available
+name = headless-chromium-frontend-secure.py
+url = ${remote-debugging-frontend:connection-secure_access}
+config-url = ${:url}
+config-http-code = 401

software/headless-chromium/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+[profile-common]
+openssl-location = {{ openssl_location }}
+nginx-location = {{ nginx_location }}
+liberation-fonts-location = {{ liberation_fonts_location }}
+fontconfig-location = {{ fontconfig_location }}
+chromium-wrapper = {{ chromium_wrapper }}
+devtools-frontend = {{ devtools_frontend }}
+template-nginx-config = {{ template_nginx_config_target }}
+template-fonts-conf = {{ template_fonts_conf_target }}
+template-monitor = {{ template_monitor }}
+template-mime-types = {{ template_mime_types_target }}
+
+[instance-headless-chromium]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_instance_headless_chromium_target }}
+rendered = ${buildout:directory}/${:filename}
+filename = instance-headless-chromium.cfg
+context =
+  section buildout buildout
+  section parameter_list profile-common
+  key partition_ipv4 slap-configuration:ipv4-random
+  key partition_ipv6 slap-configuration:ipv6-random
+  key slapparameter_dict slap-configuration:configuration
+  jsonkey default_parameter_dict :default-parameters
+default-parameters =
+  {
+    "remote-debugging-port": 8081,
+    "nginx-proxy-port": 8082,
+    "target-url": "https://www.example.com",
+    "monitor-httpd-port": 8083
+  }
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = ${:default}
+default = instance-headless-chromium:rendered
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}

software/headless-chromium/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/headless-chromium/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/fonts/buildout.cfg
+  ../../component/fontconfig/buildout.cfg
+
+parts =
+  slapos-cookbook
+  template-cfg
+
+[python]
+part = python3
+
+[template-cfg]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/template.cfg
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+context =
+  section buildout buildout
+  key openssl_location openssl:location
+  key nginx_location nginx:location
+  key liberation_fonts_location liberation-fonts:location
+  key fontconfig_location fontconfig:location
+  key chromium_wrapper headless-chromium-wrapper:rendered
+  key devtools_frontend headless-chromium:devtools-frontend
+  key template_nginx_config_target template-nginx-conf:target
+  key template_mime_types_target template-mime-types:target
+  key template_fonts_conf_target template-fonts-conf:output
+  key template_instance_headless_chromium_target instance-headless-chromium:target
+  key template_monitor monitor2-template:rendered
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 0644
+
+[instance-headless-chromium]
+<= download-base
+
+[template-nginx-conf]
+<= download-base
+
+[template-mime-types]
+<= download-base

software/headless-chromium/software.cfg.json

+{
+  "name": "Headless Chromium",
+  "description": "Headless (stripped-down) Chromium shell",
+  "serialization": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Standalone headless shell",
+      "request": "instance-headless-chromium-input-schema.json",
+      "response": "instance-headless-chromium-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/headless-chromium/templates/mime-types.in

+types {
+	text/html				html htm shtml;
+	text/css				css;
+	text/xml				xml rss;
+	image/gif				gif;
+	image/jpeg				jpeg jpg;
+	application/x-javascript		js;
+	application/atom+xml			atom;
+
+	text/mathml				mml;
+	text/plain				txt;
+	text/vnd.sun.j2me.app-descriptor	jad;
+	text/vnd.wap.wml			wml;
+	text/x-component			htc;
+
+	image/png				png;
+	image/tiff				tif tiff;
+	image/vnd.wap.wbmp			wbmp;
+	image/x-icon				ico;
+	image/x-jng				jng;
+	image/x-ms-bmp				bmp;
+	image/svg+xml				svg svgz;
+
+	application/java-archive		jar war ear;
+	application/mac-binhex40		hqx;
+	application/msword			doc;
+	application/pdf				pdf;
+	application/postscript			ps eps ai;
+	application/rtf				rtf;
+	application/vnd.ms-excel		xls;
+	application/vnd.ms-powerpoint		ppt;
+	application/vnd.wap.wmlc		wmlc;
+	application/vnd.google-earth.kml+xml	kml;
+	application/vnd.google-earth.kmz	kmz;
+	application/x-7z-compressed		7z;
+	application/x-cocoa			cco;
+	application/x-java-archive-diff		jardiff;
+	application/x-java-jnlp-file		jnlp;
+	application/x-makeself			run;
+	application/x-perl			pl pm;
+	application/x-pilot			prc pdb;
+	application/x-rar-compressed		rar;
+	application/x-redhat-package-manager	rpm;
+	application/x-sea			sea;
+	application/x-shockwave-flash		swf;
+	application/x-stuffit			sit;
+	application/x-tcl			tcl tk;
+	application/x-x509-ca-cert		der pem crt;
+	application/x-xpinstall			xpi;
+	application/xhtml+xml			xhtml;
+	application/zip				zip;
+
+	application/octet-stream		bin exe dll;
+	application/octet-stream		deb;
+	application/octet-stream		dmg;
+	application/octet-stream		eot;
+	application/octet-stream		iso img;
+	application/octet-stream		msi msp msm;
+	application/ogg				ogx;
+
+	audio/midi				mid midi kar;
+	audio/mpeg				mpga mpega mp2 mp3 m4a;
+	audio/ogg				oga ogg spx;
+	audio/x-realaudio			ra;
+	audio/webm				weba;
+
+	video/3gpp				3gpp 3gp;
+	video/mp4				mp4;
+	video/mpeg				mpeg mpg mpe;
+	video/ogg				ogv;
+	video/quicktime				mov;
+	video/webm				webm;
+	video/x-flv				flv;
+	video/x-mng				mng;
+	video/x-ms-asf				asx asf;
+	video/x-ms-wmv				wmv;
+	video/x-msvideo				avi;
+}

software/headless-chromium/templates/nginx.conf.in

+pid {{ param_headless_chromium['nginx-pid-path'] }};
+error_log {{ param_headless_chromium['nginx-error-log'] }};
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  access_log {{ param_headless_chromium['nginx-access-log'] }};
+
+  include {{ param_headless_chromium['nginx-mime-types'] }};
+  default_type application/octet-stream;
+
+  types {
+    text/html html;
+    text/css css;
+    application/javascript js;
+  }
+
+  server {
+    listen {{ param_headless_chromium['proxy-address'] }} ssl;
+
+    # Require username/password to access remote debugging port.
+    auth_basic "Remote Debugging";
+    auth_basic_user_file {{ param_headless_chromium['nginx-htpasswd-file'] }};
+
+    # Use self-signed SSL certificate.
+    ssl_certificate {{ param_headless_chromium['nginx-cert-file'] }};
+    ssl_certificate_key {{ param_headless_chromium['nginx-key-file'] }};
+
+    client_body_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    proxy_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    fastcgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    uwsgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    scgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+
+    # All websocket connections are served from /devtools.
+    location /devtools {
+      proxy_http_version 1.1;
+      proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
+      proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
+      proxy_set_header Upgrade "websocket";
+      proxy_set_header Connection "Upgrade";
+    }
+
+    # The DevTools frontend is served from /serve_file/@{version_hash}.
+    location ~ "^\/serve_file\/@[0-9a-f]{5,40}\/(.*)" {
+      alias {{ param_headless_chromium['devtools-frontend-root'] }}/$1;
+    }
+
+    location / {
+      proxy_http_version 1.1;
+
+      # The proxy must set the Host header to an IP address, since the
+      # headless Chromium shell refuses to run otherwise, for security
+      # reasons.
+      # See https://bugs.chromium.org/p/chromium/issues/detail?id=813540.
+      proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
+      proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
+
+      # The browser security policy will prevent us from loading the
+      # Websocket connection without TLS, so we have to go through the
+      # frontend CDN URL. The tricky thing is that the frontend URL is
+      # not available yet when this file is built; what we do instead is
+      # use the given Host header.
+      sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$host";
+      sub_filter_once on;
+      sub_filter_types application/json;
+
+      sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$host";
+      sub_filter_types application/json;
+
+      # We want to use our own DevTools frontend rather than
+      # https://chrome-devtools-frontend.appspot.com. There should be a
+      # --custom-devtools-frontend flag for Chromium, but it doesn't
+      # seem to work with the remote debugging port.
+      sub_filter "chrome-devtools-frontend.appspot.com" "$host";
+      sub_filter_types *;
+    }
+  }
+}

software/headless-chromium/test/README.md

+Tests for headless Chromium software release

software/headless-chromium/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.headless-chromium'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS headless Chromium",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/headless-chromium/test/test.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '../software.cfg')))
+
+class TestHeadlessChromium(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.connection_parameters = self.requestDefaultInstance().getConnectionParameterDict()
+
+  def test_remote_debugging_port(self):
+    # The headless browser should respond at /json with a nonempty list
+    # of available pages, each of which has a webSocketDebuggerUrl and a
+    # devtoolsFrontendUrl.
+    url = self.connection_parameters['remote-debug-url']
+    response = requests.get('%s/json' % url)
+
+    # Check that request was successful and the response was a nonempty
+    # list.
+    self.assertEqual(requests.codes['ok'], response.status_code)
+    self.assertTrue(len(response.json()) > 0)
+
+    # Check that the first page has the correct fields.
+    first_page = response.json()[0]
+    self.assertIn('webSocketDebuggerUrl', first_page)
+    self.assertIn('devtoolsFrontendUrl', first_page)
+
+  def test_devtools_frontend_ok(self):
+    # The proxy should serve the DevTools frontend from
+    # /serve_file/@{hash}/inspector.html, where {hash} is a 5-32 digit
+    # hash.
+    proxyURL = self.connection_parameters['proxy-url']
+    username = self.connection_parameters['username']
+    password = self.connection_parameters['password']
+    frontend = '/serve_file/@aaaaa/inspector.html'
+
+    response = requests.get(proxyURL + frontend, verify=False,
+                            auth=(username, password))
+    self.assertEqual(requests.codes['ok'], response.status_code)

software/slapos-sr-testing/software-py3.cfg

@@ -22,3 +22,4 @@ extra =
   proftpd ${slapos.test.proftpd-setup:setup}
   repman ${slapos.test.repman-setup:setup}
   restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
+  headless-chromium ${slapos.test.headless-chromium-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -194,6 +194,11 @@ setup = ${slapos-repository:location}/software/jscrawler/test/
 egg = slapos.test.galene
 setup = ${slapos-repository:location}/software/galene/test/
 
+[slapos.test.headless-chromium-setup]
+<= setup-develop-egg
+egg = slapos.test.headless-chromium
+setup = ${slapos-repository:location}/software/headless-chromium/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -254,6 +259,7 @@ extra-eggs =
   ${slapos.test.html5as-setup:egg}
   ${slapos.test.html5as-base-setup:egg}
   ${slapos.test.fluentd-setup:egg}
+  ${slapos.test.headless-chromium-setup:egg}
 
 # We don't name this interpreter `python`, so that when we run slapos node
 # software, installation scripts running `python` use a python without any