diff --git a/component/headless-chromium/buildout.cfg b/component/headless-chromium/buildout.cfg
index 2001b0fff0fd1a922c32e32b8d87b3f007d79483..4bd2d6deb03dbc6786e626d363dd10159ebbc13d 100644
--- a/component/headless-chromium/buildout.cfg
+++ b/component/headless-chromium/buildout.cfg
@@ -1,90 +1,141 @@
[buildout]
extends =
- ../binutils/buildout.cfg
- ../bison/buildout.cfg
- ../pkgconfig/buildout.cfg
- ../gperf/buildout.cfg
- ../ninja/buildout.cfg
- ../freetype/buildout.cfg
- ../cmake/buildout.cfg
- ../git/buildout.cfg
- ../pkgconfig/buildout.cfg
- ../cups/buildout.cfg
+# Build dependencies:
../coreutils/buildout.cfg
+ ../curl/buildout.cfg
../depot_tools/buildout.cfg
- ../findutils/buildout.cfg
- ../fontconfig/buildout.cfg
- ../gettext/buildout.cfg
- ../glib/buildout.cfg
- ../gtk-2/buildout.cfg
- ../libexpat/buildout.cfg
- ../libffi/buildout.cfg
- ../libpng/buildout.cfg
- ../libxml2/buildout.cfg
- ../mesa/buildout.cfg
+ ../git/buildout.cfg
+ ../gperf/buildout.cfg
+ ../pkgconfig/buildout.cfg
+# Runtime dependencies:
../nspr/buildout.cfg
../nss/buildout.cfg
- ../pcre/buildout.cfg
- ../sqlite3/buildout.cfg
- ../xorg/buildout.cfg
- ../zlib/buildout.cfg
parts =
- chromium
+ headless-chromium-wrapper
-[gconf]
-recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz
-md5sum = 2b16996d0e4b112856ee5c59130e822c
-configure-options = --disable-orbit --disable-static
-environment =
- PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s
- PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH
-
-[chromedriver]
-recipe = hexagonit.recipe.download
-url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
-md5sum = a72088c0a6b018ded2c0fff616da8f65
-
-[chromium-download]
+# The Chromium project recommends using their own `fetch' tool rather
+# than doing a `git clone', but this is a little more flexible and works
+# fine.
+#
+# Setting depth=1000 is a middle ground between cloning no history
+# (causes some scripts to break) and cloning the full history (takes a
+# really long time).
+#
+# Note: we should add a `depth' option to slapos.recipe.build:gitclone
+# and then migrate this part to use that recipe at some point.
+[chromium-source]
recipe = plone.recipe.command
-# This revision is 56.0.2924.122. Because the chromedriver only support some certain
-# version. Which version 56 is in the middle of 55-57.
-revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
-path = ${buildout:parts-directory}/${:_buildout_section_name_}
command =
- set -e
- PATH=${depot_tools:location}:${git:location}/bin:$PATH
- [ -d ${:path} ] && rm -r ${:path}
- mkdir -p ${:path}
- cd ${:path}
- # Do never use `fetch` with the `--no-history` option unless you find an
- # option to fetch directly at the wanted revision. `--no-history` could
- # reduce the download size significantly but even if it retrieves enough
- # commits at the time you test this section, development continues upstream
- # and at some point the next command (gclient) would break.
- # ...
- # This command only could work in an empty dir.
- fetch --nohooks chromium
- gclient sync --revision ${:revision} --with_branch_heads
+ export PATH=$PATH:${git:location}/bin
+ git clone ${:repository} ${:location} \
+ --branch ${:version} \
+ --depth 1000
+repository = https://chromium.googlesource.com/chromium/src.git
stop-on-error = true
-[chromium]
+# We should place the .gclient file in the parent directory of the
+# checkout/repository itself.
+location = ${:gclient-location}/${:name}
+gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+# Theoretically the checkout can be put anywhere as long as you specify
+# "name" appropriately in the .gclient file, but in practice it seems
+# that some automated tools break. It's safest to put it in a directory
+# called "src".
+name = src
+
+# There's nothing special about version 92.0.4515.107. It just happened
+# to be the current Chromium stable version at the time of writing.
+version = 92.0.4515.107
+
+
+[headless-chromium]
recipe = slapos.recipe.cmmi
-path = ${chromium-download:path}/src
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+path = ${chromium-source:location}
+location = ${:path}/out/headless
+
+# Configuration file for GN, the tool to build the actual compilation
+# configuration file.
+default-gclient-config =
+ solutions = [
+ {
+ "name": "${chromium-source:name}",
+ "url": "${chromium-source:repository}",
+ "managed": False,
+ "custom_deps": {},
+ "custom_vars": {},
+ },
+ ]
+
+# Configuration for a headless build.
+build-config-options =
+ import("//build/args/headless.gn")
+ is_debug = false
+ symbol_level = 0
+ blink_symbol_level = 0
+
+ # We need to unbundle the build toolchain in order to set our own
+ # LDFLAGS.
+ custom_toolchain = "//build/toolchain/linux/unbundle:default"
+ host_toolchain = "//build/toolchain/linux/unbundle:default"
+ current_os = "linux"
+ current_cpu = "x64"
+
+# Chromium bundles its own LLVM toolchain, so we might as well use it.
+llvm-toolchain = ${:path}/third_party/llvm-build/Release+Asserts/bin
+
configure-command =
- gclient runhooks
+# Sync build dependencies---this is a little finnicky.
+ echo '${:default-gclient-config}' \
+ > ${chromium-source:gclient-location}/.gclient
+ gclient sync --no-history
+# Generate build configuration files.
mkdir -p ${:location}
- echo 'use_udev = true
- is_debug = false
- enable_nacl = false' > ${:location}/args.gn
+ echo '${:build-config-options}' > ${:location}/args.gn
gn gen ${:location}
-# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu
-make-binary = ninja -C ${:location} chrome
+# You can run the headless Chromium shell using
+# ${:binary} --remote-debugging-port=1234
+make-binary =
+ autoninja -C ${:location} headless_shell
+# By building our own version of Chromedriver, we can ensure version
+# compatibility. The build is quite cheap compared to Chromium, anyway.
+ autoninja -C ${:location} chromedriver
environment =
- PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH
- PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s
- CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH
- LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH
+ PATH=${depot_tools:location}:${gperf:location}/bin:${pkgconfig:location}/bin:${coreutils:location}/bin:${git:location}/bin:${curl:location}/bin:%(PATH)s
+ LDFLAGS="-Wl,-rpath=${nss:location}/lib,-rpath=${nspr:location}/lib"
+ CC="${:llvm-toolchain}/clang"
+ CXX="${:llvm-toolchain}/clang++"
+ AR="${:llvm-toolchain}/llvm-ar"
+ NM="${:llvm-toolchain}/llvm-nm"
+
+# Expose devtools frontend location.
+devtools-frontend = ${:location}/gen/third_party/devtools-frontend/src/front_end
+
+binary = ${:location}/headless_shell
+chromedriver = ${:location}/chromedriver
+promises =
+ ${:binary}
+ ${:chromedriver}
+
+
+# At runtime, Chromium tries to dynamically load the NSS certificate
+# database from "libnssckbi.so". But Chromium does this through NSPR,
+# which doesn't know where SlapOS installed NSS. Since we don't want to
+# modify the NSPR component from the Chromium component, we just set
+# LD_LIBRARY_PATH in a wrapper script so that NSPR knows where NSS is.
+#
+# Alternatively, we could patch crypto/nss_util.cc in the Chromium
+# source code to use an absolute path for libnssckbi.so, but this is not
+# as future-proof against new versions of Chromium.
+[headless-chromium-wrapper]
+recipe = slapos.recipe.template:jinja2
+template =
+ inline:#!/bin/sh
+ export LD_LIBRARY_PATH="{{ nss_location }}/lib:$LD_LIBRARY_PATH"
+ exec {{ chromium_binary }} "$@"
+rendered = ${buildout:bin-directory}/headless-chromium
+context =
+ key nss_location nss:location
+ key chromium_binary headless-chromium:binary
diff --git a/component/nginx/buildout.cfg b/component/nginx/buildout.cfg
index 845e0ead180203912b73dfadf2214fae70099b55..c3e092863b68c0aac6585cb80fe84c076bf469d6 100644
--- a/component/nginx/buildout.cfg
+++ b/component/nginx/buildout.cfg
@@ -24,6 +24,7 @@ configure-options=
--with-http_v2_module
--with-http_gzip_static_module
--with-http_realip_module
+ --with-http_sub_module
--with-mail
--with-mail_ssl_module
--with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
diff --git a/software/headless-chromium/README.md b/software/headless-chromium/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..8dea379c92ea114f0d9f94c6b121d589cda6fa87
--- /dev/null
+++ b/software/headless-chromium/README.md
@@ -0,0 +1,30 @@
+# Headless Chromium
+This software release compiles and runs a headless Chromium shell and
+exposes an interface to connect to it remotely from another browser.
+
+After deployment, the instance is configured like this:
+
+```
+ Caddy frontend
+ |
+ (HTTPS, IPv6)
+ |
+ Nginx proxy, basic authentication
+ |
+ (HTTP, IPv4)
+ |
+ Chromium shell
+```
+
+The proxy is necessary because Chromium only accepts local connections
+for remote debugging.
+
+## Parameters
+The following instance parameters can be configured:
+
+- target-url: URL for Chromium to load on startup.
+- remote-debugging-port: Port for Chromium to listen on.
+- nginx-proxy-port: Port for Ningx proxy to listen on.
+- monitor-httpd-port: Port for monitor.
+
+See `instance-headless-chromium-input-schema.json` for default values.
diff --git a/software/headless-chromium/buildout.hash.cfg b/software/headless-chromium/buildout.hash.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..d6323ccc7a41b4fb78ff395f73646667ae3e1d48
--- /dev/null
+++ b/software/headless-chromium/buildout.hash.cfg
@@ -0,0 +1,15 @@
+[template-cfg]
+filename = instance.cfg.in
+md5sum = 5dfeeb5eca125dcaa5f9e537f941dd41
+
+[instance-headless-chromium]
+_update_hash_filename_ = instance-headless-chromium.cfg.in
+md5sum = fad685238b26ca20537c12ce7432e7e7
+
+[template-nginx-conf]
+_update_hash_filename_ = templates/nginx.conf.in
+md5sum = c4d09d2b819f624087ef4c38551dfe2f
+
+[template-mime-types]
+_update_hash_filename_ = templates/mime-types.in
+md5sum = 4ef94a7b458d885cd79ba0b930a5727e
diff --git a/software/headless-chromium/instance-headless-chromium-input-schema.json b/software/headless-chromium/instance-headless-chromium-input-schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..9ee95425d43736f5723ca968062c2269bcba49b8
--- /dev/null
+++ b/software/headless-chromium/instance-headless-chromium-input-schema.json
@@ -0,0 +1,31 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "title": "Input Parameters",
+ "properties": {
+ "target-url": {
+ "description": "URL for Chromium to load on startup.",
+ "title": "Target URL",
+ "type": "string",
+ "default": "https://www.example.com"
+ },
+ "remote-debugging-port": {
+ "description": "Port for Chromium to listen on.",
+ "title": "Remote Debugging Port",
+ "type": "integer",
+ "default": 8081
+ },
+ "nginx-proxy-port": {
+ "description": "Port for Nginx proxy to listen on.",
+ "title": "Nginx Proxy Port",
+ "type": "integer",
+ "default": 8082
+ },
+ "monitor-httpd-port": {
+ "description": "Port for monitor frontend.",
+ "title": "Monitor Httpd Port",
+ "type": "integer",
+ "default": 8083
+ }
+ }
+}
diff --git a/software/headless-chromium/instance-headless-chromium-output-schema.json b/software/headless-chromium/instance-headless-chromium-output-schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..5595a1798521a432ff5e7a5096f4eaf7e8f4b7b4
--- /dev/null
+++ b/software/headless-chromium/instance-headless-chromium-output-schema.json
@@ -0,0 +1,35 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-04/schema",
+ "title": "Values returned by headless Chromium instantiation",
+ "properties": {
+ "frontend-url": {
+ "description": "URL to access remote debugging interface",
+ "type": "string"
+ },
+ "username": {
+ "description": "Username for remote debugging interface",
+ "type": "string"
+ },
+ "password": {
+ "description": "Password for remote debugging interface",
+ "type": "string"
+ },
+ "monitor-base-url": {
+ "description": "Base URL used by monitor",
+ "type": "string"
+ },
+ "monitor-setup-url": {
+ "description": "One-click link to setup and monitor feeds",
+ "type": "string"
+ },
+ "proxy-url": {
+ "description": "Raw IPv6 address used by Nginx proxy",
+ "type": "string"
+ },
+ "remote-debug-url": {
+ "description": "Local IPv4 address used by Chromium",
+ "type": "string"
+ }
+ }
+}
diff --git a/software/headless-chromium/instance-headless-chromium.cfg.in b/software/headless-chromium/instance-headless-chromium.cfg.in
new file mode 100644
index 0000000000000000000000000000000000000000..36fd69d74a5d38bf8a778676346d106f464199cf
--- /dev/null
+++ b/software/headless-chromium/instance-headless-chromium.cfg.in
@@ -0,0 +1,192 @@
+{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
+
+[buildout]
+parts =
+ chromium-launcher
+ generate-passwd-file
+ nginx-config
+ nginx-mime-types
+ nginx-launcher
+ logrotate-entry-nginx
+ publish-connection-information
+ frontend-ok-promise
+ frontend-secure-promise
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+extends = {{ parameter_list['template-monitor'] }}
+
+# Create necessary directories.
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+tmp = ${:home}/tmp
+log = ${:home}/log
+etc = ${:home}/etc
+ssl = ${:etc}/ssl
+service = ${:etc}/service
+
+# Options for instance configuration. See README.md for a list of
+# options that can be configured when requesting an instance.
+[headless-chromium]
+ipv4 = {{ partition_ipv4 }}
+ipv6 = {{ partition_ipv6 }}
+remote-debugging-port = {{ parameter_dict['remote-debugging-port'] }}
+url = {{ parameter_dict['target-url'] }}
+remote-debugging-address = ${:ipv4}:${:remote-debugging-port}
+devtools-frontend-root = {{ parameter_list['devtools-frontend'] }}
+
+nginx-port = {{ parameter_dict['nginx-proxy-port'] }}
+proxy-address = [${:ipv6}]:${:nginx-port}
+nginx-config-target = ${directory:etc}/nginx.conf
+nginx-pid-path = ${directory:log}/nginx.pid
+nginx-temp-path = ${directory:tmp}
+nginx-error-log = ${directory:log}/nginx-error.log
+nginx-access-log = ${directory:log}/nginx-access.log
+nginx-htpasswd-file = ${directory:etc}/.htpasswd
+nginx-key-file = ${frontend-instance-certificate:key-file}
+nginx-cert-file = ${frontend-instance-certificate:cert-file}
+nginx-mime-types = ${directory:etc}/mime-types
+
+
+# Create a launcher script in /etc/service for the headless shell
+# executable.
+[chromium-launcher]
+recipe = slapos.recipe.template:jinja2
+template =
+ inline:#!/bin/sh
+
+ export FONTCONFIG_FILE=${font-config:rendered}
+ exec {{ parameter_list['chromium-wrapper'] }} \
+ --remote-debugging-address=${headless-chromium:ipv4} \
+ --remote-debugging-port=${headless-chromium:remote-debugging-port} \
+ ${headless-chromium:url}
+rendered = ${directory:service}/chromium
+
+
+# Configure and launch the proxy server.
+[nginx-config]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-nginx-config'] }}
+rendered = ${headless-chromium:nginx-config-target}
+mode = 700
+context =
+ section param_headless_chromium headless-chromium
+
+[nginx-mime-types]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-mime-types'] }}
+rendered = ${headless-chromium:nginx-mime-types}
+
+[nginx-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = {{ parameter_list['nginx-location'] }}/sbin/nginx -c ${headless-chromium:nginx-config-target}
+wrapper-path = ${directory:service}/nginx
+
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = ${headless-chromium:nginx-error-log} ${headless-chromium:nginx-access-log}
+
+[frontend-instance-password]
+recipe = slapos.cookbook:generate.password
+username = admin
+bytes = 12
+
+[generate-passwd-file]
+recipe = plone.recipe.command
+command =
+ echo -n '${frontend-instance-password:username}:' > ${headless-chromium:nginx-htpasswd-file}
+ openssl passwd -apr1 '${frontend-instance-password:passwd}' >> ${headless-chromium:nginx-htpasswd-file}
+environment =
+ PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
+
+# Generate a self-signed TLS certificate.
+[frontend-instance-certificate]
+recipe = plone.recipe.command
+command =
+ if [ ! -e ${:key-file} ]
+ then
+ openssl req -x509 -nodes -days 3650 \
+ -subj "/C=AA/ST=X/L=X/O=Dis/CN=${:common-name}" \
+ -newkey rsa:1024 -keyout ${:key-file} \
+ -out ${:cert-file}
+ openssl x509 -addtrust serverAuth \
+ -in ${:cert-file} \
+ -out ${:cert-file}
+ fi
+update-command = ${:command}
+key-file = ${directory:ssl}/${:_buildout_section_name_}.key
+cert-file = ${directory:ssl}/${:_buildout_section_name_}.cert
+common-name = ${headless-chromium:ipv6}
+environment =
+ PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
+
+
+# Generate a fonts.conf file.
+[font-config]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-fonts-conf'] }}
+rendered = ${directory:etc}/fonts.conf
+context =
+ key cachedir :cache-dir
+ key fonts :fonts
+ key includes :includes
+cache-dir =
+ ${directory:etc}/.fontconfig.cache
+fonts =
+ {{ parameter_list['liberation-fonts-location'] }}
+includes =
+ {{ parameter_list['fontconfig-location'] }}/etc/fonts/conf.d
+
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+<= monitor-publish
+remote-debug-url = http://${headless-chromium:remote-debugging-address}
+proxy-url = https://${headless-chromium:proxy-address}
+frontend-url = ${remote-debugging-frontend:connection-secure_access}
+username = ${frontend-instance-password:username}
+password = ${frontend-instance-password:passwd}
+
+# Request a frontend URL from the CDN for the remote debugging interface.
+[remote-debugging-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Headless Chromium Remote Debugging Frontend
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://${headless-chromium:proxy-address}
+config-https-only = true
+config-type = websocket
+config-websocket-path-list = /devtools
+return = domain secure_access
+
+
+# Monitoring: check that the Chromium process is alive and responding to
+# requests through the proxy.
+[monitor-instance-parameter]
+monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
+
+# Promise to make sure the remote debugging frontend returns 200 when
+# queried with the correct credentials.
+[frontend-ok-promise]
+<= monitor-promise-base
+module = check_url_available
+name = headless-chromium-frontend-ok.py
+url = ${remote-debugging-frontend:connection-secure_access}
+config-url = ${:url}
+config-username = ${frontend-instance-password:username}
+config-password = ${frontend-instance-password:passwd}
+
+# Promise to make sure that the remote debugging frontend returns 401
+# when queried with no credentials.
+[frontend-secure-promise]
+<= monitor-promise-base
+module = check_url_available
+name = headless-chromium-frontend-secure.py
+url = ${remote-debugging-frontend:connection-secure_access}
+config-url = ${:url}
+config-http-code = 401
diff --git a/software/headless-chromium/instance.cfg.in b/software/headless-chromium/instance.cfg.in
new file mode 100644
index 0000000000000000000000000000000000000000..51a6c6acdff5c42b6fb0c3181d2d934c31e8bb55
--- /dev/null
+++ b/software/headless-chromium/instance.cfg.in
@@ -0,0 +1,52 @@
+[buildout]
+parts =
+ switch-softwaretype
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+[profile-common]
+openssl-location = {{ openssl_location }}
+nginx-location = {{ nginx_location }}
+liberation-fonts-location = {{ liberation_fonts_location }}
+fontconfig-location = {{ fontconfig_location }}
+chromium-wrapper = {{ chromium_wrapper }}
+devtools-frontend = {{ devtools_frontend }}
+template-nginx-config = {{ template_nginx_config_target }}
+template-fonts-conf = {{ template_fonts_conf_target }}
+template-monitor = {{ template_monitor }}
+template-mime-types = {{ template_mime_types_target }}
+
+[instance-headless-chromium]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_instance_headless_chromium_target }}
+rendered = ${buildout:directory}/${:filename}
+filename = instance-headless-chromium.cfg
+context =
+ section buildout buildout
+ section parameter_list profile-common
+ key partition_ipv4 slap-configuration:ipv4-random
+ key partition_ipv6 slap-configuration:ipv6-random
+ key slapparameter_dict slap-configuration:configuration
+ jsonkey default_parameter_dict :default-parameters
+default-parameters =
+ {
+ "remote-debugging-port": 8081,
+ "nginx-proxy-port": 8082,
+ "target-url": "https://www.example.com",
+ "monitor-httpd-port": 8083
+ }
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = ${:default}
+default = instance-headless-chromium:rendered
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
diff --git a/software/headless-chromium/software.cfg b/software/headless-chromium/software.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..8950608e3a30bc02d329e2e4d99a90f1c84fd7e3
--- /dev/null
+++ b/software/headless-chromium/software.cfg
@@ -0,0 +1,50 @@
+[buildout]
+extends =
+ buildout.hash.cfg
+ ../../stack/slapos.cfg
+ ../../stack/monitor/buildout.cfg
+ ../../component/headless-chromium/buildout.cfg
+ ../../component/openssl/buildout.cfg
+ ../../component/nginx/buildout.cfg
+ ../../component/fonts/buildout.cfg
+ ../../component/fontconfig/buildout.cfg
+
+parts =
+ slapos-cookbook
+ template-cfg
+
+[python]
+part = python3
+
+[template-cfg]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/template.cfg
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+context =
+ section buildout buildout
+ key openssl_location openssl:location
+ key nginx_location nginx:location
+ key liberation_fonts_location liberation-fonts:location
+ key fontconfig_location fontconfig:location
+ key chromium_wrapper headless-chromium-wrapper:rendered
+ key devtools_frontend headless-chromium:devtools-frontend
+ key template_nginx_config_target template-nginx-conf:target
+ key template_mime_types_target template-mime-types:target
+ key template_fonts_conf_target template-fonts-conf:output
+ key template_instance_headless_chromium_target instance-headless-chromium:target
+ key template_monitor monitor2-template:rendered
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 0644
+
+[instance-headless-chromium]
+<= download-base
+
+[template-nginx-conf]
+<= download-base
+
+[template-mime-types]
+<= download-base
diff --git a/software/headless-chromium/software.cfg.json b/software/headless-chromium/software.cfg.json
new file mode 100644
index 0000000000000000000000000000000000000000..0cc8a09a5a203d6addbecd3e2e6cf3f4feda74ee
--- /dev/null
+++ b/software/headless-chromium/software.cfg.json
@@ -0,0 +1,14 @@
+{
+ "name": "Headless Chromium",
+ "description": "Headless (stripped-down) Chromium shell",
+ "serialization": "xml",
+ "software-type": {
+ "default": {
+ "title": "Default",
+ "description": "Standalone headless shell",
+ "request": "instance-headless-chromium-input-schema.json",
+ "response": "instance-headless-chromium-output-schema.json",
+ "index": 0
+ }
+ }
+}
diff --git a/software/headless-chromium/templates/mime-types.in b/software/headless-chromium/templates/mime-types.in
new file mode 100644
index 0000000000000000000000000000000000000000..f07db332b6dce975e457a8f9ba3bbd8ec1c39b49
--- /dev/null
+++ b/software/headless-chromium/templates/mime-types.in
@@ -0,0 +1,78 @@
+types {
+ text/html html htm shtml;
+ text/css css;
+ text/xml xml rss;
+ image/gif gif;
+ image/jpeg jpeg jpg;
+ application/x-javascript js;
+ application/atom+xml atom;
+
+ text/mathml mml;
+ text/plain txt;
+ text/vnd.sun.j2me.app-descriptor jad;
+ text/vnd.wap.wml wml;
+ text/x-component htc;
+
+ image/png png;
+ image/tiff tif tiff;
+ image/vnd.wap.wbmp wbmp;
+ image/x-icon ico;
+ image/x-jng jng;
+ image/x-ms-bmp bmp;
+ image/svg+xml svg svgz;
+
+ application/java-archive jar war ear;
+ application/mac-binhex40 hqx;
+ application/msword doc;
+ application/pdf pdf;
+ application/postscript ps eps ai;
+ application/rtf rtf;
+ application/vnd.ms-excel xls;
+ application/vnd.ms-powerpoint ppt;
+ application/vnd.wap.wmlc wmlc;
+ application/vnd.google-earth.kml+xml kml;
+ application/vnd.google-earth.kmz kmz;
+ application/x-7z-compressed 7z;
+ application/x-cocoa cco;
+ application/x-java-archive-diff jardiff;
+ application/x-java-jnlp-file jnlp;
+ application/x-makeself run;
+ application/x-perl pl pm;
+ application/x-pilot prc pdb;
+ application/x-rar-compressed rar;
+ application/x-redhat-package-manager rpm;
+ application/x-sea sea;
+ application/x-shockwave-flash swf;
+ application/x-stuffit sit;
+ application/x-tcl tcl tk;
+ application/x-x509-ca-cert der pem crt;
+ application/x-xpinstall xpi;
+ application/xhtml+xml xhtml;
+ application/zip zip;
+
+ application/octet-stream bin exe dll;
+ application/octet-stream deb;
+ application/octet-stream dmg;
+ application/octet-stream eot;
+ application/octet-stream iso img;
+ application/octet-stream msi msp msm;
+ application/ogg ogx;
+
+ audio/midi mid midi kar;
+ audio/mpeg mpga mpega mp2 mp3 m4a;
+ audio/ogg oga ogg spx;
+ audio/x-realaudio ra;
+ audio/webm weba;
+
+ video/3gpp 3gpp 3gp;
+ video/mp4 mp4;
+ video/mpeg mpeg mpg mpe;
+ video/ogg ogv;
+ video/quicktime mov;
+ video/webm webm;
+ video/x-flv flv;
+ video/x-mng mng;
+ video/x-ms-asf asx asf;
+ video/x-ms-wmv wmv;
+ video/x-msvideo avi;
+}
diff --git a/software/headless-chromium/templates/nginx.conf.in b/software/headless-chromium/templates/nginx.conf.in
new file mode 100644
index 0000000000000000000000000000000000000000..0b812f16ce0d8127af404a8fe24847ef0661e2b2
--- /dev/null
+++ b/software/headless-chromium/templates/nginx.conf.in
@@ -0,0 +1,81 @@
+pid {{ param_headless_chromium['nginx-pid-path'] }};
+error_log {{ param_headless_chromium['nginx-error-log'] }};
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ access_log {{ param_headless_chromium['nginx-access-log'] }};
+
+ include {{ param_headless_chromium['nginx-mime-types'] }};
+ default_type application/octet-stream;
+
+ types {
+ text/html html;
+ text/css css;
+ application/javascript js;
+ }
+
+ server {
+ listen {{ param_headless_chromium['proxy-address'] }} ssl;
+
+ # Require username/password to access remote debugging port.
+ auth_basic "Remote Debugging";
+ auth_basic_user_file {{ param_headless_chromium['nginx-htpasswd-file'] }};
+
+ # Use self-signed SSL certificate.
+ ssl_certificate {{ param_headless_chromium['nginx-cert-file'] }};
+ ssl_certificate_key {{ param_headless_chromium['nginx-key-file'] }};
+
+ client_body_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+ proxy_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+ fastcgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+ uwsgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+ scgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+
+ # All websocket connections are served from /devtools.
+ location /devtools {
+ proxy_http_version 1.1;
+ proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
+ proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
+ proxy_set_header Upgrade "websocket";
+ proxy_set_header Connection "Upgrade";
+ }
+
+ # The DevTools frontend is served from /serve_file/@{version_hash}.
+ location ~ "^\/serve_file\/@[0-9a-f]{5,40}\/(.*)" {
+ alias {{ param_headless_chromium['devtools-frontend-root'] }}/$1;
+ }
+
+ location / {
+ proxy_http_version 1.1;
+
+ # The proxy must set the Host header to an IP address, since the
+ # headless Chromium shell refuses to run otherwise, for security
+ # reasons.
+ # See https://bugs.chromium.org/p/chromium/issues/detail?id=813540.
+ proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
+ proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
+
+ # The browser security policy will prevent us from loading the
+ # Websocket connection without TLS, so we have to go through the
+ # frontend CDN URL. The tricky thing is that the frontend URL is
+ # not available yet when this file is built; what we do instead is
+ # use the given Host header.
+ sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$host";
+ sub_filter_once on;
+ sub_filter_types application/json;
+
+ sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$host";
+ sub_filter_types application/json;
+
+ # We want to use our own DevTools frontend rather than
+ # https://chrome-devtools-frontend.appspot.com. There should be a
+ # --custom-devtools-frontend flag for Chromium, but it doesn't
+ # seem to work with the remote debugging port.
+ sub_filter "chrome-devtools-frontend.appspot.com" "$host";
+ sub_filter_types *;
+ }
+ }
+}
diff --git a/software/headless-chromium/test/README.md b/software/headless-chromium/test/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..81a3023d3310b023af91489c8f40c1086bbf7a56
--- /dev/null
+++ b/software/headless-chromium/test/README.md
@@ -0,0 +1 @@
+Tests for headless Chromium software release
diff --git a/software/headless-chromium/test/setup.py b/software/headless-chromium/test/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..49cef6de2bf96cd4a31683db6dc540db32389c6c
--- /dev/null
+++ b/software/headless-chromium/test/setup.py
@@ -0,0 +1,50 @@
+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.headless-chromium'
+long_description = open("README.md").read()
+
+setup(
+ name=name,
+ version=version,
+ description="Test for SlapOS headless Chromium",
+ long_description=long_description,
+ long_description_content_type='text/markdown',
+ maintainer="Nexedi",
+ maintainer_email="info@nexedi.com",
+ url="https://lab.nexedi.com/nexedi/slapos",
+ packages=find_packages(),
+ install_requires=[
+ 'slapos.core',
+ 'slapos.libnetworkcache',
+ 'requests',
+ ],
+ zip_safe=True,
+ test_suite='test',
+)
diff --git a/software/headless-chromium/test/test.py b/software/headless-chromium/test/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..db7be48b4bbc08d39b135576fc3b8e340c3a89b2
--- /dev/null
+++ b/software/headless-chromium/test/test.py
@@ -0,0 +1,70 @@
+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+ os.path.abspath(
+ os.path.join(os.path.dirname(__file__), '../software.cfg')))
+
+class TestHeadlessChromium(SlapOSInstanceTestCase):
+ def setUp(self):
+ self.connection_parameters = self.requestDefaultInstance().getConnectionParameterDict()
+
+ def test_remote_debugging_port(self):
+ # The headless browser should respond at /json with a nonempty list
+ # of available pages, each of which has a webSocketDebuggerUrl and a
+ # devtoolsFrontendUrl.
+ url = self.connection_parameters['remote-debug-url']
+ response = requests.get('%s/json' % url)
+
+ # Check that request was successful and the response was a nonempty
+ # list.
+ self.assertEqual(requests.codes['ok'], response.status_code)
+ self.assertTrue(len(response.json()) > 0)
+
+ # Check that the first page has the correct fields.
+ first_page = response.json()[0]
+ self.assertIn('webSocketDebuggerUrl', first_page)
+ self.assertIn('devtoolsFrontendUrl', first_page)
+
+ def test_devtools_frontend_ok(self):
+ # The proxy should serve the DevTools frontend from
+ # /serve_file/@{hash}/inspector.html, where {hash} is a 5-32 digit
+ # hash.
+ proxyURL = self.connection_parameters['proxy-url']
+ username = self.connection_parameters['username']
+ password = self.connection_parameters['password']
+ frontend = '/serve_file/@aaaaa/inspector.html'
+
+ response = requests.get(proxyURL + frontend, verify=False,
+ auth=(username, password))
+ self.assertEqual(requests.codes['ok'], response.status_code)
diff --git a/software/slapos-sr-testing/software-py3.cfg b/software/slapos-sr-testing/software-py3.cfg
index 4c1f721c80c7529870ed69914b781150c937da15..4de703b57d2228aaa1d37e72a13df22d10b75694 100644
--- a/software/slapos-sr-testing/software-py3.cfg
+++ b/software/slapos-sr-testing/software-py3.cfg
@@ -22,3 +22,4 @@ extra =
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
+ headless-chromium ${slapos.test.headless-chromium-setup:setup}
diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg
index ff8c9ae38a80d2cc1b0d645066de484ed7fbeb01..403efee1386694ef35c7e08281e5767f55320349 100644
--- a/software/slapos-sr-testing/software.cfg
+++ b/software/slapos-sr-testing/software.cfg
@@ -194,6 +194,11 @@ setup = ${slapos-repository:location}/software/jscrawler/test/
egg = slapos.test.galene
setup = ${slapos-repository:location}/software/galene/test/
+[slapos.test.headless-chromium-setup]
+<= setup-develop-egg
+egg = slapos.test.headless-chromium
+setup = ${slapos-repository:location}/software/headless-chromium/test/
+
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -254,6 +259,7 @@ extra-eggs =
${slapos.test.html5as-setup:egg}
${slapos.test.html5as-base-setup:egg}
${slapos.test.fluentd-setup:egg}
+ ${slapos.test.headless-chromium-setup:egg}
# We don't name this interpreter `python`, so that when we run slapos node
# software, installation scripts running `python` use a python without any