component/headless-chromium: rewrite buildout.cfg

component/headless-chromium/buildout.cfg

 [buildout]
 extends =
-  ../binutils/buildout.cfg
-  ../bison/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../gperf/buildout.cfg
-  ../ninja/buildout.cfg
-  ../freetype/buildout.cfg
-  ../cmake/buildout.cfg
-  ../git/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../cups/buildout.cfg
+# Build dependencies: note that bash, bzip2, getconf, grep, tar, and xz are already in /bin and /usr/bin
+# TODO: I think findutils is only here to expose xargs, which is already in /usr/bin, so see if we can get away with not requiring it
+# If the Python version doesn't default to 2, it might be necessary to include Python 2 as a component and then prepend it to the PATH or some similar trick.
   ../coreutils/buildout.cfg
+  ../curl/buildout.cfg
   ../depot_tools/buildout.cfg
-  ../findutils/buildout.cfg
+  ../git/buildout.cfg
+  ../gperf/buildout.cfg
+  ../pkgconfig/buildout.cfg
+# Runtime dependencies:
+  ../fonts/buildout.cfg
   ../fontconfig/buildout.cfg
-  ../gettext/buildout.cfg
-  ../glib/buildout.cfg
-  ../gtk-2/buildout.cfg
-  ../libexpat/buildout.cfg
-  ../libffi/buildout.cfg
-  ../libpng/buildout.cfg
-  ../libxml2/buildout.cfg
-  ../mesa/buildout.cfg
   ../nspr/buildout.cfg
   ../nss/buildout.cfg
-  ../pcre/buildout.cfg
-  ../sqlite3/buildout.cfg
-  ../xorg/buildout.cfg
-  ../zlib/buildout.cfg
 
 parts =
-  chromium
+  chromium-wrapper
 
-[gconf]
-recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz
-md5sum = 2b16996d0e4b112856ee5c59130e822c
-configure-options = --disable-orbit --disable-static
-environment =
-  PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH
+# TODO: it may be more appropriate to manage our own copy of depot_tools, because I think some of the "gclient sync" commands may modify depot_tools, which could be problematic if other components or software releases depend on a shared copy, and at any rate it causes some build errors: when updating depot_tools, git complains because HEAD has been moved. Is there a way to guarantee a private copy? (This may be a moot point, since I don't think it's used by anything else, anyway, and trying again fixes the git issue.)
 
-[chromedriver]
-recipe = hexagonit.recipe.download
-url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
-md5sum = a72088c0a6b018ded2c0fff616da8f65
+# TODO: so there's a funny little issue: because we didn't clone the full repository history, some automated tools can't figure out the version number (well, actually the appropriate commit hash). This causes the frontend for the remote debugging port to crash, because it was expecting a version number. I see a few possible fixes:
+# 1. Clone the full repository. But this blows up the size, from ~25 GB to ~100 GB.
+# 2. Patch the C++ version API (see src/components/version_info) before compiling, to hard-code the number in.
+# 3. Patch the frontend JS (not sure where it is) before compiling.
+# 4. Proxy the connection and patch the server's response on the fly.
+# The correct commit hash for this particular version should be 3aad95ffffff0767e9b7d161885aa811de1f2195, which you can verify at https://github.com/chromium/chromium/commit/3aad95ffffff0767e9b7d161885aa811de1f2195 and find by searching on https://chromium-review.googlesource.com/c/chromium/src/.
 
-[chromium-download]
+# There's nothing special about version 91.0.4472.132. It just happened to be
+# the current Chromium stable version at the time of writing.
+# TODO: set source URL as a parameter to avoid duplication?
+[chromium-source]
 recipe = plone.recipe.command
-# This revision is 56.0.2924.122. Because the chromedriver only support some certain
-# version. Which version 56 is in the middle of 55-57.
-revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
 path = ${buildout:parts-directory}/${:_buildout_section_name_}
+version = 91.0.4472.132
 command =
-  set -e
-  PATH=${depot_tools:location}:${git:location}/bin:$PATH
-  [ -d ${:path} ] && rm -r ${:path}
+  PATH=$PATH:${git:location}/bin
+  rm -rf ${:path}
   mkdir -p ${:path}
   cd ${:path}
-  # Do never use `fetch` with the `--no-history` option unless you find an
-  # option to fetch directly at the wanted revision. `--no-history` could
-  # reduce the download size significantly but even if it retrieves enough
-  # commits at the time you test this section, development continues upstream
-  # and at some point the next command (gclient) would break.
-  # ...
-  # This command only could work in an empty dir.
-  fetch --nohooks chromium
-  gclient sync --revision ${:revision} --with_branch_heads
-stop-on-error = true
+  git clone \
+    --branch ${:version} \
+    --depth 1 \
+    https://chromium.googlesource.com/chromium/src.git
 
 [chromium]
 recipe = slapos.recipe.cmmi
-path = ${chromium-download:path}/src
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+path = ${chromium-source:path}/src
+location = ${:path}/out/headless
+
+# Configuration file for GN, the tool to build the actual compilation configuration file.
+default-gclient-config =
+  solutions = [
+    {
+      "name": "src",
+      "url": "https://chromium.googlesource.com/chromium/src.git",
+      "managed": False,
+      "custom_deps": {},
+      "custom_vars": {},
+    },
+  ]
+
+# Build configuration options
+build-config-options =
+  import("//build/args/headless.gn")
+  is_debug = false
+  symbol_level = 0
+  blink_symbol_level = 0
+
+# Libraries needed at runtime. When using the compiled headless shell, you should specify LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${libraries}.
+libraries = ${nspr:location}/lib:${nss:location}/lib
+
+# Create a headless build configuration.
 configure-command =
+# Sync build dependencies---this is a little finnicky.
+# NOTE: I'm not sure if a sync at the end is needed?
+  echo '${:default-gclient-config}' > ${chromium-source:path}/.gclient
+  gclient sync --revision ${chromium-source:version}
   gclient runhooks
+# Generate build configuration files.
   mkdir -p ${:location}
-  echo 'use_udev = true
-  is_debug = false
-  enable_nacl = false' > ${:location}/args.gn
+  echo '${:build-config-options}' > ${:location}/args.gn
   gn gen ${:location}
 
-# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu
-make-binary = ninja -C ${:location} chrome
+# You can run the headless Chromium shell using
+# ${:location}/headless-chromium --remote-debugging-port=1234
+make-binary =
+# This is most certainly a massive hack, but retrying possibly works...
+  autoninja -C ${:location} headless_shell || autoninja -C ${:location} headless_shell
 environment =
-  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH
-  PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s
-  CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH
-  LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH
+# Possible foot-gun: if you specify PATH=...:$PATH, it will literally insert the string '$PATH' on your PATH; buildout won't do the expansion unless you use %(PATH)s
+# TODO: make sure it's appropriate for SlapOS components to depend on things in /bin, /usr/bin, etc.
+  PATH=${depot_tools:location}:${gperf:location}/bin:${pkgconfig:location}/bin:${coreutils:location}/bin:${git:location}/bin:${curl:location}/bin:%(PATH)s
+
+promises =
+  ${:location}/headless_shell
+
+# A default font configuration to fall back on, same as the Firefox one.
+[default-font-config]
+recipe = slapos.recipe.template:jinja2
+template = ${template-fonts-conf:output}
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/fonts.conf
+context =
+  key cachedir :cache-dir
+  key fonts :fonts
+  key includes :includes
+fonts =
+  ${liberation-fonts:location}
+includes =
+  ${fontconfig:location}/etc/fonts/conf.d
+cache-dir =
+  ~/.fontconfig-chromium/
+
+# Create a wrapper around the headless Chromium shell. The wrapper sets some default values for environment variables and is based on the Firefox one.
+[chromium-wrapper]
+wrapper-name = headless-chromium
+part = ${chromium:location}
+
+recipe = slapos.recipe.build
+location = ${buildout:bin-directory}/${:wrapper-name}
+
+install =
+  import os
+  chromium = options['part']
+  part = self.buildout[os.path.basename(chromium)]
+  with open(location, 'w') as f:
+    f.write("""#!/bin/sh -e
+  cd {}
+  export LD_LIBRARY_PATH={}
+  export PATH={}:$PATH
+  [ -z $FONTCONFIG_FILE ] && export FONTCONFIG_FILE=${default-font-config:rendered}
+  exec ${chromium:location}/headless_shell "$@"
+  """.format(
+    chromium,
+    ':'.join(part['library'].split()),
+    ':'.join(part['path'].split()),
+  ))
+    os.fchmod(f.fileno(), 0o755)
+
+# Runtime libraries
+libraries =
+  ${nspr:location}/lib
+  ${nss:location}/lib
+
+# Runtime path
+# TODO: see if this part is really necessary
+path =
+  ${fontconfig:location}/bin
+