UBUNTU: [Packaging] dkms -- dkms package build packaging support

BugLink: https://bugs.launchpad.net/bugs/1861284

Add new dkms-build scripting which prepares a kernel headers tree and
then builds specified DKMS packages against those headers.  The
resulting .kos are then incorporated into the specified package,
including signing them into this kernels module signing key.  This
allows them to be loaded in a secure-boot environment.

Squashes the following commits from bionic:

  UBUNTU: update dkms package versions
  UBUNTU: [Config] wireguard -- enable for all architectures
  UBUNTU: [Packaging]: ignore wireguard modules when wireguard is disabled
  UBUNTU: [Packaging] wireguard -- add support for building signed .ko
  dkms-build: apt-cache policy elides username:password information
  UBUNTU: [Packaging] file-downloader not handling positive failures correctly
  UBUNTU: temporarily drop Built-Using data
  UBUNTU: [packaging] handle downloads from the librarian better
  UBUNTU: [Packaging] autoreconstruct -- manage executable debian files
  UBUNTU: [Packaging] Fix config file assembly
  UBUNTU: [Packaging] dkms -- dkms-build quieten wget verbiage
  UBUNTU: [Packaging] dkms -- try launchpad librarian for pool downloads
  UBUNTU: [Packaging] dkms-build -- backport latest version from disco
  UBUNTU: [Packaging] dkms-build -- add support for unversioned overrides
  UBUNTU: [Packaging] dkms-build: do not redownload files on subsequent passes
  UBUNTU: [Packaging] dkms-build -- support building against packages in PPAs
  UBUNTU: [Packaging] dkms -- switch to a consistent build prefix length and strip
  UBUNTU: [Packaging] dkms -- add per package post-process step
  UBUNTU: [Packaging] dkms -- dkms package build packaging support
Signed-off-by: Andy Whitcroft <apw@canonical.com>
Acked-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
Acked-by: Stefan Bader <stefan.bader@canonical.com
Signed-off-by: Ian May <ian.may@canonical.com>
Signed-off-by: Kelsey Skunberg <kelsey.skunberg@canonical.com>

debian.master/control.stub.in

@@ -36,6 +36,8 @@ Build-Depends:
  automake <!stage1>,
  libtool <!stage1>,
  uuid-dev <!stage1>,
+ dkms <!stage1>,
+ curl <!stage1>,
 Build-Depends-Indep:
  xmlto <!stage1>,
  docbook-utils <!stage1>,

debian/rules

@@ -138,6 +138,7 @@ clean: debian/control
 	# Remove generated intermediate files
 	rm -f $(DROOT)/control.stub $(DEBIAN)/control.stub \
 		$(DEBIAN)/d-i/kernel-versions
+	rm -f $(DROOT)/scripts/fix-filenames
 
 distclean: clean
 	rm -rf $(DROOT)/control debian/changelog \

debian/rules.d/2-binary-arch.mk

@@ -17,17 +17,20 @@ endif
 
 shlibdeps_opts = $(if $(CROSS_COMPILE),-- -l$(CROSS_COMPILE:%-=/usr/%)/lib)
 
+debian/scripts/fix-filenames: debian/scripts/fix-filenames.c
+	$(CC) -o $@ $^
+
 $(stampdir)/stamp-prepare-%: config-prepare-check-%
 	@echo Debug: $@
 	@touch $@
 $(stampdir)/stamp-prepare-tree-%: target_flavour = $*
-$(stampdir)/stamp-prepare-tree-%: $(commonconfdir)/config.common.$(family) $(archconfdir)/config.common.$(arch) $(archconfdir)/config.flavour.%
+$(stampdir)/stamp-prepare-tree-%: $(commonconfdir)/config.common.$(family) $(archconfdir)/config.common.$(arch) $(archconfdir)/config.flavour.% debian/scripts/fix-filenames
 	@echo Debug: $@
 	install -d $(builddir)/build-$*
 	touch $(builddir)/build-$*/ubuntu-build
 	[ "$(do_full_source)" != 'true' ] && true || \
 		rsync -a --exclude debian --exclude debian.master --exclude $(DEBIAN) * $(builddir)/build-$*
-	cat $^ | sed -e 's/.*CONFIG_VERSION_SIGNATURE.*/CONFIG_VERSION_SIGNATURE="Ubuntu $(release)-$(revision)-$* $(raw_kernelversion)"/' > $(builddir)/build-$*/.config
+	cat $(wordlist 1,3,$^) | sed -e 's/.*CONFIG_VERSION_SIGNATURE.*/CONFIG_VERSION_SIGNATURE="Ubuntu $(release)-$(revision)-$* $(raw_kernelversion)"/' > $(builddir)/build-$*/.config
 	find $(builddir)/build-$* -name "*.ko" | xargs rm -f
 	$(build_cd) $(kmake) $(build_O) -j1 silentoldconfig prepare scripts
 	touch $@
@@ -81,6 +84,18 @@ define install_zfs =
 		$(kmake) -C $(builddir)/build-$* SUBDIRS=`pwd` modules_install $(zfsopts)
 endef
 
+define build_dkms_sign =
+	$(shell set -x; if grep -q CONFIG_MODULE_SIG=y $(1)/.config; then
+			echo $(1)/scripts/sign-file $(MODHASHALGO) $(MODSECKEY) $(MODPUBKEY);
+		else
+			echo "-";
+		fi
+	)
+endef
+define build_dkms =
+	$(SHELL) $(DROOT)/scripts/dkms-build $(dkms_dir) $(abi_release)-$* '$(call build_dkms_sign,$(builddir)/build-$*)' $(1) $(2) $(3) $(4)
+endef
+
 define install_control =
 	for which in $(3);							\
 	do									\
@@ -93,6 +108,17 @@ define install_control =
 	done
 endef
 
+# Ensure the directory prefix is exactly 100 characters long so pathnames are the
+# exact same length in any binary files produced by the builds.  These will be
+# commonised later.
+dkms_20d=....................
+dkms_100d=$(dkms_20d)$(dkms_20d)$(dkms_20d)$(dkms_20d)$(dkms_20d)
+dkms_100c=$(shell echo '$(dkms_100d)' | sed -e 's/\./_/g')
+define dkms_dir_prefix =
+$(shell echo $(1)/$(dkms_100c) | \
+	sed -e 's/\($(dkms_100d)\).*/\1/' -e 's/^\(.*\)....$$/\1dkms/')
+endef
+
 # Install the finished build
 install-%: pkgdir_bin = $(CURDIR)/debian/$(bin_pkg_name)-$*
 install-%: pkgdir = $(CURDIR)/debian/$(mods_pkg_name)-$*
@@ -113,13 +139,14 @@ install-%: MODHASHALGO=sha512
 install-%: MODSECKEY=$(builddir)/build-$*/certs/signing_key.pem
 install-%: MODPUBKEY=$(builddir)/build-$*/certs/signing_key.x509
 install-%: build_dir=$(builddir)/build-$*
+install-%: dkms_dir=$(call dkms_dir_prefix,$(builddir)/build-$*)
 install-%: enable_zfs = $(call custom_override,do_zfs,$*)
 install-%: splopts  = INSTALL_MOD_STRIP=1
 install-%: splopts += INSTALL_MOD_PATH=$(pkgdir)/
 install-%: splopts += INSTALL_MOD_DIR=kernel/zfs
 install-%: splopts += $(conc_level)
 install-%: zfsopts  = $(splopts)
-install-%: $(stampdir)/stamp-build-%
+install-%: $(stampdir)/stamp-build-% install-headers
 	@echo Debug: $@ kernel_file $(kernel_file) kernfile $(kernfile) install_file $(install_file) instfile $(instfile)
 	dh_testdir
 	dh_testroot
@@ -397,6 +424,10 @@ ifeq ($(do_tools_hyperv),true)
 endif
 endif
 
+	# Build a temporary "installed headers" directory.
+	install -d $(dkms_dir) $(dkms_dir)/headers $(dkms_dir)/build $(dkms_dir)/source
+	cp -rp "$(hdrdir)" "$(indep_hdrdir)" "$(dkms_dir)/headers"
+
 	# Build the final ABI information.
 	install -d $(abidir)
 	sed -e 's/^\(.\+\)[[:space:]]\+\(.\+\)[[:space:]]\(.\+\)$$/\3 \2 \1/'	\

debian/scripts/dkms-build

+#!/bin/bash
+set -e
+
+dkms_dir="$1"
+abi_flavour="$2"
+sign="$3"
+pkgname="$4"
+pkgdir="$5"
+package="$6"
+shift 6
+
+here=$(dirname "$(readlink -f "${0}")")
+
+srcdir=$(pwd)
+cd "$dkms_dir" || exit 1
+
+built_using_record()
+{
+	local subst="$1"
+	local built_using="$2"
+	if [ ! -f "$subst" ]; then
+		touch "$subst"
+	fi
+	if ! grep -q -s "^linux:BuiltUsing=" "$subst"; then
+		echo "linux:BuiltUsing=" >>"$subst"
+	fi
+	sed -i -e "s/^\(linux:BuiltUsing=.*\)/\1$built_using, /" "$subst"
+}
+
+# ABI: returns present in $? and located path in lpackage_path when found.
+package_present()
+{
+	for lpackage_path in "$1"_*.deb
+	do
+		break
+	done
+	[ -f "$lpackage_path" ]
+}
+
+# Download and extract the DKMS package -- note there may be more
+# than one package to install.
+for package_path in "$@"
+do
+	package_file=$(basename "$package_path")
+	echo "II: dkms-build downloading $package ($package_file)"
+	rpackage=$( echo "$package_path" | sed -e 's@.*/@@' -e 's@_.*@@' )
+	lpackage=$( echo "$rpackage" | sed -e 's@=.*@@' )
+
+	while true
+	do
+		if package_present "$lpackage"; then
+			break
+		fi
+		case "$package_path" in
+		pool/*)
+			# Attempt download from the launchpad librarian first.
+			"$here/file-downloader" "https://launchpad.net/ubuntu/+archive/primary/+files/$package_file" || true
+			if package_present "$lpackage"; then
+				break
+			fi
+
+			# Download from the available pools.
+			for pool in $( grep -h '^deb ' /etc/apt/sources.list /etc/apt/sources.list.d/*.list | awk '{print $2}' | sort -u )
+			do
+				if package_present "$lpackage"; then
+					break
+				fi
+				url="$pool/$package_path"
+				"$here/file-downloader" "$url" && break || true
+				# No components in PPAs.
+				url=$(echo "$url" | sed -e 's@/pool/[^/]*/@/pool/main/@')
+				"$here/file-downloader" "$url" && break || true
+			done
+			;;
+		http*:*)
+			"$here/file-downloader" "$package_path"
+			;;
+		*/*)
+			cp -p "$package_path" .
+			;;
+		*)
+			apt-get download "$rpackage"
+			;;
+		esac
+		break
+	done
+	if ! package_present "$lpackage"; then
+		echo "EE: $lpackage not found"
+		exit 1
+	fi
+
+	dpkg -x "$lpackage"_*.deb "$package"
+
+	lversion=$( echo "$lpackage_path" | sed -e 's@.*/@@' -e 's@_[^_]*$@@' -e 's@.*_@@')
+	#built_using_record "$srcdir/debian/$pkgname.substvars" "$built_using$lpackage (= $lversion)"
+done
+
+# Pick out the package/version from the dkms.conf.
+for dkms_conf in "$package/usr/src"/*/"dkms.conf"
+do
+	break
+done
+# It seems some packages have a # in the name which works fine if the
+# package is installed directly, but not so much if we build it out
+# of the normal location.
+sed -i -e '/^PACKAGE_NAME=/ s/#//g' "$dkms_conf"
+cat - <<'EOF' >>"$dkms_conf"
+POST_BUILD="ubuntu-save-objects ${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build ${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/objects $POST_BUILD"
+EOF
+ubuntu_script="$(dirname "$dkms_conf")/ubuntu-save-objects"
+cat - <<'EOF' >"$ubuntu_script"
+#!/bin/sh
+from="$1"
+to="$2"
+script="$3"
+shift 2
+
+# Copy the objects.
+echo "II: copying objects to '$to'"
+mkdir -p "$to"
+(cd "$from" && find -name \*.o -o -name \*.o.ur-\* | cpio -Lpd "$to")
+
+# Call the original post_install script if there is one.
+[ "$script" = '' ] && exit 0
+
+shift
+exec "$(dirname "$0")/$script" "$@"
+EOF
+chmod +x "$ubuntu_script"
+dkms_package=$( sed -ne 's/PACKAGE_NAME="\(.*\)"/\1/p' "$dkms_conf" )
+dkms_version=$( sed -ne 's/PACKAGE_VERSION="\(.*\)"/\1/p' "$dkms_conf" )
+
+# Build the DKMS binaries.
+echo "II: dkms-build building $package"
+rc=0
+/usr/sbin/dkms build --no-prepare-kernel --no-clean-kernel \
+	-k "$abi_flavour" \
+	--sourcetree "$dkms_dir/source" \
+	--dkmstree "$dkms_dir/build" \
+	--kernelsourcedir "$dkms_dir/headers/linux-headers-$abi_flavour" \
+	"$dkms_conf" || rc=1
+
+# Find the log and add it to our own.
+for log in "$dkms_dir/build/$dkms_package/$dkms_version/$abi_flavour"/*/"log/make.log"
+do
+	break
+done
+sed -e "s@$dkms_dir@<<DKMSDIR>>@g" <"$log"
+
+# If this build failed then exit here.
+[ "$rc" != 0 ] && exit "$rc"
+
+# Install and optionally sign the modules we have built.
+pkgdir="$pkgdir/$package"
+echo "II: dkms-build installing $package into $pkgdir"
+install -d "$pkgdir"
+find "$dkms_dir/build/$dkms_package/$dkms_version/$abi_version" -name \*.ko |
+while read module; do
+	vmodule=$( basename "$module" )
+	case "$sign" in
+	--*)
+		echo "copying $vmodule"
+		cp "$module" "$pkgdir"
+		;;
+	*)
+		echo "signing $vmodule"
+		$sign "$module" "$pkgdir/$vmodule"
+		;;
+	esac
+done
+
+find "$dkms_dir/build/$dkms_package/$dkms_version/objects" -name \*.o -print | \
+while read object
+do
+	"$srcdir/debian/scripts/fix-filenames" "$object" "$dkms_dir"
+done
+
+# Finally see if there is a dkms-package specific post processor present.  Hand
+# it the original source directory, destination package directory, the objects
+# as squirreled away, and the log in case it is useful.  Finally pass a formed
+# signing command line in case we need to do that.
+dkms_build_specific="$srcdir/$0--$package"
+dkms_build_generic=$(echo "$dkms_build_specific" | sed -e 's/-[0-9][0-9]*$/-N/')
+for dkms_build in "$dkms_build_specific" "$dkms_build_generic"
+do
+	if [ ! -e "$dkms_build" ]; then
+		continue
+	fi
+	echo "II: dkms-build override $(basename "$dkms_build") found, executing"
+	$SHELL "$dkms_build" \
+		"$srcdir" \
+		"$dkms_dir/build/$dkms_package/$dkms_version/objects" \
+		"$log" \
+		"$dkms_dir" \
+		"$abi_flavour" \
+		"$sign" \
+		"$pkgname" \
+		"$pkgdir" \
+		"$package" \
+		"$@" || exit 1
+	break
+done
+
+echo "II: dkms-build build $package complete"

debian/scripts/file-downloader

+#!/bin/sh
+
+if [ "$#" -ne 1 ]; then
+	echo "Usage: $0 <url>" 1>&2
+	exit 1
+fi
+url="$1"
+
+to=$(basename "$url")
+
+count=0
+what='fetching'
+while :
+do
+	if [ "$count" -eq 20 ]; then
+		echo "EE: excessive redirects" 1>&2
+		exit 1
+	fi
+	count=$(($count+1))
+
+	echo "II: $what $url"
+
+	curl --silent --fail --show-error "$url" -o "$to" -D "$to.hdr" || exit 1
+	redirect=$(awk '/^Location: / {gsub(/^[[:space:]]+|[[:space:]]+$/,"",$2); print $2;}' "$to.hdr")
+	[ -z "$redirect" ] && break
+	what='  following'
+
+	url=$(echo "$redirect" | sed -e 's@https://launchpadlibrarian.net/@http://launchpadlibrarian.net/@')
+	if [ "$redirect" != "$url" ]; then
+		echo "II:   fixing $redirect"
+	fi
+done
+
+exit 0

debian/scripts/fix-filenames.c

+/*
+ * fix-filenames: find a specified pathname prefix and remove it from
+ *                C strings.
+ *
+ * Copyright (C) 2018 Canonical Ltd.
+ * Author: Andy Whitcroft <apw@canonical.com>
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+int
+main(int argc, char *argv[])
+{
+	int rc;
+	char *in_name;
+	char *prefix;
+	int prefix_len;
+	int in_fd;
+	struct stat in_info;
+	char *in;
+	off_t size;
+	int length;
+
+	if (argc != 3) {
+		fprintf(stderr, "Usage: %s <file> <prefix>\n", argv[0]);
+		exit(1);
+	}
+	in_name    = argv[1];
+	prefix     = argv[2];
+	prefix_len = strlen(prefix);
+
+	in_fd = open(in_name, O_RDWR);
+	if (in_fd < 0) {
+		perror("open input failed");
+		exit(1);
+	}
+
+	rc = fstat(in_fd, &in_info);
+	if (rc < 0) {
+		perror("fstat input failed");
+		exit(1);
+	}
+	size = in_info.st_size;
+	printf("%s %ld bytes\n", in_name + prefix_len + 1, size);
+
+	in = mmap((void *)0, size, PROT_READ|PROT_WRITE, MAP_SHARED, in_fd, (off_t)0);
+	if (!in) {
+		perror("mmap failed");
+		exit(1);
+	}
+
+	for (; size > 0; size--, in++) {
+		if (*in != *prefix)
+			continue;
+		if (strncmp(in, prefix, prefix_len) != 0)
+			continue;
+		length = strlen(in + prefix_len + 1) + 1;
+
+		/*
+		 * Copy the suffix portion down to the start and clear
+		 * the remainder of the space to 0.
+		 */
+		memmove(in, in + prefix_len + 1, length);
+		memset(in + length, '_', prefix_len);
+	}
+}