Deb: Add a customized salsa-ci.yml for easy extra testing

As initially most tests fail, they have allow_failures defined so that
testing anyway proceeds all the way to the final 'upgrade extras' stage.

All of these tests work for downstream Debian packaging of MariaDB 10.4
and should eventually pass on upstream MariaDB 10.5 as well.

Also upstream the Debian autopkgtests from MariaDB 10.4 in Debian so that
pipeline includes running mtr.

debian/autobake-deb.sh

@@ -12,12 +12,17 @@ set -e
 # building the deb packages here.
 export DEB_BUILD_OPTIONS="nocheck $DEB_BUILD_OPTIONS"
 
-# Travis-CI optimizations
-if [[ $TRAVIS ]]
+# General CI optimizations to keep build output smaller
+if [[ $TRAVIS ]] || [[ $GITLAB_CI ]]
 then
-  # On Travis-CI, the log must stay under 4MB so make the build less verbose
+  # On both Travis and Gitlab the output log must stay under 4MB so make the
+  # build less verbose
   sed -i -e '/Add support for verbose builds/,/^$/d' debian/rules
+fi
 
+# Travis-CI optimizations to keep build small (in both duration and disk space)
+if [[ $TRAVIS ]]
+then
   # Don't include test suite package on Travis-CI to make the build time shorter
   sed '/Package: mariadb-test-data/,/^$/d' -i debian/control
   sed '/Package: mariadb-test$/,/^$/d' -i debian/control
@@ -112,12 +117,12 @@ LOGSTRING="MariaDB build"
 CODENAME="$(lsb_release -sc)"
 EPOCH="1:"
 
-dch -b -D ${CODENAME} -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automatic build with ${LOGSTRING}."
+dch -b -D "${CODENAME}" -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automatic build with ${LOGSTRING}."
 
 echo "Creating package version ${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... "
 
-# On Travis CI, use -b to build binary only packages as there is no need to
-# waste time on generating the source package.
+# On Travis CI and Gitlab-CI, use -b to build binary only packages as there is
+# no need to waste time on generating the source package.
 if [[ $TRAVIS ]]
 then
   BUILDPACKAGE_FLAGS="-b"
@@ -131,15 +136,15 @@ fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS
 # If the step above fails due to missing dependencies, you can manually run
 #   sudo mk-build-deps debian/control -r -i
 
-# Don't log package contents on Travis-CI to save time and log size
-if [[ ! $TRAVIS ]]
+# Don't log package contents on Travis-CI or Gitlab-CI to save time and log size
+if [[ ! $TRAVIS ]] && [[ ! $GITLAB_CI ]]
 then
   echo "List package contents ..."
   cd ..
-  for package in `ls *.deb`
+  for package in *.deb
   do
-    echo $package | cut -d '_' -f 1
-    dpkg-deb -c $package | awk '{print $1 " " $2 " " $6 " " $7 " " $8}' | sort -k 3
+    echo "$package" | cut -d '_' -f 1
+    dpkg-deb -c "$package" | awk '{print $1 " " $2 " " $6 " " $7 " " $8}' | sort -k 3
     echo "------------------------------------------------"
   done
 fi

debian/tests/control

+Tests: smoke
+# RocksDB is not built for all archs. Rather than duplicating the condition
+# for its existence (see the list in debian/control), install it if available
+# and check in the test if it's funcational when it should be.
+# The plugin package also already depends on the other one.
+Depends: mariadb-plugin-rocksdb | mariadb-server-10.5
+Restrictions: allow-stderr needs-root isolation-container
+
+Tests: upstream
+Depends: mariadb-test
+Restrictions: allow-stderr breaks-testbed

debian/tests/smoke

+#!/bin/sh
+# dep8 smoke test for mysql-server
+# Author: Robie Basak <robie.basak at canonical.com>
+#
+# This test should be declared in debian/tests/control with a dependency
+# on the package that provides a configured MariaDB server (eg.
+# mariadb-server-10.5).
+#
+# This test should be declared in debian/tests/control with the
+# following restrictions:
+#
+# needs-root (to be able to log into the database)
+# allow-stderr
+#
+# This test:
+#
+# 1) Creates a test database and test user as the root user.
+#
+# 2) Creates a test table and checks it appears to operate normally
+# using the test user and test database.
+#
+# 3) Checks compression support for InnoDB & RocksDB engine.
+
+echo "Running test 'smoke'"
+set -ex
+
+# Start the deamon if it was not running. For example in Docker testing
+# environments there might not be any systemd et al and the service needs to
+# be started manually.
+if ! which systemctl
+then
+  if ! /etc/init.d/mysql status
+  then
+    echo "Did not find systemctl and deamon was not running, starting it.."
+    /etc/init.d/mysql start
+  fi
+else
+  # If systemd (and systemctl) is available, but the service did not start, then
+  # this smoke test is supposed to fail if next commands don't work.
+  echo "Found systemctl, continuing smoke test.."
+fi
+
+mysql <<EOT
+CREATE DATABASE testdatabase;
+CREATE USER 'testuser'@'localhost' identified by 'testpassword';
+GRANT ALL ON testdatabase.* TO 'testuser'@'localhost';
+EOT
+
+mysql testdatabase <<EOT
+CREATE TABLE foo (bar INTEGER);
+INSERT INTO foo (bar) VALUES (41);
+EOT
+
+result=$(echo 'SELECT bar+1 FROM foo;'|mysql --batch --skip-column-names --user=testuser --password=testpassword testdatabase)
+if [ "$result" != "42" ]; then
+       echo "Unexpected result" >&2
+       exit 1
+fi
+
+mysql --user=testuser --password=testpassword testdatabase <<EOT
+DROP TABLE foo;
+EOT
+
+mysql <<EOT
+DROP DATABASE testdatabase;
+DROP USER 'testuser'@'localhost';
+EOT
+
+mysql <<EOT
+SET GLOBAL innodb_compression_algorithm=lz4;
+SET GLOBAL innodb_compression_algorithm=snappy;
+SET GLOBAL innodb_compression_algorithm=zlib;
+SET GLOBAL innodb_compression_algorithm=none;
+EOT
+
+# Check whether RocksDB should be installed or not.
+plugin=mariadb-plugin-rocksdb
+if [ "$(dpkg-architecture -qDEB_HOST_ARCH_BITS)" != 32 ] &&
+   [ "$(dpkg-architecture -qDEB_HOST_ARCH_ENDIAN)" = little ]; then
+  dpkg-query -W $plugin
+
+  LOG=/var/lib/mysql/#rocksdb/LOG
+  # XXX: The server may only be started during the install of
+  #      mariadb-server-10.5, which happens before that of the plugin.
+  [ -e $LOG ] || mysql -e "INSTALL PLUGIN RocksDB SONAME 'ha_rocksdb';"
+  # XXX: rocksdb_supported_compression_types variable does not report ZSTD.
+  for a in LZ4 Snappy Zlib ZSTD; do
+    grep -qE "k$a(Compression)? supported: 1" $LOG
+  done
+else
+  ! dpkg-query -W $plugin
+fi

debian/tests/upstream

+#!/bin/sh
+# autopkgtest check: Build and run the upstream test suite.
+# (C) 2012 Canonical Ltd.
+# Author: Daniel Kessel <d.kessel@gmx.de>
+
+# running the mysql testsuite as described in:
+# https://bugs.launchpad.net/ubuntu/+source/mysql-5.5/+bug/959683
+
+echo "Running test 'testsuite'"
+set -e
+
+SKIP_TEST_LST="/tmp/skip-test.lst"
+WORKDIR=$(mktemp -d)
+trap 'rm -rf $WORKDIR $SKIP_TEST_LST' 0 INT QUIT ABRT PIPE TERM
+cd "$WORKDIR"
+
+mkdir var
+mkdir tmp
+
+echo "using vardir: $WORKDIR/var"
+echo "using tmpdir: $WORKDIR/tmp"
+
+echo "Setting up skip-tests-list"
+
+# Use unstable-tests list as base to skip all tests  considered unstable
+cp /usr/share/mysql/mysql-test/unstable-tests $SKIP_TEST_LST
+
+# Also use arch specific skiplists if such files exist
+for filename in /usr/share/mysql/mysql-test/unstable-tests.*
+do
+  # Check for case that no files matched and glob is returned
+  [ -e "$filename" ] || continue
+  # Append file to the main skip test list file
+  cat "$filename" >> $SKIP_TEST_LST
+done
+
+# Skip tests that cannot run properly on ci.debian.net / autopkgtests.ubuntu.com
+cat >> $SKIP_TEST_LST << EOF
+binlog.binlog_server_start_options : Requires writable /usr
+main.ctype_uca : Requires writable /usr
+rpl.rpl_gtid_mode : Requires starting server as root ref http://bugs.mysql.com/bug.php?id=70517
+EOF
+
+# Skip tests that cannot run properly on Gitlab-CI
+if [ ! -z "$GITLAB_CI" ]
+then
+  cat >> $SKIP_TEST_LST << EOF
+main.mysqld--help : For unknown reason table-cache is 4000 instead of default 421
+EOF
+fi
+
+ARCH=$(dpkg --print-architecture)
+if [ "$ARCH" = "s390x" ]; then
+    echo "main.func_regexp_pcre : recursion fails on s390x https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1723947" >> $SKIP_TEST_LST
+fi
+
+cd /usr/share/mysql/mysql-test
+echo "starting mysql-test-tun.pl..."
+perl -I. ./mysql-test-run.pl --suite=main --vardir="$WORKDIR/var" --tmpdir="$WORKDIR/tmp" \
+    --parallel=auto --skip-rpl \
+    --force --skip-test-list=$SKIP_TEST_LST $@ 2>&1
+echo "run: OK"