Deb: Misc ColumnStore Debian packaging improvements

- Add 'libboost-all-dev' and 'libreadline-gplv2-dev' as they were was found
  to be a compulsory build dependency for columnstore plugin.
- Add 'expect' as run-time dependencey for columnstore plugin as scripts
  use it:
    usr/bin/mcs_module_installer.sh: #!/usr/bin/expect
    usr/bin/remote_command.sh: #!/usr/bin/expect
    usr/bin/remote_command_verify.sh: #!/usr/bin/expect
    usr/bin/remote_scp_get.sh: #!/usr/bin/expect
    usr/bin/remote_scp_put.sh: #!/usr/bin/expect
    usr/bin/rsync.sh: #!/usr/bin/expect
- Properly define depends on Python. No Python 2 support needs to be
  considered, Python 3 has been around long enough. Fixes Lintian errors
    E: mariadb-plugin-columnstore: python-script-but-no-python-dep
    usr/bin/mcs-loadbrm.py #!python
    E: mariadb-plugin-columnstore: python-script-but-no-python-dep
    usr/bin/mcs-start-storagemanager.py #!python
- Partially revert undocumented and thus unjustified changes in commits
  d69a79da63287089efdc5f90a11ecd66ce55b471 and
  c0565666cfe6528b76bc53ce50d3690d13c92cf6.
- Trigger ldconfig, otherwise Lintian complains:
    E: mariadb-plugin-columnstore: package-must-activate-ldconfig-trigger
    usr/lib/x86_64-linux-gnu/libwriteengineredistribute.so
- Update postinst to be compatible with new server binary mariadbd name.
- Properly detect systemd or fallback to sysv init in postrm script.
- Only attempt to build ColumnStore on amd64 and i386. Test builds on
  Launchpad.net showed the CMake plugin configure step will prevent even
  attempts to build on other platforms.
- Clean up and unify cmake build command in debian/rules.
- Explicitly list files not installed.
- Run 'wrap-and-sort -a -v'.
- Truncate build logs on Salsa-CI to keep under 4 MB. This is now needed
  as the ColumnStore build is so verbose.
  See https://jira.mariadb.org/browse/MCOL-4111.
- Update Travis-CI dependencies to match new debian/control.

.travis.yml

@@ -40,7 +40,14 @@ addons:
       - g++-9 # ubuntu-toolchain-r/test (xenial and bionic)
       - gdb
       - libaio-dev
+      - libboost-atomic-dev
+      - libboost-chrono-dev
+      - libboost-date-time-dev
       - libboost-dev
+      - libboost-filesystem-dev
+      - libboost-regex-dev
+      - libboost-system-dev
+      - libboost-thread-dev
       - libcrack2-dev
       - libcurl3-dev
       - libdbd-mysql
@@ -52,6 +59,7 @@ addons:
       - libnuma-dev
       - libpam0g-dev
       - libpcre2-dev
+      - libreadline-gplv2-dev
       - libsnappy-dev
       - libssl-dev
       - libstemmer-dev

debian/autobake-deb.sh

@@ -31,7 +31,7 @@ then
   sed 's|DINSTALL_MYSQLTESTDIR=share/mysql/mysql-test|DINSTALL_MYSQLTESTDIR=false|' -i debian/rules
 
   # Also skip building RocksDB, Mroonga etc to save even more time and disk space
-  sed 's|-DDEB|-DPLUGIN_MROONGA=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO -DDEB|' -i debian/rules
+  sed 's|-DDEB|-DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO -DDEB|' -i debian/rules
 fi
 
 # Convert gcc version to numberical value. Format is Mmmpp where M is Major
@@ -75,7 +75,6 @@ fi
 if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]] || [[ $TRAVIS ]]
 then
   sed '/Package: mariadb-plugin-rocksdb/,/^$/d' -i debian/control
-  sed -i 's|-DPLUGIN_ROCKSDB=YES|-DPLUGIN_ROCKSDB=NO|' debian/rules
 fi
 
 # If libpcre2-dev is not available (before Debian Stretch and Ubuntu Xenial)
@@ -95,11 +94,13 @@ then
   sed -i -e "/Package: libmariadbd-dev/,/^$/d" debian/control
 fi
 
-if [[ $TRAVIS ]] || ! [[ $(arch) =~ 86 ]]
+# Don't build nor package ColumnStore on Travis-CI
+if [[ $TRAVIS ]]
 then
-  sed -i -e "/Package: mariadb-plugin-columnstore/,/^$/d" debian/control
-  sed -i '/flex/d' debian/control
-  sed -i 's|-DPLUGIN_COLUMNSTORE=YES|-DPLUGIN_COLUMNSTORE=NO|' debian/rules
+  sed "/Package: mariadb-plugin-columnstore/,/^$/d" -i debian/control
+  sed '/flex/d' -i debian/control
+  sed -r '/libboost-[a-z-]+-dev/d' -i debian/control
+  sed 's|-DPLUGIN_COLUMNSTORE=YES|-DPLUGIN_COLUMNSTORE=NO|' -i debian/rules
 fi
 
 # Adjust changelog, add new version

debian/control

@@ -9,10 +9,17 @@ Build-Depends: bison,
                dh-apparmor,
                dh-exec,
                dh-systemd,
-               flex,
+               flex [i386 amd64],
                gdb,
                libaio-dev [linux-any],
+               libboost-atomic-dev [i386 amd64],
+               libboost-chrono-dev [i386 amd64],
+               libboost-date-time-dev [i386 amd64],
                libboost-dev,
+               libboost-filesystem-dev [i386 amd64],
+               libboost-regex-dev [i386 amd64],
+               libboost-system-dev [i386 amd64],
+               libboost-thread-dev [i386 amd64],
                libcrack2-dev (>= 2.9.0),
                libcurl4-openssl-dev | libcurl4-dev,
                libedit-dev,
@@ -24,6 +31,7 @@ Build-Depends: bison,
                libnuma-dev [!armhf],
                libpam0g-dev,
                libpcre2-dev,
+               libreadline-gplv2-dev [i386 amd64],
                libsnappy-dev,
                libssl-dev | libssl1.0-dev,
                libsystemd-dev [linux-any],
@@ -653,11 +661,12 @@ Description: RocksDB storage engine for MariaDB
 
 Package: mariadb-plugin-columnstore
 Architecture: amd64 i386
-Depends: mariadb-server-10.5 (= ${binary:Version}),
-         net-tools,
-         binutils,
-         libsnappy1 | libsnappy1v5,
+Depends: binutils,
+         expect,
          libjemalloc1 | libjemalloc2,
+         libsnappy1 | libsnappy1v5,
+         mariadb-server-10.5 (= ${binary:Version}),
+         net-tools,
          python3,
          ${misc:Depends},
          ${shlibs:Depends}

debian/mariadb-plugin-columnstore.install

@@ -18,7 +18,6 @@ usr/bin/ProcMon
 usr/bin/ServerMonitor
 usr/bin/StorageManager
 usr/bin/WriteEngineServer
-usr/bin/dbmsReport.sh
 usr/bin/autoConfigure
 usr/bin/bulklogReport.sh
 usr/bin/clearShm
@@ -42,6 +41,7 @@ usr/bin/cpimport
 usr/bin/cpimport.bin
 usr/bin/cplogger
 usr/bin/dbbuilder
+usr/bin/dbmsReport.sh
 usr/bin/dbrmctl
 usr/bin/ddlcleanup
 usr/bin/disable-rep-columnstore.sh
@@ -51,8 +51,12 @@ usr/bin/hardwareReport.sh
 usr/bin/idbmeminfo
 usr/bin/load_brm
 usr/bin/logReport.sh
+usr/bin/mariadb-columnstore-start.sh
+usr/bin/mariadb-columnstore-stop.sh
 usr/bin/mariadb-command-line.sh
 usr/bin/master-rep-columnstore.sh
+usr/bin/mcs-loadbrm.py
+usr/bin/mcs-start-storagemanager.py
 usr/bin/mcs-stop-controllernode.sh
 usr/bin/mcsGetConfig
 usr/bin/mcsSetConfig
@@ -82,10 +86,6 @@ usr/bin/smrm
 usr/bin/startupTests.sh
 usr/bin/viewtablelock
 usr/bin/workernode
-usr/bin/mcs-start-storagemanager.py
-usr/bin/mcs-loadbrm.py
-usr/bin/mariadb-columnstore-start.sh
-usr/bin/mariadb-columnstore-stop.sh
 usr/lib/*/libalarmmanager.so
 usr/lib/*/libbatchloader.so
 usr/lib/*/libbrm.so
@@ -146,9 +146,9 @@ usr/share/columnstore/mcs-dmlproc.service
 usr/share/columnstore/mcs-exemgr.service
 usr/share/columnstore/mcs-loadbrm.service
 usr/share/columnstore/mcs-primproc.service
+usr/share/columnstore/mcs-storagemanager.service
 usr/share/columnstore/mcs-workernode.service
 usr/share/columnstore/mcs-writeengineserver.service
-usr/share/columnstore/mcs-storagemanager.service
 usr/share/columnstore/mcstest-001.sh
 usr/share/columnstore/mcstest-002.sh
 usr/share/columnstore/mcstest-003.sh

debian/mariadb-plugin-columnstore.postinst

@@ -5,6 +5,4 @@ set -e
 # Install ColumnStore
 columnstore-post-install --rpmmode=install
 
-echo "MariaDB ColumnStore install completed"
-
 #DEBHELPER#

debian/mariadb-plugin-columnstore.postrm

@@ -8,22 +8,17 @@ if [ "$1" = "purge" ]; then
     rm -f /etc/mysql/mariadb.conf.d/columnstore.cnf.rpmsave
 fi
 
-running_systemd() {
-   if  [ "$(ps --no-headers -o comm 1)" = "systemd" ]; then
-       echo 0
-   else
-       echo 1
-   fi
-}
-
-if [ ! -z "$(pgrep -x mysqld)" ];then
-    systemctl cat mariadb.service > /dev/null 2>&1
-    if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then
-        systemctl restart mariadb.service > /dev/null 2>&1
-    else
-        pkill mysqld > /dev/null 2>&1
-        /usr/bin/mysqld_safe &
+# Automatically restart MariaDB after ColumnStore plugin has been removed
+case "$1" in
+  purge|remove|disappear)
+    if [ -d /run/systemd/system ]; then
+      # If systemd
+      deb-systemd-invoke restart mariadb.service >/dev/null
+    elif [ -x "/etc/init.d/mariadb" ]; then
+      # Fall-back to SysV init
+      invoke-rc.d mariadb restart || exit $?
     fi
-fi
+  ;;
+esac
 
 #DEBHELPER#

debian/mariadb-plugin-columnstore.prerm

@@ -4,6 +4,5 @@ set -e
 
 columnstore-pre-uninstall
 
-exit 0
 
 #DEBHELPER#

debian/mariadb-plugin-columnstore.triggers

+activate-noawait ldconfig

debian/mariadb-plugin-cracklib-password-check.install

-etc/mysql/mariadb.conf.d/cracklib_password_check.cnf 
+etc/mysql/mariadb.conf.d/cracklib_password_check.cnf
 usr/lib/mysql/plugin/cracklib_password_check.so

debian/mariadb-plugin-gssapi-server.install

-etc/mysql/mariadb.conf.d/auth_gssapi.cnf 
+etc/mysql/mariadb.conf.d/auth_gssapi.cnf
 usr/lib/mysql/plugin/auth_gssapi.so

debian/not-installed

+etc/columnstore/storagemanager.cnf.example # Copy of etc/columnstore/storagemanager.cnf that is installed
 lib/systemd/system/mariadb.service # Installed by rules file
 lib/systemd/system/mariadb@.service # Installed by rules file
 usr/bin/mysql_config # We already have the MariaDB variant
 usr/bin/mytop # Mytop is distributed from a separate source package
 usr/bin/sst_dump # Use the one from rocksdb-tools package
-usr/lib/sysusers.d/sysusers.conf
-usr/lib/tmpfiles.d/tmpfiles.conf
 usr/lib/mysql/plugin/JavaWrappers.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless
 usr/lib/mysql/plugin/JdbcInterface.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless
+usr/lib/sysusers.d/sysusers.conf
+usr/lib/tmpfiles.d/tmpfiles.conf
+usr/lib/*/libdbbc.a # ColumnStore header file
+usr/lib/*/libidbboot.a # ColumnStore header file
+usr/lib/*/libprocessor.a # ColumnStore header file
+usr/lib/*/libwe_xml.a # ColumnStore header file
 usr/share/doc/mariadb-server-10.5/COPYING
 usr/share/doc/mariadb-server-10.5/COPYING.AGPLv3
 usr/share/doc/mariadb-server-10.5/COPYING.GPLv2

debian/rules

@@ -39,6 +39,17 @@ else
     TESTSUITE_FAIL_CMD:=exit 1
 endif
 
+# RocksDB cannot build on 32-bit platforms
+ifeq (32,$(DEB_HOST_ARCH_BITS))
+    CMAKEFLAGS += -DWITHOUT_ROCKSDB=true
+endif
+
+# ColumnStore only attempts to build on a few platforms as dictated by CMake checks
+# Also note in debian/control the CS-only build deps marked '[i386 amd64]'
+ifeq ($(DEB_HOST_ARCH),$(filter $(DEB_HOST_ARCH),amd64 i386))
+    CMAKEFLAGS += -DPLUGIN_COLUMNSTORE=YES
+endif
+
 # Add extra flag to avoid WolfSSL code crashing the entire mariadbd on s390x. This
 # can be removed once upstream has made the code s390x compatible, see
 # https://jira.mariadb.org/browse/MDEV-21705 and
@@ -83,10 +94,7 @@ endif
 	    -DBUILD_CONFIG=mysql_release \
 	    -DPLUGIN_TOKUDB=NO \
 	    -DPLUGIN_CASSANDRA=NO \
-	    -DPLUGIN_COLUMNSTORE=YES \
-	    -DPLUGIN_ROCKSDB=YES \
 	    -DPLUGIN_AWS_KEY_MANAGEMENT=NO \
-	    -WITH_EMBEDDED_SERVER=OFF \
 	    -DDEB=$(DEB_VENDOR) ..'
 
 # This is needed, otherwise 'make test' will run before binaries have been built

debian/salsa-ci.yml

@@ -33,7 +33,8 @@ build:
     - export CCACHE_DIR="${CCACHE_TMP_DIR}"
     - update-ccache-symlinks; ccache -z # Zero out ccache counters
     - mk-build-deps debian/control -t "apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends" -r -i
-    - debian/autobake-deb.sh
+    - while true; do sleep 600; echo "10 minutes passed" >&2; done & # Progress keeper since build is long and silent
+    - debian/autobake-deb.sh |& tail -n 10000 # Keep Gitlab-CI output under 4 MB
     - cd ..; rm -rfv *.tmp # Clean away build files not to store as artifacts
     - cp -v *.* ${WORKING_DIR}/
     - du -shc ${WORKING_DIR}/* # Show total file size of artifacts. Must stay are under 100 MB.