MDEV-20928 mtr test galera.galera_var_innodb_disallow_writes test failure

The sporadic test hangs happen because of mutex dealock between innodb
background threads and two test connection executions.
The test sets variable innodb_disallow_writes, which blocks all writes
to filesyste. The test logic is to execute an INSERT, which should hang
because of filesytstem writes are blocked, and through another session
verify by SELECT that this hanging happens. The SELECT session will then
release innodb_disallow_writes blocking.

However, filesystem write  blocking affects also innodb background threads
and they may hang while keeping some other resources locked.
As an example, in one test hang situation, buffer pool access was blocked.
And, if buffer pool is blocked, the test connections will be blocked as well,
and the SELECT session will not be able to continue to release the
innodb_disallow_writes.

The fix in this commit is refactoring of the test logic.
The test will now set first innodb_disallow_writes blocking, and then record
a hash of data directory's filesystem contents. This works as checksum of the
state of data on the datadirectory.

Then some SQL load is tried on both nodes, these sessions will be blocking
due to frozen file system state. The test will have a short sleep to allow
innodb background threads to loop and possibly encounter innodb_disallow_writes
blocking as well.

After the sleep, the test will record file system checksun for the second time,
and then release the innodb_disallow-writes blocking.

Finally, the two checksums are compared, they should be identical to verify that
nothing was written on datadirectory during the test execution.

The checksum is implemented by md5sum hash over all files found in datadirectory
by find command. all these file hashes are hashed together by one more md5sum.

The test therefore depends on md5sum and find. find may work differently with some
OS distributions, e.g. freebsd may be problematic.

mysql-test/suite/galera/r/galera_var_innodb_disallow_writes.result

 connection node_2;
 connection node_1;
-connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 connection node_1a;
 SET SESSION wsrep_sync_wait = 0;
 connection node_1;
-CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
+CREATE TABLE t1 (f1 INTEGER, f2 varchar(1024)) Engine=InnoDB;
+CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 SET GLOBAL innodb_disallow_writes=ON;
-INSERT INTO t1 VALUES (1);;
+INSERT INTO t1 (f2) SELECT 'abcde ' FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;;
+connection node_2;
+INSERT INTO t1 (f2) SELECT 'fghij ' FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;
+SELECT COUNT(*) AS EXPECT_10000 FROM t1;
+EXPECT_10000
+10000
 connection node_1a;
-SELECT COUNT(*) AS EXPECT_0 FROM t1;
-EXPECT_0
-0
-SELECT COUNT(*) AS EXPECT_0 FROM t1;
-EXPECT_0
-0
 SET GLOBAL innodb_disallow_writes=OFF;
 connection node_1;
-SELECT COUNT(*) AS EXPECT_1 FROM t1;
-EXPECT_1
-1
+SELECT COUNT(*) AS EXPECT_20000 FROM t1;
+EXPECT_20000
+20000
+connection node_2;
+SELECT COUNT(*) AS EXPECT_20000 FROM t1;
+EXPECT_20000
+20000
+connection node_1;
 connection node_2;
-SELECT COUNT(*) AS EXPECT_1 FROM t1;
-EXPECT_1
-1
 DROP TABLE t1;
+DROP TABLE ten;
 disconnect node_1a;

mysql-test/suite/galera/t/galera_var_innodb_disallow_writes.test

@@ -11,40 +11,62 @@
 #
 
 --source include/galera_cluster.inc
+--source include/have_innodb.inc
 --source include/have_log_bin.inc
 
+--let $datadir= `SELECT @@datadir`
+
+
 # Open a separate connection to be used to run SHOW PROCESSLIST
---connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--let $galera_connection_name = node_1a
+--let $galera_server_number = 1
+--source include/galera_connect.inc
 --connection node_1a
 SET SESSION wsrep_sync_wait = 0;
 
 --connection node_1
-CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
+CREATE TABLE t1 (f1 INTEGER, f2 varchar(1024)) Engine=InnoDB;
+CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
+
 SET GLOBAL innodb_disallow_writes=ON;
---send INSERT INTO t1 VALUES (1);
+--exec find $datadir -type f-exec md5sum {} \; | md5sum >$MYSQLTEST_VARDIR/tmp/innodb_before
+
+#
+# This insert has no effect before innodb_disallow_writes is OFF
+#
+--send INSERT INTO t1 (f2) SELECT 'abcde ' FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;
+
+--connection node_2
+INSERT INTO t1 (f2) SELECT 'fghij ' FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;
+SELECT COUNT(*) AS EXPECT_10000 FROM t1;
 
 --connection node_1a
---let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1';
---source include/wait_condition.inc
-SELECT COUNT(*) AS EXPECT_0 FROM t1;
-let $wait_condition = SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO = 'INSERT INTO t1 VALUES (1)';
---source include/wait_condition.inc
-SELECT COUNT(*) AS EXPECT_0 FROM t1;
+--sleep 5
+
+--exec find $datadir -type f-exec md5sum {} \; | md5sum >$MYSQLTEST_VARDIR/tmp/innodb_after
 
 SET GLOBAL innodb_disallow_writes=OFF;
 
 --connection node_1
 --reap
-SELECT COUNT(*) AS EXPECT_1 FROM t1;
+--let $wait_condition = SELECT COUNT(*) = 20000 FROM t1;
+--source include/wait_condition.inc
+
+SELECT COUNT(*) AS EXPECT_20000 FROM t1;
 
 --connection node_2
---let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1';
---source include/wait_condition.inc
---let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--let $wait_condition = SELECT COUNT(*) = 20000 FROM t1;
 --source include/wait_condition.inc
-SELECT COUNT(*) AS EXPECT_1 FROM t1;
+SELECT COUNT(*) AS EXPECT_20000 FROM t1;
+
+--connection node_1
+--diff_files $MYSQLTEST_VARDIR/tmp/innodb_before $MYSQLTEST_VARDIR/tmp/innodb_after
+
+--connection node_2
 
 DROP TABLE t1;
+DROP TABLE ten;
 
 --disconnect node_1a
 

storage/innobase/srv/srv0srv.cc

@@ -75,7 +75,6 @@ Created 10/8/1995 Heikki Tuuri
 #include "btr0scrub.h"
 
 #include <my_service_manager.h>
-
 /* The following is the maximum allowed duration of a lock wait. */
 UNIV_INTERN ulong	srv_fatal_semaphore_wait_threshold =  DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT;