MDEV-25962: binlog.binlog_truncate_multi_log_unsafe test fails in buildbot

1. sync_binlog=1 is needed to ensure that on flush binlog is available on
   disk.
2. Insert on 'connection master2' should wait for 'master1_ready' from
   'connection master1'.

Added sync_binlog=1 to all tests that got added as part of MDEV-21117 fix

@mysql-test/suite/binlog/t/binlog_truncate_multi_log_unsafe.test
@mysql-test/suite/binlog/t/binlog_truncate_multi_engine.test
@mysql-test/suite/binlog/t/binlog_truncate_active_log.test
@mysql-test/suite/binlog/t/binlog_truncate_multi_log.test

mysql-test/suite/binlog/r/binlog_truncate_active_log.result

 call mtr.add_suppression("Can.t init tc log");
 call mtr.add_suppression("Aborting");
 RESET MASTER;
+SET @@global.sync_binlog=1;
 CREATE TABLE t (f INT) ENGINE=INNODB;
 CREATE TABLE t2 (f INT) ENGINE=INNODB;
 CREATE TABLE tm (f INT) ENGINE=Aria;
@@ -28,7 +29,7 @@ connection default;
 disconnect master1;
 disconnect master2;
 disconnect master3;
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-6/ in mysqld.1.err
 Pre-crash binlog file content:
 include/show_binlog_events.inc
@@ -77,7 +78,7 @@ connection default;
 disconnect master1;
 disconnect master2;
 disconnect master3;
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-10/ in mysqld.1.err
 Pre-crash binlog file content:
 include/show_binlog_events.inc
@@ -133,7 +134,7 @@ connection default;
 disconnect master1;
 disconnect master2;
 disconnect master3;
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-14/ in mysqld.1.err
 Pre-crash binlog file content:
 include/show_binlog_events.inc
@@ -195,7 +196,7 @@ connection default;
 disconnect master1;
 disconnect master2;
 disconnect master3;
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-20/ in mysqld.1.err
 Pre-crash binlog file content:
 include/show_binlog_events.inc
@@ -234,4 +235,5 @@ DELETE FROM t;
 DROP PROCEDURE sp_xa;
 # Cleanup
 DROP TABLE t,t2,tm;
+SET @@global.sync_binlog= default;
 # End of the tests

mysql-test/suite/binlog/r/binlog_truncate_multi_engine.result

 call mtr.add_suppression("Can.t init tc log");
 call mtr.add_suppression("Aborting");
+SET @@global.sync_binlog= 1;
 CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
 CREATE TABLE t2 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=rocksdb;
 #
@@ -30,9 +31,9 @@ Log_name	File_size
 master-bin.000001	#
 master-bin.000002	#
 master-bin.000003	#
-# restart the server with --rpl-semi-sync-slave-enabled=1
+# restart the server with --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 # the server is restarted
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 connection default;
 #
 # *** Summary: 1 row should be present in both tables; binlog is truncated; number of binlogs at reconnect - 3:
@@ -97,7 +98,7 @@ INSERT INTO t2 VALUES (2, REPEAT("x", 4100));
 INSERT INTO t1 VALUES (2, REPEAT("x", 4100));
 COMMIT;
 connection default;
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 connection default;
 #
 # *** Summary: 2 rows should be present in both tables; no binlog truncation; one extra binlog file compare with A; number of binlogs at reconnect - 4:
@@ -154,9 +155,9 @@ Log_name	File_size
 master-bin.000001	#
 master-bin.000002	#
 master-bin.000003	#
-# restart the server with --rpl-semi-sync-slave-enabled=1
+# restart the server with --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 # the server is restarted
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 connection default;
 #
 # *** Summary: 2 rows should be present in both tables; no binlog truncation; the same # of binlog files as in B; number of binlogs at reconnect - 4:
@@ -186,4 +187,5 @@ DELETE FROM t2;
 disconnect con1;
 #
 DROP TABLE t1, t2;
+SET @@global.sync_binlog= default;
 # End of the tests

mysql-test/suite/binlog/r/binlog_truncate_multi_log.result

 call mtr.add_suppression("Can.t init tc log");
 call mtr.add_suppression("Aborting");
 SET @@global.max_binlog_size= 4096;
+SET @@global.sync_binlog= 1;
 RESET MASTER;
 FLUSH LOGS;
 CREATE TABLE ti (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
@@ -13,11 +14,11 @@ master-bin.000001	#
 master-bin.000002	#
 INSERT INTO ti VALUES(1,"I am gonna survive");
 INSERT INTO tm VALUES(1,"me too!");
-SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR con1_go";
+SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go_never_arrives";
 INSERT INTO ti VALUES (2, REPEAT("x", 4100));
 connect master2,localhost,root,,;
 SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
-SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go";
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go_never_arrives";
 INSERT INTO ti VALUES (3, "not gonna survive");
 connection default;
 SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
@@ -35,7 +36,7 @@ connection default;
 # Kill the server
 disconnect master1;
 disconnect master2;
-# restart: --rpl-semi-sync-slave-enabled=1
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 FOUND 1 /truncated binlog file:.*master.*000002/ in mysqld.1.err
 "One record should be present in table"
 SELECT * FROM ti;
@@ -50,4 +51,5 @@ SELECT @@global.gtid_binlog_pos;
 0-1-4
 # Cleanup
 DROP TABLE ti;
+SET @@global.sync_binlog= default;
 # End of the tests

mysql-test/suite/binlog/r/binlog_truncate_multi_log_unsafe.result

@@ -12,6 +12,7 @@ call mtr.add_suppression("mysqld: Table.*tm.*is marked as crashed");
 call mtr.add_suppression("Checking table.*tm");
 RESET MASTER;
 FLUSH LOGS;
+SET @@global.sync_binlog=1;
 CREATE TABLE ti (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
 CREATE TABLE tm (f INT) ENGINE=MYISAM;
 INSERT INTO tm VALUES(1);
@@ -19,10 +20,11 @@ connect master1,localhost,root,,;
 connect master2,localhost,root,,;
 connect master3,localhost,root,,;
 connection master1;
-SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go";
+SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go_never_arrives";
 INSERT INTO ti VALUES (5 - 1, REPEAT("x", 4100));
 connection master2;
-SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go";
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go_never_arrives";
 INSERT INTO ti VALUES (5, REPEAT("x", 1));
 connection master3;
 SET DEBUG_SYNC= "now WAIT_FOR master2_ready";

mysql-test/suite/binlog/t/binlog_truncate_active_log.inc

@@ -31,7 +31,7 @@ SELECT @@global.gtid_binlog_pos as 'Before the crash';
 #
 # Server restart
 #
---let $restart_parameters= --rpl-semi-sync-slave-enabled=1
+--let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 --source include/start_mysqld.inc
 
 # Check error log for a successful truncate message.
@@ -39,7 +39,7 @@ SELECT @@global.gtid_binlog_pos as 'Before the crash';
 
 --let SEARCH_FILE=$log_error_
 --let SEARCH_PATTERN=Successfully truncated.*to remove transactions starting from GTID $truncate_gtid_pos
- 
+
 --source include/search_pattern_in_file.inc
 
 --echo Pre-crash binlog file content:

mysql-test/suite/binlog/t/binlog_truncate_active_log.test

@@ -22,6 +22,7 @@ call mtr.add_suppression("Aborting");
 # C. Similarly to A, with the XA blank transaction
 
 RESET MASTER;
+SET @@global.sync_binlog=1;
 CREATE TABLE t (f INT) ENGINE=INNODB;
 CREATE TABLE t2 (f INT) ENGINE=INNODB;
 CREATE TABLE tm (f INT) ENGINE=Aria;
@@ -98,5 +99,5 @@ DROP PROCEDURE sp_xa;
 
 --echo # Cleanup
 DROP TABLE t,t2,tm;
-
+SET @@global.sync_binlog= default;
 --echo # End of the tests

mysql-test/suite/binlog/t/binlog_truncate_multi_engine.test

@@ -18,6 +18,7 @@ call mtr.add_suppression("Can.t init tc log");
 call mtr.add_suppression("Aborting");
 --let $MYSQLD_DATADIR= `SELECT @@datadir`
 
+SET @@global.sync_binlog= 1;
 CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
 CREATE TABLE t2 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=rocksdb;
 
@@ -27,7 +28,7 @@ CREATE TABLE t2 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=rocksdb;
 # The transaction is killed along with the server after that.
 --let $shutdown_timeout=0
 --let $debug_sync_action = "commit_after_release_LOCK_log SIGNAL con1_ready WAIT_FOR signal_no_signal"
---let $restart_parameters = --rpl-semi-sync-slave-enabled=1
+--let $restart_parameters = --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 --let $test_outcome= 1 row should be present in both tables; binlog is truncated; number of binlogs at reconnect - 3
 --source binlog_truncate_multi_engine.inc
 --echo Proof of the truncated binlog file is readable (two transactions must be seen):
@@ -40,7 +41,7 @@ CREATE TABLE t2 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=rocksdb;
 --let $debug_sync_action = ""
 # Both debug_sync and debug-dbug are required to make sure Engines remember the commit state
 # debug_sync alone will not help.
---let $restart_parameters = --rpl-semi-sync-slave-enabled=1
+--let $restart_parameters = --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 --let $test_outcome= 2 rows should be present in both tables; no binlog truncation; one extra binlog file compare with A; number of binlogs at reconnect - 4
 --source binlog_truncate_multi_engine.inc
 
@@ -49,12 +50,10 @@ CREATE TABLE t2 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=rocksdb;
 --let $debug_sync_action = "commit_after_run_commit_ordered SIGNAL con1_ready WAIT_FOR signal_no_signal"
 # Hold off after both engines have committed. The server is shut down.
 --let $shutdown_timeout=
---let $restart_parameters = --rpl-semi-sync-slave-enabled=1
+--let $restart_parameters = --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 --let $test_outcome= 2 rows should be present in both tables; no binlog truncation; the same # of binlog files as in B; number of binlogs at reconnect - 4
 --source binlog_truncate_multi_engine.inc
 
-
-
 DROP TABLE t1, t2;
-
+SET @@global.sync_binlog= default;
 --echo # End of the tests

mysql-test/suite/binlog/t/binlog_truncate_multi_log.test

@@ -13,7 +13,7 @@ call mtr.add_suppression("Can.t init tc log");
 call mtr.add_suppression("Aborting");
 
 SET @@global.max_binlog_size= 4096;
-
+SET @@global.sync_binlog= 1;
 RESET MASTER;
 FLUSH LOGS;
 CREATE TABLE ti (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
@@ -29,13 +29,13 @@ INSERT INTO ti VALUES(1,"I am gonna survive");
 INSERT INTO tm VALUES(1,"me too!");
 
 # hold on near engine commit
-SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR con1_go";
+SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go_never_arrives";
 --send INSERT INTO ti VALUES (2, REPEAT("x", 4100))
 
 connect(master2,localhost,root,,);
 # The 2nd trx for recovery, it does not rotate binlog
 SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
-SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go";
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go_never_arrives";
 --send INSERT INTO ti VALUES (3, "not gonna survive")
 
 --connection default
@@ -53,7 +53,7 @@ SELECT @@global.gtid_binlog_state;
 #
 # Server restart
 #
---let $restart_parameters= --rpl-semi-sync-slave-enabled=1
+--let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1
 --source include/start_mysqld.inc
 
 # Check error log for a successful truncate message.
@@ -73,5 +73,5 @@ SELECT @@global.gtid_binlog_pos;
 
 --echo # Cleanup
 DROP TABLE ti;
-
+SET @@global.sync_binlog= default;
 --echo # End of the tests

mysql-test/suite/binlog/t/binlog_truncate_multi_log_unsafe.test

@@ -45,6 +45,7 @@ call mtr.add_suppression("Checking table.*tm");
 
 RESET MASTER;
 FLUSH LOGS;
+SET @@global.sync_binlog=1;
 CREATE TABLE ti (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
 CREATE TABLE tm (f INT) ENGINE=MYISAM;
 
@@ -66,13 +67,13 @@ connect(master3,localhost,root,,);
 --connection master1
 
 # The 1st trx binlogs, rotate binlog and hold on before committing at engine
-SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go";
+SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go_never_arrives";
 --send_eval INSERT INTO ti VALUES ($row_count - 1, REPEAT("x", 4100))
 
 --connection master2
-
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 # The 2nd trx for recovery, it does not rotate binlog
-SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go";
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go_never_arrives";
 --send_eval INSERT INTO ti VALUES ($row_count, REPEAT("x", 1))
 
 --connection master3