MDEV-29775 : Assertion `0' failed in void Protocol::end_statement() when...

MDEV-29775 : Assertion `0' failed in void Protocol::end_statement() when adding data to the MyISAM table after setting wsrep_mode=replicate_myisam

Firstly, variables wsrep_forced_binlog_format, wsrep_replicate_myisam
should not be able to set if wsrep_on=OFF.

Secondly, if wsrep_replicate_myisam=ON we allow wsrep_forced_binlog_format
to be [DEFAULT|ROW].
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

mysql-test/suite/galera/r/mdev-29775.result

+connection node_2;
+connection node_1;
+SET GLOBAL wsrep_replicate_myisam=ON;
+CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM;
+INSERT INTO t VALUES();
+SELECT * FROM t;
+f0
+NULL
+connection node_2;
+SELECT * FROM t;
+f0
+NULL
+DROP TABLE t;
+connection node_1;
+SET GLOBAL wsrep_replicate_myisam=ON;
+SET GLOBAL wsrep_forced_binlog_format=ROW;
+CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM;
+INSERT INTO t VALUES();
+SELECT * FROM t;
+f0
+NULL
+connection node_2;
+SELECT * FROM t;
+f0
+NULL
+DROP TABLE t;
+connection node_1;
+SET GLOBAL wsrep_forced_binlog_format=DEFAULT;
+SET GLOBAL wsrep_replicate_myisam=OFF;
+SET GLOBAL wsrep_forced_binlog_format=MIXED;
+SET GLOBAL wsrep_replicate_myisam=ON;
+ERROR HY000: wsrep_replicate_myisam=ON can't be enabled if wsrep_forced_binlog != [NONE|ROW]
+SET GLOBAL wsrep_forced_binlog_format=STATEMENT;
+SET GLOBAL wsrep_replicate_myisam=ON;
+ERROR HY000: wsrep_replicate_myisam=ON can't be enabled if wsrep_forced_binlog != [NONE|ROW]
+SET GLOBAL wsrep_forced_binlog_format=NONE;
+SET GLOBAL wsrep_replicate_myisam=OFF;
+SET GLOBAL wsrep_replicate_myisam=ON;
+SET GLOBAL wsrep_forced_binlog_format=MIXED;
+ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_replicate_myisam=ON
+SET GLOBAL wsrep_forced_binlog_format=STATEMENT;
+ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_replicate_myisam=ON
+SET GLOBAL wsrep_forced_binlog_format=NONE;
+SET GLOBAL wsrep_replicate_myisam=OFF;

mysql-test/suite/galera/t/mdev-29775.test

+--source include/galera_cluster.inc
+
+#
+# MDEV-29775 : Assertion `0' failed in void Protocol::end_statement() when adding data to the MyISAM table after setting wsrep_mode=replicate_myisam
+#
+SET GLOBAL wsrep_replicate_myisam=ON;
+CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM;
+INSERT INTO t VALUES();
+SELECT * FROM t;
+--connection node_2
+SELECT * FROM t;
+DROP TABLE t;
+
+--connection node_1
+SET GLOBAL wsrep_replicate_myisam=ON;
+SET GLOBAL wsrep_forced_binlog_format=ROW;
+CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM;
+INSERT INTO t VALUES();
+SELECT * FROM t;
+--connection node_2
+SELECT * FROM t;
+DROP TABLE t;
+
+--connection node_1
+SET GLOBAL wsrep_forced_binlog_format=DEFAULT;
+SET GLOBAL wsrep_replicate_myisam=OFF;
+
+SET GLOBAL wsrep_forced_binlog_format=MIXED;
+--error ER_WRONG_ARGUMENTS
+SET GLOBAL wsrep_replicate_myisam=ON;
+SET GLOBAL wsrep_forced_binlog_format=STATEMENT;
+--error ER_WRONG_ARGUMENTS
+SET GLOBAL wsrep_replicate_myisam=ON;
+
+SET GLOBAL wsrep_forced_binlog_format=NONE;
+SET GLOBAL wsrep_replicate_myisam=OFF;
+
+SET GLOBAL wsrep_replicate_myisam=ON;
+--error ER_WRONG_ARGUMENTS
+SET GLOBAL wsrep_forced_binlog_format=MIXED;
+--error ER_WRONG_ARGUMENTS
+SET GLOBAL wsrep_forced_binlog_format=STATEMENT;
+
+SET GLOBAL wsrep_forced_binlog_format=NONE;
+SET GLOBAL wsrep_replicate_myisam=OFF;

mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.cnf

+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW

mysql-test/suite/sys_vars/t/wsrep_forced_binlog_format_basic.test → mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.test

---source include/have_wsrep.inc
+--source include/have_innodb.inc
+--source include/have_wsrep_provider.inc
+--source include/have_binlog_format_row.inc
+
 
 --echo #
 --echo # wsrep_forced_binlog_format

sql/sys_vars.cc

@@ -6016,7 +6016,9 @@ static const char *wsrep_binlog_format_names[]=
 static Sys_var_enum Sys_wsrep_forced_binlog_format(
        "wsrep_forced_binlog_format", "binlog format to take effect over user's choice",
        GLOBAL_VAR(wsrep_forced_binlog_format), CMD_LINE(REQUIRED_ARG),
-       wsrep_binlog_format_names, DEFAULT(BINLOG_FORMAT_UNSPEC));
+       wsrep_binlog_format_names, DEFAULT(BINLOG_FORMAT_UNSPEC),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_forced_binlog_format_check));
 
 static Sys_var_mybool Sys_wsrep_recover_datadir(
        "wsrep_recover", "Recover database state after crash and exit",
@@ -6025,7 +6027,9 @@ static Sys_var_mybool Sys_wsrep_recover_datadir(
 
 static Sys_var_mybool Sys_wsrep_replicate_myisam(
        "wsrep_replicate_myisam", "To enable myisam replication",
-       GLOBAL_VAR(wsrep_replicate_myisam), CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+       GLOBAL_VAR(wsrep_replicate_myisam), CMD_LINE(OPT_ARG), DEFAULT(FALSE),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_replicate_myisam_check));
 
 static Sys_var_mybool Sys_wsrep_log_conflicts(
        "wsrep_log_conflicts", "To log multi-master conflicts",

sql/wsrep_var.cc

-/* Copyright 2008-2022 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2023 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1124,3 +1124,47 @@ bool wsrep_gtid_domain_id_update(sys_var* self, THD *thd, enum_var_type)
   wsrep_gtid_server.domain_id= wsrep_gtid_domain_id;
   return false;
 }
+
+bool wsrep_replicate_myisam_check(sys_var *self, THD* thd, set_var* var)
+{
+  bool new_replicate_myisam= (bool)var->save_result.ulonglong_value;
+
+  if (new_replicate_myisam && !WSREP_ON_)
+  {
+    my_message(ER_WRONG_ARGUMENTS, "wsrep_replicate_myisam=ON can't be enabled "
+               "if wsrep_on=OFF", MYF(0));
+    return true;
+  }
+  if (new_replicate_myisam &&
+      !(wsrep_forced_binlog_format == BINLOG_FORMAT_UNSPEC ||
+        wsrep_forced_binlog_format == BINLOG_FORMAT_ROW))
+  {
+    my_message(ER_WRONG_ARGUMENTS, "wsrep_replicate_myisam=ON can't be enabled "
+               "if wsrep_forced_binlog != [NONE|ROW]", MYF(0));
+    return true;
+  }
+  return false;
+}
+
+bool wsrep_forced_binlog_format_check(sys_var *self, THD* thd, set_var* var)
+{
+  ulonglong new_forced_binlog_format= var->save_result.ulonglong_value;
+  if (new_forced_binlog_format != BINLOG_FORMAT_UNSPEC && !WSREP_ON)
+  {
+    my_message(ER_WRONG_ARGUMENTS, "wsrep_forced_binlog_format can't be set "
+               "if wsrep_on=OFF", MYF(0));
+    return true;
+  }
+  if (!(new_forced_binlog_format == BINLOG_FORMAT_UNSPEC ||
+        new_forced_binlog_format == BINLOG_FORMAT_ROW))
+  {
+    if (wsrep_replicate_myisam)
+    {
+      my_message(ER_WRONG_ARGUMENTS, "wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set "
+                 "if wsrep_replicate_myisam=ON", MYF(0));
+      return true;
+    }
+  }
+
+  return false;
+}

sql/wsrep_var.h

-/* Copyright (C) 2013 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2023 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -107,6 +107,9 @@ extern bool wsrep_debug_update               UPDATE_ARGS;
 extern bool wsrep_gtid_seq_no_check          CHECK_ARGS;
 
 extern bool wsrep_gtid_domain_id_update      UPDATE_ARGS;
+
+extern bool wsrep_replicate_myisam_check     CHECK_ARGS;
+extern bool wsrep_forced_binlog_format_check CHECK_ARGS;
 #else  /* WITH_WSREP */
 
 #define wsrep_provider_init(X)