MDEV-12398 All cluster nodes stop due to a foreign key constraint failure

Comment from Codership:-
To fix the problem, we changed the certification logic in galera to treat insert
on child table row as exclusive to prevent any operation on referenced
parent table row. At the same time, update and delete on
 child table row were demoted to "shared", which makes it possible to
update/delete referenced parent table row, but only in a later transaction.
 This change allows somewhat more concurrency for foreign key constrained
 transactions, but is still safe for correct certification end result.

mysql-test/suite/galera/r/MW-369.result

+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+DELETE FROM p WHERE f1 = 1;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+INSERT INTO c VALUES (1, 1);
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+COMMIT;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SELECT * FROM p;
+f1	f2
+1	0
+2	0
+SELECT * FROM c;
+f1	p_id
+1	1
+DROP TABLE c;
+DROP TABLE p;
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+INSERT INTO c VALUES (1, 1, 0);
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+UPDATE p SET f2 = 1 WHERE f1 = 1;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+UPDATE c SET f2 = 1 WHERE f1 = 1;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+COMMIT;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SELECT * FROM p;
+f1	f2
+1	1
+2	0
+SELECT * FROM c;
+f1	p_id	f2
+1	1	1
+DROP TABLE c;
+DROP TABLE p;
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+INSERT INTO c VALUES (1, 1);
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+UPDATE p SET f2 = 1 WHERE f1 = 1;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+DELETE FROM c WHERE f1 = 1;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+COMMIT;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SELECT * FROM p;
+f1	f2
+1	1
+2	0
+SELECT * FROM c;
+f1	p_id
+DROP TABLE c;
+DROP TABLE p;
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER UNIQUE KEY) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f2))  ;
+INSERT INTO p VALUES (1, 0);
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+UPDATE p SET f2 = 1 WHERE f1 = 1;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+INSERT INTO c VALUES (1, 0);;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+COMMIT;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SELECT * FROM p;
+f1	f2
+1	0
+SELECT * FROM c;
+f1	p_id
+1	0
+DROP TABLE c;
+DROP TABLE p;
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1)
+ON DELETE CASCADE)  ;
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+INSERT INTO c VALUES (1, 1, 0);
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+DELETE FROM p WHERE f1 = 1;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+UPDATE c SET f2 = 1 WHERE f1 = 1;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+COMMIT;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SELECT * FROM p;
+f1	f2
+1	0
+2	0
+SELECT * FROM c;
+f1	p_id	f2
+1	1	1
+DROP TABLE c;
+DROP TABLE p;

mysql-test/suite/galera/t/MW-369.inc

+#
+# This file should be included from tests for MW-369 to run concurrent
+# transaction from node_1 with autocommit query from node_2.
+#
+# The parameters:
+# * $mw_369_parent_query - the parent query to be run inside transaction
+# * $mw_369_child_query - the child query
+#
+# The operations are the following:
+#
+# node_1:
+# START TRANSACTION;
+# $mw_369_parent_query
+# node_2
+# $mw_369_child_query - will be blocked on node_1 in apply monitor
+# node_1:
+# COMMIT;             - will be blocked on node_1 in local monitor
+#
+# The $mw_369_child_query is always expected to succeed. The caller is
+# responsible for checking if the final COMMIT on connection node_1
+# succeeds.
+#
+
+--connection node_1
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+
+--eval $mw_369_parent_query
+
+#
+# Block the $mw_369_child_query from node_2
+#
+# --connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--connection node_1a
+SET SESSION wsrep_sync_wait = 0;
+--let $galera_sync_point = apply_monitor_slave_enter_sync
+--source include/galera_set_sync_point.inc
+
+#
+# insert client row, which will make it impossible to replay the
+# delete on parent
+#
+--connection node_2
+--eval $mw_369_child_query
+
+#
+# Wait until $mw_369_child_query from node_2 reaches the sync point and
+# block the 'COMMIT' from node_1 before it certifies.
+#
+--connection node_1a
+--source include/galera_wait_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+--let $galera_sync_point = local_monitor_enter_sync
+--source include/galera_set_sync_point.inc
+
+--connection node_1
+--send COMMIT
+
+#
+# Wait until both sync points have been reached
+#
+--connection node_1a
+--let $galera_sync_point = apply_monitor_slave_enter_sync local_monitor_enter_sync
+--source include/galera_wait_sync_point.inc
+
+#
+# both threads are now parked in sync points, signal them to continue
+#
+--let $galera_sync_point = apply_monitor_slave_enter_sync
+--source include/galera_signal_sync_point.inc
+
+--let $galera_sync_point = local_monitor_enter_sync
+--source include/galera_signal_sync_point.inc
+--source include/galera_clear_sync_point.inc

mysql-test/suite/galera/t/MW-369.test

+#
+# Test A Outline:
+# ===============
+#
+# This test tests the scenario for MW-369 where a new child table
+# row referring to parent table row is inserted concurrently from
+# another node while the transaction which tries to delete a
+# referred row from the parent table is committing.
+#
+# The p table will originally have rows (1, 0), (2, 0).
+# The c table will be empty.
+#
+# A new row (1, 1) pointing to parent row (1, 0) is inserted from
+# connection node_2, the transaction which tries to remove the
+# parent row (1, 0) is run from connection node_1.
+#
+# Expected outcome:
+# ================
+#
+# The transaction on node_1 will fail. The parent table will contain
+# rows (1, 0), (2, 0) and the child table will contain row (1, 1).
+#
+
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_debug_sync.inc
+--source suite/galera/include/galera_have_debug_sync.inc
+
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
+
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+
+--let $mw_369_parent_query = DELETE FROM p WHERE f1 = 1
+--let $mw_369_child_query = INSERT INTO c VALUES (1, 1)
+
+#
+# we must open connection node_1a here, MW-369.inc will use it later
+#
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--source MW-369.inc
+
+# Commit fails
+--connection node_1
+--error ER_LOCK_DEADLOCK
+--reap
+
+--connection node_2
+SELECT * FROM p;
+SELECT * FROM c;
+
+DROP TABLE c;
+DROP TABLE p;
+
+#
+# Test B Outline:
+# ===============
+#
+# This test tests the scenario for MW-369 where a existing
+# child table row is updated concurrently from another node
+# with a transaction which updates the parent table.
+#
+# The p table will originally have rows (1, 0), (2, 0).
+# The c table will originally have rows (1, 1, 0) which points
+# to parent table row (1, 0).
+#
+# Expected outcome:
+# ================
+#
+# Both updates should succeed since they are done to separate tables and
+# rows. The parent table will contain rows (1, 1), (2, 0). The child
+# table will contain row (1, 1, 1).
+#
+
+--connection node_1
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+                f2 INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
+
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+INSERT INTO c VALUES (1, 1, 0);
+
+--let mw_369_parent_query = UPDATE p SET f2 = 1 WHERE f1 = 1
+--let $mw_369_child_query = UPDATE c SET f2 = 1 WHERE f1 = 1
+--source MW-369.inc
+
+# Commit succeeds
+--connection node_1
+--reap
+
+--connection node_2
+SELECT * FROM p;
+SELECT * FROM c;
+
+DROP TABLE c;
+DROP TABLE p;
+
+#
+# Test C Outline:
+# ===============
+#
+# This test tests the scenario for MW-369 where a child table row is
+# deleted concurrently from the other node while a transaction updates
+# the parent table referred by the child table row.
+#
+# The p table will originally have rows (1, 0), (2, 0)
+# The c table will originally have row (1, 1) which points to parent
+# table row (1, 0).
+#
+# A row (1, 1) pointing to parent row (1, 0) is deleted from
+# connection node_2, the transaction which tries to update the
+# parent row (1, 0) is run from connection node_1.
+#
+# Expected Outcome:
+# ================
+# Both operations on node_1 and node_2 should succeed without conflicts.
+# The parent table should contain values (1, 1), (2, 0) and the child
+# table should be empty.
+
+--connection node_1
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
+
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+INSERT INTO c VALUES (1, 1);
+
+--let $mw_369_parent_query = UPDATE p SET f2 = 1 WHERE f1 = 1
+--let $mw_369_child_query = DELETE FROM c WHERE f1 = 1
+--source MW-369.inc
+
+# Commit succeeds
+--connection node_1
+--reap
+
+--connection node_2
+SELECT * FROM p;
+SELECT * FROM c;
+
+DROP TABLE c;
+DROP TABLE p;
+
+
+#
+# Test D Outline:
+# ===============
+#
+# This test is similar to test A, where parent row is deleted while a child row
+# is inserted simultaneously on node 2. However, in this test case the FK
+# constraint's target column is a unique key, and parent row is not delete,
+# but this key value is changed so that insert on node 2 will cause FK
+# violation
+#
+# The p table will originally have rows (1, 0)
+# The c table will originally be empty 
+#
+# in node_1, parent row is updated to value (1,1)
+# A row (1, 0) pointing to the old version of parent row (1, 0) is inserted
+# in connection node_2
+#
+# Expected Outcome:
+# ================
+# This is a true conflict and one transaciton must abort. In this case it is node_1
+# transaction, which was scheduled later.
+#    Parent table should have row (1,0)
+#    child table should have row (1,0)
+#
+
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER UNIQUE KEY) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f2))  ;
+
+INSERT INTO p VALUES (1, 0);
+
+--let $mw_369_parent_query = UPDATE p SET f2 = 1 WHERE f1 = 1
+--let $mw_369_child_query = INSERT INTO c VALUES (1, 0);
+--source MW-369.inc
+
+# Commit fails
+--connection node_1
+--error ER_LOCK_DEADLOCK
+--reap
+
+--connection node_2
+SELECT * FROM p;
+SELECT * FROM c;
+
+DROP TABLE c;
+DROP TABLE p;
+
+#
+# Test E Outline:
+# ===============
+#
+# This test is similar to test B, where parent row is deleted while a child row
+# is updated simultaneously on node 2. However, in this test case the FK
+# constraint has ON DELETE CASCADE option, and the delete on parent row will
+# cascade a delete on child row as well. This will cause true conflict with 
+# connection node_2, which tries to update unrelated column on child table.
+#
+# The p table will originally have rows (1, 0), (2,0)
+# The c table will originally have row (1,1,0)
+#
+# in node_1, parent row (1,0) is deleted and cascaded delete will happen on
+# child table row (1,1,0).
+# in connection node_2 child table row is update to value (1,1,1)
+#
+# Expected Outcome:
+# ================
+# This is a true conflict and one transaciton must abort. In this case it is node_1
+# transaction, which was scheduled later.
+#    Parent table should have rows (1,0), (2,0)
+#    child table should have row (1,1,1)
+#
+
+
+CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER,
+                CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1)
+                ON DELETE CASCADE)  ;
+
+INSERT INTO p VALUES (1, 0);
+INSERT INTO p VALUES (2, 0);
+INSERT INTO c VALUES (1, 1, 0);
+
+--let $mw_369_parent_query = DELETE FROM p WHERE f1 = 1
+--let $mw_369_child_query = UPDATE c SET f2 = 1 WHERE f1 = 1
+--source MW-369.inc
+
+# Commit fails
+--connection node_1
+--error ER_LOCK_DEADLOCK
+--reap
+
+--connection node_2
+SELECT * FROM p;
+SELECT * FROM c;
+
+DROP TABLE c;
+DROP TABLE p;
+
+--sleep 3

storage/innobase/row/row0ins.cc

@@ -1290,11 +1290,12 @@ row_ins_foreign_check_on_constraint(
 
 #ifdef WITH_WSREP
 	err = wsrep_append_foreign_key(
-				       thr_get_trx(thr),
-				       foreign,
-				       clust_rec,
-				       clust_index,
-				       FALSE, FALSE);
+					thr_get_trx(thr),
+					foreign,
+					clust_rec,
+					clust_index,
+					FALSE,
+					(node) ? TRUE : FALSE);
 	if (err != DB_SUCCESS) {
 		fprintf(stderr,
 			"WSREP: foreign key append failed: %d\n", err);
@@ -1455,6 +1456,9 @@ row_ins_check_foreign_constraint(
 	ulint*		offsets		= offsets_;
 	rec_offs_init(offsets_);
 
+#ifdef WITH_WSREP
+	upd_node= NULL;
+#endif /* WITH_WSREP */
 run_again:
 #ifdef UNIV_SYNC_DEBUG
 	ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_SHARED));
@@ -1638,9 +1642,10 @@ row_ins_check_foreign_constraint(
 					err = wsrep_append_foreign_key(
 						thr_get_trx(thr),
 						foreign,
-						rec, 
-						check_index, 
-						check_ref, TRUE);
+						rec,
+						check_index,
+						check_ref,
+						(upd_node) ? TRUE : FALSE);
 #endif /* WITH_WSREP */
 					goto end_scan;
 				} else if (foreign->type != 0) {

storage/xtradb/row/row0ins.cc

@@ -1296,11 +1296,12 @@ row_ins_foreign_check_on_constraint(
 
 #ifdef WITH_WSREP
 	err = wsrep_append_foreign_key(
-				       thr_get_trx(thr),
-				       foreign,
-				       clust_rec,
-				       clust_index,
-				       FALSE, FALSE);
+					thr_get_trx(thr),
+					foreign,
+					clust_rec,
+					clust_index,
+					FALSE,
+					(node) ? TRUE : FALSE);
 	if (err != DB_SUCCESS) {
 		fprintf(stderr,
 			"WSREP: foreign key append failed: %d\n", err);
@@ -1461,6 +1462,9 @@ row_ins_check_foreign_constraint(
 	ulint*		offsets		= offsets_;
 	rec_offs_init(offsets_);
 
+#ifdef WITH_WSREP
+       upd_node= NULL;
+#endif /* WITH_WSREP */
 run_again:
 #ifdef UNIV_SYNC_DEBUG
 	ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_SHARED));
@@ -1651,8 +1655,9 @@ row_ins_check_foreign_constraint(
 						thr_get_trx(thr),
 						foreign,
 						rec, 
-						check_index, 
-						check_ref, TRUE);
+						check_index,
+						check_ref,
+						(upd_node) ? TRUE : FALSE);
 #endif /* WITH_WSREP */
 					goto end_scan;
 				} else if (foreign->type != 0) {