MDEV-32589 FULL_NODUP mode for binlog_row_image

This patch provides a new mode FULL_NODUP to binlog_row_image system
variable. With FULL_NODUP mode, all columns are included in before
image, but only updated columns are included in after image for UPDATE.
While all columns are included in the after image for INSERT.

FULL_NODUP is for replacing FULL mode. It includes all data of
the before and after image as FULL mode, but it uses less storage
especially in the case that only a few columns are updated.

Note: It will binlog full before and after image for all modes if the
      table has no primary key. FULL_NODUP follows the behavior.

mysql-test/main/mysqld--help.result

@@ -122,15 +122,17 @@ The following specify which files/extra groups are read (specified before remain
  size if possible. The value has to be a multiple of 256.
  --binlog-row-image=name 
  Controls whether rows should be logged in 'FULL',
- 'NOBLOB' or 'MINIMAL' formats. 'FULL', means that all
- columns in the before and after image are logged.
- 'NOBLOB', means that mysqld avoids logging blob columns
- whenever possible (eg, blob column was not changed or is
- not part of primary key). 'MINIMAL', means that a PK
- equivalent (PK columns or full row if there is no PK in
- the table) is logged in the before image, and only
- changed columns are logged in the after image. (Default:
- FULL).
+ 'FULL_NODUP', 'NOBLOB' or 'MINIMAL' formats. 'FULL',
+ means that all columns in the before and after image are
+ logged. 'FULL_NODUP', means that all columns are logged
+ in before image, but only changed columns or all columns
+ of inserted record are logged in after image, 'NOBLOB',
+ means that mysqld avoids logging blob columns whenever
+ possible (eg, blob column was not changed or is not part
+ of primary key). 'MINIMAL', means that a PK equivalent
+ (PK columns or full row if there is no PK in the table)
+ is logged in the before image, and only changed columns
+ are logged in the after image. (Default: FULL).
  --binlog-row-metadata=name 
  Controls whether metadata is logged using FULL , MINIMAL
  format and NO_LOG.FULL causes all metadata to be logged;

mysql-test/suite/binlog/include/row_img.test

+# Auxaliary script for test binlog_row_image
+#
+CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 varchar(100),
+                 c3 INT DEFAULT 1000, c4 TEXT);
+CREATE TABLE t2 (c1 INT, c2 char(100), c3 INT DEFAULT 1000, c4 TEXT);
+
+FLUSH BINARY LOGS;
+--let $binlog= query_get_value(SHOW MASTER STATUS, File, 1)
+--let $datadir= `SELECT @@datadir`
+
+--let $img_mode= `SELECT @@binlog_row_image`
+--echo
+--echo #########################################################################
+--echo # binlog_row_image = $img_mode
+--echo #########################################################################
+--echo
+INSERT INTO t1 VALUES(1, "insert_to_t1", 1, repeat('a', 20)),
+                     (2, "insert_to_t1", 2, repeat('a', 20)),
+                     (3, "insert_to_t1", 3, repeat('a', 20));
+INSERT INTO t1(c1) VALUES(4);
+UPDATE t1 SET c2 = "only_c2_changed";
+UPDATE t1 SET c3 = 1, c4 = "c3_c4_changed";
+DELETE FROM t1 WHERE c1 = 1;
+
+--echo
+--echo # Verify that rows events are binlogged as expeced.
+--echo
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--replace_regex /^\# at .*// /.*SET (@@|TIMESTAMP).*// /.* end_log_pos .*// /xid=\d*/xid=<xid>/
+--exec $MYSQL_BINLOG --force-if-open --verbose --verbose --base64-output=DECODE-ROWS $datadir/$binlog
+
+FLUSH BINARY LOGS;
+--let $binlog= query_get_value(SHOW MASTER STATUS, File, 1)
+--let $datadir= `SELECT @@datadir`
+
+--echo
+--echo # t2 has no primary key.
+--echo # It will binlog full before and after image for all modes if the
+--echo # table has no primary key. FULL_NODUP follows the behavior.
+--echo
+INSERT INTO t2 VALUES(1, "insert_to_t2", 1, repeat('a', 20)),
+                     (2, "insert_to_t2", 2, repeat('a', 20)),
+                     (3, "insert_to_t2", 3, repeat('a', 20));
+INSERT INTO t2(c1) VALUES(4);
+UPDATE t2 SET c2 = "only_c2_changed";
+UPDATE t2 SET c3 = 1, c4 = "c3_c4_changed";
+DELETE FROM t2 WHERE c1 > 2;
+
+--echo
+--echo # Verify that rows events are binlogged as expeced.
+--echo
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--replace_regex /^\# at .*// /.*SET (@@|TIMESTAMP).*// /.* end_log_pos .*// /xid=\d*/xid=<xid>/
+--exec $MYSQL_BINLOG --force-if-open --verbose --verbose --base64-output=DECODE-ROWS $datadir/$binlog
+
+DROP TABLE t1, t2;

mysql-test/suite/binlog/t/binlog_row_img.test

+################################################################################
+#  Support binlog_row_image FULL_NODUP mode
+#
+#  In FULL_NODUP:
+#  - before image includes all columns.
+#  - after image includes only changed columns.
+################################################################################
+
+--source include/have_binlog_format_row.inc
+--echo #
+--echo # FULL_NODUP mode works as expected
+--echo #
+SET binlog_row_image = "FULL_NODUP";
+--source suite/binlog/include/row_img.test
+
+--echo #
+--echo # MINIMAL mode works as expected
+--echo #
+SET binlog_row_image = "MINIMAL";
+--source suite/binlog/include/row_img.test
+
+--echo #
+--echo # NOBLOB mode works as expected
+--echo #
+SET binlog_row_image = "NOBLOB";
+--source suite/binlog/include/row_img.test
+
+--echo #
+--echo # FULL mode works as expected
+--echo #
+SET binlog_row_image = "FULL";
+--source suite/binlog/include/row_img.test
+
+RESET MASTER;
+

mysql-test/suite/rpl/t/rpl_row_img_blobs.test

@@ -47,6 +47,14 @@
 -- let $row_img_test_script= include/rpl_row_img_blobs.test
 -- source include/rpl_row_img_general_loop.inc
 
+## FULL_NODUP
+
+-- let $row_img_set=server_1:FULL_NODUP:N,server_2:FULL_NODUP:Y,server_3:FULL_NODUP:Y
+-- source include/rpl_row_img_set.inc
+
+-- let $row_img_test_script= include/rpl_row_img_blobs.test
+-- source include/rpl_row_img_general_loop.inc
+
 ## FULL
 
 -- let $row_img_set=server_1:FULL:N,server_2:FULL:Y,server_3:FULL:Y

mysql-test/suite/rpl/t/rpl_row_img_eng_full_nodup.cnf

+!include suite/rpl/t/rpl_row_img.cnf

mysql-test/suite/rpl/t/rpl_row_img_eng_full_nodup.test

+#Want to skip this test from daily Valgrind execution
+--source include/no_valgrind_without_big.inc
+#
+# This file contains tests for FULL_NODUP row image mode
+#
+
+--let $rpl_topology= 1->2->3
+--source include/rpl_init.inc
+-- source include/have_binlog_format_row.inc
+
+-- connection server_1
+-- source include/have_innodb.inc
+-- connection server_2
+-- source include/have_innodb.inc
+-- connection server_3
+-- source include/have_innodb.inc
+-- connection server_1
+
+#  Tests for different storage engines on each server,
+#  but same index structure on tables. The tests are conducted
+#  using FULL_NODUP binlog-row-image on all servers.
+#
+
+-- let $row_img_set=server_1:FULL_NODUP:N,server_2:FULL_NODUP:Y,server_3:FULL_NODUP:Y
+-- source include/rpl_row_img_set.inc
+
+-- let $row_img_test_script= include/rpl_row_img.test
+-- source include/rpl_row_img_general_loop.inc
+
+-- let $row_img_set=server_1:FULL:N,server_2:FULL:Y,server_3:FULL:Y
+-- source include/rpl_row_img_set.inc
+
+
+--source include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_row_img_full_nodup.cnf

+!include suite/rpl/t/rpl_row_img.cnf

mysql-test/suite/rpl/t/rpl_row_img_full_nodup.test

+#Want to skip this test from daily Valgrind execution
+--source include/no_valgrind_without_big.inc
+#
+# This file contains tests for FULL_NODUP row image mode
+#
+
+--let $rpl_topology= 1->2->3
+--source include/rpl_init.inc
+-- source include/have_binlog_format_row.inc
+
+-- connection server_1
+-- source include/have_innodb.inc
+-- connection server_2
+-- source include/have_innodb.inc
+-- connection server_3
+-- source include/have_innodb.inc
+-- connection server_1
+
+#  Tests for different storage engines on each server,
+#  but same index structure on tables. The tests are conducted
+#  using FULL_NODUP binlog-row-image on all servers.
+#
+
+-- let $row_img_set=server_1:FULL_NODUP:N,server_2:FULL_NODUP:Y,server_3:FULL_NODUP:Y
+-- source include/rpl_row_img_set.inc
+
+-- let $row_img_test_script= include/rpl_row_img.test
+-- source include/rpl_row_img_general_loop.inc
+
+-- let $row_img_set=server_1:FULL:N,server_2:FULL:Y,server_3:FULL:Y
+-- source include/rpl_row_img_set.inc
+
+
+--source include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_row_img_sequence_full_nodup.cnf

+!include include/default_mysqld.cnf
+
+[mysqld.1]
+log-slave-updates
+innodb
+gtid_domain_id=0
+
+[mysqld.2]
+log-slave-updates
+innodb
+gtid_domain_id=1
+
+[mysqld.3]
+log-slave-updates
+innodb
+gtid_domain_id=2
+
+[ENV]
+SERVER_MYPORT_1= @mysqld.1.port
+SERVER_MYPORT_2= @mysqld.2.port
+SERVER_MYPORT_3= @mysqld.3.port

mysql-test/suite/rpl/t/rpl_row_img_sequence_full_nodup.test

+#
+# Purpose:
+#   The rpl_row_img_sequence group of tests verify that sequence MDL updates,
+# i.e. NEXTVAL and SETVAL, respect the binlog_row_image variable value when
+# written into the binary log. In particular, it ensures that only changed
+# columns are written with MINIMAL image mode, and all columns are written
+# otherwise. This test focuses on validating the behavior of
+# binlog_row_img=FULL_NODUP.
+#
+# Methodology
+#   After issuing a sequence update, ensure that both 1) it was replicated
+# correctly, and 2) it was binlogged respective to the binlog_row_image value.
+# The sequence table does not use caching to ensure each update is immediately
+# binlogged. Each command is binlogged into its own unique log file, and the
+# entirety of the file is analyzed for correctness of its sequence event.
+# Specifically, mysqlbinlog is used in verbose mode so it outputs the columns
+# which belong to the event, and the columns are analyzed to ensure the correct
+# ones were logged. rpl_row_img_general_loop.inc is used to test with multiple
+# chained replicas, varying engines between InnoDB and MyISAM.
+#
+# References:
+#   MDEV-28487: sequences not respect value of binlog_row_image with select
+#               nextval(seq_gen)
+#
+
+--let $rpl_topology= 1->2->3
+--source include/rpl_init.inc
+--source include/have_binlog_format_row.inc
+
+--connection server_1
+--source include/have_innodb.inc
+--connection server_2
+--source include/have_innodb.inc
+--connection server_3
+--source include/have_innodb.inc
+--connection server_1
+
+--echo #
+--echo # binlog_row_image=FULL_NODUP should write all columns to the binary log
+--echo #
+--let $row_img_set=server_1:FULL_NODUP:N,server_2:FULL_NODUP:Y,server_3:FULL_NODUP:Y
+--source include/rpl_row_img_set.inc
+--let $expected_columns=(1,2,3,4,5,6,7,8)
+--let row_img_test_script= include/rpl_row_img_sequence.inc
+--source include/rpl_row_img_general_loop.inc
+
+--let $row_img_set=server_1:FULL:N,server_2:FULL:Y,server_3:FULL:Y
+--source include/rpl_row_img_set.inc
+
+--source include/rpl_end.inc
+--echo # End of tests

mysql-test/suite/sys_vars/r/sysvars_server_embedded.result

@@ -445,11 +445,11 @@ COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	BINLOG_ROW_IMAGE
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	ENUM
-VARIABLE_COMMENT	Controls whether rows should be logged in 'FULL', 'NOBLOB' or 'MINIMAL' formats. 'FULL', means that all columns in the before and after image are logged. 'NOBLOB', means that mysqld avoids logging blob columns whenever possible (eg, blob column was not changed or is not part of primary key). 'MINIMAL', means that a PK equivalent (PK columns or full row if there is no PK in the table) is logged in the before image, and only changed columns are logged in the after image. (Default: FULL).
+VARIABLE_COMMENT	Controls whether rows should be logged in 'FULL', 'FULL_NODUP', 'NOBLOB' or 'MINIMAL' formats. 'FULL', means that all columns in the before and after image are logged. 'FULL_NODUP', means that all columns are logged in before image, but only changed columns or all columns of inserted record are logged in after image, 'NOBLOB', means that mysqld avoids logging blob columns whenever possible (eg, blob column was not changed or is not part of primary key). 'MINIMAL', means that a PK equivalent (PK columns or full row if there is no PK in the table) is logged in the before image, and only changed columns are logged in the after image. (Default: FULL).
 NUMERIC_MIN_VALUE	NULL
 NUMERIC_MAX_VALUE	NULL
 NUMERIC_BLOCK_SIZE	NULL
-ENUM_VALUE_LIST	MINIMAL,NOBLOB,FULL
+ENUM_VALUE_LIST	MINIMAL,NOBLOB,FULL,FULL_NODUP
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	BINLOG_ROW_METADATA

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -495,11 +495,11 @@ COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	BINLOG_ROW_IMAGE
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	ENUM
-VARIABLE_COMMENT	Controls whether rows should be logged in 'FULL', 'NOBLOB' or 'MINIMAL' formats. 'FULL', means that all columns in the before and after image are logged. 'NOBLOB', means that mysqld avoids logging blob columns whenever possible (eg, blob column was not changed or is not part of primary key). 'MINIMAL', means that a PK equivalent (PK columns or full row if there is no PK in the table) is logged in the before image, and only changed columns are logged in the after image. (Default: FULL).
+VARIABLE_COMMENT	Controls whether rows should be logged in 'FULL', 'FULL_NODUP', 'NOBLOB' or 'MINIMAL' formats. 'FULL', means that all columns in the before and after image are logged. 'FULL_NODUP', means that all columns are logged in before image, but only changed columns or all columns of inserted record are logged in after image, 'NOBLOB', means that mysqld avoids logging blob columns whenever possible (eg, blob column was not changed or is not part of primary key). 'MINIMAL', means that a PK equivalent (PK columns or full row if there is no PK in the table) is logged in the before image, and only changed columns are logged in the after image. (Default: FULL).
 NUMERIC_MIN_VALUE	NULL
 NUMERIC_MAX_VALUE	NULL
 NUMERIC_BLOCK_SIZE	NULL
-ENUM_VALUE_LIST	MINIMAL,NOBLOB,FULL
+ENUM_VALUE_LIST	MINIMAL,NOBLOB,FULL,FULL_NODUP
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	BINLOG_ROW_METADATA

sql/sql_class.h

@@ -149,7 +149,9 @@ enum enum_binlog_row_image {
   /** Whenever possible, before and after image contain all columns except blobs. */
   BINLOG_ROW_IMAGE_NOBLOB= 1,
   /** All columns in both before and after image. */
-  BINLOG_ROW_IMAGE_FULL= 2
+  BINLOG_ROW_IMAGE_FULL= 2,
+  /** All columns in before image, but only updated columns in after image */
+  BINLOG_ROW_IMAGE_FULL_NODUP= 3
 };
 
 

sql/sys_vars.cc

@@ -6798,15 +6798,21 @@ static Sys_var_mybool Sys_binlog_encryption(
        READ_ONLY GLOBAL_VAR(encrypt_binlog), CMD_LINE(OPT_ARG),
        DEFAULT(FALSE));
 
-static const char *binlog_row_image_names[]= {"MINIMAL", "NOBLOB", "FULL", NullS};
+static const char *binlog_row_image_names[]=
+{
+  "MINIMAL", "NOBLOB", "FULL", "FULL_NODUP", NullS
+};
 static Sys_var_on_access<Sys_var_enum,
                          PRIV_SET_SYSTEM_VAR_BINLOG_ROW_IMAGE,
                          PRIV_SET_SYSTEM_VAR_BINLOG_ROW_IMAGE>
 Sys_binlog_row_image(
        "binlog_row_image",
-       "Controls whether rows should be logged in 'FULL', 'NOBLOB' or "
-       "'MINIMAL' formats. 'FULL', means that all columns in the before "
-       "and after image are logged. 'NOBLOB', means that mysqld avoids logging "
+       "Controls whether rows should be logged in 'FULL', 'FULL_NODUP', "
+       "'NOBLOB' or 'MINIMAL' formats. 'FULL', means that all columns in the "
+       "before and after image are logged. 'FULL_NODUP', means that all "
+       "columns are logged in before image, but only changed columns or all "
+       "columns of inserted record are logged in after image, "
+       "'NOBLOB', means that mysqld avoids logging "
        "blob columns whenever possible (eg, blob column was not changed or "
        "is not part of primary key). 'MINIMAL', means that a PK equivalent (PK "
        "columns or full row if there is no PK in the table) is logged in the "

sql/table.cc

@@ -7911,6 +7911,11 @@ void TABLE::mark_columns_needed_for_insert()
   if (vfield)
     mark_virtual_columns_for_write(TRUE);
   mark_columns_per_binlog_row_image();
+
+  /* FULL_NODUP is for replacing FULL mode, insert includes all columns. */
+  if (in_use->variables.binlog_row_image == BINLOG_ROW_IMAGE_FULL_NODUP)
+    rpl_write_set= read_set;
+
   if (check_constraints)
     mark_check_constraint_columns_for_read();
   DBUG_VOID_RETURN;
@@ -7981,6 +7986,11 @@ void TABLE::mark_columns_per_binlog_row_image()
         /* Set of columns that should be written (all) */
         rpl_write_set= read_set;
         break;
+      case BINLOG_ROW_IMAGE_FULL_NODUP:
+        bitmap_set_all(read_set);
+        // TODO: After MDEV-18432 we don't pass history rows, so remove this:
+        rpl_write_set= versioned() ? &s->all_set : write_set;
+        break;
       case BINLOG_ROW_IMAGE_NOBLOB:
         /* Only write changed columns + not blobs */
         rpl_write_set= &def_rpl_write_set;