MDEV-31273: Precompute binlog checksums

Compute binlog checksums (when enabled) already when writing events
into the statement or transaction caches, where before it was done
when the caches are copied to the real binlog file. This moves the
checksum computation outside of holding LOCK_log, improving
scalabitily.

At stmt/trx cache write time, the final end_log_pos values are not
known, so with this patch these will be set to 0. Events that are
written directly to the binlog file (not through stmt/trx cache) keep
the correct end_log_pos value. The GTID and COMMIT/XID events at the
start and end of event groups are written directly, so the zero
end_log_pos is only for events in the middle of event groups, which
do not negatively affect replication.

An option --binlog-legacy-event-pos, off by default, is provided to
disable this behavior to provide backwards compatibility with any
external applications that might rely on end_log_pos in events in the
middle of event groups.

Checksums cannot be pre-computed when binlog encryption is enabled, as
encryption relies on correct end_log_pos to provide part of the
nonce/IV.

Checksum pre-computation is also disabled for WSREP/Galera, as it uses
events differently in its write-sets and so on. Extending pre-computation of
checksums to Galera where it makes sense could be added in a future patch.

The current --binlog-checksum configuration is saved in
binlog_cache_data at transaction start and used to pre-compute
checksums in cache, if applicable. When the cache is later copied to
the binlog, a check is made if the saved value still matches the
configured global value; if so, the events are block-copied directly
into the binlog file. If --binlog-checksum was changed during the
transaction, events are re-written to the binlog file one-by-one and
the checksums recomputed/discarded as appropriate.
Reviewed-by: Monty <monty@mariadb.org>
Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>

include/my_sys.h

@@ -607,6 +607,8 @@ static inline size_t my_b_bytes_in_cache(const IO_CACHE *info)
 
 int my_b_copy_to_file    (IO_CACHE *cache, FILE *file, size_t count);
 int my_b_copy_all_to_file(IO_CACHE *cache, FILE *file);
+int my_b_copy_to_cache(IO_CACHE *from_cache, IO_CACHE *to_cache, size_t count);
+int my_b_copy_all_to_cache(IO_CACHE *from_cache, IO_CACHE *to_cache);
 
 my_off_t my_b_append_tell(IO_CACHE* info);
 my_off_t my_b_safe_tell(IO_CACHE* info); /* picks the correct tell() */

mysql-test/main/mysqlbinlog_row_compressed.result

@@ -57,11 +57,11 @@ START TRANSACTION
 /*!*/;
 # at 787
 # at 861
-#<date> server id 1  end_log_pos 861 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (10, 1, 2, 3, 4, 5, 6, 7, "")
-#<date> server id 1  end_log_pos 917 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 917
-#<date> server id 1  end_log_pos 985 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -86,11 +86,11 @@ START TRANSACTION
 /*!*/;
 # at 1100
 # at 1176
-#<date> server id 1  end_log_pos 1176 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (11, 1, 2, 3, 4, 5, 6, 7, NULL)
-#<date> server id 1  end_log_pos 1232 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 1232
-#<date> server id 1  end_log_pos 1299 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=11 /* INT meta=0 nullable=0 is_null=0 */
@@ -115,11 +115,11 @@ START TRANSACTION
 /*!*/;
 # at 1414
 # at 1492
-#<date> server id 1  end_log_pos 1492 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (12, 1, 2, 3, NULL, 5, 6, 7, "A")
-#<date> server id 1  end_log_pos 1548 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 1548
-#<date> server id 1  end_log_pos 1614 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=12 /* INT meta=0 nullable=0 is_null=0 */
@@ -144,11 +144,11 @@ START TRANSACTION
 /*!*/;
 # at 1729
 # at 1804
-#<date> server id 1  end_log_pos 1804 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (13, 1, 2, 3, 0, 5, 6, 7, "A")
-#<date> server id 1  end_log_pos 1860 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 1860
-#<date> server id 1  end_log_pos 1927 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_compressed_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=13 /* INT meta=0 nullable=0 is_null=0 */
@@ -173,11 +173,11 @@ START TRANSACTION
 /*!*/;
 # at 2042
 # at 2096
-#<date> server id 1  end_log_pos 2096 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t2 SELECT * FROM t1
-#<date> server id 1  end_log_pos 2152 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
 # at 2152
-#<date> server id 1  end_log_pos 2243 CRC32 XXX 	Write_compressed_rows: table id 33 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_compressed_rows: table id 33 flags: STMT_END_F
 ### INSERT INTO `test`.`t2`
 ### SET
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -235,11 +235,11 @@ START TRANSACTION
 /*!*/;
 # at 2358
 # at 2424
-#<date> server id 1  end_log_pos 2424 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> UPDATE t2 SET f4=5 WHERE f4>0 or f4 is NULL
-#<date> server id 1  end_log_pos 2480 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
 # at 2480
-#<date> server id 1  end_log_pos 2579 CRC32 XXX 	Update_compressed_rows: table id 33 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Update_compressed_rows: table id 33 flags: STMT_END_F
 ### UPDATE `test`.`t2`
 ### WHERE
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -316,11 +316,11 @@ START TRANSACTION
 /*!*/;
 # at 2694
 # at 2731
-#<date> server id 1  end_log_pos 2731 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> DELETE FROM t1
-#<date> server id 1  end_log_pos 2787 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 2787
-#<date> server id 1  end_log_pos 2879 CRC32 XXX 	Delete_compressed_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Delete_compressed_rows: table id 32 flags: STMT_END_F
 ### DELETE FROM `test`.`t1`
 ### WHERE
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -378,11 +378,11 @@ START TRANSACTION
 /*!*/;
 # at 2994
 # at 3031
-#<date> server id 1  end_log_pos 3031 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> DELETE FROM t2
-#<date> server id 1  end_log_pos 3087 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
 # at 3087
-#<date> server id 1  end_log_pos 3172 CRC32 XXX 	Delete_compressed_rows: table id 33 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Delete_compressed_rows: table id 33 flags: STMT_END_F
 ### DELETE FROM `test`.`t2`
 ### WHERE
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */

mysql-test/main/mysqlbinlog_row_minimal.result

@@ -55,11 +55,11 @@ START TRANSACTION
 /*!*/;
 # at 834
 # at 908
-#<date> server id 1  end_log_pos 908 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (10, 1, 2, 3, 4, 5, 6, 7, "")
-#<date> server id 1  end_log_pos 964 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 964
-#<date> server id 1  end_log_pos 1033 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -84,11 +84,11 @@ START TRANSACTION
 /*!*/;
 # at 1148
 # at 1224
-#<date> server id 1  end_log_pos 1224 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (11, 1, 2, 3, 4, 5, 6, 7, NULL)
-#<date> server id 1  end_log_pos 1280 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 1280
-#<date> server id 1  end_log_pos 1348 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=11 /* INT meta=0 nullable=0 is_null=0 */
@@ -113,11 +113,11 @@ START TRANSACTION
 /*!*/;
 # at 1463
 # at 1541
-#<date> server id 1  end_log_pos 1541 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (12, 1, 2, 3, NULL, 5, 6, 7, "A")
-#<date> server id 1  end_log_pos 1597 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 1597
-#<date> server id 1  end_log_pos 1664 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=12 /* INT meta=0 nullable=0 is_null=0 */
@@ -142,11 +142,11 @@ START TRANSACTION
 /*!*/;
 # at 1779
 # at 1854
-#<date> server id 1  end_log_pos 1854 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t1 VALUES (13, 1, 2, 3, 0, 5, 6, 7, "A")
-#<date> server id 1  end_log_pos 1910 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 1910
-#<date> server id 1  end_log_pos 1980 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_rows: table id 32 flags: STMT_END_F
 ### INSERT INTO `test`.`t1`
 ### SET
 ###   @1=13 /* INT meta=0 nullable=0 is_null=0 */
@@ -171,11 +171,11 @@ START TRANSACTION
 /*!*/;
 # at 2095
 # at 2149
-#<date> server id 1  end_log_pos 2149 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> INSERT INTO t2 SELECT * FROM t1
-#<date> server id 1  end_log_pos 2205 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
 # at 2205
-#<date> server id 1  end_log_pos 2372 CRC32 XXX 	Write_rows: table id 33 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Write_rows: table id 33 flags: STMT_END_F
 ### INSERT INTO `test`.`t2`
 ### SET
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -233,11 +233,11 @@ START TRANSACTION
 /*!*/;
 # at 2487
 # at 2553
-#<date> server id 1  end_log_pos 2553 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> UPDATE t2 SET f4=5 WHERE f4>0 or f4 is NULL
-#<date> server id 1  end_log_pos 2609 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
 # at 2609
-#<date> server id 1  end_log_pos 2675 CRC32 XXX 	Update_rows: table id 33 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Update_rows: table id 33 flags: STMT_END_F
 ### UPDATE `test`.`t2`
 ### WHERE
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -266,11 +266,11 @@ START TRANSACTION
 /*!*/;
 # at 2790
 # at 2827
-#<date> server id 1  end_log_pos 2827 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> DELETE FROM t1
-#<date> server id 1  end_log_pos 2883 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t1` mapped to number num
 # at 2883
-#<date> server id 1  end_log_pos 2937 CRC32 XXX 	Delete_rows: table id 32 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Delete_rows: table id 32 flags: STMT_END_F
 ### DELETE FROM `test`.`t1`
 ### WHERE
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */
@@ -296,11 +296,11 @@ START TRANSACTION
 /*!*/;
 # at 3052
 # at 3089
-#<date> server id 1  end_log_pos 3089 CRC32 XXX 	Annotate_rows:
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Annotate_rows:
 #Q> DELETE FROM t2
-#<date> server id 1  end_log_pos 3145 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Table_map: `test`.`t2` mapped to number num
 # at 3145
-#<date> server id 1  end_log_pos 3199 CRC32 XXX 	Delete_rows: table id 33 flags: STMT_END_F
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Delete_rows: table id 33 flags: STMT_END_F
 ### DELETE FROM `test`.`t2`
 ### WHERE
 ###   @1=10 /* INT meta=0 nullable=0 is_null=0 */

mysql-test/main/mysqlbinlog_stmt_compressed.result

@@ -56,7 +56,7 @@ CREATE TABLE t2 (pk INT PRIMARY KEY, f1 INT, f2 INT, f3 INT, f4 INT, f5 MEDIUMIN
 START TRANSACTION
 /*!*/;
 # at 787
-#<date> server id 1  end_log_pos 915 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 INSERT INTO t1 VALUES (10, 1, 2, 3, 4, 5, 6, 7, "")
 /*!*/;
@@ -71,7 +71,7 @@ COMMIT
 START TRANSACTION
 /*!*/;
 # at 1030
-#<date> server id 1  end_log_pos 1158 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 INSERT INTO t1 VALUES (11, 1, 2, 3, 4, 5, 6, 7, NULL)
 /*!*/;
@@ -86,7 +86,7 @@ COMMIT
 START TRANSACTION
 /*!*/;
 # at 1273
-#<date> server id 1  end_log_pos 1403 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 INSERT INTO t1 VALUES (12, 1, 2, 3, NULL, 5, 6, 7, "A")
 /*!*/;
@@ -101,7 +101,7 @@ COMMIT
 START TRANSACTION
 /*!*/;
 # at 1518
-#<date> server id 1  end_log_pos 1645 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 INSERT INTO t1 VALUES (13, 1, 2, 3, 0, 5, 6, 7, "A")
 /*!*/;
@@ -116,7 +116,7 @@ COMMIT
 START TRANSACTION
 /*!*/;
 # at 1760
-#<date> server id 1  end_log_pos 1868 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 INSERT INTO t2 SELECT * FROM t1
 /*!*/;
@@ -131,7 +131,7 @@ COMMIT
 START TRANSACTION
 /*!*/;
 # at 1983
-#<date> server id 1  end_log_pos 2100 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 UPDATE t2 SET f4=5 WHERE f4>0 or f4 is NULL
 /*!*/;
@@ -146,7 +146,7 @@ COMMIT
 START TRANSACTION
 /*!*/;
 # at 2215
-#<date> server id 1  end_log_pos 2306 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 DELETE FROM t1
 /*!*/;
@@ -161,7 +161,7 @@ COMMIT
 START TRANSACTION
 /*!*/;
 # at 2421
-#<date> server id 1  end_log_pos 2512 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
+#<date> server id 1  end_log_pos 0 CRC32 XXX 	Query_compressed	thread_id=5	exec_time=x	error_code=0	xid=<xid>
 SET TIMESTAMP=X/*!*/;
 DELETE FROM t2
 /*!*/;

mysql-test/main/mysqld--help.result

@@ -101,6 +101,12 @@ The following specify which files/extra groups are read (specified before remain
  --binlog-ignore-db=name 
  Tells the master that updates to the given database
  should not be logged to the binary log.
+ --binlog-legacy-event-pos 
+ Fill in the end_log_pos field of _all_ events in the
+ binlog, even when doing so costs performance. Can be used
+ in case some old application needs it for backwards
+ compatibility. Setting this option can hurt binlog
+ scalability.
  --binlog-optimize-thread-scheduling 
  Run fast part of group commit in a single thread, to
  optimize kernel thread scheduling. On by default. Disable
@@ -1565,6 +1571,7 @@ binlog-direct-non-transactional-updates FALSE
 binlog-expire-logs-seconds 0
 binlog-file-cache-size 16384
 binlog-format MIXED
+binlog-legacy-event-pos FALSE
 binlog-optimize-thread-scheduling TRUE
 binlog-row-event-max-size 8192
 binlog-row-image FULL

mysql-test/suite/binlog/include/binlog_ioerr.inc

@@ -17,11 +17,14 @@ CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=innodb;
 INSERT INTO t1 VALUES(0);
 SET @saved_dbug = @@SESSION.debug_dbug;
 SET SESSION debug_dbug='+d,fail_binlog_write_1';
+# The error injection is in the "legacy" code path.
+SET GLOBAL binlog_legacy_event_pos= 1;
 --error ER_ERROR_ON_WRITE
 INSERT INTO t1 VALUES(1);
 --error ER_ERROR_ON_WRITE
 INSERT INTO t1 VALUES(2);
 SET SESSION debug_dbug=@saved_dbug;
+SET GLOBAL binlog_legacy_event_pos= 0;
 INSERT INTO t1 VALUES(3);
 SELECT * FROM t1;
 

mysql-test/suite/binlog/r/binlog_ioerr.result

@@ -4,11 +4,13 @@ CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=innodb;
 INSERT INTO t1 VALUES(0);
 SET @saved_dbug = @@SESSION.debug_dbug;
 SET SESSION debug_dbug='+d,fail_binlog_write_1';
+SET GLOBAL binlog_legacy_event_pos= 1;
 INSERT INTO t1 VALUES(1);
 ERROR HY000: Error writing file 'master-bin' (errno: 28 "No space left on device")
 INSERT INTO t1 VALUES(2);
 ERROR HY000: Error writing file 'master-bin' (errno: 28 "No space left on device")
 SET SESSION debug_dbug=@saved_dbug;
+SET GLOBAL binlog_legacy_event_pos= 0;
 INSERT INTO t1 VALUES(3);
 SELECT * FROM t1;
 a

mysql-test/suite/binlog/t/binlog_killed.test

@@ -67,7 +67,7 @@ let $rows= `select count(*) from t2  /* must be 2 or 0 */`;
 
 let $MYSQLD_DATADIR= `select @@datadir`;
 --let $binlog_killed_pos=query_get_value(SHOW BINLOG EVENTS, Pos, 6)
---let $binlog_killed_end_log_pos=query_get_value(SHOW BINLOG EVENTS, End_log_pos, 6)
+--let $binlog_killed_end_log_pos=query_get_value(SHOW BINLOG EVENTS, Pos, 7)
 --exec $MYSQL_BINLOG --force-if-open --start-position=$binlog_killed_pos --stop-position=$binlog_killed_end_log_pos $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/kill_query_calling_sp.binlog
 --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
 --disable_result_log

mysql-test/suite/binlog/t/binlog_table_map_optional_metadata.test

@@ -275,7 +275,7 @@ INSERT INTO t1 VALUES(2, "b");
 
 # The invalid metadata will case assertion failure on Write_rows_log_event
 # So we need to stop mysqlbinlog before reading Write_rows_log_event.
---let $stop_position= query_get_value(SHOW BINLOG EVENTS FROM $start_pos LIMIT 3, End_log_pos, 3)
+--let $stop_position= query_get_value(SHOW BINLOG EVENTS FROM $start_pos LIMIT 4, Pos, 4)
 --source include/print_optional_metadata.inc
 
 --echo #
@@ -291,7 +291,7 @@ INSERT INTO t1(c_point) VALUES(ST_PointFromText('POINT(10 10)'));
 
 # The invalid metadata will case assertion failure on Write_rows_log_event
 # So we need to stop mysqlbinlog before reading Write_rows_log_event.
---let $stop_position= query_get_value(SHOW BINLOG EVENTS FROM $start_pos LIMIT 3, End_log_pos, 3)
+--let $stop_position= query_get_value(SHOW BINLOG EVENTS FROM $start_pos LIMIT 4, Pos, 4)
 --source include/print_optional_metadata.inc
 
 DROP TABLE t1;

mysql-test/suite/binlog_encryption/binlog_ioerr.result

@@ -4,11 +4,13 @@ CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=innodb;
 INSERT INTO t1 VALUES(0);
 SET @saved_dbug = @@SESSION.debug_dbug;
 SET SESSION debug_dbug='+d,fail_binlog_write_1';
+SET GLOBAL binlog_legacy_event_pos= 1;
 INSERT INTO t1 VALUES(1);
 ERROR HY000: Error writing file 'master-bin' (errno: 28 "No space left on device")
 INSERT INTO t1 VALUES(2);
 ERROR HY000: Error writing file 'master-bin' (errno: 28 "No space left on device")
 SET SESSION debug_dbug=@saved_dbug;
+SET GLOBAL binlog_legacy_event_pos= 0;
 INSERT INTO t1 VALUES(3);
 SELECT * FROM t1;
 a

mysql-test/suite/rpl/r/rpl_checksum_cache.result

@@ -121,13 +121,54 @@ connection slave;
 include/diff_tables.inc [master:test.t1, slave:test.t1]
 include/diff_tables.inc [master:test.t2, slave:test.t2]
 include/diff_tables.inc [master:test.t3, slave:test.t3]
+*** Test switching checksum algorithm while ongoing transactions have pre-computed checksum in their binlog cache ***
+connection master;
+CREATE TABLE t4 (a INT, b INT, c VARCHAR(1024), PRIMARY KEY (a,b)) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t4 VALUES (1, 1, "small, pre-computed checksums");
+connection server_1;
+BEGIN;
+INSERT INTO t4 VALUES (2, 1, "big, pre-computed checksums");
+set @@global.binlog_checksum = NONE;
+connection master;
+INSERT INTO t4 VALUES (1, 2, "done");
+COMMIT;
+connection server_1;
+INSERT INTO t4 VALUES (2, 22, "done");
+COMMIT;
+connection master;
+BEGIN;
+INSERT INTO t4 VALUES (3, 1, "small, no pre-computed checksums");
+connection server_1;
+BEGIN;
+INSERT INTO t4 VALUES (4, 1, "big, no pre-computed checksums");
+set @@global.binlog_checksum = CRC32;
+connection master;
+INSERT INTO t4 VALUES (3, 2, "done");
+COMMIT;
+connection server_1;
+INSERT INTO t4 VALUES (4, 22, "done");
+COMMIT;
+connection slave;
+*** Test the --binlog-legacy-event-pos option.
+connection master;
+FLUSH BINARY LOGS;
+BEGIN;
+INSERT INTO t4 VALUES (5, 1, "Zero end_log_pos");
+COMMIT;
+set @@global.binlog_legacy_event_pos= 1;
+BEGIN;
+INSERT INTO t4 VALUES (6, 1, "Non-zero end_log_pos");
+COMMIT;
+set @@global.binlog_legacy_event_pos= 0;
+connection slave;
 connection master;
 begin;
 delete from t1;
 delete from t2;
 delete from t3;
 commit;
-drop table t1, t2, t3;
+drop table t1, t2, t3, t4;
 set @@global.binlog_cache_size = @save_binlog_cache_size;
 set @@global.binlog_checksum = @save_binlog_checksum;
 set @@global.master_verify_checksum = @save_master_verify_checksum;

mysql-test/suite/rpl/t/rpl_checksum_cache.test

@@ -243,6 +243,102 @@ let $diff_tables=master:test.t3, slave:test.t3;
 source include/diff_tables.inc;
 
 
+--echo *** Test switching checksum algorithm while ongoing transactions have pre-computed checksum in their binlog cache ***
+
+--connection master
+CREATE TABLE t4 (a INT, b INT, c VARCHAR(1024), PRIMARY KEY (a,b)) ENGINE=InnoDB;
+
+# Create a couple transactions that will precompute checksums but commit
+# without them.
+
+BEGIN;
+INSERT INTO t4 VALUES (1, 1, "small, pre-computed checksums");
+
+--connection server_1
+BEGIN;
+INSERT INTO t4 VALUES (2, 1, "big, pre-computed checksums");
+--let $i= 20
+--disable_query_log
+while ($i) {
+  eval INSERT INTO t4 VALUES (2, 22-$i, REPEAT("x", FLOOR(RAND()*100) + 831));
+  dec $i;
+}
+--enable_query_log
+
+# Disable checksums dynamically, so MYSQL_BIN_LOG::write_cache() will have
+# to drop the pre-computed checksums.
+set @@global.binlog_checksum = NONE;
+
+--connection master
+INSERT INTO t4 VALUES (1, 2, "done");
+COMMIT;
+--connection server_1
+INSERT INTO t4 VALUES (2, 22, "done");
+COMMIT;
+
+# Create a couple transactions that will not precompute checksums but commit
+# with them.
+
+--connection master
+BEGIN;
+INSERT INTO t4 VALUES (3, 1, "small, no pre-computed checksums");
+
+--connection server_1
+BEGIN;
+INSERT INTO t4 VALUES (4, 1, "big, no pre-computed checksums");
+--let $i= 20
+--disable_query_log
+while ($i) {
+  eval INSERT INTO t4 VALUES (4, 22-$i, REPEAT("x", FLOOR(RAND()*100) + 853));
+  dec $i;
+}
+--enable_query_log
+
+# Ebable checksums dynamically, so MYSQL_BIN_LOG::write_cache() will have
+# to recompute the checksums.
+set @@global.binlog_checksum = CRC32;
+
+--connection master
+INSERT INTO t4 VALUES (3, 2, "done");
+COMMIT;
+--connection server_1
+INSERT INTO t4 VALUES (4, 22, "done");
+COMMIT;
+
+sync_slave_with_master;
+
+
+--echo *** Test the --binlog-legacy-event-pos option.
+--connection master
+FLUSH BINARY LOGS;
+--source include/wait_for_binlog_checkpoint.inc
+
+--let $query_file= query_get_value(SHOW MASTER STATUS, File, 1)
+--let $query_pos= query_get_value(SHOW MASTER STATUS, Position, 1)
+BEGIN;
+INSERT INTO t4 VALUES (5, 1, "Zero end_log_pos");
+COMMIT;
+--let $end_log_pos= query_get_value(SHOW BINLOG EVENTS IN "$query_file" FROM $query_pos LIMIT 3, End_log_pos, 2)
+if ($end_log_pos!=0) {
+  eval SHOW BINLOG EVENTS IN "$query_file";
+  --die Wrong End_log_pos=$end_log_pos, expected zero.
+}
+
+set @@global.binlog_legacy_event_pos= 1;
+--let $query_pos= query_get_value(SHOW MASTER STATUS, Position, 1)
+BEGIN;
+INSERT INTO t4 VALUES (6, 1, "Non-zero end_log_pos");
+COMMIT;
+--let $end_log_pos= query_get_value(SHOW BINLOG EVENTS IN "$query_file" FROM $query_pos LIMIT 3, End_log_pos, 2)
+if ($end_log_pos==0) {
+  eval SHOW BINLOG EVENTS IN "$query_file";
+  --die Wrong End_log_pos=$end_log_pos, expected non-zero.
+}
+set @@global.binlog_legacy_event_pos= 0;
+
+sync_slave_with_master;
+
+
 connection master;
 
 begin;
@@ -251,7 +347,7 @@ delete from t2;
 delete from t3;
 commit;
 
-drop table t1, t2, t3;
+drop table t1, t2, t3, t4;
 set @@global.binlog_cache_size = @save_binlog_cache_size;
 set @@global.binlog_checksum = @save_binlog_checksum;
 set @@global.master_verify_checksum = @save_master_verify_checksum;

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -462,6 +462,16 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	NULL
 READ_ONLY	YES
 COMMAND_LINE_ARGUMENT	NULL
+VARIABLE_NAME	BINLOG_LEGACY_EVENT_POS
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	Fill in the end_log_pos field of _all_ events in the binlog, even when doing so costs performance. Can be used in case some old application needs it for backwards compatibility. Setting this option can hurt binlog scalability.
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	BINLOG_OPTIMIZE_THREAD_SCHEDULING
 VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	BOOLEAN

mysys/mf_iocache2.c

@@ -74,6 +74,41 @@ int my_b_copy_all_to_file(IO_CACHE *cache, FILE *file)
   DBUG_RETURN(my_b_copy_to_file(cache, file, SIZE_T_MAX));
 }
 
+/**
+   Similar to above my_b_copy_to_file(), but destination is another IO_CACHE.
+*/
+int
+my_b_copy_to_cache(IO_CACHE *from_cache, IO_CACHE *to_cache,
+                  size_t count)
+{
+  size_t curr_write, bytes_in_cache;
+  DBUG_ENTER("my_b_copy_to_cache");
+
+  bytes_in_cache= my_b_bytes_in_cache(from_cache);
+  do
+  {
+    curr_write= MY_MIN(bytes_in_cache, count);
+    if (my_b_write(to_cache, from_cache->read_pos, curr_write))
+      DBUG_RETURN(1);
+
+    from_cache->read_pos += curr_write;
+    count -= curr_write;
+  } while (count && (bytes_in_cache= my_b_fill(from_cache)));
+  if(from_cache->error == -1)
+    DBUG_RETURN(1);
+  DBUG_RETURN(0);
+}
+
+int my_b_copy_all_to_cache(IO_CACHE *from_cache, IO_CACHE *to_cache)
+{
+  DBUG_ENTER("my_b_copy_all_to_cache");
+  /* Reinit the cache to read from the beginning of the cache */
+  if (reinit_io_cache(from_cache, READ_CACHE, 0L, FALSE, FALSE))
+    DBUG_RETURN(1);
+  DBUG_RETURN(my_b_copy_to_cache(from_cache, to_cache,
+                                 from_cache->end_of_file));
+}
+
 my_off_t my_b_append_tell(IO_CACHE* info)
 {
   /*

sql/log.h

@@ -402,7 +402,7 @@ class Event_log: public MYSQL_LOG
                                        bool is_transactional);
   void set_write_error(THD *thd, bool is_transactional);
   static bool check_write_error(THD *thd);
-  int write_cache(THD *thd, IO_CACHE *cache);
+  int write_cache(THD *thd, binlog_cache_data *cache_data);
   int write_cache_raw(THD *thd, IO_CACHE *cache);
   char* get_name() { return name; }
   void cleanup()
@@ -427,6 +427,7 @@ class Event_log: public MYSQL_LOG
                                    bool encrypt, bool dont_set_created,
                                    bool is_relay_log);
 
+  bool write_event(Log_event *ev, binlog_cache_data *data, IO_CACHE *file);
   bool write_event(Log_event *ev, enum enum_binlog_checksum_alg checksum_alg,
                    binlog_cache_data *data, IO_CACHE *file);
 };
@@ -994,7 +995,6 @@ class MYSQL_BIN_LOG: public TC_LOG, private Event_log
 
   using Event_log::write_event;
 
-  bool write_event(Log_event *ev, binlog_cache_data *data, IO_CACHE *file);
   bool write_event(Log_event *ev, enum enum_binlog_checksum_alg checksum_alg)
   {
     return write_event(ev, checksum_alg, 0, &log_file);

sql/log_event.cc

@@ -720,7 +720,7 @@ Log_event::Log_event(const uchar *buf,
                      const Format_description_log_event* description_event)
   :temp_buf(0), exec_time(0), cache_type(Log_event::EVENT_INVALID_CACHE)
 #ifndef MYSQL_CLIENT
-    , slave_exec_mode(SLAVE_EXEC_MODE_STRICT),
+    , slave_exec_mode(SLAVE_EXEC_MODE_STRICT)
 #endif
 {
 #ifndef MYSQL_CLIENT

sql/log_event.h

@@ -1473,7 +1473,7 @@ class Log_event
   bool write_footer(Log_event_writer *writer)
   { return writer->write_footer(); }
 
-  enum_binlog_checksum_alg select_checksum_alg();
+  enum_binlog_checksum_alg select_checksum_alg(const binlog_cache_data *data);
 
   virtual bool write(Log_event_writer *writer)
   {

sql/log_event_server.cc

@@ -687,24 +687,6 @@ void Log_event::init_show_field_list(THD *thd, List<Item>* field_list)
                         mem_root);
 }
 
-/**
-   Select if and how to write checksum for an event written to the binlog.
-   It returns the actively configured binlog checksum option, unless the event
-   is being written to a cache (in which case the checksum, if any, is added
-   later when the cache is copied to the real binlog).
-*/
-enum_binlog_checksum_alg Log_event::select_checksum_alg()
-{
-  if (cache_type == Log_event::EVENT_NO_CACHE)
-    return (enum_binlog_checksum_alg)binlog_checksum_options;
-
-  DBUG_ASSERT(get_type_code() != ROTATE_EVENT &&
-              get_type_code() != STOP_EVENT &&
-              get_type_code() != FORMAT_DESCRIPTION_EVENT);
-
-  return BINLOG_CHECKSUM_ALG_OFF;
-}
-
 int Log_event_writer::write_internal(const uchar *pos, size_t len)
 {
   DBUG_ASSERT(!ctx || encrypt_or_write == &Log_event_writer::encrypt_and_write);
@@ -857,11 +839,17 @@ bool Log_event::write_header(Log_event_writer *writer, size_t event_data_length)
     change the position
   */
 
-  if (is_artificial_event())
+  if (is_artificial_event() ||
+      cache_type == Log_event::EVENT_STMT_CACHE ||
+      cache_type == Log_event::EVENT_TRANSACTIONAL_CACHE)
   {
     /*
       Artificial events are automatically generated and do not exist
       in master's binary log, so log_pos should be set to 0.
+
+      Events written through transaction or statement cache have log_pos set
+      to 0 so that they can be copied directly to the binlog without having
+      to compute the real end_log_pos.
     */
     log_pos= 0;
   }

sql/mysqld.cc

@@ -764,6 +764,7 @@ char *relay_log_info_file, *report_user, *report_password, *report_host;
 char *opt_relay_logname = 0, *opt_relaylog_index_name=0;
 char *opt_logname, *opt_slow_logname, *opt_bin_logname;
 char *opt_binlog_index_name=0;
+my_bool opt_binlog_legacy_event_pos= FALSE;
 
 /* Static variables */
 

sql/mysqld.h

@@ -173,6 +173,7 @@ extern ulong delay_key_write_options;
 extern char *opt_logname, *opt_slow_logname, *opt_bin_logname, 
             *opt_relay_logname;
 extern char *opt_binlog_index_name;
+extern my_bool opt_binlog_legacy_event_pos;
 extern char *opt_backup_history_logname, *opt_backup_progress_logname,
             *opt_backup_settings_name;
 extern const char *log_output_str;

sql/privilege.h

@@ -363,6 +363,9 @@ constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_COMMIT_WAIT_USEC=
 constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_ROW_METADATA=
   BINLOG_ADMIN_ACL;
 
+constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_LEGACY_EVENT_POS=
+  SUPER_ACL | BINLOG_ADMIN_ACL;
+
 constexpr privilege_t PRIV_SET_SYSTEM_GLOBAL_VAR_EXPIRE_LOGS_DAYS=
   BINLOG_ADMIN_ACL;
 

sql/rpl_record.cc

@@ -349,7 +349,7 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const colcnt,
   st.next_null_byte();
   if (!rpl_data.is_online_alter())
   {
-    Field *result_field;
+    Field *result_field= NULL;
     for (; i < colcnt && (result_field= table->field[i]); i++)
     {
       /*
@@ -439,7 +439,10 @@ int unpack_row(const rpl_group_info *rgi, TABLE *table, uint const colcnt,
     {
       DBUG_ASSERT(bitmap_is_set(cols, i));
       Field *f= conv_table->field[i];
-      bool result= unpack_field(tabledef, f, &st, i);
+#ifndef DBUG_OFF
+      bool result=
+#endif
+        unpack_field(tabledef, f, &st, i);
       DBUG_ASSERT(result);
     }
 

sql/sys_vars.cc

@@ -3505,6 +3505,19 @@ Sys_master_verify_checksum(
        GLOBAL_VAR(opt_master_verify_checksum), CMD_LINE(OPT_ARG),
        DEFAULT(FALSE));
 
+
+static Sys_var_on_access_global<Sys_var_mybool,
+                           PRIV_SET_SYSTEM_GLOBAL_VAR_BINLOG_LEGACY_EVENT_POS>
+Sys_binlog_legacy_event_pos(
+       "binlog_legacy_event_pos",
+       "Fill in the end_log_pos field of _all_ events in the binlog, even when "
+       "doing so costs performance. Can be used in case some old application needs "
+       "it for backwards compatibility. Setting this option can hurt binlog "
+       "scalability.",
+       GLOBAL_VAR(opt_binlog_legacy_event_pos), CMD_LINE(OPT_ARG),
+       DEFAULT(FALSE));
+
+
 /* These names must match RPL_SKIP_XXX #defines in slave.h. */
 static const char *replicate_events_marked_for_skip_names[]= {
   "REPLICATE", "FILTER_ON_SLAVE", "FILTER_ON_MASTER", 0