MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to...

MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to never be null in my_strnncoll_binary on SELECT ... COUNT or GROUP_CONCAT

Also fixes:
  MDEV-30982 UBSAN: runtime error: null pointer passed as argument 2, which is declared to never be null in my_strnncoll_binary on DELETE

Calling memcmp() with a NULL pointer is undefined behaviour
according to the C standard, even if the length argument is 0.

Adding tests for length==0 before calling memcmp() into:
- my_strnncoll_binary()
- my_strnncoll_8bit_bin

mysql-test/main/ctype_binary.result

@@ -3362,3 +3362,37 @@ DROP FUNCTION f1;
 #
 # End of 10.3 tests
 #
+#
+# Start of 10.4 tests
+#
+#
+# MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to never be null in my_strnncoll_binary on SELECT ... COUNT or GROUP_CONCAT
+#
+CREATE TABLE t (c BLOB NOT NULL);
+INSERT IGNORE INTO t VALUES (0);
+SELECT COUNT(*) FROM t WHERE EXTRACTVALUE(c,'a')='a';
+COUNT(*)
+0
+DROP TABLE t;
+SET sql_mode='';
+CREATE TABLE t (c TEXT NOT NULL);
+INSERT INTO t VALUES();
+Warnings:
+Warning	1364	Field 'c' doesn't have a default value
+INSERT IGNORE INTO t VALUES (NULL);
+Warnings:
+Warning	1048	Column 'c' cannot be null
+SELECT GROUP_CONCAT(c ORDER BY BINARY c) FROM t GROUP BY c;
+GROUP_CONCAT(c ORDER BY BINARY c)
+,
+DROP TABLE t;
+#
+# MDEV-30982 UBSAN: runtime error: null pointer passed as argument 2, which is declared to never be null in my_strnncoll_binary on DELETE
+#
+CREATE TABLE t (c1 SET('1','2','3'),c2 BINARY);
+INSERT INTO t VALUES (0,0);
+DELETE FROM t WHERE c2<c1;
+DROP TABLE t;
+#
+# End of 10.4 tests
+#

mysql-test/main/ctype_binary.test

@@ -211,3 +211,36 @@ DROP FUNCTION f1;
 --echo #
 --echo # End of 10.3 tests
 --echo #
+
+--echo #
+--echo # Start of 10.4 tests
+--echo #
+
+--echo #
+--echo # MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to never be null in my_strnncoll_binary on SELECT ... COUNT or GROUP_CONCAT
+--echo #
+
+CREATE TABLE t (c BLOB NOT NULL);
+INSERT IGNORE INTO t VALUES (0);
+SELECT COUNT(*) FROM t WHERE EXTRACTVALUE(c,'a')='a';
+DROP TABLE t;
+
+SET sql_mode='';
+CREATE TABLE t (c TEXT NOT NULL);
+INSERT INTO t VALUES();
+INSERT IGNORE INTO t VALUES (NULL);
+SELECT GROUP_CONCAT(c ORDER BY BINARY c) FROM t GROUP BY c;
+DROP TABLE t;
+
+--echo #
+--echo # MDEV-30982 UBSAN: runtime error: null pointer passed as argument 2, which is declared to never be null in my_strnncoll_binary on DELETE
+--echo #
+
+CREATE TABLE t (c1 SET('1','2','3'),c2 BINARY);
+INSERT INTO t VALUES (0,0);
+DELETE FROM t WHERE c2<c1;
+DROP TABLE t;
+
+--echo #
+--echo # End of 10.4 tests
+--echo #

mysql-test/main/ctype_latin1.result

@@ -8891,3 +8891,30 @@ DROP TABLE t1;
 #
 # End of 10.2 tests
 #
+#
+# Start of 10.4 tests
+#
+#
+# MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to never be null in my_strnncoll_binary on SELECT ... COUNT or GROUP_CONCAT
+#
+CREATE TABLE t (c TEXT CHARACTER SET latin1 COLLATE latin1_bin NOT NULL);
+INSERT IGNORE INTO t VALUES (0);
+SELECT COUNT(*) FROM t WHERE EXTRACTVALUE(c,'a')='a';
+COUNT(*)
+0
+DROP TABLE t;
+SET sql_mode='';
+CREATE TABLE t (c TEXT CHARACTER SET latin1 COLLATE latin1_bin NOT NULL);
+INSERT INTO t VALUES();
+Warnings:
+Warning	1364	Field 'c' doesn't have a default value
+INSERT IGNORE INTO t VALUES (NULL);
+Warnings:
+Warning	1048	Column 'c' cannot be null
+SELECT GROUP_CONCAT(c ORDER BY BINARY c) FROM t GROUP BY c;
+GROUP_CONCAT(c ORDER BY BINARY c)
+,
+DROP TABLE t;
+#
+# End of 10.4 tests
+#

mysql-test/main/ctype_latin1.test

@@ -441,3 +441,27 @@ SET NAMES latin1;
 --echo #
 --echo # End of 10.2 tests
 --echo #
+
+--echo #
+--echo # Start of 10.4 tests
+--echo #
+
+--echo #
+--echo # MDEV-28384 UBSAN: null pointer passed as argument 1, which is declared to never be null in my_strnncoll_binary on SELECT ... COUNT or GROUP_CONCAT
+--echo #
+
+CREATE TABLE t (c TEXT CHARACTER SET latin1 COLLATE latin1_bin NOT NULL);
+INSERT IGNORE INTO t VALUES (0);
+SELECT COUNT(*) FROM t WHERE EXTRACTVALUE(c,'a')='a';
+DROP TABLE t;
+
+SET sql_mode='';
+CREATE TABLE t (c TEXT CHARACTER SET latin1 COLLATE latin1_bin NOT NULL);
+INSERT INTO t VALUES();
+INSERT IGNORE INTO t VALUES (NULL);
+SELECT GROUP_CONCAT(c ORDER BY BINARY c) FROM t GROUP BY c;
+DROP TABLE t;
+
+--echo #
+--echo # End of 10.4 tests
+--echo #

strings/ctype-bin.c

@@ -82,7 +82,7 @@ static int my_strnncoll_binary(CHARSET_INFO * cs __attribute__((unused)),
                                my_bool t_is_prefix)
 {
   size_t len=MY_MIN(slen,tlen);
-  int cmp= memcmp(s,t,len);
+  int cmp= len ? memcmp(s, t, len) : 0;
   return cmp ? cmp : (int)((t_is_prefix ? len : slen) - tlen);
 }
 
@@ -143,7 +143,7 @@ static int my_strnncoll_8bit_bin(CHARSET_INFO * cs __attribute__((unused)),
                                  my_bool t_is_prefix)
 {
   size_t len=MY_MIN(slen,tlen);
-  int cmp= memcmp(s,t,len);
+  int cmp= len ? memcmp(s, t, len) : 0;
   return cmp ? cmp : (int)((t_is_prefix ? len : slen) - tlen);
 }