MDEV-9711 NO PAD collations

Based on the patch from Daniil Medvedev (a Google Summer of Code task)

include/m_ctype.h

@@ -362,6 +362,8 @@ struct my_collation_handler_st
 
 extern MY_COLLATION_HANDLER my_collation_8bit_bin_handler;
 extern MY_COLLATION_HANDLER my_collation_8bit_simple_ci_handler;
+extern MY_COLLATION_HANDLER my_collation_8bit_nopad_bin_handler;
+extern MY_COLLATION_HANDLER my_collation_8bit_simple_nopad_ci_handler;
 extern MY_COLLATION_HANDLER my_collation_ucs2_uca_handler;
 
 /* Some typedef to make it easy for C++ to make function pointers */
@@ -585,49 +587,81 @@ struct charset_info_st
 
 extern MYSQL_PLUGIN_IMPORT struct charset_info_st my_charset_bin;
 extern MYSQL_PLUGIN_IMPORT struct charset_info_st my_charset_latin1;
+extern MYSQL_PLUGIN_IMPORT struct charset_info_st my_charset_latin1_nopad;
 extern MYSQL_PLUGIN_IMPORT struct charset_info_st my_charset_filename;
 extern MYSQL_PLUGIN_IMPORT struct charset_info_st my_charset_utf8_general_ci;
 
 extern struct charset_info_st my_charset_big5_bin;
 extern struct charset_info_st my_charset_big5_chinese_ci;
+extern struct charset_info_st my_charset_big5_nopad_bin;
+extern struct charset_info_st my_charset_big5_chinese_nopad_ci;
 extern struct charset_info_st my_charset_cp1250_czech_ci;
 extern struct charset_info_st my_charset_cp932_bin;
 extern struct charset_info_st my_charset_cp932_japanese_ci;
+extern struct charset_info_st my_charset_cp932_nopad_bin;
+extern struct charset_info_st my_charset_cp932_japanese_nopad_ci;
 extern struct charset_info_st my_charset_eucjpms_bin;
 extern struct charset_info_st my_charset_eucjpms_japanese_ci;
+extern struct charset_info_st my_charset_eucjpms_nopad_bin;
+extern struct charset_info_st my_charset_eucjpms_japanese_nopad_ci;
 extern struct charset_info_st my_charset_euckr_bin;
 extern struct charset_info_st my_charset_euckr_korean_ci;
+extern struct charset_info_st my_charset_euckr_nopad_bin;
+extern struct charset_info_st my_charset_euckr_korean_nopad_ci;
 extern struct charset_info_st my_charset_gb2312_bin;
 extern struct charset_info_st my_charset_gb2312_chinese_ci;
+extern struct charset_info_st my_charset_gb2312_nopad_bin;
+extern struct charset_info_st my_charset_gb2312_chinese_nopad_ci;
 extern struct charset_info_st my_charset_gbk_bin;
 extern struct charset_info_st my_charset_gbk_chinese_ci;
+extern struct charset_info_st my_charset_gbk_nopad_bin;
+extern struct charset_info_st my_charset_gbk_chinese_nopad_ci;
 extern struct charset_info_st my_charset_latin1_bin;
+extern struct charset_info_st my_charset_latin1_nopad_bin;
 extern struct charset_info_st my_charset_latin1_german2_ci;
 extern struct charset_info_st my_charset_latin2_czech_ci;
 extern struct charset_info_st my_charset_sjis_bin;
 extern struct charset_info_st my_charset_sjis_japanese_ci;
+extern struct charset_info_st my_charset_sjis_nopad_bin;
+extern struct charset_info_st my_charset_sjis_japanese_nopad_ci;
 extern struct charset_info_st my_charset_tis620_bin;
 extern struct charset_info_st my_charset_tis620_thai_ci;
+extern struct charset_info_st my_charset_tis620_nopad_bin;
+extern struct charset_info_st my_charset_tis620_thai_nopad_ci;
 extern struct charset_info_st my_charset_ucs2_bin;
 extern struct charset_info_st my_charset_ucs2_general_ci;
+extern struct charset_info_st my_charset_ucs2_nopad_bin;
+extern struct charset_info_st my_charset_ucs2_general_nopad_ci;
 extern struct charset_info_st my_charset_ucs2_general_mysql500_ci;
 extern struct charset_info_st my_charset_ucs2_unicode_ci;
 extern struct charset_info_st my_charset_ucs2_general_mysql500_ci;
 extern struct charset_info_st my_charset_ujis_bin;
 extern struct charset_info_st my_charset_ujis_japanese_ci;
+extern struct charset_info_st my_charset_ujis_nopad_bin;
+extern struct charset_info_st my_charset_ujis_japanese_nopad_ci;
 extern struct charset_info_st my_charset_utf16_bin;
 extern struct charset_info_st my_charset_utf16_general_ci;
 extern struct charset_info_st my_charset_utf16_unicode_ci;
 extern struct charset_info_st my_charset_utf16le_bin;
 extern struct charset_info_st my_charset_utf16le_general_ci;
+extern struct charset_info_st my_charset_utf16_general_nopad_ci;
+extern struct charset_info_st my_charset_utf16_nopad_bin;
+extern struct charset_info_st my_charset_utf16le_nopad_bin;
+extern struct charset_info_st my_charset_utf16le_general_nopad_ci;
 extern struct charset_info_st my_charset_utf32_bin;
 extern struct charset_info_st my_charset_utf32_general_ci;
 extern struct charset_info_st my_charset_utf32_unicode_ci;
+extern struct charset_info_st my_charset_utf32_nopad_bin;
+extern struct charset_info_st my_charset_utf32_general_nopad_ci;
 extern struct charset_info_st my_charset_utf8_bin;
+extern struct charset_info_st my_charset_utf8_nopad_bin;
+extern struct charset_info_st my_charset_utf8_general_nopad_ci;
 extern struct charset_info_st my_charset_utf8_general_mysql500_ci;
 extern struct charset_info_st my_charset_utf8_unicode_ci;
 extern struct charset_info_st my_charset_utf8mb4_bin;
 extern struct charset_info_st my_charset_utf8mb4_general_ci;
+extern struct charset_info_st my_charset_utf8mb4_nopad_bin;
+extern struct charset_info_st my_charset_utf8mb4_general_nopad_ci;
 extern struct charset_info_st my_charset_utf8mb4_unicode_ci;
 
 #define MY_UTF8MB3                 "utf8"
@@ -653,6 +687,11 @@ extern int  my_strnncollsp_simple(CHARSET_INFO *, const uchar *, size_t,
 extern void my_hash_sort_simple(CHARSET_INFO *cs,
 				const uchar *key, size_t len,
 				ulong *nr1, ulong *nr2); 
+
+extern void my_hash_sort_simple_nopad(CHARSET_INFO *cs,
+				      const uchar *key, size_t len,
+				      ulong *nr1, ulong *nr2);
+
 extern void my_hash_sort_bin(CHARSET_INFO *cs,
                              const uchar *key, size_t len, ulong *nr1,
                              ulong *nr2);
@@ -824,18 +863,38 @@ int my_strcasecmp_mb_bin(CHARSET_INFO * cs __attribute__((unused)),
 void my_hash_sort_mb_bin(CHARSET_INFO *cs __attribute__((unused)),
                          const uchar *key, size_t len,ulong *nr1, ulong *nr2);
 
+void my_hash_sort_mb_nopad_bin(CHARSET_INFO *cs __attribute__((unused)),
+                               const uchar *key, size_t len,
+                               ulong *nr1, ulong *nr2);
+
 size_t my_strnxfrm_mb(CHARSET_INFO *,
                       uchar *dst, size_t dstlen, uint nweights,
                       const uchar *src, size_t srclen, uint flags);
 
+size_t my_strnxfrm_mb_nopad(CHARSET_INFO *,
+			    uchar *dst, size_t dstlen, uint nweights,
+			    const uchar *src, size_t srclen, uint flags);
+
 size_t my_strnxfrm_unicode(CHARSET_INFO *,
                            uchar *dst, size_t dstlen, uint nweights,
                            const uchar *src, size_t srclen, uint flags);
+
+size_t my_strnxfrm_unicode_nopad(CHARSET_INFO *,
+				 uchar *dst, size_t dstlen, uint nweights,
+				 const uchar *src, size_t srclen, uint flags);
+
 size_t  my_strnxfrmlen_unicode(CHARSET_INFO *, size_t); 
 
 size_t my_strnxfrm_unicode_full_bin(CHARSET_INFO *,
-                                    uchar *dst, size_t dstlen, uint nweights,
-                                    const uchar *src, size_t srclen, uint flags);
+                                    uchar *dst, size_t dstlen,
+                                    uint nweights, const uchar *src,
+                                    size_t srclen, uint flags);
+
+size_t my_strnxfrm_unicode_full_nopad_bin(CHARSET_INFO *,
+					  uchar *dst, size_t dstlen,
+					  uint nweights, const uchar *src,
+					  size_t srclen, uint flags);
+
 size_t  my_strnxfrmlen_unicode_full_bin(CHARSET_INFO *, size_t); 
 
 int my_wildcmp_unicode(CHARSET_INFO *cs,
@@ -873,6 +932,10 @@ void my_strxfrm_desc_and_reverse(uchar *str, uchar *strend,
 size_t my_strxfrm_pad_desc_and_reverse(CHARSET_INFO *cs,
                                        uchar *str, uchar *frmend, uchar *strend,
                                        uint nweights, uint flags, uint level);
+size_t my_strxfrm_pad_desc_and_reverse_nopad(CHARSET_INFO *cs,
+					     uchar *str, uchar *frmend,
+					     uchar *strend, uint nweights,
+					     uint flags, uint level);
 
 const MY_CONTRACTIONS *my_charset_get_contractions(CHARSET_INFO *cs,
                                                    int level);

mysql-test/include/ctype_pad.inc

+--echo #
+--echo # Start of ctype_pad.inc
+--echo #
+
+--echo #
+--echo # Unique indexes
+--echo #
+
+eval CREATE TABLE t1 (a VARCHAR(10) PRIMARY KEY) COLLATE $coll;
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES ('abc'),('abc  '),(' a'),(' a '),('a   ');
+SELECT HEX(a), a FROM t1 ORDER BY a;
+SELECT HEX(a), a FROM t1 IGNORE INDEX(PRIMARY) ORDER BY a;
+SELECT HEX(a), a FROM t1 IGNORE INDEX(PRIMARY) ORDER BY a DESC;
+
+--echo #
+--echo # UNION
+--echo #
+
+eval CREATE TABLE t2 (a VARCHAR(10)) COLLATE $coll;
+INSERT INTO t2 VALUES ('abc '),('abc  '),(' a'),('a ');
+SELECT HEX(a),a FROM (SELECT * FROM t1 UNION SELECT * FROM t2 ORDER BY a) td;
+DROP TABLE t1;
+DROP TABLE t2;
+
+--echo #
+--echo # DISTINCT, COUNT, MAX
+--echo #
+
+eval CREATE TABLE t1 (a VARCHAR(10)) COLLATE $coll;
+INSERT INTO t1 VALUES ('a'),('a  '),(' a'),(' a '),('a   ');
+SELECT HEX(a), a FROM (SELECT DISTINCT a FROM t1 ORDER BY a) td;
+SELECT COUNT(DISTINCT a) FROM t1 ORDER BY a;
+SELECT HEX(MAX(a)), MAX(a) FROM t1;
+
+--echo #
+--echo # GROUP BY
+--echo #
+
+eval CREATE TABLE t2 (a VARCHAR(10), b int, c varchar(10)) COLLATE $coll;
+INSERT t2 values('ab', 12, 'cd'), ('ab', 2, 'ed'), ('aa', 20, 'er'), ('aa  ', 0, 'er ');
+SELECT HEX(a), cnt FROM (SELECT a, COUNT(a) AS cnt FROM t2 GROUP BY a ORDER BY a) AS td;
+DROP TABLE t2;
+
+--echo #
+--echo # Weights
+--echo #
+
+SELECT HEX(WEIGHT_STRING(a AS CHAR(10))) FROM t1;
+DROP TABLE t1;
+
+--echo #
+--echo # IF, CASE, LEAST
+--echo #
+
+eval SELECT IF('abc' COLLATE $coll = 'abc ', 'pad', 'nopad');
+eval SELECT CASE 'abc' COLLATE $coll WHEN 'abc ' THEN 'pad' ELSE 'nopad' END;
+eval SELECT CASE WHEN 'abc' COLLATE $coll = 'abc ' THEN 'pad' ELSE 'nopad' END;
+eval SELECT HEX(LEAST('abc ' COLLATE $coll, 'abc  '));
+eval SELECT HEX(GREATEST('abc ' COLLATE $coll, 'abc  '));
+
+--echo #
+--echo # Collation mix
+--echo #
+
+eval CREATE TABLE t1 (a VARCHAR(10)) COLLATE $coll_pad;
+INSERT INTO t1 VALUES ('a'),('a ');
+SELECT COUNT(*) FROM t1 WHERE a='a';
+eval SELECT COUNT(*) FROM t1 WHERE a='a' COLLATE $coll_pad;
+eval SELECT COUNT(*) FROM t1 WHERE a='a' COLLATE $coll;
+eval ALTER TABLE t1 MODIFY a VARCHAR(10) COLLATE $coll;
+SELECT COUNT(*) FROM t1 WHERE a='a';
+eval SELECT COUNT(*) FROM t1 WHERE a='a' COLLATE $coll_pad;
+eval SELECT COUNT(*) FROM t1 WHERE a='a' COLLATE $coll;
+DROP TABLE t1;
+
+--echo #
+--echo # End of ctype_pad.inc
+--echo #

mysql-test/include/ctype_pad_all_engines.inc

+SET STORAGE_ENGINE=MyISAM;
+--source include/ctype_pad.inc
+
+SET STORAGE_ENGINE=HEAP;
+--source include/ctype_pad.inc
+
+SET STORAGE_ENGINE=Default;

mysql-test/std_data/ldml/Index.xml

@@ -328,6 +328,11 @@
       - ascii2_general_inherited_ci inherits sort order in ascii2.xml
       - ascii2_genegal_inherited2_ci inherits sort order directly in this file.
     -->
+    <collation name="ascii2_general_nopad_ci" id="318" flag="nopad">
+      <rules>
+        <import source="ascii2_general_ci"/>
+      </rules>
+    </collation>
     <collation name="ascii2_bin2" id="319" flag="binary"/>
     <collation name="ascii2_general_ci"	id="320" flag="primary"/>
     <collation name="ascii2_bin" id="321" flag="binary"/>
@@ -342,6 +347,7 @@
         <import source="ascii2_non_existing_ci"/>
       </rules>
     </collation>
+    <collation name="ascii2_nopad_bin" id="325" flag="binary" flag="nopad"/>
   </charset>
 
   <charset name="latin1">
@@ -357,6 +363,11 @@
     <collation name="latin1_test" id="99" order="test"/>
     <collation name="latin1_test2" id="332" order="test"/>
     <collation name="latin1_test2_cs" id="333"/>
+    <collation name="latin1_swedish_nopad2_ci" id="334" flag="nopad">
+      <rules>
+        <import source="latin1_swedish_ci"/>
+      </rules>
+    </collation>
   </charset>
 
   <charset name="utf8">

mysql-test/suite/innodb/t/innodb_ctype_big5.test

+--source include/have_innodb.inc
+--source include/have_big5.inc
+
+SET STORAGE_ENGINE=InnoDB;
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9711 NO PAD Collatons
+--echo #
+
+SET NAMES big5;
+let $coll='big5_chinese_nopad_ci';
+let $coll_pad='big5_chinese_ci';
+--source include/ctype_pad.inc
+
+let $coll='big5_nopad_bin';
+let $coll_pad='big5_bin';
+--source include/ctype_pad.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/suite/innodb/t/innodb_ctype_latin1.test

+--source include/have_innodb.inc
+
+SET STORAGE_ENGINE=InnoDB;
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9711 NO PAD Collatons
+--echo #
+
+SET NAMES latin1;
+
+let $coll='latin1_swedish_nopad_ci';
+let $coll_pad='latin1_swedish_ci';
+--source include/ctype_pad.inc
+
+let $coll='latin1_nopad_bin';
+let $coll_pad='latin1_bin';
+--source include/ctype_pad.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/suite/innodb/t/innodb_ctype_utf8.test

+--source include/have_innodb.inc
+
+SET STORAGE_ENGINE=InnoDB;
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9711 NO PAD Collatons
+--echo #
+
+SET NAMES utf8;
+
+let $coll='utf8_general_nopad_ci';
+let $coll_pad='utf8_general_ci';
+--source include/ctype_pad.inc
+
+let $coll='utf8_nopad_bin';
+let $coll_pad='utf8_bin';
+--source include/ctype_pad.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/t/ctype_big5.test

@@ -249,3 +249,23 @@ DROP TABLE t1;
 --echo #
 --echo # End of 10.0 tests
 --echo #
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9711 NO PAD Collatons
+--echo #
+SET character_set_connection=big5;
+let $coll='big5_chinese_nopad_ci';
+let $coll_pad='big5_chinese_ci';
+--source include/ctype_pad_all_engines.inc
+
+let $coll='big5_nopad_bin';
+let $coll_pad='big5_bin';
+--source include/ctype_pad_all_engines.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/t/ctype_cp932.test

@@ -27,3 +27,23 @@ DROP TABLE t1;
 SET @@character_set_client= @old_character_set_client;
 SET @@character_set_connection= @old_character_set_connection;
 SET @@character_set_results= @old_character_set_results;
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9711 NO PAD Collatons
+--echo #
+SET character_set_connection=cp932;
+let $coll='cp932_japanese_nopad_ci';
+let $coll_pad='cp932_japanese_ci';
+--source include/ctype_pad_all_engines.inc
+
+let $coll='cp932_nopad_bin';
+let $coll_pad='cp932_bin';
+--source include/ctype_pad_all_engines.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

strings/ctype.c

@@ -618,6 +618,8 @@ static int cs_value(MY_XML_PARSER *st,const char *attr, size_t len)
       i->cs.state|= MY_CS_BINSORT;
     else if (!strncmp("compiled",attr,len))
       i->cs.state|= MY_CS_COMPILED;
+    else if (!strncmp("nopad",attr,len))
+      i->cs.state|= MY_CS_NOPAD;
     break;
   case _CS_UPPERMAP:
     fill_uchar(i->to_upper,MY_CS_TO_UPPER_TABLE_SIZE,attr,len);