MDEV-10877 xxx_unicode_nopad_ci collations

include/m_ctype.h

@@ -634,6 +634,7 @@ extern struct charset_info_st my_charset_ucs2_nopad_bin;
 extern struct charset_info_st my_charset_ucs2_general_nopad_ci;
 extern struct charset_info_st my_charset_ucs2_general_mysql500_ci;
 extern struct charset_info_st my_charset_ucs2_unicode_ci;
+extern struct charset_info_st my_charset_ucs2_unicode_nopad_ci;
 extern struct charset_info_st my_charset_ucs2_general_mysql500_ci;
 extern struct charset_info_st my_charset_ujis_bin;
 extern struct charset_info_st my_charset_ujis_japanese_ci;
@@ -642,6 +643,7 @@ extern struct charset_info_st my_charset_ujis_japanese_nopad_ci;
 extern struct charset_info_st my_charset_utf16_bin;
 extern struct charset_info_st my_charset_utf16_general_ci;
 extern struct charset_info_st my_charset_utf16_unicode_ci;
+extern struct charset_info_st my_charset_utf16_unicode_nopad_ci;
 extern struct charset_info_st my_charset_utf16le_bin;
 extern struct charset_info_st my_charset_utf16le_general_ci;
 extern struct charset_info_st my_charset_utf16_general_nopad_ci;
@@ -651,6 +653,7 @@ extern struct charset_info_st my_charset_utf16le_general_nopad_ci;
 extern struct charset_info_st my_charset_utf32_bin;
 extern struct charset_info_st my_charset_utf32_general_ci;
 extern struct charset_info_st my_charset_utf32_unicode_ci;
+extern struct charset_info_st my_charset_utf32_unicode_nopad_ci;
 extern struct charset_info_st my_charset_utf32_nopad_bin;
 extern struct charset_info_st my_charset_utf32_general_nopad_ci;
 extern struct charset_info_st my_charset_utf8_bin;
@@ -658,11 +661,13 @@ extern struct charset_info_st my_charset_utf8_nopad_bin;
 extern struct charset_info_st my_charset_utf8_general_nopad_ci;
 extern struct charset_info_st my_charset_utf8_general_mysql500_ci;
 extern struct charset_info_st my_charset_utf8_unicode_ci;
+extern struct charset_info_st my_charset_utf8_unicode_nopad_ci;
 extern struct charset_info_st my_charset_utf8mb4_bin;
 extern struct charset_info_st my_charset_utf8mb4_general_ci;
 extern struct charset_info_st my_charset_utf8mb4_nopad_bin;
 extern struct charset_info_st my_charset_utf8mb4_general_nopad_ci;
 extern struct charset_info_st my_charset_utf8mb4_unicode_ci;
+extern struct charset_info_st my_charset_utf8mb4_unicode_nopad_ci;
 
 #define MY_UTF8MB3                 "utf8"
 #define MY_UTF8MB4                 "utf8mb4"

mysql-test/std_data/ldml/Index.xml

@@ -170,6 +170,8 @@
       <rules>
       </rules>
     </collation>
+    <collation name="utf8mb4_test_520_nopad_ci" id="329" version="5.2.0" flag="nopad">
+    </collation>
   </charset>
 
   <charset name="utf16">

mysql-test/suite/engines/funcs/r/db_alter_collate_ascii.result

@@ -129,6 +129,8 @@ utf8_myanmar_ci	utf8	577		#	#
 utf8_thai_520_w2	utf8	578		#	#
 utf8_general_nopad_ci	utf8	1057		#	#
 utf8_nopad_bin	utf8	1107		#	#
+utf8_unicode_nopad_ci	utf8	1216		#	#
+utf8_unicode_520_nopad_ci	utf8	1238		#	#
 ucs2_general_ci	ucs2	35	Yes	#	#
 ucs2_bin	ucs2	90		#	#
 ucs2_unicode_ci	ucs2	128		#	#
@@ -161,6 +163,8 @@ ucs2_myanmar_ci	ucs2	641		#	#
 ucs2_thai_520_w2	ucs2	642		#	#
 ucs2_general_nopad_ci	ucs2	1059		#	#
 ucs2_nopad_bin	ucs2	1114		#	#
+ucs2_unicode_nopad_ci	ucs2	1152		#	#
+ucs2_unicode_520_nopad_ci	ucs2	1174		#	#
 cp866_general_ci	cp866	36	Yes	#	#
 cp866_bin	cp866	68		#	#
 cp866_general_nopad_ci	cp866	1060		#	#
@@ -218,6 +222,8 @@ utf8mb4_myanmar_ci	utf8mb4	609		#	#
 utf8mb4_thai_520_w2	utf8mb4	610		#	#
 utf8mb4_general_nopad_ci	utf8mb4	1069		#	#
 utf8mb4_nopad_bin	utf8mb4	1070		#	#
+utf8mb4_unicode_nopad_ci	utf8mb4	1248		#	#
+utf8mb4_unicode_520_nopad_ci	utf8mb4	1270		#	#
 cp1251_bulgarian_ci	cp1251	14		#	#
 cp1251_ukrainian_ci	cp1251	23		#	#
 cp1251_bin	cp1251	50		#	#
@@ -256,6 +262,8 @@ utf16_myanmar_ci	utf16	673		#	#
 utf16_thai_520_w2	utf16	674		#	#
 utf16_general_nopad_ci	utf16	1078		#	#
 utf16_nopad_bin	utf16	1079		#	#
+utf16_unicode_nopad_ci	utf16	1125		#	#
+utf16_unicode_520_nopad_ci	utf16	1147		#	#
 utf16le_general_ci	utf16le	56	Yes	#	#
 utf16le_bin	utf16le	62		#	#
 utf16le_general_nopad_ci	utf16le	1080		#	#
@@ -300,6 +308,8 @@ utf32_myanmar_ci	utf32	737		#	#
 utf32_thai_520_w2	utf32	738		#	#
 utf32_general_nopad_ci	utf32	1084		#	#
 utf32_nopad_bin	utf32	1085		#	#
+utf32_unicode_nopad_ci	utf32	1184		#	#
+utf32_unicode_520_nopad_ci	utf32	1206		#	#
 binary	binary	63	Yes	#	#
 geostd8_general_ci	geostd8	92	Yes	#	#
 geostd8_bin	geostd8	93		#	#

mysql-test/suite/engines/funcs/r/db_alter_collate_utf8.result

@@ -129,6 +129,8 @@ utf8_myanmar_ci	utf8	577		#	#
 utf8_thai_520_w2	utf8	578		#	#
 utf8_general_nopad_ci	utf8	1057		#	#
 utf8_nopad_bin	utf8	1107		#	#
+utf8_unicode_nopad_ci	utf8	1216		#	#
+utf8_unicode_520_nopad_ci	utf8	1238		#	#
 ucs2_general_ci	ucs2	35	Yes	#	#
 ucs2_bin	ucs2	90		#	#
 ucs2_unicode_ci	ucs2	128		#	#
@@ -161,6 +163,8 @@ ucs2_myanmar_ci	ucs2	641		#	#
 ucs2_thai_520_w2	ucs2	642		#	#
 ucs2_general_nopad_ci	ucs2	1059		#	#
 ucs2_nopad_bin	ucs2	1114		#	#
+ucs2_unicode_nopad_ci	ucs2	1152		#	#
+ucs2_unicode_520_nopad_ci	ucs2	1174		#	#
 cp866_general_ci	cp866	36	Yes	#	#
 cp866_bin	cp866	68		#	#
 cp866_general_nopad_ci	cp866	1060		#	#
@@ -218,6 +222,8 @@ utf8mb4_myanmar_ci	utf8mb4	609		#	#
 utf8mb4_thai_520_w2	utf8mb4	610		#	#
 utf8mb4_general_nopad_ci	utf8mb4	1069		#	#
 utf8mb4_nopad_bin	utf8mb4	1070		#	#
+utf8mb4_unicode_nopad_ci	utf8mb4	1248		#	#
+utf8mb4_unicode_520_nopad_ci	utf8mb4	1270		#	#
 cp1251_bulgarian_ci	cp1251	14		#	#
 cp1251_ukrainian_ci	cp1251	23		#	#
 cp1251_bin	cp1251	50		#	#
@@ -256,6 +262,8 @@ utf16_myanmar_ci	utf16	673		#	#
 utf16_thai_520_w2	utf16	674		#	#
 utf16_general_nopad_ci	utf16	1078		#	#
 utf16_nopad_bin	utf16	1079		#	#
+utf16_unicode_nopad_ci	utf16	1125		#	#
+utf16_unicode_520_nopad_ci	utf16	1147		#	#
 utf16le_general_ci	utf16le	56	Yes	#	#
 utf16le_bin	utf16le	62		#	#
 utf16le_general_nopad_ci	utf16le	1080		#	#
@@ -300,6 +308,8 @@ utf32_myanmar_ci	utf32	737		#	#
 utf32_thai_520_w2	utf32	738		#	#
 utf32_general_nopad_ci	utf32	1084		#	#
 utf32_nopad_bin	utf32	1085		#	#
+utf32_unicode_nopad_ci	utf32	1184		#	#
+utf32_unicode_520_nopad_ci	utf32	1206		#	#
 binary	binary	63	Yes	#	#
 geostd8_general_ci	geostd8	92	Yes	#	#
 geostd8_bin	geostd8	93		#	#

mysql-test/t/ctype_ldml.test

@@ -482,3 +482,20 @@ SELECT a, a='a', a='a ', HEX(a), LOWER(a), UPPER(a), CONVERT(a USING utf8) FROM
 ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_swedish_nopad2_ci;
 SELECT a, a='a', a='a ', HEX(a), LOWER(a), UPPER(a), CONVERT(a USING utf8) FROM t1;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-10877 xxx_unicode_nopad_ci collations
+--echo #
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_test_520_nopad_ci);
+INSERT INTO t1 VALUES ('a'),('b'),('a '),('b ');
+SELECT
+  HEX(a),
+  WEIGHT_STRING(a)=WEIGHT_STRING(a COLLATE utf8mb4_unicode_ci) AS is_400,
+  WEIGHT_STRING(a)=WEIGHT_STRING(a COLLATE utf8mb4_unicode_520_ci) AS is_520
+FROM t1
+ORDER BY a;
+SELECT COUNT(DISTINCT a) FROM t1;
+SELECT HEX(a), REPLACE(a,' ','<SP>') FROM t1 WHERE a='a';
+SELECT HEX(a), REPLACE(a,' ','<SP>') FROM t1 ORDER BY a;
+SELECT HEX(a), REPLACE(a,' ','<SP>') FROM t1 ORDER BY a DESC;
+DROP TABLE t1;

mysql-test/t/ctype_ucs2_uca.test

+-- source include/have_ucs2.inc
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-10877 xxx_unicode_nopad_ci collations
+--echo #
+SET NAMES utf8, character_set_connection=ucs2;
+let $coll='ucs2_unicode_nopad_ci';
+let $coll_pad='ucs2_unicode_ci';
+--source include/ctype_pad_all_engines.inc
+
+let $coll='ucs2_unicode_520_nopad_ci';
+let $coll_pad='ucs2_unicode_520_ci';
+--source include/ctype_pad_all_engines.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/t/ctype_utf16_uca.test

@@ -221,3 +221,23 @@ SET collation_connection=utf16_thai_520_w2;
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-10877 xxx_unicode_nopad_ci collations
+--echo #
+SET NAMES utf8, character_set_connection=utf16;
+let $coll='utf16_unicode_nopad_ci';
+let $coll_pad='utf16_unicode_ci';
+--source include/ctype_pad_all_engines.inc
+
+let $coll='utf16_unicode_520_nopad_ci';
+let $coll_pad='utf16_unicode_520_ci';
+--source include/ctype_pad_all_engines.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/t/ctype_utf32_uca.test

@@ -242,3 +242,24 @@ SET collation_connection=utf32_thai_520_w2;
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-10877 xxx_unicode_nopad_ci collations
+--echo #
+SET NAMES utf8, character_set_connection=utf32;
+let $coll='utf32_unicode_nopad_ci';
+let $coll_pad='utf32_unicode_ci';
+--source include/ctype_pad_all_engines.inc
+
+let $coll='utf32_unicode_520_nopad_ci';
+let $coll_pad='utf32_unicode_520_ci';
+--source include/ctype_pad_all_engines.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/t/ctype_utf8_uca.test

+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-10877 xxx_unicode_nopad_ci collations
+--echo #
+SET NAMES utf8;
+let $coll='utf8_unicode_nopad_ci';
+let $coll_pad='utf8_unicode_ci';
+--source include/ctype_pad_all_engines.inc
+
+let $coll='utf8_unicode_520_nopad_ci';
+let $coll_pad='utf8_unicode_520_ci';
+--source include/ctype_pad_all_engines.inc
+
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysql-test/t/ctype_utf8mb4_uca.test

@@ -83,3 +83,23 @@ SET collation_connection=utf8mb4_thai_520_w2;
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9711 NO PAD Collatons
+--echo #
+SET NAMES utf8mb4;
+let $coll='utf8mb4_unicode_nopad_ci';
+let $coll_pad='utf8mb4_unicode_ci';
+--source include/ctype_pad_all_engines.inc
+
+let $coll='utf8mb4_unicode_520_nopad_ci';
+let $coll_pad='utf8mb4_unicode_520_ci';
+--source include/ctype_pad_all_engines.inc
+
+--echo #
+--echo # End of 10.2 tests
+--echo #

mysys/charset-def.c

@@ -50,6 +50,7 @@ extern struct charset_info_st my_charset_ucs2_vietnamese_ci;
 extern struct charset_info_st my_charset_ucs2_croatian_uca_ci;
 extern struct charset_info_st my_charset_ucs2_myanmar_uca_ci;
 extern struct charset_info_st my_charset_ucs2_thai_520_w2;
+extern struct charset_info_st my_charset_ucs2_unicode_520_nopad_ci;
 #endif
 
 
@@ -80,6 +81,7 @@ extern struct charset_info_st my_charset_utf32_vietnamese_ci;
 extern struct charset_info_st my_charset_utf32_croatian_uca_ci;
 extern struct charset_info_st my_charset_utf32_myanmar_uca_ci;
 extern struct charset_info_st my_charset_utf32_thai_520_w2;
+extern struct charset_info_st my_charset_utf32_unicode_520_nopad_ci;
 #endif /* HAVE_CHARSET_utf32 */
 
 
@@ -110,6 +112,7 @@ extern struct charset_info_st my_charset_utf16_vietnamese_ci;
 extern struct charset_info_st my_charset_utf16_croatian_uca_ci;
 extern struct charset_info_st my_charset_utf16_myanmar_uca_ci;
 extern struct charset_info_st my_charset_utf16_thai_520_w2;
+extern struct charset_info_st my_charset_utf16_unicode_520_nopad_ci;
 #endif  /* HAVE_CHARSET_utf16 */
 
 
@@ -143,6 +146,7 @@ extern struct charset_info_st my_charset_utf8_thai_520_w2;
 #ifdef HAVE_UTF8_GENERAL_CS
 extern struct charset_info_st my_charset_utf8_general_cs;
 #endif
+extern struct charset_info_st my_charset_utf8_unicode_520_nopad_ci;
 #endif
 
 #ifdef HAVE_CHARSET_utf8mb4
@@ -172,6 +176,7 @@ extern struct charset_info_st my_charset_utf8mb4_vietnamese_ci;
 extern struct charset_info_st my_charset_utf8mb4_croatian_uca_ci;
 extern struct charset_info_st my_charset_utf8mb4_myanmar_uca_ci;
 extern struct charset_info_st my_charset_utf8mb4_thai_520_w2;
+extern struct charset_info_st my_charset_utf8mb4_unicode_520_nopad_ci;
 #endif /* HAVE_CHARSET_utf8mb4 */
 
 #endif /* HAVE_UCA_COLLATIONS */
@@ -287,6 +292,8 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused)))
   add_compiled_collation(&my_charset_ucs2_croatian_uca_ci);
   add_compiled_collation(&my_charset_ucs2_myanmar_uca_ci);
   add_compiled_collation(&my_charset_ucs2_thai_520_w2);
+  add_compiled_collation(&my_charset_ucs2_unicode_nopad_ci);
+  add_compiled_collation(&my_charset_ucs2_unicode_520_nopad_ci);
 #endif
 #endif
 
@@ -334,6 +341,8 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused)))
   add_compiled_collation(&my_charset_utf8_croatian_uca_ci);
   add_compiled_collation(&my_charset_utf8_myanmar_uca_ci);
   add_compiled_collation(&my_charset_utf8_thai_520_w2);
+  add_compiled_collation(&my_charset_utf8_unicode_nopad_ci);
+  add_compiled_collation(&my_charset_utf8_unicode_520_nopad_ci);
 #endif
 #endif /* HAVE_CHARSET_utf8 */
 
@@ -371,6 +380,8 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused)))
   add_compiled_collation(&my_charset_utf8mb4_croatian_uca_ci);
   add_compiled_collation(&my_charset_utf8mb4_myanmar_uca_ci);
   add_compiled_collation(&my_charset_utf8mb4_thai_520_w2);
+  add_compiled_collation(&my_charset_utf8mb4_unicode_nopad_ci);
+  add_compiled_collation(&my_charset_utf8mb4_unicode_520_nopad_ci);
 #endif /* HAVE_UCA_COLLATIONS  */
 #endif /* HAVE_CHARSET_utf8mb4 */
 
@@ -412,6 +423,8 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused)))
   add_compiled_collation(&my_charset_utf16_croatian_uca_ci);
   add_compiled_collation(&my_charset_utf16_myanmar_uca_ci);
   add_compiled_collation(&my_charset_utf16_thai_520_w2);
+  add_compiled_collation(&my_charset_utf16_unicode_nopad_ci);
+  add_compiled_collation(&my_charset_utf16_unicode_520_nopad_ci);
 #endif /* HAVE_UCA_COLLATIONS */
 #endif /* HAVE_CHARSET_utf16 */
 
@@ -449,6 +462,8 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused)))
   add_compiled_collation(&my_charset_utf32_croatian_uca_ci);
   add_compiled_collation(&my_charset_utf32_myanmar_uca_ci);
   add_compiled_collation(&my_charset_utf32_thai_520_w2);
+  add_compiled_collation(&my_charset_utf32_unicode_nopad_ci);
+  add_compiled_collation(&my_charset_utf32_unicode_520_nopad_ci);
 #endif /* HAVE_UCA_COLLATIONS */
 #endif /* HAVE_CHARSET_utf32 */
 

mysys/charset.c

@@ -310,14 +310,18 @@ static int add_collation(struct charset_info_st *cs)
       if (!strcmp(cs->csname,"ucs2") )
       {
 #if defined(HAVE_CHARSET_ucs2) && defined(HAVE_UCA_COLLATIONS)
-        copy_uca_collation(newcs, &my_charset_ucs2_unicode_ci);
+        copy_uca_collation(newcs, newcs->state & MY_CS_NOPAD ?
+                                  &my_charset_ucs2_unicode_nopad_ci :
+                                  &my_charset_ucs2_unicode_ci);
         newcs->state|= MY_CS_AVAILABLE | MY_CS_LOADED | MY_CS_NONASCII;
 #endif        
       }
       else if (!strcmp(cs->csname, "utf8") || !strcmp(cs->csname, "utf8mb3"))
       {
 #if defined (HAVE_CHARSET_utf8) && defined(HAVE_UCA_COLLATIONS)
-        copy_uca_collation(newcs, &my_charset_utf8_unicode_ci);
+        copy_uca_collation(newcs, newcs->state & MY_CS_NOPAD ?
+                                  &my_charset_utf8_unicode_nopad_ci :
+                                  &my_charset_utf8_unicode_ci);
         newcs->ctype= my_charset_utf8_unicode_ci.ctype;
         if (init_state_maps(newcs))
           return MY_XML_ERROR;
@@ -326,7 +330,9 @@ static int add_collation(struct charset_info_st *cs)
       else if (!strcmp(cs->csname, "utf8mb4"))
       {
 #if defined (HAVE_CHARSET_utf8mb4) && defined(HAVE_UCA_COLLATIONS)
-        copy_uca_collation(newcs, &my_charset_utf8mb4_unicode_ci);
+        copy_uca_collation(newcs, newcs->state & MY_CS_NOPAD ?
+                                  &my_charset_utf8mb4_unicode_nopad_ci :
+                                  &my_charset_utf8mb4_unicode_ci);
         newcs->ctype= my_charset_utf8mb4_unicode_ci.ctype;
         newcs->state|= MY_CS_AVAILABLE | MY_CS_LOADED;
 #endif
@@ -334,14 +340,18 @@ static int add_collation(struct charset_info_st *cs)
       else if (!strcmp(cs->csname, "utf16"))
       {
 #if defined (HAVE_CHARSET_utf16) && defined(HAVE_UCA_COLLATIONS)
-        copy_uca_collation(newcs, &my_charset_utf16_unicode_ci);
+        copy_uca_collation(newcs, newcs->state & MY_CS_NOPAD ?
+                                  &my_charset_utf16_unicode_nopad_ci :
+                                  &my_charset_utf16_unicode_ci);
         newcs->state|= MY_CS_AVAILABLE | MY_CS_LOADED | MY_CS_NONASCII;
 #endif
       }
       else if (!strcmp(cs->csname, "utf32"))
       {
 #if defined (HAVE_CHARSET_utf32) && defined(HAVE_UCA_COLLATIONS)
-        copy_uca_collation(newcs, &my_charset_utf32_unicode_ci);
+        copy_uca_collation(newcs, newcs->state & MY_CS_NOPAD ?
+                                  &my_charset_utf32_unicode_nopad_ci :
+                                  &my_charset_utf32_unicode_ci);
         newcs->state|= MY_CS_AVAILABLE | MY_CS_LOADED | MY_CS_NONASCII;
 #endif
       }