MDEV-11685: sql_mode can't be set with non-ascii connection charset

The supplied sql_mode(s) should be converted to ASCII first,
before comparing it with the sql_mode set.

mysql-test/r/ctype_ucs.result

@@ -4357,5 +4357,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
 CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061))
 1
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=ucs2;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #

mysql-test/r/ctype_utf16.result

@@ -1207,5 +1207,15 @@ DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 Warnings:
 Warning	1300	Invalid utf16 character string: 'DE9899'
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=utf16;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #

mysql-test/r/ctype_utf32.result

@@ -1273,5 +1273,15 @@ select hex(lower(cast(0xffff0000 as char character set utf32))) as c;
 c
 FFFF0000
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=utf32;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #

mysql-test/t/ctype_ucs.test

@@ -853,6 +853,14 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061));
 SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061));
 SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=ucs2;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
 
 --echo #
 --echo # End of 5.5 tests

mysql-test/t/ctype_utf16.test

@@ -792,6 +792,15 @@ SELECT 'a' AS id, REPEAT('bla bla', 100) AS body) t1;
 DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=utf16;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

mysql-test/t/ctype_utf32.test

@@ -881,6 +881,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061));
 #
 select hex(lower(cast(0xffff0000 as char character set utf32))) as c;
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=utf32;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

sql/sys_vars.h

@@ -1141,7 +1141,7 @@ class Sys_var_set: public Sys_var_typelib
 
     if (var->value->result_type() == STRING_RESULT)
     {
-      if (!(res=var->value->val_str(&str)))
+      if (!(res=var->value->val_str_ascii(&str)))
         return true;
       else
       {