Bug#30986 Character set introducer followed by a HEX string can return bad result(addon)

issue an error if string has illegal characters

mysql-test/r/ctype_utf8.result

@@ -1742,25 +1742,13 @@ NULL
 Warnings:
 Error	1300	Invalid utf8 character string: 'FF'
 select hex(_utf8 0x616263FF);
-hex(_utf8 0x616263FF)
-NULL
-Warnings:
-Error	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'
 select hex(_utf8 X'616263FF');
-hex(_utf8 X'616263FF')
-NULL
-Warnings:
-Error	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'
 select hex(_utf8 B'001111111111');
-hex(_utf8 B'001111111111')
-NULL
-Warnings:
-Error	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'
 select (_utf8 X'616263FF');
-(_utf8 X'616263FF')
-NULL
-Warnings:
-Error	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'
 set sql_mode=default;
 select hex(char(0xFF using utf8));
 hex(char(0xFF using utf8))
@@ -1773,22 +1761,10 @@ hex(convert(0xFF using utf8))
 Warnings:
 Warning	1300	Invalid utf8 character string: 'FF'
 select hex(_utf8 0x616263FF);
-hex(_utf8 0x616263FF)
-616263
-Warnings:
-Warning	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'
 select hex(_utf8 X'616263FF');
-hex(_utf8 X'616263FF')
-616263
-Warnings:
-Warning	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'
 select hex(_utf8 B'001111111111');
-hex(_utf8 B'001111111111')
-03
-Warnings:
-Warning	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'
 select (_utf8 X'616263FF');
-(_utf8 X'616263FF')
-abc
-Warnings:
-Warning	1300	Invalid utf8 character string: 'FF'
+ERROR HY000: Invalid utf8 character string: 'FF'

mysql-test/t/ctype_utf8.test

@@ -1411,14 +1411,22 @@ DROP TABLE t1, t2;
 set sql_mode=traditional;
 select hex(char(0xFF using utf8));
 select hex(convert(0xFF using utf8));
+--error ER_INVALID_CHARACTER_STRING
 select hex(_utf8 0x616263FF);
+--error ER_INVALID_CHARACTER_STRING
 select hex(_utf8 X'616263FF');
+--error ER_INVALID_CHARACTER_STRING
 select hex(_utf8 B'001111111111');
+--error ER_INVALID_CHARACTER_STRING
 select (_utf8 X'616263FF');
 set sql_mode=default;
 select hex(char(0xFF using utf8));
 select hex(convert(0xFF using utf8));
+--error ER_INVALID_CHARACTER_STRING
 select hex(_utf8 0x616263FF);
+--error ER_INVALID_CHARACTER_STRING
 select hex(_utf8 X'616263FF');
+--error ER_INVALID_CHARACTER_STRING
 select hex(_utf8 B'001111111111');
+--error ER_INVALID_CHARACTER_STRING
 select (_utf8 X'616263FF');

sql/item.cc

@@ -4247,7 +4247,7 @@ bool Item::is_datetime()
 }
 
 
-String *Item::check_well_formed_result(String *str)
+String *Item::check_well_formed_result(String *str, bool send_error)
 {
   /* Check whether we got a well-formed string */
   CHARSET_INFO *cs= str->charset();
@@ -4263,8 +4263,14 @@ String *Item::check_well_formed_result(String *str)
     uint diff= str->length() - wlen;
     set_if_smaller(diff, 3);
     octet2hex(hexbuf, str->ptr() + wlen, diff);
-    if (thd->variables.sql_mode &
-        (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES))
+    if (send_error)
+    {
+      my_error(ER_INVALID_CHARACTER_STRING, MYF(0),
+               cs->csname,  hexbuf);
+      return 0;
+    }
+    if ((thd->variables.sql_mode &
+         (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES)))
     {
       level= MYSQL_ERROR::WARN_LEVEL_ERROR;
       null_value= 1;

sql/item.h

@@ -870,7 +870,7 @@ class Item {
   */
   virtual bool result_as_longlong() { return FALSE; }
   bool is_datetime();
-  String *check_well_formed_result(String *str);
+  String *check_well_formed_result(String *str, bool send_error= 0);
 };
 
 

sql/sql_yacc.yy

@@ -7720,15 +7720,11 @@ literal:
                                 str ? str->ptr() : "",
                                 str ? str->length() : 0,
                                 Lex->underscore_charset);
-            if ($$)
+            if (!$$ || !$$->check_well_formed_result(&$$->str_value, TRUE))
             {
-              ((Item_string *) $$)->set_repertoire_from_value();
-              if (!$$->check_well_formed_result(&$$->str_value))
-              {
-                $$= new Item_null();
-                $$->set_name(NULL, 0, system_charset_info);
-              }
+              MYSQL_YYABORT;
             }
+            ((Item_string *) $$)->set_repertoire_from_value();
           }
 	| UNDERSCORE_CHARSET BIN_NUM
           {
@@ -7744,10 +7740,9 @@ literal:
                                 str ? str->ptr() : "",
                                 str ? str->length() : 0,
                                 Lex->underscore_charset);
-            if ($$ && !$$->check_well_formed_result(&$$->str_value))
+            if (!$$ || !$$->check_well_formed_result(&$$->str_value, TRUE))
             {
-              $$= new Item_null();
-              $$->set_name(NULL, 0, system_charset_info);
+              MYSQL_YYABORT;
             }
           }
 	| DATE_SYM text_literal { $$ = $2; }