Bug#10504

  Character set does not support traditional mode
ctype_utf8.result, ctype_utf8.test:
  adding test case.
password.c, mysql_com.h
  Changeing octet2hex availability from static to public.
item_strfunc.cc:
  Result string is now checked to be well-formed.
  Warning/error is generated, depending on sql_mode.

include/mysql_com.h

@@ -409,6 +409,7 @@ my_bool check_scramble(const char *reply, const char *message,
                        const unsigned char *hash_stage2);
 void get_salt_from_password(unsigned char *res, const char *password);
 void make_password_from_salt(char *to, const unsigned char *hash_stage2);
+void octet2hex(char *to, const unsigned char *str, unsigned int len);
 
 /* end of password.c */
 

mysql-test/r/ctype_utf8.result

@@ -1028,6 +1028,39 @@ xxx
 yyy
 DROP TABLE t1;
 set names utf8;
+select hex(char(1));
+hex(char(1))
+01
+select char(0xd1,0x8f);
+char(0xd1,0x8f)
+я
+select char(0xff,0x8f);
+char(0xff,0x8f)
+
+Warnings:
+Warning	1300	Invalid utf8 character string: 'FF8F'
+set sql_mode=traditional;
+select char(0xff,0x8f);
+char(0xff,0x8f)
+NULL
+Warnings:
+Error	1300	Invalid utf8 character string: 'FF8F'
+select char(195);
+char(195)
+NULL
+Warnings:
+Error	1300	Invalid utf8 character string: 'C3'
+select char(196);
+char(196)
+NULL
+Warnings:
+Error	1300	Invalid utf8 character string: 'C4'
+select char(2557);
+char(2557)
+NULL
+Warnings:
+Error	1300	Invalid utf8 character string: 'FD'
+set names utf8;
 create table t1 (a char(1)) default character set utf8;
 create table t2 (a char(1)) default character set utf8;
 insert into t1 values('a'),('a'),(0xE38182),(0xE38182);

mysql-test/t/ctype_utf8.test

@@ -864,6 +864,22 @@ SELECT DISTINCT id FROM t1 ORDER BY id;
 
 DROP TABLE t1;
 
+#
+#  Bugs#10504: Character set does not support traditional mode
+#
+set names utf8;
+# correct value
+select hex(char(1));
+select char(0xd1,0x8f);
+# incorrect value: return with warning
+select char(0xff,0x8f);
+# incorrect value in strict mode: return NULL with "Error" level warning
+set sql_mode=traditional;
+select char(0xff,0x8f);
+select char(195);
+select char(196);
+select char(2557);
+
 #
 # Bug#12891: UNION doesn't return DISTINCT result for multi-byte characters
 #

sql/item_strfunc.cc

@@ -1980,6 +1980,33 @@ b1:        str->append((char)(num>>8));
   }
   str->set_charset(collation.collation);
   str->realloc(str->length());			// Add end 0 (for Purify)
+
+  /* Check whether we got a well-formed string */
+  CHARSET_INFO *cs= collation.collation;
+  int well_formed_error;
+  uint wlen= cs->cset->well_formed_len(cs,
+                                       str->ptr(), str->ptr() + str->length(),
+                                       str->length(), &well_formed_error);
+  if (wlen < str->length())
+  {
+    THD *thd= current_thd;
+    char hexbuf[7];
+    enum MYSQL_ERROR::enum_warning_level level;
+    uint diff= str->length() - wlen;
+    set_if_smaller(diff, 3);
+    octet2hex(hexbuf, (const uchar*) str->ptr() + wlen, diff);
+    if (thd->variables.sql_mode &
+        (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES))
+    {
+      level= MYSQL_ERROR::WARN_LEVEL_ERROR;
+      null_value= 1;
+      str= 0;
+    }
+    else
+      level= MYSQL_ERROR::WARN_LEVEL_WARN;
+    push_warning_printf(thd, level, ER_INVALID_CHARACTER_STRING,
+                        ER(ER_INVALID_CHARACTER_STRING), cs->csname, hexbuf);
+  }
   return str;
 }
 

sql/password.c

@@ -318,8 +318,8 @@ void create_random_string(char *to, uint length, struct rand_struct *rand_st)
     str, len  IN  the beginning and the length of the input string
 */
 
-static void
-octet2hex(char *to, const uint8 *str, uint len)
+void
+octet2hex(char *to, const unsigned char *str, uint len)
 {
   const uint8 *str_end= str + len; 
   for (; str != str_end; ++str)