Bug #11765023: 57934: DOS POSSIBLE SINCE BINARY CASTING DOESN'T

ADHERE TO MAX_ALLOWED_PACKET

Added a check for max_packet_length in CONVERT(, BINARY|CHAR).
Added a test case.

mysql-test/r/cast.result

@@ -451,4 +451,19 @@ SELECT CONVERT(t2.a USING UTF8) FROM t1, t1 t2 LIMIT 1
 1
 1
 DROP TABLE t1;
+#
+# Bug #11765023: 57934: DOS POSSIBLE SINCE BINARY CASTING 
+#   DOESN'T ADHERE TO MAX_ALLOWED_PACKET
+SET @@GLOBAL.max_allowed_packet=2048;
+SELECT CONVERT('a', BINARY(2049));
+CONVERT('a', BINARY(2049))
+NULL
+Warnings:
+Warning	1301	Result of cast_as_binary() was larger than max_allowed_packet (2048) - truncated
+SELECT CONVERT('a', CHAR(2049));
+CONVERT('a', CHAR(2049))
+NULL
+Warnings:
+Warning	1301	Result of cast_as_char() was larger than max_allowed_packet (2048) - truncated
+SET @@GLOBAL.max_allowed_packet=default;
 End of 5.1 tests

mysql-test/t/cast.test

@@ -282,5 +282,19 @@ SELECT 1 FROM
 ) AS s LIMIT 1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug #11765023: 57934: DOS POSSIBLE SINCE BINARY CASTING 
+--echo #   DOESN'T ADHERE TO MAX_ALLOWED_PACKET
+
+SET @@GLOBAL.max_allowed_packet=2048;
+# reconnect to make the new max packet size take effect
+--connect (newconn, localhost, root,,)
+
+SELECT CONVERT('a', BINARY(2049));  
+SELECT CONVERT('a', CHAR(2049));  
+
+connection default;
+disconnect newconn;
+SET @@GLOBAL.max_allowed_packet=default;
 
 --echo End of 5.1 tests

sql/item_timefunc.cc

@@ -2444,6 +2444,19 @@ String *Item_char_typecast::val_str(String *str)
   String *res;
   uint32 length;
 
+  if (cast_length >= 0 &&
+      ((unsigned) cast_length) > current_thd->variables.max_allowed_packet)
+  {
+    push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
+			ER_WARN_ALLOWED_PACKET_OVERFLOWED,
+			ER(ER_WARN_ALLOWED_PACKET_OVERFLOWED),
+			cast_cs == &my_charset_bin ?
+                        "cast_as_binary" : func_name(),
+                        current_thd->variables.max_allowed_packet);
+    null_value= 1;
+    return 0;
+  }
+
   if (!charset_conversion)
   {
     if (!(res= args[0]->val_str(str)))