A fix (bug #5497: COMPRESS() returns NULL for large strings).

mysql-test/r/func_compress.result

@@ -68,3 +68,7 @@ Warnings:
 Error	1259	ZLIB: Input data corrupted
 Error	1256	Uncompressed data size too large; the maximum size is 1048576 (probably, length of uncompressed data was corrupted)
 drop table t1;
+set @@max_allowed_packet=1048576*100;
+select length(compress(repeat('aaaaaaaaaa', 10000000)));
+length(compress(repeat('aaaaaaaaaa', 10000000)))
+97214

mysql-test/t/func_compress.test

@@ -35,3 +35,10 @@ select length(a) from t1;
 select length(uncompress(a)) from t1;
 drop table t1;
 
+
+#
+# Bug #5497: a problem with large strings
+#
+
+set @@max_allowed_packet=1048576*100;
+select length(compress(repeat('aaaaaaaaaa', 10000000)));

sql/item_strfunc.cc

@@ -2723,11 +2723,17 @@ String *Item_func_compress::val_str(String *str)
    compress(compress(compress(...)))
    I.e. zlib give number 'at least'..
   */
-  ulong new_size= (ulong)((res->length()*120)/100)+12;
+  ulong new_size= res->length() + res->length() / 5 + 12;
 
-  buffer.realloc((uint32)new_size + 4 + 1);
-  Byte *body= ((Byte*)buffer.ptr()) + 4;
+  // Will check new_size overflow: new_size <= res->length()
+  if (((uint32) new_size <= res->length()) || 
+      buffer.realloc((uint32) new_size + 4 + 1))
+  {
+    null_value= 1;
+    return 0;
+  }
 
+  Byte *body= ((Byte*)buffer.ptr()) + 4;
 
   // As far as we have checked res->is_empty() we can use ptr()
   if ((err= compress(body, &new_size,