Merge jbruehe@bk-internal.mysql.com:/home/bk/mysql-4.1

into mysql.com:/M41/mysql-4.1

mysql-test/r/func_compress.result

@@ -68,3 +68,7 @@ Warnings:
 Error	1259	ZLIB: Input data corrupted
 Error	1256	Uncompressed data size too large; the maximum size is 1048576 (probably, length of uncompressed data was corrupted)
 drop table t1;
+set @@max_allowed_packet=1048576*100;
+select length(compress(repeat('aaaaaaaaaa', 10000000)));
+length(compress(repeat('aaaaaaaaaa', 10000000)))
+97214

mysql-test/t/func_compress.test

@@ -35,3 +35,10 @@ select length(a) from t1;
 select length(uncompress(a)) from t1;
 drop table t1;
 
+
+#
+# Bug #5497: a problem with large strings
+#
+
+set @@max_allowed_packet=1048576*100;
+select length(compress(repeat('aaaaaaaaaa', 10000000)));

sql/item_strfunc.cc

@@ -2731,11 +2731,17 @@ String *Item_func_compress::val_str(String *str)
    compress(compress(compress(...)))
    I.e. zlib give number 'at least'..
   */
-  ulong new_size= (ulong)((res->length()*120)/100)+12;
+  ulong new_size= res->length() + res->length() / 5 + 12;
 
-  buffer.realloc((uint32)new_size + 4 + 1);
-  Byte *body= ((Byte*)buffer.ptr()) + 4;
+  // Will check new_size overflow: new_size <= res->length()
+  if (((uint32) new_size <= res->length()) || 
+      buffer.realloc((uint32) new_size + 4 + 1))
+  {
+    null_value= 1;
+    return 0;
+  }
 
+  Byte *body= ((Byte*)buffer.ptr()) + 4;
 
   // As far as we have checked res->is_empty() we can use ptr()
   if ((err= compress(body, &new_size,

strings/ctype-ucs2.c

@@ -1345,10 +1345,10 @@ my_bool my_like_range_ucs2(CHARSET_INFO *cs,
     }
     if (ptr[0] == '\0' && ptr[1] == w_one)	/* '_' in SQL */
     {
-      *min_str++= (char) cs->min_sort_char >> 8;
-      *min_str++= (char) cs->min_sort_char & 255;
-      *max_str++= (char) cs->max_sort_char >> 8;
-      *max_str++= (char) cs->max_sort_char & 255;
+      *min_str++= (char) (cs->min_sort_char >> 8);
+      *min_str++= (char) (cs->min_sort_char & 255);
+      *max_str++= (char) (cs->max_sort_char >> 8);
+      *max_str++= (char) (cs->max_sort_char & 255);
       continue;
     }
     if (ptr[0] == '\0' && ptr[1] == w_many)	/* '%' in SQL */
@@ -1358,8 +1358,8 @@ my_bool my_like_range_ucs2(CHARSET_INFO *cs,
       do {
         *min_str++ = 0;
 	*min_str++ = 0;
-	*max_str++ = (char) cs->max_sort_char >>8;
-	*max_str++ = (char) cs->max_sort_char & 255;
+	*max_str++ = (char) (cs->max_sort_char >> 8);
+	*max_str++ = (char) (cs->max_sort_char & 255);
       } while (min_str + 1 < min_end);
       return 0;
     }