sql_select.cc, item_buff.cc, item.h:

  Fixed bug #11088: a crash for queries with GROUP BY a BLOB column
  + COUNT(DISTINCT...) due to an attempt to allocate a too large
  buffer for the BLOB field.
  Now the size of the buffer is limited by max_sort_length.
group_by.test, group_by.result:
  Added a test case for bug #11088.

mysql-test/r/group_by.result

@@ -711,3 +711,14 @@ select min(b) from t1;
 min(b)
 3000000000
 drop table t1;
+CREATE TABLE t1 (id int PRIMARY KEY, user_id int, hostname longtext);
+INSERT INTO t1 VALUES
+(1, 7, 'cache-dtc-af05.proxy.aol.com'),
+(2, 3, 'what.ever.com'),
+(3, 7, 'cache-dtc-af05.proxy.aol.com'),
+(4, 7, 'cache-dtc-af05.proxy.aol.com');
+SELECT hostname, COUNT(DISTINCT user_id) as no FROM t1
+WHERE hostname LIKE '%aol%'
+    GROUP BY hostname;
+hostname	no
+cache-dtc-af05.proxy.aol.com	1

mysql-test/t/group_by.test

@@ -522,3 +522,20 @@ insert into t1 values(3000000000);
 select * from t1;
 select min(b) from t1;
 drop table t1;
+
+#
+# Test for bug #11088: GROUP BY a BLOB colimn with COUNT(DISTINCT column1) 
+#
+
+CREATE TABLE t1 (id int PRIMARY KEY, user_id int, hostname longtext);
+
+INSERT INTO t1 VALUES
+  (1, 7, 'cache-dtc-af05.proxy.aol.com'),
+  (2, 3, 'what.ever.com'),
+  (3, 7, 'cache-dtc-af05.proxy.aol.com'),
+  (4, 7, 'cache-dtc-af05.proxy.aol.com');
+
+SELECT hostname, COUNT(DISTINCT user_id) as no FROM t1
+  WHERE hostname LIKE '%aol%'
+    GROUP BY hostname;
+

sql/item.h

@@ -1118,7 +1118,7 @@ class Item_str_buff :public Item_buff
   Item *item;
   String value,tmp_value;
 public:
-  Item_str_buff(Item *arg) :item(arg),value(arg->max_length) {}
+  Item_str_buff(THD *thd, Item *arg);
   bool cmp(void);
   ~Item_str_buff();				// Deallocate String:s
 };
@@ -1385,7 +1385,7 @@ public:
 };
 
 
-extern Item_buff *new_Item_buff(Item *item);
+extern Item_buff *new_Item_buff(THD *thd, Item *item);
 extern Item_result item_cmp_type(Item_result a,Item_result b);
 extern void resolve_const_item(THD *thd, Item **ref, Item *cmp_item);
 extern bool field_is_equal_to_item(Field *field,Item *item);

sql/item_buff.cc

@@ -23,13 +23,13 @@
 ** Create right type of item_buffer for an item
 */
 
-Item_buff *new_Item_buff(Item *item)
+Item_buff *new_Item_buff(THD *thd, Item *item)
 {
   if (item->type() == Item::FIELD_ITEM &&
       !(((Item_field *) item)->field->flags & BLOB_FLAG))
     return new Item_field_buff((Item_field *) item);
   if (item->result_type() == STRING_RESULT)
-    return new Item_str_buff((Item_field *) item);
+    return new Item_str_buff(thd, (Item_field *) item);
   if (item->result_type() == INT_RESULT)
     return new Item_int_buff((Item_field *) item);
   return new Item_real_buff(item);
@@ -42,12 +42,17 @@ Item_buff::~Item_buff() {}
 ** Return true if values have changed
 */
 
+Item_str_buff::Item_str_buff(THD *thd, Item *arg)
+  :item(arg), value(min(arg->max_length, thd->variables. max_sort_length))
+{}
+
 bool Item_str_buff::cmp(void)
 {
   String *res;
   bool tmp;
 
   res=item->val_str(&tmp_value);
+  res->length(min(res->length(), value.alloced_length()));
   if (null_value != item->null_value)
   {
     if ((null_value= item->null_value))

sql/sql_select.cc

@@ -8656,7 +8656,7 @@ alloc_group_fields(JOIN *join,ORDER *group)
   {
     for (; group ; group=group->next)
     {
-      Item_buff *tmp=new_Item_buff(*group->item);
+      Item_buff *tmp=new_Item_buff(join->thd, *group->item);
       if (!tmp || join->group_fields.push_front(tmp))
 	return TRUE;
     }