Merge epotemkin@bk-internal.mysql.com:/home/bk/mysql-5.0

into moonbone.local:/work/12887-bug-5.0-mysql

mysql-test/r/olap.result

@@ -580,6 +580,18 @@ a	LENGTH(a)	COUNT(*)
 2	1	1
 NULL	NULL	2
 DROP TABLE t1;
+create table t1 ( a varchar(9), b int );
+insert into t1 values('a',1),(null,2);
+select a, max(b) from t1 group by a with rollup;
+a	max(b)
+NULL	2
+a	1
+NULL	2
+select distinct a, max(b) from t1 group by a with rollup;
+a	max(b)
+NULL	2
+a	1
+drop table t1;
 CREATE TABLE t1(id int, type char(1));
 INSERT INTO t1 VALUES
 (1,"A"),(2,"C"),(3,"A"),(4,"A"),(5,"B"),

mysql-test/t/olap.test

@@ -274,6 +274,15 @@ SELECT * FROM (SELECT a, LENGTH(a), COUNT(*) FROM t1 GROUP BY a WITH ROLLUP) t;
 
 DROP TABLE t1;
 
+#
+# Bug #12887 Distinct is not always applied after rollup
+#
+create table t1 ( a varchar(9), b int );
+insert into t1 values('a',1),(null,2);
+select a, max(b) from t1 group by a with rollup;
+select distinct a, max(b) from t1 group by a with rollup;
+drop table t1;
+
 # End of 4.1 tests
 
 #
@@ -309,3 +318,4 @@ SELECT * FROM v1;
 
 DROP VIEW v1;
 DROP TABLE t1;
+# End of 4.1 tests

sql/sql_select.cc

@@ -755,7 +755,7 @@ JOIN::optimize()
   }
   if (group_list || tmp_table_param.sum_func_count)
   {
-    if (! hidden_group_fields)
+    if (! hidden_group_fields && rollup.state == ROLLUP::STATE_NONE)
       select_distinct=0;
   }
   else if (select_distinct && tables - const_tables == 1)

vio/viosocket.c

@@ -145,7 +145,7 @@ int vio_blocking(Vio * vio __attribute__((unused)), my_bool set_blocking_mode,
 #endif /* !defined(NO_FCNTL_NONBLOCK) */
 #else /* !defined(__WIN__) && !defined(__EMX__) */
 #ifndef __EMX__
-  if (vio->type != VIO_TYPE_NAMEDPIPE)  
+  if (vio->type != VIO_TYPE_NAMEDPIPE && vio->type != VIO_TYPE_SHARED_MEMORY)
 #endif
   { 
     ulong arg;