merge 5.0-opt -> 5.1-opt

mysql-test/r/subselect.result

@@ -4139,6 +4139,107 @@ SELECT (SELECT SUM(t1.a) FROM t2 WHERE a=1) FROM t1;
 (SELECT SUM(t1.a) FROM t2 WHERE a=1)
 3
 DROP TABLE t1,t2;
+create table t1(a int,b int,key(a),key(b));
+insert into t1(a,b) values (1,2),(2,1),(2,3),(3,4),(5,4),(5,5),
+(6,7),(7,4),(5,3);
+select sum(a),a from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1
+)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1) 
+group by a;
+sum(a)	a
+select sum(a),a from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1) 
+group by a;
+ERROR HY000: Thread stack overrun detected
+explain select sum(a),a from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 
+)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1) 
+group by a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	index	a	a	5	NULL	9	Using where; Using index
+2	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+3	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+4	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+5	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+6	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+7	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+8	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+9	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+10	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+11	SUBQUERY	t1	index	a	b	5	NULL	9	Using where; Using filesort
+12	SUBQUERY	t1	range	a	a	5	NULL	1	Using where; Using temporary; Using filesort
+13	SUBQUERY	t1	index	NULL	a	5	NULL	9	Using index
+explain select sum(a),a from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 where a> ( select sum(a) from t1 where a> (
+select sum(a) from t1 
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1
+)group by b limit 1)group by b limit 1)group by b limit 1) 
+group by a;
+ERROR HY000: Thread stack overrun detected
+drop table t1;
 CREATE TABLE t1 (a1 INT, a2 INT);
 CREATE TABLE t2 (b1 INT, b2 INT);
 INSERT INTO t1 VALUES (100, 200);

sql/opt_range.cc

@@ -2162,7 +2162,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
   if (!keys_to_use.is_clear_all())
   {
 #ifndef EMBEDDED_LIBRARY                      // Avoid compiler warning
-    char buff[STACK_BUFF_ALLOC];
+    uchar buff[STACK_BUFF_ALLOC];
 #endif
     MEM_ROOT alloc;
     SEL_TREE *tree= NULL;

sql/sql_select.cc

@@ -2367,7 +2367,7 @@ static ha_rows get_quick_record_count(THD *thd, SQL_SELECT *select,
   int error;
   DBUG_ENTER("get_quick_record_count");
 #ifndef EMBEDDED_LIBRARY                      // Avoid compiler warning
-  char buff[STACK_BUFF_ALLOC];
+  uchar buff[STACK_BUFF_ALLOC];
 #endif
   if (check_stack_overrun(thd, STACK_MIN_SIZE, buff))
     DBUG_RETURN(0);                           // Fatal error flag is set