BUG#29582: huge memory consumption with union, subselect, joins:

- Don't call mysql_select() several times for the select that enumerates
  a temporary table with the results of the UNION. Making this call for
  every subquery execution caused O(#enumerated-rows-in-the-outer-query) 
  memory allocations.
- Instead, call join->reinit() and join->exec(), and
  = disable constant table detection for such joins,  
  = provide special handling for table-less constant subqueries.


sql/sql_select.cc:
  BUG#29582: huge memory consumption with union, subselect, joins:
   - Don't mark tables as constant if JOIN::no_const_tables flag is set
sql/sql_select.h:
  BUG#29582: huge memory consumption with union, subselect, joins:
   - Don't mark tables as constant if JOIN::no_const_tables flag is set
sql/sql_union.cc:
  BUG#29582: huge memory consumption with union, subselect, joins:
  - Don't call mysql_select() several times for the select that enumerates
    a temporary table with UNION results. 
  - Instead, call join->reinit() and join->exec().
  - Provide special handling for table-less constant subqueries.

sql/sql_select.cc

@@ -2416,7 +2416,7 @@ make_join_statistics(JOIN *join, TABLE_LIST *tables, COND *conds,
 
     if ((table->s->system || table->file->records <= 1) && ! s->dependent &&
 	!(table->file->table_flags() & HA_NOT_EXACT_COUNT) &&
-        !table->fulltext_searched)
+        !table->fulltext_searched && !join->no_const_tables)
     {
       set_position(join,const_count++,s,(KEYUSE*) 0);
     }

sql/sql_select.h

@@ -277,6 +277,14 @@ public:
   SELECT_LEX_UNIT *unit;
   // select that processed
   SELECT_LEX *select_lex;
+  /* 
+    TRUE <=> optimizer must not mark any table as a constant table.
+    This is needed for subqueries in form "a IN (SELECT .. UNION SELECT ..):
+    when we optimize the select that reads the results of the union from a
+    temporary table, we must not mark the temp. table as constant because
+    the number of rows in it may vary from one subquery execution to another.
+  */
+  bool no_const_tables; 
   
   JOIN *tmp_join; // copy of this JOIN to be used with temporary tables
   ROLLUP rollup;				// Used with rollup
@@ -397,6 +405,8 @@ public:
     tmp_table_param.init();
     tmp_table_param.end_write_records= HA_POS_ERROR;
     rollup.state= ROLLUP::STATE_NONE;
+
+    no_const_tables= FALSE;
   }
 
   int prepare(Item ***rref_pointer_array, TABLE_LIST *tables, uint wind_num,

sql/sql_union.cc

@@ -545,6 +545,10 @@ bool st_select_lex_unit::exec()
 	/*
 	  allocate JOIN for fake select only once (prevent
 	  mysql_select automatic allocation)
+          TODO: The above is nonsense. mysql_select() will not allocate the
+          join if one already exists. There must be some other reason why we
+          don't let it allocate the join. Perhaps this is because we need
+          some special parameter values passed to join constructor?
 	*/
 	if (!(fake_select_lex->join= new JOIN(thd, item_list,
 					      fake_select_lex->options, result)))
@@ -552,25 +556,36 @@ bool st_select_lex_unit::exec()
 	  fake_select_lex->table_list.empty();
 	  DBUG_RETURN(TRUE);
 	}
+        fake_select_lex->join->no_const_tables= TRUE;
 
 	/*
 	  Fake st_select_lex should have item list for correctref_array
 	  allocation.
 	*/
 	fake_select_lex->item_list= item_list;
+        saved_error= mysql_select(thd, &fake_select_lex->ref_pointer_array,
+                              &result_table_list,
+                              0, item_list, NULL,
+                              global_parameters->order_list.elements,
+                              (ORDER*)global_parameters->order_list.first,
+                              (ORDER*) NULL, NULL, (ORDER*) NULL,
+                              fake_select_lex->options | SELECT_NO_UNLOCK,
+                              result, this, fake_select_lex);
       }
       else
       {
-	JOIN_TAB *tab,*end;
-	for (tab=join->join_tab, end=tab+join->tables ;
-	     tab && tab != end ;
-	     tab++)
+        if (describe)
         {
-	  delete tab->select;
-	  delete tab->quick;
-	}
+          /*
+            In EXPLAIN command, constant subqueries that do not use any
+            tables are executed two times:
+             - 1st time is a real evaluation to get the subquery value
+             - 2nd time is to produce EXPLAIN output rows.
+            1st execution sets certain members (e.g. select_result) to perform
+            subquery execution rather than EXPLAIN line production. In order 
+            to reset them back, we re-do all of the actions (yes it is ugly):
+          */
 	  join->init(thd, item_list, fake_select_lex->options, result);
-      }
           saved_error= mysql_select(thd, &fake_select_lex->ref_pointer_array,
                                 &result_table_list,
                                 0, item_list, NULL,
@@ -579,6 +594,14 @@ bool st_select_lex_unit::exec()
                                 (ORDER*) NULL, NULL, (ORDER*) NULL,
                                 fake_select_lex->options | SELECT_NO_UNLOCK,
                                 result, this, fake_select_lex);
+        }
+        else
+        {
+          join->examined_rows= 0;
+          join->reinit();
+          saved_error= join->exec();
+        }
+      }
 
       fake_select_lex->table_list.empty();
       if (!saved_error)