BUG#836523: Crash in JOIN::get_partial_cost_and_fanout with semijoin+materialization

- Make JOIN::get_partial_cost_and_fanout() be able to handle join plans with 
  semi-join-materialization nests.

mysql-test/r/subselect_mat.result

@@ -3,7 +3,7 @@ set @subselect_sj_mat_tmp= @@optimizer_switch;
 set optimizer_switch=ifnull(@subselect_mat_test_optimizer_switch_value, 'semijoin=on,firstmatch=on,loosescan=on');
 set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on';
 set @optimizer_switch_local_default= @@optimizer_switch;
-drop table if exists t1, t2, t3, t1i, t2i, t3i;
+drop table if exists t1, t2, t3, t4, t1i, t2i, t3i;
 drop table if exists columns;
 drop table if exists t1_16, t2_16, t3_16;
 drop view if exists v1, v2, v1m, v2m;
@@ -1579,6 +1579,28 @@ GROUP BY 1 , 2;
 a	c
 1	2
 drop table t1,t2,t3;
+#
+# BUG#836523: Crash in JOIN::get_partial_cost_and_fanout with semijoin+materialization 
+#
+CREATE TABLE t1 (a varchar(1));
+INSERT INTO t1 VALUES ('a'),('a');
+CREATE TABLE t2 (a varchar(1));
+CREATE TABLE t3 (a int);
+INSERT INTO t3 VALUES (1),(2);
+CREATE TABLE t4 (a varchar(1));
+INSERT INTO t4 VALUES ('a'),('a');
+SELECT t1.a
+FROM t1
+WHERE t1.a IN (
+SELECT t2.a
+FROM t2, t3
+)
+HAVING a IN (
+SELECT a
+FROM t4
+);
+a
+DROP TABLE t1, t2, t3, t4;
 set optimizer_switch=@subselect_sj_mat_tmp;
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';

mysql-test/r/subselect_sj_mat.result

@@ -2,7 +2,7 @@ set @subselect_sj_mat_tmp= @@optimizer_switch;
 set optimizer_switch=ifnull(@subselect_mat_test_optimizer_switch_value, 'semijoin=on,firstmatch=on,loosescan=on');
 set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on';
 set @optimizer_switch_local_default= @@optimizer_switch;
-drop table if exists t1, t2, t3, t1i, t2i, t3i;
+drop table if exists t1, t2, t3, t4, t1i, t2i, t3i;
 drop table if exists columns;
 drop table if exists t1_16, t2_16, t3_16;
 drop view if exists v1, v2, v1m, v2m;
@@ -1617,4 +1617,26 @@ GROUP BY 1 , 2;
 a	c
 1	2
 drop table t1,t2,t3;
+#
+# BUG#836523: Crash in JOIN::get_partial_cost_and_fanout with semijoin+materialization 
+#
+CREATE TABLE t1 (a varchar(1));
+INSERT INTO t1 VALUES ('a'),('a');
+CREATE TABLE t2 (a varchar(1));
+CREATE TABLE t3 (a int);
+INSERT INTO t3 VALUES (1),(2);
+CREATE TABLE t4 (a varchar(1));
+INSERT INTO t4 VALUES ('a'),('a');
+SELECT t1.a
+FROM t1
+WHERE t1.a IN (
+SELECT t2.a
+FROM t2, t3
+)
+HAVING a IN (
+SELECT a
+FROM t4
+);
+a
+DROP TABLE t1, t2, t3, t4;
 set optimizer_switch=@subselect_sj_mat_tmp;

mysql-test/t/subselect_sj_mat.test

@@ -9,7 +9,7 @@ set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on';
 set @optimizer_switch_local_default= @@optimizer_switch;
 
 --disable_warnings
-drop table if exists t1, t2, t3, t1i, t2i, t3i;
+drop table if exists t1, t2, t3, t4, t1i, t2i, t3i;
 drop table if exists columns;
 drop table if exists t1_16, t2_16, t3_16;
 drop view if exists v1, v2, v1m, v2m;
@@ -1263,5 +1263,32 @@ GROUP BY 1 , 2;
 
 drop table t1,t2,t3;
 
+--echo #
+--echo # BUG#836523: Crash in JOIN::get_partial_cost_and_fanout with semijoin+materialization 
+--echo #
+CREATE TABLE t1 (a varchar(1));
+INSERT INTO t1 VALUES ('a'),('a');
+
+CREATE TABLE t2 (a varchar(1));
+
+CREATE TABLE t3 (a int);
+INSERT INTO t3 VALUES (1),(2);
+
+CREATE TABLE t4 (a varchar(1));
+INSERT INTO t4 VALUES ('a'),('a');
+
+SELECT t1.a
+FROM t1
+WHERE t1.a IN (
+        SELECT t2.a
+        FROM t2, t3
+)
+HAVING a IN (
+        SELECT a
+        FROM t4
+);
+DROP TABLE t1, t2, t3, t4;
+
+
 set optimizer_switch=@subselect_sj_mat_tmp;
 

sql/opt_subselect.cc

@@ -4478,20 +4478,6 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
     outer_join= unit->outer_select() ? unit->outer_select()->join : NULL;
     if (outer_join && outer_join->table_count > 0)
     {
-      /*
-        The index of the last JOIN_TAB in the outer JOIN where in_subs is
-        attached (pushed to).
-      */
-      uint max_outer_join_tab_idx;
-      /*
-        Make_cond_for_table is called for predicates only in the WHERE/ON
-        clauses. In all other cases, predicates are not pushed to any
-        JOIN_TAB, and their join_tab_idx remains MAX_TABLES. Such predicates
-        are evaluated for each complete row of the outer join.
-      */
-      max_outer_join_tab_idx= (in_subs->get_join_tab_idx() == MAX_TABLES) ?
-                               outer_join->table_count - 1:
-                               in_subs->get_join_tab_idx();
       /*
         TODO:
         Currently outer_lookup_keys is computed as the number of rows in
@@ -4504,7 +4490,7 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
         If the join order: t1, t2, the number of unique lookup keys is ~ to
         the number of unique values t2.c2 in the partial join t1 join t2.
       */
-      outer_join->get_partial_cost_and_fanout(max_outer_join_tab_idx,
+      outer_join->get_partial_cost_and_fanout(in_subs->get_join_tab_idx(),
                                               table_map(-1),
                                               &dummy,
                                               &outer_lookup_keys);

sql/sql_select.cc

@@ -175,14 +175,14 @@ int join_read_always_key_or_null(JOIN_TAB *tab);
 int join_read_next_same_or_null(READ_RECORD *info);
 static COND *make_cond_for_table(THD *thd, Item *cond,table_map table,
                                  table_map used_table,
-                                 uint join_tab_idx_arg,
+                                 int join_tab_idx_arg,
                                  bool exclude_expensive_cond,
                                  bool retain_ref_cond);
 static COND *make_cond_for_table_from_pred(THD *thd, Item *root_cond,
                                            Item *cond,
                                            table_map tables,
                                            table_map used_table,
-                                           uint join_tab_idx_arg,
+                                           int join_tab_idx_arg,
                                            bool exclude_expensive_cond,
                                            bool retain_ref_cond);
 
@@ -1089,7 +1089,7 @@ JOIN::optimize()
       if (conds && !(thd->lex->describe & DESCRIBE_EXTENDED))
       {
         COND *table_independent_conds=
-          make_cond_for_table(thd, conds, PSEUDO_TABLE_BITS, 0, MAX_TABLES,
+          make_cond_for_table(thd, conds, PSEUDO_TABLE_BITS, 0, -1,
                               FALSE, FALSE);
         DBUG_EXECUTE("where",
                      print_where(table_independent_conds,
@@ -2536,7 +2536,7 @@ JOIN::exec()
 
       Item* sort_table_cond= make_cond_for_table(thd, curr_join->tmp_having,
 						 used_tables,
-						 (table_map)0, MAX_TABLES,
+						 (table_map)0, -1,
 						 FALSE, FALSE);
       if (sort_table_cond)
       {
@@ -2574,7 +2574,7 @@ JOIN::exec()
                                          QT_ORDINARY););
 	curr_join->tmp_having= make_cond_for_table(thd, curr_join->tmp_having,
 						   ~ (table_map) 0,
-						   ~used_tables, MAX_TABLES,
+						   ~used_tables, -1,
 						   FALSE, FALSE);
 	DBUG_EXECUTE("where",print_where(curr_join->tmp_having,
                                          "having after sort",
@@ -6048,7 +6048,7 @@ greedy_search(JOIN      *join,
     read_time_arg and record_count_arg contain the computed cost and fanout
 */
 
-void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
+void JOIN::get_partial_cost_and_fanout(int end_tab_idx,
                                        table_map filter_map,
                                        double *read_time_arg, 
                                        double *record_count_arg)
@@ -6058,14 +6058,14 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
   double sj_inner_fanout= 1.0;
   JOIN_TAB *end_tab= NULL;
   JOIN_TAB *tab;
-  uint i;
-  uint last_sj_table= MAX_TABLES;
+  int i;
+  int last_sj_table= MAX_TABLES;
 
   /* 
     Handle a special case where the join is degenerate, and produces no
     records
   */
-  if (table_count == 0)
+  if (table_count == const_tables)
   {
     *read_time_arg= 0.0;
     /*
@@ -6075,6 +6075,7 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
          calculations.
     */
     *record_count_arg=1.0;
+    return;
   }
 
   for (tab= first_depth_first_tab(this), i= const_tables;
@@ -6087,19 +6088,17 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
   }
 
   for (tab= first_depth_first_tab(this), i= const_tables;
-       (i <= end_tab_idx && tab);
+       ;
        tab= next_depth_first_tab(this, tab), i++)
   {
-    /* 
-      We've entered the SJM nest that contains the end_tab. The caller is
-      actually 
-      - interested in fanout inside the nest (because that's how many times 
-        we'll invoke the attached WHERE conditions)
-      - not interested in cost
-    */
     if (end_tab->bush_root_tab && end_tab->bush_root_tab == tab)
     {
-      /* Ok, end_tab is inside SJM nest and we're entering that nest now */
+      /* 
+        We've entered the SJM nest that contains the end_tab. The caller is
+        - interested in fanout inside the nest (because that's how many times 
+          we'll invoke the attached WHERE conditions)
+        - not interested in cost
+      */
       record_count= 1.0;
       read_time= 0.0;
     }
@@ -6113,8 +6112,18 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
       sj_inner_fanout= 1.0;
       last_sj_table= i + tab->n_sj_tables;
     }
-
-    if (tab->records_read && (tab->table->map & filter_map))
+    
+    table_map cur_table_map;
+    if (tab->table)
+      cur_table_map= tab->table->map;
+    else
+    {
+      /* This is a SJ-Materialization nest. Check all of its tables */
+      TABLE *first_child= tab->bush_children->start->table;
+      TABLE_LIST *sjm_nest= first_child->pos_in_table_list->embedding;
+      cur_table_map= sjm_nest->nested_join->used_tables;
+    }
+    if (tab->records_read && (cur_table_map & filter_map))
     {
       record_count *= tab->records_read;
       read_time += tab->read_time;
@@ -6128,6 +6137,9 @@ void JOIN::get_partial_cost_and_fanout(uint end_tab_idx,
       sj_inner_fanout= 1.0;
       last_sj_table= MAX_TABLES;
     }
+
+    if (tab == end_tab)
+      break;
   }
   *read_time_arg= read_time;// + record_count / TIME_FOR_COMPARE;
   *record_count_arg= record_count;
@@ -6616,7 +6628,7 @@ int JOIN_TAB::make_scan_filter()
   if (cond &&
       (tmp= make_cond_for_table(join->thd, cond,
                                join->const_table_map | table->map,
-			       table->map, MAX_TABLES, FALSE, TRUE)))
+			       table->map, -1, FALSE, TRUE)))
   {
      DBUG_EXECUTE("where",print_where(tmp,"cache", QT_ORDINARY););
      if (!(cache_select=
@@ -7902,7 +7914,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
         join->exec_const_cond=
 	  make_cond_for_table(thd, cond,
                               join->const_table_map,
-                              (table_map) 0, MAX_TABLES, FALSE, FALSE);
+                              (table_map) 0, -1, FALSE, FALSE);
         /* Add conditions added by add_not_null_conds(). */
         for (uint i= 0 ; i < join->const_tables ; i++)
           add_cond_and_fix(thd, &join->exec_const_cond,
@@ -7921,7 +7933,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
         COND *outer_ref_cond= make_cond_for_table(thd, cond, 
                                                   OUTER_REF_TABLE_BIT,
                                                   OUTER_REF_TABLE_BIT,
-                                                  MAX_TABLES, FALSE, FALSE);
+                                                  -1, FALSE, FALSE);
         if (outer_ref_cond)
 	{
           add_cond_and_fix(thd, &outer_ref_cond, join->outer_ref_cond);
@@ -8091,7 +8103,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
             {
               COND *push_cond= 
               make_cond_for_table(thd, tmp, current_map, current_map,
-                                  MAX_TABLES, FALSE, FALSE);
+                                  -1, FALSE, FALSE);
               if (push_cond)
               {
                 /* Push condition to handler */
@@ -8263,7 +8275,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
           JOIN_TAB *cond_tab= join_tab->first_inner;
           COND *tmp= make_cond_for_table(thd, *join_tab->on_expr_ref,
                                          join->const_table_map,
-                                         (table_map) 0, MAX_TABLES, FALSE, FALSE);
+                                         (table_map) 0, -1, FALSE, FALSE);
           if (!tmp)
             continue;
           tmp= new Item_func_trig_cond(tmp, &cond_tab->not_null_compl);
@@ -8309,10 +8321,10 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
           current_map= tab->table->map;
           used_tables2|= current_map;
           /*
-            psergey: have put the MAX_TABLES below. It's bad, will need to fix it.
+            psergey: have put the -1 below. It's bad, will need to fix it.
           */
           COND *tmp_cond= make_cond_for_table(thd, on_expr, used_tables2,
-                                              current_map, /*(tab - first_tab)*/ MAX_TABLES,
+                                              current_map, /*(tab - first_tab)*/ -1,
 					      FALSE, FALSE);
           if (tab == first_inner_tab && tab->on_precond)
             add_cond_and_fix(thd, &tmp_cond, tab->on_precond);
@@ -16683,7 +16695,7 @@ bool test_if_ref(Item *root_cond, Item_field *left_item,Item *right_item)
 static Item *
 make_cond_for_table(THD *thd, Item *cond, table_map tables,
                     table_map used_table,
-                    uint join_tab_idx_arg,
+                    int join_tab_idx_arg,
                     bool exclude_expensive_cond __attribute__((unused)),
 		    bool retain_ref_cond)
 {
@@ -16697,7 +16709,7 @@ make_cond_for_table(THD *thd, Item *cond, table_map tables,
 static Item *
 make_cond_for_table_from_pred(THD *thd, Item *root_cond, Item *cond,
                               table_map tables, table_map used_table,
-                              uint join_tab_idx_arg,
+                              int join_tab_idx_arg,
                               bool exclude_expensive_cond __attribute__
                               ((unused)),
                               bool retain_ref_cond)

sql/sql_select.h

@@ -1154,7 +1154,7 @@ class JOIN :public Sql_alloc
            max_allowed_join_cache_level > JOIN_CACHE_HASHED_BIT;
   }
   bool choose_subquery_plan(table_map join_tables);
-  void get_partial_cost_and_fanout(uint end_tab_idx,
+  void get_partial_cost_and_fanout(int end_tab_idx,
                                    table_map filter_map,
                                    double *read_time_arg, 
                                    double *record_count_arg);