MWL#90, code movearound to unify merged and non-merged semi-join materialization processing

- First code, needs cleanup.

sql/item_cmpfunc.cc

@@ -5733,6 +5733,8 @@ Item_field* Item_equal::get_first(Item_field *field)
       It's a field from an materialized semi-join. We can substitute it only
       for a field from the same semi-join.
     */
+#if 0
+    psergey3:remove:
     JOIN_TAB *first;
     JOIN *join= field_tab->join;
     int tab_idx= field_tab - field_tab->join->join_tab;
@@ -5746,10 +5748,12 @@ Item_field* Item_equal::get_first(Item_field *field)
         // Found first tab that doesn't belong to current SJ.
         break;
     }
+#endif    
     /* Find an item to substitute for. */
     while ((item= it++))
     {
-      if (item->field->table->reginfo.join_tab >= first)
+      //if (item->field->table->reginfo.join_tab >= first)
+      if (item->field->table->pos_in_table_list->embedding == emb_nest)
       {
         /*
           If we found given field then return NULL to avoid unnecessary

sql/opt_subselect.cc

@@ -2607,6 +2607,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
   }
 }
 
+enum_nested_loop_state 
+end_sj_materialize(JOIN *join, JOIN_TAB *join_tab, bool end_of_records);
+
 /*
   Setup semi-join materialization strategy for one semi-join nest
   
@@ -2628,10 +2631,11 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
     TRUE   Error
 */
 
-bool setup_sj_materialization(JOIN_TAB *tab)
+bool setup_sj_materialization(JOIN_TAB *sjm_tab)
 {
   uint i;
   DBUG_ENTER("setup_sj_materialization");
+  JOIN_TAB *tab= sjm_tab->bush_children->start;
   TABLE_LIST *emb_sj_nest= tab->table->pos_in_table_list->embedding;
   SJ_MATERIALIZATION_INFO *sjm= emb_sj_nest->sj_mat_info;
   THD *thd= tab->join->thd;
@@ -2659,10 +2663,13 @@ bool setup_sj_materialization(JOIN_TAB *tab)
     DBUG_RETURN(TRUE); /* purecov: inspected */
   sjm->table->file->extra(HA_EXTRA_WRITE_CACHE);
   sjm->table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
+  //psergey2-todo: need this or can take advantage of re-init functionality?
   tab->join->sj_tmp_tables.push_back(sjm->table);
   tab->join->sjm_info_list.push_back(sjm);
   
   sjm->materialized= FALSE;
+  sjm_tab->table= sjm->table;
+
   if (!sjm->is_sj_scan)
   {
     KEY           *tmp_key; /* The only index on the temporary table. */
@@ -2675,8 +2682,9 @@ bool setup_sj_materialization(JOIN_TAB *tab)
       temptable.
     */
     TABLE_REF *tab_ref;
-    if (!(tab_ref= (TABLE_REF*) thd->alloc(sizeof(TABLE_REF))))
-      DBUG_RETURN(TRUE); /* purecov: inspected */
+    //if (!(tab_ref= (TABLE_REF*) thd->alloc(sizeof(TABLE_REF))))
+    //  DBUG_RETURN(TRUE); /* purecov: inspected */
+    tab_ref= &sjm_tab->ref;
     tab_ref->key= 0; /* The only temp table index. */
     tab_ref->key_length= tmp_key->key_length;
     if (!(tab_ref->key_buff=
@@ -2733,6 +2741,7 @@ bool setup_sj_materialization(JOIN_TAB *tab)
     if (!(sjm->in_equality= create_subq_in_equalities(thd, sjm,
                                                       emb_sj_nest->sj_subq_pred)))
       DBUG_RETURN(TRUE); /* purecov: inspected */
+    sjm_tab->type= JT_EQ_REF;
   }
   else
   {
@@ -2818,8 +2827,18 @@ bool setup_sj_materialization(JOIN_TAB *tab)
       /* The write_set for source tables must be set up to allow the copying */
       bitmap_set_bit(copy_to->table->write_set, copy_to->field_index);
     }
+    sjm_tab->type= JT_ALL;
+
+    /* Initialize full scan */
+    sjm_tab->read_first_record= join_read_record_no_init;
+    sjm_tab->read_record.copy_field= sjm->copy_field;
+    sjm_tab->read_record.copy_field_end= sjm->copy_field +
+                                         sjm->sjm_table_cols.elements;
+    sjm_tab->read_record.read_record= rr_sequential_and_unpack;
   }
 
+  sjm_tab->bush_children->end[-1].next_select= end_sj_materialize;
+
   DBUG_RETURN(FALSE);
 }
 
@@ -3919,8 +3938,26 @@ static void remove_subq_pushed_predicates(JOIN *join, Item **where)
 }
 
 
-bool do_jtbm_materialization_if_needed(JOIN_TAB *tab)
+/*
+  Join tab execution startup function. 
+
+  DESCRIPTION
+    Join tab execution startup function. This is different from
+    tab->read_first_record in the regard that this has actions that are to be
+    done once per join execution.
+
+    Currently there are only two possible startup functions, so we have them
+    both here inside if (...) branches. In future we could switch to function
+    pointers.
+  
+  RETURN 
+    FALSE  Ok
+    TRUE   Error, join execution is not possible.
+*/
+
+bool join_tab_execution_startup(JOIN_TAB *tab)
 {
+  DBUG_ENTER("join_tab_execution_startup");
   Item_in_subselect *in_subs;
   if (tab->table->pos_in_table_list && 
       (in_subs= tab->table->pos_in_table_list->jtbm_subselect))
@@ -3936,9 +3973,54 @@ bool do_jtbm_materialization_if_needed(JOIN_TAB *tab)
       hash_sj_engine->is_materialized= TRUE; 
 
       if (hash_sj_engine->materialize_join->error || tab->join->thd->is_fatal_error)
-        return TRUE;
+        DBUG_RETURN(TRUE);
     }
   }
-  return FALSE;
+  else if (tab->bush_children)
+  {
+    /* It's a merged SJM nest */
+    int rc; // psergey3: todo: error codes!
+    JOIN *join= tab->join;
+    SJ_MATERIALIZATION_INFO *sjm= tab->bush_children->start->emb_sj_nest->sj_mat_info;
+    JOIN_TAB *join_tab= tab->bush_children->start;
+
+    if (!sjm->materialized)
+    {
+      /*
+        Now run the join for the inner tables. The first call is to run the
+        join, the second one is to signal EOF (this is essential for some
+        join strategies, e.g. it will make join buffering flush the records)
+      */
+      if ((rc= sub_select(join, join_tab, FALSE/* no EOF */)) < 0 ||
+          (rc= sub_select(join, join_tab, TRUE/* now EOF */)) < 0)
+      {
+        //psergey3-todo: set sjm->materialized=TRUE here, too??
+        DBUG_RETURN(rc); /* it's NESTED_LOOP_(ERROR|KILLED)*/
+      }
+      /*
+        Ok, materialization finished. Initialize the access to the temptable
+      */
+      sjm->materialized= TRUE;
+#if 0 
+      psergey3: already done at setup:
+      if (sjm->is_sj_scan)
+      {
+        /* Initialize full scan */
+        JOIN_TAB *last_tab= join_tab + (sjm->tables - 1);
+        init_read_record(&last_tab->read_record, join->thd,
+                         sjm->table, NULL, TRUE, TRUE, FALSE);
+
+        DBUG_ASSERT(last_tab->read_record.read_record == rr_sequential);
+        last_tab->read_first_record= join_read_record_no_init;
+        last_tab->read_record.copy_field= sjm->copy_field;
+        last_tab->read_record.copy_field_end= sjm->copy_field +
+                                              sjm->sjm_table_cols.elements;
+        last_tab->read_record.read_record= rr_sequential_and_unpack;
+      }
+#endif       
+    }
+  }
+
+  DBUG_RETURN(0);
 }
 

sql/opt_subselect.h

@@ -372,5 +372,5 @@ void get_delayed_table_estimates(TABLE *table,
                                  double *scan_time,
                                  double *startup_cost);
 
-bool do_jtbm_materialization_if_needed(JOIN_TAB *tab);
+bool join_tab_execution_startup(JOIN_TAB *tab);
 

sql/sql_join_cache.cc

@@ -1778,7 +1778,7 @@ enum_nested_loop_state JOIN_CACHE_BNL::join_matching_records(bool skip_last)
 
   /* Start retrieving all records of the joined table */
   
-  if (do_jtbm_materialization_if_needed(join_tab))
+  if (join_tab_execution_startup(join_tab))
   {
     rc= NESTED_LOOP_ERROR;
     goto finish;

sql/sql_select.h

@@ -142,13 +142,25 @@ enum enum_nested_loop_state
 
 typedef enum_nested_loop_state
 (*Next_select_func)(JOIN *, struct st_join_table *, bool);
+
+/*
+  Function prototype for reading first record for a join tab
+
+  RETURN
+     0     - OK
+    -1     - Record not found
+    Other  - Error
+*/
 typedef int (*Read_record_func)(struct st_join_table *tab);
+
 Next_select_func setup_end_select_func(JOIN *join);
 int rr_sequential(READ_RECORD *info);
+int rr_sequential_and_unpack(READ_RECORD *info);
 
 
 class JOIN_CACHE;
 class SJ_TMP_TABLE;
+class JOIN_TAB_RANGE;
 
 typedef struct st_join_table {
   st_join_table() {}                          /* Remove gcc warning */
@@ -174,6 +186,14 @@ typedef struct st_join_table {
   st_join_table *first_upper;  /**< first inner table for embedding outer join */
   st_join_table *first_unmatched; /**< used for optimization purposes only     */
 
+  /*
+    psergey2:  for join tabs that are inside a bush: root of this bush.
+  */
+  st_join_table *bush_root_tab;
+  bool          last_leaf_in_bush;
+
+  JOIN_TAB_RANGE *bush_children;
+  
   /* Special content for EXPLAIN 'Extra' column or NULL if none */
   const char	*info;
   /* 
@@ -212,6 +232,8 @@ typedef struct st_join_table {
   */
   double        read_time;
 
+  ha_rows       records_read;
+  
   /* Startup cost for execution */
   double        startup_cost;
 
@@ -292,7 +314,7 @@ typedef struct st_join_table {
   /*
     Semi-join strategy to be used for this join table. This is a copy of
     POSITION::sj_strategy field. This field is set up by the
-    fix_semijion_strategies_for_picked_join_order.
+    fix_semijoin_strategies_for_picked_join_order.
   */
   uint sj_strategy;
 
@@ -1365,17 +1387,29 @@ inline bool sj_is_materialize_strategy(uint strategy)
 }
 
 
+class JOIN_TAB_RANGE: public Sql_alloc
+{
+public:
+  JOIN_TAB *start;
+  JOIN_TAB *end;
+};
+
+
 class JOIN :public Sql_alloc
 {
   JOIN(const JOIN &rhs);                        /**< not implemented */
   JOIN& operator=(const JOIN &rhs);             /**< not implemented */
 public:
-  JOIN_TAB *join_tab,**best_ref;
+  JOIN_TAB *join_tab, **best_ref;
   JOIN_TAB **map2table;    ///< mapping between table indexes and JOIN_TABs
   JOIN_TAB *join_tab_save; ///< saved join_tab for subquery reexecution
+
+  List<JOIN_TAB_RANGE> join_tab_ranges;
+  
   /*
     Base tables participating in the join. After join optimization is done, the
-    tables are stored in the join order.
+    tables are stored in the join order (but the only really important part is 
+    that const tables are first).
   */
   TABLE    **table;
   /**
@@ -1387,6 +1421,13 @@ class JOIN :public Sql_alloc
   uint	   tables;        /**< Number of tables in the join */
   uint     outer_tables;  /**< Number of tables that are not inside semijoin */
   uint     const_tables;
+  /* 
+    Number of tables in the top join_tab array. Normally this matches
+    (join_tab_ranges.head()->end - join_tab_ranges.head()->start). 
+    
+    We keep it here so that it is saved/restored with JOIN::restore_tmp.
+  */
+  uint     top_jtrange_tables;
   uint	   send_group_parts;
   bool	   group;          /**< If query contains GROUP BY clause */
   /**
@@ -1595,6 +1636,7 @@ class JOIN :public Sql_alloc
     join_tab= join_tab_save= 0;
     table= 0;
     tables= 0;
+    top_jtrange_tables= 0;
     const_tables= 0;
     eliminated_tables= 0;
     join_list= 0;
@@ -1933,6 +1975,7 @@ COND *remove_eq_conds(THD *thd, COND *cond, Item::cond_result *cond_value);
 int test_if_item_cache_changed(List<Cached_item> &list);
 void calc_used_field_length(THD *thd, JOIN_TAB *join_tab);
 int join_init_read_record(JOIN_TAB *tab);
+int join_read_record_no_init(JOIN_TAB *tab);
 void set_position(JOIN *join,uint idx,JOIN_TAB *table,KEYUSE *key);
 inline Item * and_items(Item* cond, Item *item)
 {

sql/sql_test.cc

@@ -165,17 +165,25 @@ void TEST_filesort(SORT_FIELD *sortorder,uint s_length)
 void
 TEST_join(JOIN *join)
 {
-  uint i,ref;
+  uint ref;
+  int i;
+  List_iterator<JOIN_TAB_RANGE> it(join->join_tab_ranges);
+  JOIN_TAB_RANGE *jt_range;
   DBUG_ENTER("TEST_join");
 
+  DBUG_LOCK_FILE;
+  VOID(fputs("\nInfo about JOIN\n",DBUG_FILE));
+
+  while ((jt_range= it++))
+  {
     /*
       Assemble results of all the calls to full_name() first,
       in order not to garble the tabular output below.
     */
     String ref_key_parts[MAX_TABLES];
-  for (i= 0; i < join->tables; i++)
+    for (i= 0; i < (jt_range->end - jt_range->start); i++)
     {
-    JOIN_TAB *tab= join->join_tab + i;
+      JOIN_TAB *tab= jt_range->start + i;
       for (ref= 0; ref < tab->ref.key_parts; ref++)
       {
         ref_key_parts[i].append(tab->ref.items[ref]->full_name());
@@ -183,11 +191,9 @@ TEST_join(JOIN *join)
       }
     }
 
-  DBUG_LOCK_FILE;
-  VOID(fputs("\nInfo about JOIN\n",DBUG_FILE));
-  for (i=0 ; i < join->tables ; i++)
+    for (i= 0; i < (jt_range->end - jt_range->start); i++)
     {
-    JOIN_TAB *tab=join->join_tab+i;
+      JOIN_TAB *tab= jt_range->start + i;
       TABLE *form=tab->table;
       char key_map_buff[128];
       fprintf(DBUG_FILE,"%-16.16s  type: %-7s  q_keys: %s  refs: %d  key: %d  len: %d\n",
@@ -218,6 +224,8 @@ TEST_join(JOIN *join)
                 "                  refs:  %s\n", ref_key_parts[i].ptr());
       }
     }
+    VOID(fputs("\n",DBUG_FILE));
+  }
   DBUG_UNLOCK_FILE;
   DBUG_VOID_RETURN;
 }

sql/sql_union.cc

@@ -703,6 +703,7 @@ bool st_select_lex_unit::cleanup()
     {
       join->tables_list= 0;
       join->tables= 0;
+      join->top_jtrange_tables= 0;
     }
     error|= fake_select_lex->cleanup();
     /*