Merging index_merge/Unique into 5.0

BitKeeper/etc/logging_ok:
  auto-union
sql/sql_list.h:
  Auto merged

sql/filesort.cc

@@ -48,7 +48,8 @@ static int merge_index(SORTPARAM *param,uchar *sort_buffer,
 		       BUFFPEK *buffpek,
 		       uint maxbuffer,IO_CACHE *tempfile,
 		       IO_CACHE *outfile);
-static bool save_index(SORTPARAM *param,uchar **sort_keys, uint count);
+static bool save_index(SORTPARAM *param,uchar **sort_keys, uint count, 
+                       FILESORT_INFO *table_sort);
 static uint sortlength(SORT_FIELD *sortorder, uint s_length,
 		       bool *multi_byte_charset);
 static SORT_ADDON_FIELD *get_addon_fields(THD *thd, Field **ptabfield,
@@ -85,6 +86,8 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 #ifdef SKIP_DBUG_IN_FILESORT
   DBUG_PUSH("");		/* No DBUG here */
 #endif
+  FILESORT_INFO table_sort;
+  bzero(&table_sort, sizeof(FILESORT_INFO));
   
   outfile= table->sort.io_cache;
   my_b_clear(&tempfile);
@@ -107,14 +110,15 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
                                         param.sort_length,
                                         &param.addon_length);
   }
-  table->sort.addon_buf= 0;
-  table->sort.addon_length= param.addon_length;
-  table->sort.addon_field= param.addon_field;
-  table->sort.unpack= unpack_addon_fields;
+
+  table_sort.addon_buf= 0;
+  table_sort.addon_length= param.addon_length;
+  table_sort.addon_field= param.addon_field;
+  table_sort.unpack= unpack_addon_fields;
   if (param.addon_field)
   {
     param.res_length= param.addon_length;
-    if (!(table->sort.addon_buf= (byte *) my_malloc(param.addon_length,
+    if (!(table_sort.addon_buf= (byte *) my_malloc(param.addon_length,
                                                     MYF(MY_WME))))
       goto err;
   }
@@ -193,7 +197,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 
   if (maxbuffer == 0)			// The whole set is in memory
   {
-    if (save_index(&param,sort_keys,(uint) records))
+    if (save_index(&param,sort_keys,(uint) records, &table_sort))
       goto err;
   }
   else
@@ -256,6 +260,8 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 #ifdef SKIP_DBUG_IN_FILESORT
   DBUG_POP();			/* Ok to DBUG */
 #endif
+  memcpy(&table->sort, &table_sort, sizeof(FILESORT_INFO));
+  table->sort.io_cache= outfile;
   DBUG_PRINT("exit",("records: %ld",records));
   DBUG_RETURN(error ? HA_POS_ERROR : records);
 } /* filesort */
@@ -359,12 +365,24 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
 		    current_thd->variables.read_buff_size);
   }
 
+  READ_RECORD read_record_info;
+  if (quick_select)
+  {
+    if (select->quick->reset())
+      DBUG_RETURN(HA_POS_ERROR);
+    init_read_record(&read_record_info, current_thd, select->quick->head,
+                     select, 1, 1);
+  }
+
   for (;;)
   {
     if (quick_select)
     {
-      if ((error=select->quick->get_next()))
+      if ((error= read_record_info.read_record(&read_record_info)))
+      {
+        error= HA_ERR_END_OF_FILE;
         break;
+      }
       file->position(sort_form->record[0]);
     }
     else					/* Not quick-select */
@@ -392,6 +410,7 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
       if (error && error != HA_ERR_RECORD_DELETED)
 	break;
     }
+
     if (*killed)
     {
       DBUG_PRINT("info",("Sort killed by user"));
@@ -425,8 +444,14 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
     else
       file->unlock_row();
   }
+  if (quick_select)
+    end_read_record(&read_record_info);
+  else
+  {
     (void) file->extra(HA_EXTRA_NO_CACHE);	/* End cacheing of records */
     file->rnd_end();
+  }
+
   DBUG_PRINT("test",("error: %d  indexpos: %d",error,indexpos));
   if (error != HA_ERR_END_OF_FILE)
   {
@@ -664,8 +689,8 @@ static void make_sortkey(register SORTPARAM *param,
   return;
 }
 
-
-static bool save_index(SORTPARAM *param, uchar **sort_keys, uint count)
+static bool save_index(SORTPARAM *param, uchar **sort_keys, uint count, 
+                       FILESORT_INFO *table_sort)
 {
   uint offset,res_length;
   byte *to;
@@ -676,7 +701,7 @@ static bool save_index(SORTPARAM *param, uchar **sort_keys, uint count)
   offset= param->rec_length-res_length;
   if ((ha_rows) count > param->max_rows)
     count=(uint) param->max_rows;
-  if (!(to= param->sort_form->sort.record_pointers=
+  if (!(to= table_sort->record_pointers= 
         (byte*) my_malloc(res_length*count, MYF(MY_WME))))
     DBUG_RETURN(1);                 /* purecov: inspected */
   for (uchar **end= sort_keys+count ; sort_keys != end ; sort_keys++)

sql/opt_range.cc

@@ -649,27 +649,32 @@ QUICK_RANGE_SELECT::~QUICK_RANGE_SELECT()
 }
 
 
-QUICK_INDEX_MERGE_SELECT::QUICK_INDEX_MERGE_SELECT(THD *thd, TABLE *table)
-  :cur_quick_it(quick_selects), index_merge(thd)
+QUICK_INDEX_MERGE_SELECT::QUICK_INDEX_MERGE_SELECT(THD *thd_param, TABLE *table)
+  :cur_quick_it(quick_selects), thd(thd_param), unique(NULL)
 {
   index= MAX_KEY;
   head= table;
+  reset_called= false;
   init_sql_alloc(&alloc,1024,0);
 }
 
 int QUICK_INDEX_MERGE_SELECT::init()
 {
-  int error;
   cur_quick_it.rewind();
   cur_quick_select= cur_quick_it++;
-  if ((error= index_merge.init(head)))
-    return error;
   return cur_quick_select->init();
 }
 
-void QUICK_INDEX_MERGE_SELECT::reset()
+int QUICK_INDEX_MERGE_SELECT::reset()
 {
-  cur_quick_select->reset();
+  int result;
+  DBUG_ENTER("QUICK_INDEX_MERGE_SELECT::reset");
+  if (reset_called)
+    DBUG_RETURN(0);
+
+  reset_called= true;
+  result = cur_quick_select->reset() && prepare_unique();  
+  DBUG_RETURN(result);
 }
 
 bool 
@@ -1150,12 +1155,6 @@ imerge_fail:;
                        ("Failed to allocate index merge structures,"
                        "falling back to full scan."));
           }
-          else
-          {
-            /* with 'using filesort' quick->reset() is not called */
-            quick->reset();
-          }
-
           goto end;
         }
       }
@@ -1170,9 +1169,9 @@ imerge_fail:;
   DBUG_EXECUTE("info",
     {
       if (quick_imerge)
-        print_quick_sel_imerge(quick_imerge, &needed_reg);
+        print_quick_sel_imerge(quick_imerge, needed_reg);
       else
-        print_quick_sel_range((QUICK_RANGE_SELECT*)quick, &needed_reg);
+        print_quick_sel_range((QUICK_RANGE_SELECT*)quick, needed_reg);
     }
   );
 
@@ -1721,6 +1720,7 @@ tree_and(PARAM *param,SEL_TREE *tree1,SEL_TREE *tree2)
     uint flag=0;
     if (*key1 || *key2)
     {
+      trees_have_key = true;
       if (*key1 && !(*key1)->simple_key())
 	flag|=CLONE_KEY1_MAYBE;
       if (*key2 && !(*key2)->simple_key())
@@ -3079,165 +3079,24 @@ QUICK_RANGE_SELECT *get_quick_select_for_ref(THD *thd, TABLE *table,
   return 0;
 }
 
-INDEX_MERGE::INDEX_MERGE(THD *thd_arg) :
-  dont_save(false), thd(thd_arg)  
-{}
-
-String *INDEX_MERGE::Item_rowid::val_str(String *str)
-{
-  str->set_quick((char*)head->file->ref, head->file->ref_length, collation.collation);
-  return str;
-}
-
 
+#define MEM_STRIP_BUF_SIZE current_thd->variables.sortbuff_size
 /*
-  Initialize index_merge operation.
-  RETURN 
-    0     - OK 
-    other - error.
+  Fetch all row ids into unique.
 */
-
-int INDEX_MERGE::init(TABLE *table)
+int QUICK_INDEX_MERGE_SELECT::prepare_unique()
 {
-  DBUG_ENTER("INDEX_MERGE::init");
-  
-  head= table;
-  if (!(rowid_item= new Item_rowid(table)))
-    DBUG_RETURN(1);  
-
-  tmp_table_param.copy_field= 0;
-  tmp_table_param.end_write_records= HA_POS_ERROR;
-  tmp_table_param.group_length= table->file->ref_length;
-  tmp_table_param.group_parts= 1;
-  tmp_table_param.group_null_parts= 0;
-  tmp_table_param.hidden_field_count= 0;
-  tmp_table_param.field_count= 0;
-  tmp_table_param.func_count= 1;
-  tmp_table_param.sum_func_count= 0;
-  tmp_table_param.quick_group= 1;
-
-  bzero(&order, sizeof(ORDER));
-  order.item= (Item**)&rowid_item;
-  order.asc= 1;
-
-  fields.push_back(rowid_item);
-
-  temp_table= create_tmp_table(thd,
-                               &tmp_table_param,
-                               fields,
-                               &order,
-                               false,
-                               0,
-                               SELECT_DISTINCT,
-                               HA_POS_ERROR,
-                               (char *)"");
-  DBUG_RETURN(!temp_table);
-}
-
-/*
-  Check if record with ROWID record_pos has already been processed and 
-  if not - store the ROWID value.
-
-  RETURN
-    0 - record has not been processed yet
-    1 - record has already been processed.
-   -1 - an error occurred and query processing should be terminated.
-        Error code is stored in INDEX_MERGE::error
-*/
-
-int INDEX_MERGE::check_record_in()
-{ 
-  return (dont_save)? 
-           check_record() : 
-           put_record();
-}
-
-
-/*
-  Stop remembering records in check(). 
-  (this should be called just before the last key scan)
-
-  RETURN 
-    0 - OK
-    1 - error occurred initializing table index. 
-*/
-
-int INDEX_MERGE::start_last_quick_select()
-{
-  int result= 0;
-  if (!temp_table->uniques)
-  {
-    dont_save= true;
-    result= temp_table->file->index_init(0);
-  }
-  return result;
-}
-
-
-inline int INDEX_MERGE::put_record()
-{
-  DBUG_ENTER("INDEX_MERGE::put_record");
-  
-  copy_funcs(tmp_table_param.items_to_copy);
-  
-  if ((error= temp_table->file->write_row(temp_table->record[0])))
-  {
-    if (error == HA_ERR_FOUND_DUPP_KEY ||
-	error == HA_ERR_FOUND_DUPP_UNIQUE)
-      DBUG_RETURN(1);
-
-    DBUG_PRINT("info", 
-               ("Error writing row to temp. table: %d, converting to myisam", 
-               error));
-    if (create_myisam_from_heap(current_thd, temp_table, &tmp_table_param,
-				error,1))
-    {
-      DBUG_PRINT("info", ("Table conversion failed, bailing out"));
-      DBUG_RETURN(-1);
-    }
-  }
-
-  DBUG_RETURN(0);
-}
+  int result;
+  DBUG_ENTER("QUICK_INDEX_MERGE_SELECT::prepare_unique");
   
-inline int INDEX_MERGE::check_record()
-{
-  int result= 1;
-  DBUG_ENTER("INDEX_MERGE::check_record");  
+  /* we're going to just read rowids */
+  head->file->extra(HA_EXTRA_KEYREAD);
 
-  if ((error= temp_table->file->index_read(temp_table->record[0],
-                                           head->file->ref,
+  unique= new Unique(refposcmp2, (void *) &head->file->ref_length,
                      head->file->ref_length,
-                                           HA_READ_KEY_EXACT)))
-  {
-    if (error != HA_ERR_KEY_NOT_FOUND)
-      result= -1;
-    else
-      result= 0;
-  }
-
-  DBUG_RETURN(result);
-}
-
-INDEX_MERGE::~INDEX_MERGE()
-{
-  if (temp_table)
-  {
-    DBUG_PRINT("info", ("Freeing temp. table"));
-    free_tmp_table(current_thd, temp_table);
-  }
-  /* rowid_item is freed automatically */
-  list_node* node;
-  node= fields.first_node();
-  fields.remove(&node);
-}
-
-int QUICK_INDEX_MERGE_SELECT::get_next()
-{
-  int       result;
-  int       put_result;
-  DBUG_ENTER("QUICK_INDEX_MERGE_SELECT::get_next");
-
+                     MEM_STRIP_BUF_SIZE);  
+  if (!unique)
+    DBUG_RETURN(1);
   do
   { 
     while ((result= cur_quick_select->get_next()) == HA_ERR_END_OF_FILE)
@@ -3245,15 +3104,9 @@ int QUICK_INDEX_MERGE_SELECT::get_next()
       cur_quick_select= cur_quick_it++;
       if (!cur_quick_select)
         break;
-
       cur_quick_select->init();
-      cur_quick_select->reset();
-      
-      if (last_quick_select == cur_quick_select)
-      {
-        if ((result= index_merge.start_last_quick_select()))
-          DBUG_RETURN(result);
-      }
+      if (cur_quick_select->reset())
+        DBUG_RETURN(1);
     }
 
     if (result)
@@ -3262,14 +3115,39 @@ int QUICK_INDEX_MERGE_SELECT::get_next()
         table read error (including HA_ERR_END_OF_FILE on last quick select
         in index_merge)
       */
+      if (result != HA_ERR_END_OF_FILE)
+      {
         DBUG_RETURN(result);
       }
+      else
+        break;
+    }
+    
+    if (thd->killed)
+      DBUG_RETURN(1);
 
     cur_quick_select->file->position(cur_quick_select->record);
-    put_result= index_merge.check_record_in();
-  }while(put_result == 1); /* While record is processed */
+    if (unique->unique_add((char*)cur_quick_select->file->ref))
+      DBUG_RETURN(1);
+
+  }while(true);  
+
+  /* ok, all row ids are in Unique */
+  result= unique->get(head);
+
+  /* index_merge currently doesn't support "using index" at all */
+  head->file->extra(HA_EXTRA_NO_KEYREAD);
+  DBUG_RETURN(result);
+}
+
+int QUICK_INDEX_MERGE_SELECT::get_next()
+{
+  DBUG_ENTER("QUICK_INDEX_MERGE_SELECT::get_next");
   
-  DBUG_RETURN((put_result != -1) ? result : index_merge.error);
+  DBUG_PRINT("QUICK_INDEX_MERGE_SELECT", 
+             ("ERROR: index merge error: get_next should not be called "));
+  DBUG_ASSERT(0);
+  DBUG_RETURN(HA_ERR_END_OF_FILE);
 }
 
 	/* get next possible record using quick-struct */

sql/opt_range.h

@@ -65,7 +65,6 @@ class QUICK_RANGE :public Sql_alloc {
     }
 };
 
-class INDEX_MERGE; 
 
 /*
   Quick select interface. 
@@ -149,64 +148,6 @@ class QUICK_RANGE_SELECT : public QUICK_SELECT_I
   int get_type() { return QS_TYPE_RANGE; }
 };
 
-/*
-  Helper class for keeping track of rows that have been passed to output 
-  in index_merge access method. 
-
-  NOTES
-    Current implementation uses a temporary table to store ROWIDs of rows that
-    have been passed to output. In the future it might be changed to use more 
-    efficient mechanisms, like Unique class.
-*/
-
-class INDEX_MERGE
-{
-public:
-  INDEX_MERGE(THD *thd_arg);
-  ~INDEX_MERGE(); 
-  
-  int init(TABLE *table);
-  int check_record_in();
-  int start_last_quick_select();
-  int error;
-private:
-  /* The only field in temporary table */
-  class Item_rowid : public Item_str_func
-  {
-    TABLE *head; /* source table */
-  public:
-    Item_rowid(TABLE *table) : head(table) 
-    {
-      max_length= table->file->ref_length;
-      collation.set(&my_charset_bin);
-    };
-    const char *func_name() const { return "rowid"; }
-    bool const_item() const { return 0; }
-    String *val_str(String *);
-    void fix_length_and_dec()
-    {}
-  };
-
-  /* Check if record has been processed and save it if it wasn't  */
-  inline int put_record(); 
-  
-  /* Check if record has been processed without saving it         */
-  inline int check_record();
-  
-  /* If true, check_record_in does't store ROWIDs it is passed.   */
-  bool  dont_save;
-
-  THD *thd;
-  TABLE *head;                     /* source table                        */
-  TABLE *temp_table;               /* temp. table used for values storage */
-  TMP_TABLE_PARAM tmp_table_param; /* temp. table creation parameters     */
-  Item_rowid *rowid_item;          /* the only field in temp. table       */
-  List<Item> fields;               /* temp. table fields list 
-                                      (the only element is rowid_item)    */
-  ORDER order;                     /* key for temp. table (rowid_item)    */
-};
-
-
 /*
   Index merge quick select. 
   It is implemented as a container for several QUICK_RANGE_SELECTs.
@@ -219,7 +160,7 @@ class QUICK_INDEX_MERGE_SELECT : public QUICK_SELECT_I
   ~QUICK_INDEX_MERGE_SELECT();
 
   int  init();
-  void reset(void);
+  int  reset(void);
   int  get_next();
   bool reverse_sorted() { return false; }
   bool unique_key_range() { return false; }
@@ -234,20 +175,15 @@ class QUICK_INDEX_MERGE_SELECT : public QUICK_SELECT_I
   List_iterator_fast<QUICK_RANGE_SELECT> cur_quick_it;
   QUICK_RANGE_SELECT* cur_quick_select;
   
-  /*
-    Last element in quick_selects list. 
-    INDEX_MERGE::start_last_quick_select is called before retrieving
-    rows for it. 
-  */
+  /* last element in quick_selects list. */
   QUICK_RANGE_SELECT* last_quick_select;
   
-  /*
-    Used to keep track of what records have been already passed to output 
-    when doing index_merge access (NULL means no index_merge) 
-  */
-  INDEX_MERGE index_merge;
-
+  Unique  *unique;
   MEM_ROOT alloc;
+
+  THD *thd;
+  int prepare_unique();
+  bool reset_called;
 };
 
 class QUICK_SELECT_DESC: public QUICK_RANGE_SELECT
@@ -263,7 +199,7 @@ class QUICK_SELECT_DESC: public QUICK_RANGE_SELECT
 #ifdef NOT_USED
   bool test_if_null_range(QUICK_RANGE *range, uint used_key_parts);
 #endif
-  void reset(void) { next=0; rev_it.rewind(); }
+  int reset(void) { next=0; rev_it.rewind(); return 0; }
   List<QUICK_RANGE> rev_ranges;
   List_iterator<QUICK_RANGE> rev_it;
 };

sql/records.cc

@@ -97,7 +97,8 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
       }
     }
   }
-  else if (select && select->quick)
+  else if (select && select->quick && 
+           (select->quick->get_type() != QUICK_SELECT_I::QS_TYPE_INDEX_MERGE))
   {
     DBUG_PRINT("info",("using rr_quick"));
     info->read_record=rr_quick;

sql/sql_select.cc

@@ -5998,8 +5998,8 @@ test_if_quick_select(JOIN_TAB *tab)
 static int
 join_init_read_record(JOIN_TAB *tab)
 {
-  if (tab->select && tab->select->quick)
-    tab->select->quick->reset();
+  if (tab->select && tab->select->quick && tab->select->quick->reset())
+    return 1;
   init_read_record(&tab->read_record, tab->join->thd, tab->table,
 		   tab->select,1,1);
   return (*tab->read_record.read_record)(&tab->read_record);

sql/sql_select.h

@@ -316,6 +316,9 @@ uint find_shortest_key(TABLE *table, const key_map *usable_keys);
 /* functions from opt_sum.cc */
 int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds);
 
+/* from sql_delete.cc, used by opt_range.cc */
+extern "C" int refposcmp2(void* arg, const void *a,const void *b);
+
 /* class to copying an field/item to a key struct */
 
 class store_key :public Sql_alloc

sql/sql_union.cc

@@ -192,7 +192,7 @@ int st_select_lex_unit::prepare(THD *thd, select_result *sel_result,
   union_result->not_describe=1;
   union_result->tmp_table_param=tmp_table_param;
 
-  for (;sl; sl= sl->next_select())
+  for (sl= select_cursor; sl; sl= sl->next_select())
   {
     JOIN *join= new JOIN(thd, sl->item_list, 
 			 sl->options | thd->options | SELECT_NO_UNLOCK,