MWL#68 Subquery optimization: Efficient NOT IN execution with NULLs

* Implemented a second partial matching strategy via table scan.
  This strategy is a fallback when there is no memory for rowid merging.

* Refactored the selection and creation of partial matching strategies,
  so that the choice of strategy is encapsulated in a separate method
  choose_partial_match_strategy().

* Refactored the representation of partial match strategies so that:
  - each strategy is represented by a polymorphic class, and
  - the base class for all partial match strategies contains common
    execution code.

* Added an estimate of the memory needed for the rowid merge strategy,
  and the system variable "rowid_merge_buff_size" to control the maximum
  memory to be used by the rowid merge algorithm.

* Added two optimizer_switch system variables to control the choice of
  partial match strategy:
  "partial_match_rowid_merge", "partial_match_table_scan".

* Fixed multiple problems with deallocation of resources by the partial
  match strategies.


sql/mysql_priv.h:
  * Added two optimizer_switch system variables to control the choice of
    partial match strategy:
    "partial_match_rowid_merge", "partial_match_table_scan".
sql/mysqld.cc:
  * Added two optimizer_switch system variables to control the choice of
    partial match strategy:
    "partial_match_rowid_merge", "partial_match_table_scan".
  * Added a system variable "rowid_merge_buff_size" to control the maximum
    memory to be used by the rowid merge algorithm.
sql/set_var.cc:
  * Added a system variable "rowid_merge_buff_size" to control the maximum
    memory to be used by the rowid merge algorithm.
sql/sql_class.h:
  * Added a system variable "rowid_merge_buff_size" to control the maximum
    memory to be used by the rowid merge algorithm.
support-files/build-tags:
  Newer versions of BZR require the recursive flag in order to list all files.

sql/mysql_priv.h

@@ -552,12 +552,14 @@ protected:
 #define OPTIMIZER_SWITCH_LOOSE_SCAN 64
 #define OPTIMIZER_SWITCH_MATERIALIZATION 128
 #define OPTIMIZER_SWITCH_SEMIJOIN 256
+#define OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE 512
+#define OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN 1024
 
 #ifdef DBUG_OFF
-#  define OPTIMIZER_SWITCH_LAST 512
+#  define OPTIMIZER_SWITCH_LAST 2048
 #else
-#  define OPTIMIZER_SWITCH_TABLE_ELIMINATION 512
-#  define OPTIMIZER_SWITCH_LAST 1024
+#  define OPTIMIZER_SWITCH_TABLE_ELIMINATION 2048
+#  define OPTIMIZER_SWITCH_LAST 4096
 #endif
 
 #ifdef DBUG_OFF 
@@ -570,7 +572,9 @@ protected:
                                     OPTIMIZER_SWITCH_FIRSTMATCH | \
                                     OPTIMIZER_SWITCH_LOOSE_SCAN | \
                                     OPTIMIZER_SWITCH_MATERIALIZATION | \
-                                    OPTIMIZER_SWITCH_SEMIJOIN)
+                                    OPTIMIZER_SWITCH_SEMIJOIN | \
+                                    OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE|\
+                                    OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN)
 #else
 #  define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \
                                     OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \
@@ -581,7 +585,9 @@ protected:
                                     OPTIMIZER_SWITCH_FIRSTMATCH | \
                                     OPTIMIZER_SWITCH_LOOSE_SCAN | \
                                     OPTIMIZER_SWITCH_MATERIALIZATION | \
-                                    OPTIMIZER_SWITCH_SEMIJOIN)
+                                    OPTIMIZER_SWITCH_SEMIJOIN | \
+                                    OPTIMIZER_SWITCH_PARTIAL_MATCH_ROWID_MERGE|\
+                                    OPTIMIZER_SWITCH_PARTIAL_MATCH_TABLE_SCAN)
 #endif
 
 /*

sql/mysqld.cc

@@ -302,6 +302,8 @@ static const char *optimizer_switch_names[]=
   "index_merge_intersection",
   "index_condition_pushdown",
   "firstmatch","loosescan","materialization", "semijoin",
+  "partial_match_rowid_merge",
+  "partial_match_table_scan",
 #ifndef DBUG_OFF
   "table_elimination",
 #endif
@@ -320,6 +322,8 @@ static const unsigned int optimizer_switch_names_len[]=
   sizeof("loosescan") - 1,
   sizeof("materialization") - 1,
   sizeof("semijoin") - 1,
+  sizeof("partial_match_rowid_merge") - 1,
+  sizeof("partial_match_table_scan") - 1,
 #ifndef DBUG_OFF
   sizeof("table_elimination") - 1,
 #endif
@@ -5794,7 +5798,8 @@ enum options_mysqld
   OPT_RECORD_RND_BUFFER, OPT_DIV_PRECINCREMENT, OPT_RELAY_LOG_SPACE_LIMIT,
   OPT_RELAY_LOG_PURGE,
   OPT_SLAVE_NET_TIMEOUT, OPT_SLAVE_COMPRESSED_PROTOCOL, OPT_SLOW_LAUNCH_TIME,
-  OPT_SLAVE_TRANS_RETRIES, OPT_READONLY, OPT_DEBUGGING, OPT_DEBUG_FLUSH,
+  OPT_SLAVE_TRANS_RETRIES, OPT_READONLY, OPT_ROWID_MERGE_BUFF_SIZE,
+  OPT_DEBUGGING, OPT_DEBUG_FLUSH,
   OPT_SORT_BUFFER, OPT_TABLE_OPEN_CACHE, OPT_TABLE_DEF_CACHE,
   OPT_THREAD_CONCURRENCY, OPT_THREAD_CACHE_SIZE,
   OPT_TMP_TABLE_SIZE, OPT_THREAD_STACK,
@@ -7130,6 +7135,11 @@ The minimum value for this variable is 4096.",
    (uchar**) &max_system_variables.range_alloc_block_size, 0, GET_ULONG,
    REQUIRED_ARG, RANGE_ALLOC_BLOCK_SIZE, RANGE_ALLOC_BLOCK_SIZE,
    (longlong) ULONG_MAX, 0, 1024, 0},
+  {"rowid_merge_buff_size", OPT_ROWID_MERGE_BUFF_SIZE,
+   "The size of the buffers used [NOT] IN evaluation via partial matching.",
+   (uchar**) &global_system_variables.rowid_merge_buff_size,
+   (uchar**) &max_system_variables.rowid_merge_buff_size, 0, GET_ULONG,
+   REQUIRED_ARG, 8*1024*1024L, 0, MAX_MEM_TABLE_SIZE/2, 0, 1, 0},
   {"read_buffer_size", OPT_RECORD_BUFFER,
    "Each thread that does a sequential scan allocates a buffer of this size for each table it scans. If you do many sequential scans, you may want to increase this value.",
    (uchar**) &global_system_variables.read_buff_size,

sql/set_var.cc

@@ -540,6 +540,9 @@ static sys_var_long_ptr	sys_query_cache_size(&vars, "query_cache_size",
 
 static sys_var_thd_ulong	sys_range_alloc_block_size(&vars, "range_alloc_block_size",
 						   &SV::range_alloc_block_size);
+static sys_var_thd_ulong	sys_rowid_merge_buff_size(&vars, "rowid_merge_buff_size",
+					   &SV::rowid_merge_buff_size);
+
 static sys_var_thd_ulong	sys_query_alloc_block_size(&vars, "query_alloc_block_size",
 						   &SV::query_alloc_block_size,
 						   0, fix_thd_mem_root);

sql/sql_class.h

@@ -343,6 +343,8 @@ struct system_variables
   ulong mrr_buff_size;
   ulong div_precincrement;
   ulong sortbuff_size;
+  /* Total size of all buffers used by the subselect_rowid_merge_engine. */
+  ulong rowid_merge_buff_size;
   ulong thread_handling;
   ulong tx_isolation;
   ulong completion_type;

support-files/build-tags

@@ -4,7 +4,7 @@ rm -f TAGS
 filter='\.cc$\|\.c$\|\.h$\|\.yy$'
 
 list="find . -type f"
-bzr root >/dev/null 2>/dev/null && list="bzr ls --from-root --kind=file --versioned"
+bzr root >/dev/null 2>/dev/null && list="bzr ls --from-root -R --kind=file --versioned"
 
 $list |grep $filter |while read f; 
 do