Implementation of WL#1469 (Greedy algorithm to search for an optimal execution plan).

mysql-test/r/subselect.result:
  Table pre-sorting before optimization makes the optimizer select a different plan, this change fixes the plan.
sql/mysql_priv.h:
  Added new status variable last_query_cost that contains the cost of the last compiled query.
sql/mysqld.cc:
  * Added new system variable 'plan_search_depth' to control the exhaustiveness of the search for an optimal query plan.
  * Added new system variable 'heuristic' to control the use of query optimization heuristic.
  * Added new status variable 'last_query_cost' that contains the cost of the last compiled query.
sql/set_var.cc:
  * Added new system variable 'plan_search_depth' to control the exhaustiveness of the search for an optimal query plan.
  * Added new system variable 'heuristic' to control the use of query optimization heuristic.
sql/sql_class.h:
  * Added new system variable 'plan_search_depth' to control the exhaustiveness of the search for an optimal query plan.
  * Added new system variable 'heuristic' to control the use of query optimization heuristic.
sql/sql_select.cc:
  Added a combined greedy/exhaustive query optimization algorithm.
  * The greedy search algorithm is implemented in function 'greedy_search'.
  * The exhaustive search with controlled search depth is implemented in function 'find_best_limited_depth', a modification of 'find_best'.
  * The selection of the best access path and its cost computation is factored out from 'find_best' into function 'best_access_path'.
  * In addition:
    - added pre-sorting for the query tables before they get optimized
    - factored out the optimization of STRAIGHT_JOIN into a separate procedure
sql/sql_select.h:
  Added new field to st_position to support the greedy optimizer.
  Added a comment to class JOIN.
sql/sql_show.cc:
  Added a case to 'mysqld_show' to print double status variables.
sql/structs.h:
  Added a new type of status variables SHOW_DOUBLE
BitKeeper/etc/logging_ok:
  Logging to logging@openlogging.org accepted

BitKeeper/etc/logging_ok

@@ -141,6 +141,7 @@ tim@hundin.mysql.fi
 tim@threads.polyesthetic.msg
 tim@white.box
 tim@work.mysql.com
+timour@mysql.com
 tom@basil-firewall.home.com
 tonu@hundin.mysql.fi
 tonu@volk.internalnet

mysql-test/r/subselect.result

@@ -1348,8 +1348,8 @@ a
 explain extended select * from t2 where t2.a in (select t1.a from t1,t3 where t1.b=t3.a);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	index	NULL	a	5	NULL	4	Using where; Using index
-2	DEPENDENT SUBQUERY	t1	ref	a	a	5	func	1001	Using where; Using index
-2	DEPENDENT SUBQUERY	t3	index	a	a	5	NULL	3	Using where; Using index
+2	DEPENDENT SUBQUERY	t3	index	a	a	5	NULL	3	Using index
+2	DEPENDENT SUBQUERY	t1	ref	a	a	10	func,test.t3.a	1000	Using where; Using index
 Warnings:
 Note	1003	select high_priority test.t2.a AS `a` from test.t2 where <in_optimizer>(test.t2.a,<exists>(select 1 AS `Not_used` from test.t1 join test.t3 where ((test.t1.b = test.t3.a) and (<cache>(test.t2.a) = test.t1.a)) limit 1))
 insert into t1 values (3,31);

sql/mysql_priv.h

@@ -836,6 +836,7 @@ extern char glob_hostname[FN_REFLEN], mysql_home[FN_REFLEN];
 extern char pidfile_name[FN_REFLEN], time_zone[30], *opt_init_file;
 extern char log_error_file[FN_REFLEN];
 extern double log_10[32];
+extern double last_query_cost;
 extern ulonglong keybuff_size;
 extern ulong refresh_version,flush_version, thread_id,query_id,opened_tables;
 extern ulong created_tmp_tables, created_tmp_disk_tables;

sql/mysqld.cc

@@ -311,6 +311,7 @@ ulong my_bind_addr;			/* the address we bind to */
 volatile ulong cached_thread_count= 0;
 
 double log_10[32];			/* 10 potences */
+double last_query_cost= -1; /* -1 denotes that no query was compiled yet */
 
 time_t start_time;
 
@@ -3706,7 +3707,9 @@ enum options_mysqld
   OPT_DATE_FORMAT,
   OPT_TIME_FORMAT,
   OPT_DATETIME_FORMAT,
-  OPT_LOG_QUERIES_NOT_USING_INDEXES
+  OPT_LOG_QUERIES_NOT_USING_INDEXES,
+  OPT_PLAN_SEARCH_DEPTH,
+  OPT_HEURISTIC
 };
 
 
@@ -4308,6 +4311,11 @@ log and this option does nothing anymore.",
     "Use stopwords from this file instead of built-in list.",
     (gptr*) &ft_stopword_file, (gptr*) &ft_stopword_file, 0, GET_STR,
     REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"heuristic", OPT_HEURISTIC,
+   "Controls the heuristic(s) applied during query optimization to prune less-promising partial plans from the optimizer search space. Meaning: 0 - do not apply any heuristic, thus perform exhaustive search; 1 - prune plans based on rows and read time.",
+   (gptr*) &global_system_variables.heuristic,
+   (gptr*) &max_system_variables.heuristic,
+   0, GET_ULONG, OPT_ARG, 1, 0, 1, 0, 1, 0},
 #ifdef HAVE_INNOBASE_DB
   {"innodb_mirrored_log_groups", OPT_INNODB_MIRRORED_LOG_GROUPS,
    "Number of identical copies of log groups we keep for the database. Currently this should be set to 1.",
@@ -4542,6 +4550,11 @@ The minimum value for this variable is 4096.",
    "If this is not 0, then mysqld will use this value to reserve file descriptors to use with setrlimit(). If this value is 0 then mysqld will reserve max_connections*5 or max_connections + table_cache*2 (whichever is larger) number of files.",
    (gptr*) &open_files_limit, (gptr*) &open_files_limit, 0, GET_ULONG,
    REQUIRED_ARG, 0, 0, 65535, 0, 1, 0},
+  {"plan_search_depth", OPT_PLAN_SEARCH_DEPTH,
+   "Maximum depth of search performed by the query optimizer. Values larger than the number of relations in a query result in better query plans, but take longer to compile a query. Smaller values than the number of tables in a relation result in faster optimization, but may produce very bad query plans. If set to 0, the system will automatically pick a reasonable value; if set to MAX_TABLES+2, the optimizer will switch to the original find_best (used for testing/comparison).",
+   (gptr*) &global_system_variables.plan_search_depth,
+   (gptr*) &max_system_variables.plan_search_depth,
+   0, GET_ULONG, OPT_ARG, MAX_TABLES+1, 0, MAX_TABLES+2, 0, 1, 0},
    {"preload_buffer_size", OPT_PRELOAD_BUFFER_SIZE,
     "The size of the buffer that is allocated when preloading indexes",
     (gptr*) &global_system_variables.preload_buff_size,
@@ -4887,6 +4900,7 @@ struct show_var_st status_vars[]= {
   {"Threads_connected",        (char*) &thread_count,           SHOW_INT_CONST},
   {"Threads_running",          (char*) &thread_running,         SHOW_INT_CONST},
   {"Uptime",                   (char*) 0,                       SHOW_STARTTIME},
+  {"Last_query_cost",          (char*) &last_query_cost,        SHOW_DOUBLE},
   {NullS, NullS, SHOW_LONG}
 };
 

sql/set_var.cc

@@ -149,6 +149,7 @@ sys_var_long_ptr	sys_expire_logs_days("expire_logs_days",
 					     &expire_logs_days);
 sys_var_bool_ptr	sys_flush("flush", &myisam_flush);
 sys_var_long_ptr	sys_flush_time("flush_time", &flush_time);
+sys_var_thd_ulong       sys_heuristic("heuristic", &SV::heuristic);
 sys_var_thd_ulong	sys_interactive_timeout("interactive_timeout",
 						&SV::net_interactive_timeout);
 sys_var_thd_ulong	sys_join_buffer_size("join_buffer_size",
@@ -241,6 +242,8 @@ sys_var_thd_ulong	sys_net_retry_count("net_retry_count",
 					    fix_net_retry_count);
 sys_var_thd_bool	sys_new_mode("new", &SV::new_mode);
 sys_var_thd_bool	sys_old_passwords("old_passwords", &SV::old_passwords);
+sys_var_thd_ulong       sys_plan_search_depth("plan_search_depth",
+                                              &SV::plan_search_depth);
 sys_var_thd_ulong       sys_preload_buff_size("preload_buffer_size",
                                               &SV::preload_buff_size);
 sys_var_thd_ulong	sys_read_buff_size("read_buffer_size",
@@ -447,6 +450,7 @@ sys_var *sys_variables[]=
   &sys_flush_time,
   &sys_foreign_key_checks,
   &sys_group_concat_max_len,
+  &sys_heuristic,
   &sys_identity,
   &sys_init_connect,
   &sys_init_slave,
@@ -492,6 +496,7 @@ sys_var *sys_variables[]=
   &sys_net_write_timeout,
   &sys_new_mode,
   &sys_old_passwords,
+  &sys_plan_search_depth,
   &sys_preload_buff_size,
   &sys_pseudo_thread_id,
   &sys_query_alloc_block_size,
@@ -760,6 +765,8 @@ struct show_var_st init_vars[]= {
   {"version_compile_machine", (char*) MACHINE_TYPE,		    SHOW_CHAR},
   {"version_compile_os",      (char*) SYSTEM_TYPE,		    SHOW_CHAR},
   {sys_net_wait_timeout.name, (char*) &sys_net_wait_timeout,	    SHOW_SYS},
+  {sys_heuristic.name,        (char*) &sys_heuristic,               SHOW_SYS},
+  {sys_plan_search_depth.name,(char*) &sys_plan_search_depth,       SHOW_SYS},
   {NullS, NullS, SHOW_LONG}
 };
 

sql/sql_class.h

@@ -390,6 +390,8 @@ struct system_variables
   ulong table_type;
   ulong tmp_table_size;
   ulong tx_isolation;
+  ulong heuristic;
+  ulong plan_search_depth;
   /* Determines which non-standard SQL behaviour should be enabled */
   ulong sql_mode;
   ulong default_week_format;

sql/sql_select.h

@@ -116,6 +116,7 @@ typedef struct st_join_table {
 typedef struct st_position			/* Used in find_best */
 {
   double records_read;
+  double read_time;
   JOIN_TAB *table;
   KEYUSE *key;
 } POSITION;
@@ -133,8 +134,9 @@ typedef struct st_rollup
 class JOIN :public Sql_alloc
 {
  public:
-  JOIN_TAB *join_tab,**best_ref,**map2table;
-  JOIN_TAB *join_tab_save; //saved join_tab for subquery reexecution
+  JOIN_TAB *join_tab,**best_ref;
+  JOIN_TAB **map2table;    // mapping between table indexes and JOIN_TABs
+  JOIN_TAB *join_tab_save; // saved join_tab for subquery reexecution
   TABLE    **table,**all_tables,*sort_by_table;
   uint	   tables,const_tables;
   uint	   send_group_parts;

sql/sql_show.cc

@@ -1831,6 +1831,11 @@ int mysqld_show(THD *thd, const char *wild, show_var_st *variables,
         end= strend(pos);
         break;
       }
+      case SHOW_DOUBLE:
+      {
+        end= buff + sprintf(buff, "%f", *(double*) value);
+        break;
+      }
 #ifdef HAVE_OPENSSL
 	/* First group - functions relying on CTX */
       case SHOW_SSL_CTX_SESS_ACCEPT:

sql/structs.h

@@ -156,8 +156,8 @@ typedef struct st_known_date_time_format {
 enum SHOW_TYPE
 {
   SHOW_UNDEF,
-  SHOW_LONG, SHOW_LONGLONG, SHOW_INT, SHOW_CHAR, SHOW_CHAR_PTR, SHOW_BOOL,
-  SHOW_MY_BOOL, SHOW_OPENTABLES, SHOW_STARTTIME, SHOW_QUESTION,
+  SHOW_LONG, SHOW_LONGLONG, SHOW_INT, SHOW_CHAR, SHOW_CHAR_PTR, SHOW_DOUBLE,
+  SHOW_BOOL, SHOW_MY_BOOL, SHOW_OPENTABLES, SHOW_STARTTIME, SHOW_QUESTION,
   SHOW_LONG_CONST, SHOW_INT_CONST, SHOW_HAVE, SHOW_SYS, SHOW_HA_ROWS,
 #ifdef HAVE_OPENSSL
   SHOW_SSL_CTX_SESS_ACCEPT, 	SHOW_SSL_CTX_SESS_ACCEPT_GOOD,