BUG#778434 Wrong result with in_to_exists=on in maria-5.3-mwl89

- Make {ha_myisam,ha_maria}::index_read_idx_map check pushed index condition.
- Address review feedback (added comments)

mysql-test/include/icp_tests.inc

@@ -225,3 +225,32 @@ SELECT COUNT(*) FROM t3;
 
 DROP PROCEDURE insert_data;
 DROP TABLE t1, t2, t3;
+
+--echo #
+--echo # BUG#778434 Wrong result with in_to_exists=on in maria-5.3-mwl89
+--echo #
+CREATE TABLE t1 ( f11 int) ;
+INSERT IGNORE INTO t1 VALUES (0);
+
+CREATE TABLE t2 ( f10 int) ;
+INSERT IGNORE INTO t2 VALUES (0);
+
+CREATE TABLE t3 ( f1 int NOT NULL , f10 int, PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t3 VALUES (6,0),(10,0);
+
+CREATE TABLE t4 ( f11 int) ;
+INSERT IGNORE INTO t4 VALUES
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(NULL),
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0);
+
+set @tmp_778434=@@optimizer_switch;
+SET optimizer_switch='materialization=off,in_to_exists=on,subquery_cache=off,semijoin=off';
+
+SELECT * FROM t1 INNER JOIN t2 ON t2.f10 = t1.f11
+WHERE (6, 234) IN (
+        SELECT t3.f1, t3.f1
+        FROM t3 JOIN t4 ON t4.f11 = t3.f10
+);
+
+DROP TABLE t1,t2,t3,t4;
+set optimizer_switch= @tmp_778434;

mysql-test/r/innodb_icp.result

@@ -202,5 +202,28 @@ COUNT(*)
 12
 DROP PROCEDURE insert_data;
 DROP TABLE t1, t2, t3;
+#
+# BUG#778434 Wrong result with in_to_exists=on in maria-5.3-mwl89
+#
+CREATE TABLE t1 ( f11 int) ;
+INSERT IGNORE INTO t1 VALUES (0);
+CREATE TABLE t2 ( f10 int) ;
+INSERT IGNORE INTO t2 VALUES (0);
+CREATE TABLE t3 ( f1 int NOT NULL , f10 int, PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t3 VALUES (6,0),(10,0);
+CREATE TABLE t4 ( f11 int) ;
+INSERT IGNORE INTO t4 VALUES
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(NULL),
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0);
+set @tmp_778434=@@optimizer_switch;
+SET optimizer_switch='materialization=off,in_to_exists=on,subquery_cache=off,semijoin=off';
+SELECT * FROM t1 INNER JOIN t2 ON t2.f10 = t1.f11
+WHERE (6, 234) IN (
+SELECT t3.f1, t3.f1
+FROM t3 JOIN t4 ON t4.f11 = t3.f10
+);
+f11	f10
+DROP TABLE t1,t2,t3,t4;
+set optimizer_switch= @tmp_778434;
 set optimizer_switch=@innodb_icp_tmp;
 set storage_engine= @save_storage_engine;

mysql-test/r/maria_icp.result

@@ -202,5 +202,28 @@ COUNT(*)
 12
 DROP PROCEDURE insert_data;
 DROP TABLE t1, t2, t3;
+#
+# BUG#778434 Wrong result with in_to_exists=on in maria-5.3-mwl89
+#
+CREATE TABLE t1 ( f11 int) ;
+INSERT IGNORE INTO t1 VALUES (0);
+CREATE TABLE t2 ( f10 int) ;
+INSERT IGNORE INTO t2 VALUES (0);
+CREATE TABLE t3 ( f1 int NOT NULL , f10 int, PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t3 VALUES (6,0),(10,0);
+CREATE TABLE t4 ( f11 int) ;
+INSERT IGNORE INTO t4 VALUES
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(NULL),
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0);
+set @tmp_778434=@@optimizer_switch;
+SET optimizer_switch='materialization=off,in_to_exists=on,subquery_cache=off,semijoin=off';
+SELECT * FROM t1 INNER JOIN t2 ON t2.f10 = t1.f11
+WHERE (6, 234) IN (
+SELECT t3.f1, t3.f1
+FROM t3 JOIN t4 ON t4.f11 = t3.f10
+);
+f11	f10
+DROP TABLE t1,t2,t3,t4;
+set optimizer_switch= @tmp_778434;
 set storage_engine= @save_storage_engine;
 set optimizer_switch=@maria_icp_tmp;

mysql-test/r/myisam_icp.result

@@ -198,6 +198,29 @@ COUNT(*)
 12
 DROP PROCEDURE insert_data;
 DROP TABLE t1, t2, t3;
+#
+# BUG#778434 Wrong result with in_to_exists=on in maria-5.3-mwl89
+#
+CREATE TABLE t1 ( f11 int) ;
+INSERT IGNORE INTO t1 VALUES (0);
+CREATE TABLE t2 ( f10 int) ;
+INSERT IGNORE INTO t2 VALUES (0);
+CREATE TABLE t3 ( f1 int NOT NULL , f10 int, PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t3 VALUES (6,0),(10,0);
+CREATE TABLE t4 ( f11 int) ;
+INSERT IGNORE INTO t4 VALUES
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(NULL),
+(0),(0),(0),(0),(0),(0),(0),(0),(0),(0);
+set @tmp_778434=@@optimizer_switch;
+SET optimizer_switch='materialization=off,in_to_exists=on,subquery_cache=off,semijoin=off';
+SELECT * FROM t1 INNER JOIN t2 ON t2.f10 = t1.f11
+WHERE (6, 234) IN (
+SELECT t3.f1, t3.f1
+FROM t3 JOIN t4 ON t4.f11 = t3.f10
+);
+f11	f10
+DROP TABLE t1,t2,t3,t4;
+set optimizer_switch= @tmp_778434;
 set @myisam_icp_tmp=@@optimizer_switch;
 set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on';
 drop table if exists t0, t1, t1i, t1m;

sql/handler.h

@@ -2462,6 +2462,7 @@ class handler :public Sql_alloc
   */
 
   virtual int open(const char *name, int mode, uint test_if_locked)=0;
+  /* Note: ha_index_read_idx_map() may buypass index_init() */
   virtual int index_init(uint idx, bool sorted) { return 0; }
   virtual int index_end() { return 0; }
   /**

sql/sql_class.h

@@ -3627,11 +3627,19 @@ inline int handler::ha_index_read_map(uchar * buf, const uchar * key,
   return error;
 }
 
+
+/*
+  @note: Other index lookup/navigation functions require prior
+  handler->index_init() call. This function is different, it requires
+  that the scan is not initialized, and accepts "uint index" as an argument.
+*/
+
 inline int handler::ha_index_read_idx_map(uchar * buf, uint index,
                                           const uchar * key,
                                           key_part_map keypart_map,
                                           enum ha_rkey_function find_flag)
 {
+  DBUG_ASSERT(inited==NONE);
   increment_statistics(&SSV::ha_read_key_count);
   int error= index_read_idx_map(buf, index, key, keypart_map, find_flag);
   if (!error)

storage/maria/ha_maria.cc

@@ -2259,7 +2259,14 @@ int ha_maria::index_read_idx_map(uchar * buf, uint index, const uchar * key,
 				 key_part_map keypart_map,
 				 enum ha_rkey_function find_flag)
 {
-  int error= maria_rkey(file, buf, index, key, keypart_map, find_flag);
+  int error;
+  /* Use the pushed index condition if it matches the index we're scanning */
+  if (index == pushed_idx_cond_keyno)
+    ma_set_index_cond_func(file, index_cond_func_maria, this);
+  
+  error= maria_rkey(file, buf, index, key, keypart_map, find_flag);
+   
+  ma_set_index_cond_func(file, NULL, 0);
   table->status= error ? STATUS_NOT_FOUND : 0;
   return error;
 }

storage/myisam/ha_myisam.cc

@@ -1790,7 +1790,13 @@ int ha_myisam::index_read_idx_map(uchar *buf, uint index, const uchar *key,
                                   key_part_map keypart_map,
                                   enum ha_rkey_function find_flag)
 {
-  return mi_rkey(file, buf, index, key, keypart_map, find_flag);
+  int res;
+  /* Use the pushed index condition if it matches the index we're scanning */
+  if (index == pushed_idx_cond_keyno)
+    mi_set_index_cond_func(file, index_cond_func_myisam, this);
+  res= mi_rkey(file, buf, index, key, keypart_map, find_flag);
+  mi_set_index_cond_func(file, NULL, 0);
+  return res;
 }
 
 int ha_myisam::index_next(uchar *buf)