Fix for LP BUG#714808 and LP BUG#719280.

The patch also adjusts several instable test results
to order the result.

Analysis:

The function prev_record_reads() may skip (jump over)
some query plan nodes where record_count < 1. At the
same time, even though get_partial_join_cost() uses
all first N plan nodes after the last constant table,
it may produce a smaller record_count than
prev_record_reads(), because the record count for
some plan nodes may be < 1, and these nodes may not
participate in prev_record_reads.

Solution:
The current solution is to treat the result of
get_partial_join_cost() as the upper bound for the
total number of unique lookup keys.

mysql-test/r/subselect_mat_cost.result

@@ -3985,3 +3985,18 @@ ORDER BY field1;
 field1
 28
 drop table t1,t2;
+#
+# LP BUG#601124 Bug in eliminate_item_equal
+# leads to crash in Item_func::Item_func
+CREATE TABLE t1 ( f1 int(11), f3 varchar(1)) ;
+INSERT INTO t1 VALUES (5,'m'),(NULL,'c');
+CREATE TABLE t2 ( f2 int(11), f3 varchar(1)) ;
+INSERT INTO t2 VALUES (6,'f'),(2,'d');
+CREATE TABLE t3 ( f2 int(11), f3 varchar(1)) ;
+INSERT INTO t3 VALUES (6,'f'),(2,'d');
+SELECT * FROM t3
+WHERE ( f2 ) IN (SELECT t1.f1
+FROM t1 STRAIGHT_JOIN t2 ON t2.f3 = t1.f3
+WHERE t2.f3 = 'c');
+f2	f3
+drop table t1,t2,t3;

mysql-test/suite/pbxt/r/subselect.result

@@ -3112,8 +3112,8 @@ ORDER BY IFNULL((SELECT b FROM t2 WHERE b > 2),
 (SELECT c FROM t2 WHERE c=a AND b > 2 ORDER BY b));
 a
 1
-3
 2
+3
 4
 SELECT a FROM t1
 ORDER BY IFNULL((SELECT b FROM t2 WHERE b > 1),
@@ -3124,8 +3124,8 @@ ORDER BY IFNULL((SELECT b FROM t2 WHERE b > 4),
 (SELECT c FROM t2 WHERE c=a AND b > 2 ORDER BY b));
 a
 1
-3
 2
+3
 4
 SELECT a FROM t1
 ORDER BY IFNULL((SELECT b FROM t2 WHERE b > 4),

mysql-test/suite/pbxt/t/subselect.test

@@ -2052,7 +2052,7 @@ SELECT b, MAX(c) FROM t2 GROUP BY b, (SELECT c FROM t2 WHERE b > 2);
 --error 1242
 SELECT b, MAX(c) FROM t2 GROUP BY b, (SELECT c FROM t2 WHERE b > 1);
 
-
+--sorted_result
 SELECT a FROM t1 GROUP BY a
   HAVING IFNULL((SELECT b FROM t2 WHERE b > 2),
                 (SELECT c FROM t2 WHERE c=a AND b > 2 ORDER BY b)) > 3;
@@ -2060,7 +2060,7 @@ SELECT a FROM t1 GROUP BY a
 SELECT a FROM t1 GROUP BY a
   HAVING IFNULL((SELECT b FROM t2 WHERE b > 1),
                 (SELECT c FROM t2 WHERE c=a AND b > 2 ORDER BY b)) > 3;
-
+--sorted_result
 SELECT a FROM t1 GROUP BY a
   HAVING IFNULL((SELECT b FROM t2 WHERE b > 4),
                 (SELECT c FROM t2 WHERE c=a AND b > 2 ORDER BY b)) > 3;
@@ -2068,7 +2068,7 @@ SELECT a FROM t1 GROUP BY a
 SELECT a FROM t1 GROUP BY a
   HAVING IFNULL((SELECT b FROM t2 WHERE b > 4),
                 (SELECT c FROM t2 WHERE c=a AND b > 1 ORDER BY b)) > 3;
-
+--sorted_result
 SELECT a FROM t1
   ORDER BY IFNULL((SELECT b FROM t2 WHERE b > 2),
                   (SELECT c FROM t2 WHERE c=a AND b > 2 ORDER BY b));
@@ -2076,7 +2076,7 @@ SELECT a FROM t1
 SELECT a FROM t1
   ORDER BY IFNULL((SELECT b FROM t2 WHERE b > 1),
                   (SELECT c FROM t2 WHERE c=a AND b > 1 ORDER BY b));
-
+--sorted_result
 SELECT a FROM t1
   ORDER BY IFNULL((SELECT b FROM t2 WHERE b > 4),
                   (SELECT c FROM t2 WHERE c=a AND b > 2 ORDER BY b));

mysql-test/t/subselect_mat_cost.test

@@ -366,7 +366,7 @@ EXPLAIN SELECT f2 FROM t3 WHERE (
         )
 ) IS NULL ;
 
-drop table t1, t2;
+drop table t1, t2, t3;
 
 --echo #
 --echo # LP BUG#715034 Item_sum_distinct::clear(): Assertion `tree != 0' failed
@@ -437,3 +437,22 @@ WHERE (SELECT t1.f1
 ORDER BY field1;
 
 drop table t1,t2;
+
+--echo #
+--echo # LP BUG#601124 Bug in eliminate_item_equal
+--echo # leads to crash in Item_func::Item_func
+
+CREATE TABLE t1 ( f1 int(11), f3 varchar(1)) ;
+INSERT INTO t1 VALUES (5,'m'),(NULL,'c');
+
+CREATE TABLE t2 ( f2 int(11), f3 varchar(1)) ;
+INSERT INTO t2 VALUES (6,'f'),(2,'d');
+
+CREATE TABLE t3 ( f2 int(11), f3 varchar(1)) ;
+INSERT INTO t3 VALUES (6,'f'),(2,'d');
+
+SELECT * FROM t3
+WHERE ( f2 ) IN (SELECT t1.f1
+                 FROM t1 STRAIGHT_JOIN t2 ON t2.f3 = t1.f3
+                 WHERE t2.f3 = 'c');
+drop table t1,t2,t3;

sql/opt_subselect.cc

@@ -3738,13 +3738,13 @@ bool JOIN::choose_subquery_plan(table_map join_tables)
       outer_lookup_keys=1;
     }
     /*
-      Due to imprecise cost calculations, record/key counts my be < 1, while
-      an IN predicate will be executed at least once.
+      There cannot be more lookup keys than the total number of records.
+      TODO: this a temporary solution until we find a better way to compute
+      get_partial_join_cost() and prev_record_reads() in a consitent manner,
+      where it is guaranteed that (outer_lookup_keys <= outer_record_count).
     */
-    set_if_bigger(outer_record_count, 1);
-    set_if_bigger(outer_lookup_keys, 1);
-
-    DBUG_ASSERT(outer_lookup_keys <= outer_record_count);
+    if (outer_lookup_keys > outer_record_count)
+      outer_lookup_keys= outer_record_count;
 
     /*
       B. Estimate the cost and number of records of the subquery both