LP BUG#813418 fix.

The problem was that optimization code did not take into account later feature when instad of NOT before BETWEEN it has negated flag into the Item_func_between inherited from Item_func_neg_opt. So optimizer tried process NOT BETWEEN as BETWEEN.

The patch just switches off the optimisation for NOT BETWEEN as it was before when NOT function was really used.

mysql-test/r/func_group.result

@@ -1758,3 +1758,13 @@ MAX(f1)
 DROP TABLE t1;
 #
 End of 5.1 tests
+# LP BUG#813418 - incorrect optimisation of max/min by index for
+# negated BETWEEN
+CREATE TABLE t1 (a int, KEY (a));
+INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
+SELECT MAX(a) FROM t1 WHERE a NOT BETWEEN 3 AND 9;
+MAX(a)
+10
+drop table t1;
+#
+End of 5.1 tests

mysql-test/t/func_group.test

@@ -1142,3 +1142,12 @@ DROP TABLE t1;
 --echo #
 --echo End of 5.1 tests
 
+--echo # LP BUG#813418 - incorrect optimisation of max/min by index for
+--echo # negated BETWEEN
+CREATE TABLE t1 (a int, KEY (a));
+INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
+SELECT MAX(a) FROM t1 WHERE a NOT BETWEEN 3 AND 9;
+drop table t1;
+
+--echo #
+--echo End of 5.1 tests

sql/opt_sum.cc

@@ -657,6 +657,8 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
   case Item_func::GE_FUNC:
     break;
   case Item_func::BETWEEN:
+    if (((Item_func_between*) cond)->negated)
+      DBUG_RETURN(FALSE);
     between= 1;
     break;
   case Item_func::MULT_EQUAL_FUNC: