btr0cur.c:

  Do not let range estimator to return over 1 / 2 of total rows in table; use longlong in range estimation
btr0cur.h, ha_innobase.cc:
  Use longlong in range estimation, in case there are > 4 billion rows

innobase/btr/btr0cur.c

@@ -2531,7 +2531,7 @@ btr_cur_add_path_info(
 /***********************************************************************
 Estimates the number of rows in a given index range. */
 
-ulint
+ib_longlong
 btr_estimate_n_rows_in_range(
 /*=========================*/
 				/* out: estimated number of rows */
@@ -2548,7 +2548,7 @@ btr_estimate_n_rows_in_range(
 	btr_path_t*	slot2;
 	ibool		diverged;
 	ulint           divergence_level;           
-	ulint		n_rows;
+	ib_longlong	n_rows;
 	ulint		i;
 	mtr_t		mtr;
 
@@ -2608,6 +2608,22 @@ btr_estimate_n_rows_in_range(
 
 		                n_rows = n_rows * 2;
 		        }
+
+			/* Do not estimate the number of rows in the range
+		        to over 1 / 2 of the estimated rows in the whole
+			table */
+
+			if (n_rows > index->table->stat_n_rows / 2) {
+			        n_rows = index->table->stat_n_rows / 2;
+
+				/* If there are just 0 or 1 rows in the table,
+				then we estimate all rows are in the range */
+			  
+			        if (n_rows == 0) {
+			                n_rows = index->table->stat_n_rows;
+			        }
+			}
+
 			return(n_rows);
 		}
 

innobase/include/btr0cur.h

@@ -417,7 +417,7 @@ btr_cur_parse_del_mark_set_sec_rec(
 /***********************************************************************
 Estimates the number of rows in a given index range. */
 
-ulint
+ib_longlong
 btr_estimate_n_rows_in_range(
 /*=========================*/
 				/* out: estimated number of rows */

sql/ha_innobase.cc

@@ -3031,7 +3031,7 @@ ha_innobase::records_in_range(
 								MYF(MY_WME));
 	dtuple_t*	range_start;
 	dtuple_t*	range_end;
-	ulint		n_rows;
+	ib_longlong	n_rows;
 	ulint		mode1;
 	ulint		mode2;
 	void*           heap1;