BUG#29740: Wrong query results for index_merge/union over HEAP table.

- return HA_KEY_SCAN_NOT_ROR flag for HASH indexes;
- Fix ha_heap::cmp_ref() to work with BTREE index scans.

mysql-test/r/index_merge.result

@@ -455,3 +455,64 @@ a
 1
 UNLOCK TABLES;
 DROP TABLE t1, t2;
+CREATE TABLE `t1` (
+`a` int(11) DEFAULT NULL,
+`filler` char(200) DEFAULT NULL,
+`b` int(11) DEFAULT NULL,
+KEY `a` (`a`),
+KEY `b` (`b`)
+) ENGINE=MEMORY DEFAULT CHARSET=latin1;
+insert into t1 values
+(0, 'filler', 0), (1, 'filler', 1), (2, 'filler', 2), (3, 'filler', 3), 
+(4, 'filler', 4), (5, 'filler', 5), (6, 'filler', 6), (7, 'filler', 7), 
+(8, 'filler', 8), (9, 'filler', 9), (0, 'filler', 0), (1, 'filler', 1), 
+(2, 'filler', 2), (3, 'filler', 3), (4, 'filler', 4), (5, 'filler', 5), 
+(6, 'filler', 6), (7, 'filler', 7), (8, 'filler', 8), (9, 'filler', 9), 
+(10, 'filler', 10), (11, 'filler', 11), (12, 'filler', 12), (13, 'filler', 13),
+(14, 'filler', 14), (15, 'filler', 15), (16, 'filler', 16), (17, 'filler', 17), 
+(18, 'filler', 18), (19, 'filler', 19), (4, '5      ', 0), (5, '4      ', 0), 
+(4, '4      ', 0), (4, 'qq     ', 5), (5, 'qq     ', 4), (4, 'zz     ', 4);
+create table t2(
+`a` int(11) DEFAULT NULL,
+`filler` char(200) DEFAULT NULL,
+`b` int(11) DEFAULT NULL,
+KEY USING BTREE (`a`),
+KEY USING BTREE (`b`)
+) ENGINE=MEMORY DEFAULT CHARSET=latin1;
+insert into t2 select * from t1;
+must use sort-union rather than union:
+explain select * from t1 where a=4 or b=4;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index_merge	a,b	a,b	5,5	NULL	4	Using sort_union(a,b); Using where
+select * from t1 where a=4 or b=4;
+a	filler	b
+4	zz	4
+5	qq	4
+4	filler	4
+4	qq	5
+4	4	0
+4	filler	4
+4	5	0
+select * from t1 ignore index(a,b) where a=4 or b=4;
+a	filler	b
+4	filler	4
+4	filler	4
+4	5	0
+4	4	0
+4	qq	5
+5	qq	4
+4	zz	4
+must use union, not sort-union:
+explain select * from t2 where a=4 or b=4;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	index_merge	a,b	a,b	5,5	NULL	7	Using union(a,b); Using where
+select * from t2 where a=4 or b=4;
+a	filler	b
+4	5	0
+4	zz	4
+5	qq	4
+4	filler	4
+4	qq	5
+4	4	0
+4	filler	4
+drop table t1, t2;

mysql-test/t/index_merge.test

@@ -415,3 +415,46 @@ INSERT INTO t2(a,b) VALUES(1,2);
 SELECT t2.a FROM t1,t2 WHERE t2.b=2 AND t2.a=1;
 UNLOCK TABLES;
 DROP TABLE t1, t2;
+
+#
+# BUG#29740: HA_KEY_SCAN_NOT_ROR wasn't set for HEAP engine
+# 
+CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `filler` char(200) DEFAULT NULL,
+  `b` int(11) DEFAULT NULL,
+  KEY `a` (`a`),
+  KEY `b` (`b`)
+) ENGINE=MEMORY DEFAULT CHARSET=latin1;
+
+insert into t1 values
+(0, 'filler', 0), (1, 'filler', 1), (2, 'filler', 2), (3, 'filler', 3), 
+(4, 'filler', 4), (5, 'filler', 5), (6, 'filler', 6), (7, 'filler', 7), 
+(8, 'filler', 8), (9, 'filler', 9), (0, 'filler', 0), (1, 'filler', 1), 
+(2, 'filler', 2), (3, 'filler', 3), (4, 'filler', 4), (5, 'filler', 5), 
+(6, 'filler', 6), (7, 'filler', 7), (8, 'filler', 8), (9, 'filler', 9), 
+(10, 'filler', 10), (11, 'filler', 11), (12, 'filler', 12), (13, 'filler', 13),
+(14, 'filler', 14), (15, 'filler', 15), (16, 'filler', 16), (17, 'filler', 17), 
+(18, 'filler', 18), (19, 'filler', 19), (4, '5      ', 0), (5, '4      ', 0), 
+(4, '4      ', 0), (4, 'qq     ', 5), (5, 'qq     ', 4), (4, 'zz     ', 4);
+
+create table t2(
+  `a` int(11) DEFAULT NULL,
+  `filler` char(200) DEFAULT NULL,
+  `b` int(11) DEFAULT NULL,
+  KEY USING BTREE (`a`),
+  KEY USING BTREE (`b`)
+) ENGINE=MEMORY DEFAULT CHARSET=latin1;
+insert into t2 select * from t1;
+
+--echo must use sort-union rather than union:
+explain select * from t1 where a=4 or b=4;
+select * from t1 where a=4 or b=4;
+select * from t1 ignore index(a,b) where a=4 or b=4;
+
+--echo must use union, not sort-union:
+explain select * from t2 where a=4 or b=4;
+select * from t2 where a=4 or b=4;
+
+drop table t1, t2;
+

sql/ha_heap.h

@@ -55,7 +55,7 @@ public:
   {
     return ((table->key_info[inx].algorithm == HA_KEY_ALG_BTREE) ?
             HA_READ_NEXT | HA_READ_PREV | HA_READ_ORDER | HA_READ_RANGE :
-	    HA_ONLY_WHOLE_INDEX);
+            HA_ONLY_WHOLE_INDEX | HA_KEY_SCAN_NOT_ROR);
   }
   const key_map *keys_to_use_for_scanning() { return &btree_keys; }
   uint max_supported_keys()          const { return MAX_KEY; }
@@ -101,9 +101,7 @@ public:
 			     enum thr_lock_type lock_type);
   int cmp_ref(const byte *ref1, const byte *ref2)
   {
-    HEAP_PTR ptr1=*(HEAP_PTR*)ref1;
-    HEAP_PTR ptr2=*(HEAP_PTR*)ref2;
-    return ptr1 < ptr2? -1 : (ptr1 > ptr2? 1 : 0);
+    return memcmp(ref1, ref2, sizeof(HEAP_PTR));
   }
 private:
   void update_key_stats();

sql/opt_range.cc

@@ -6007,27 +6007,24 @@ check_quick_keys(PARAM *param,uint idx,SEL_ARG *key_tree,
     ROR (Rowid Ordered Retrieval) key scan is a key scan that produces
     ordered sequence of rowids (ha_xxx::cmp_ref is the comparison function)
 
-    An index scan is a ROR scan if it is done using a condition in form
+    This function is needed to handle a practically-important special case:
+    an index scan is a ROR scan if it is done using a condition in form
 
-        "key1_1=c_1 AND ... AND key1_n=c_n"  (1)
+        "key1_1=c_1 AND ... AND key1_n=c_n"
 
     where the index is defined on (key1_1, ..., key1_N [,a_1, ..., a_n])
 
-    and the table has a clustered Primary Key
+    and the table has a clustered Primary Key defined as 
 
-    PRIMARY KEY(a_1, ..., a_n, b1, ..., b_k) with first key parts being
-    identical to uncovered parts ot the key being scanned (2)
+      PRIMARY KEY(a_1, ..., a_n, b1, ..., b_k) 
     
-    Scans on HASH indexes are not ROR scans,
-    any range scan on clustered primary key is ROR scan  (3)
-
-    Check (1) is made in check_quick_keys()
-    Check (3) is made check_quick_select()
-    Check (2) is made by this function.
+    i.e. the first key parts of it are identical to uncovered parts ot the 
+    key being scanned. This function assumes that the index flags do not
+    include HA_KEY_SCAN_NOT_ROR flag (that is checked elsewhere).
 
   RETURN
-    TRUE  If the scan is ROR-scan
-    FALSE otherwise
+    TRUE   The scan is ROR-scan
+    FALSE  Otherwise
 */
 
 static bool is_key_scan_ror(PARAM *param, uint keynr, uint8 nparts)