MDEV-5817 MySQL BUG#11825482: Broken key length calculation for btree index

just as in 5.6 fix - copy the correct null-handling code from MyISAM

mysql-test/suite/heap/heap.result

@@ -758,6 +758,40 @@ SELECT * from t1;
 id	color	ts
 7	GREEN	2
 DROP TABLE t1;
+#
+# BUG#11825482: Broken key length calculation for btree index
+#
+CREATE TABLE h1 (f1 VARCHAR(1), f2 INT NOT NULL,
+UNIQUE KEY h1i (f1,f2) USING BTREE ) ENGINE=HEAP;
+INSERT INTO h1 VALUES(NULL,0),(NULL,1);
+SELECT 'wrong' as 'result' FROM dual WHERE ('h', 0) NOT IN (SELECT * FROM h1);
+result
+CREATE TABLE t1 (  
+pk int NOT NULL,  
+col_int_nokey INT,  
+col_varchar_nokey VARCHAR(1),  
+PRIMARY KEY (pk)  
+);
+INSERT INTO t1 VALUES (19,5,'h'),(20,5,'h');
+CREATE TABLE t2 (col_int_nokey INT);
+INSERT INTO t2 VALUES (1),(2);
+CREATE VIEW v1 AS
+SELECT col_varchar_nokey, COUNT( col_varchar_nokey )
+FROM t1
+WHERE col_int_nokey <= 141 AND pk <= 4
+;
+SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+col_int_nokey
+# shouldn't crash
+EXPLAIN SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	<derived3>	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	DERIVED	t1	range	PRIMARY	PRIMARY	4	NULL	1	Using index condition; Using where
+DROP TABLE t1,t2,h1;
+DROP VIEW v1;
 CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
 insert into t1 values(1);
 select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";

mysql-test/suite/heap/heap.test

@@ -510,6 +510,41 @@ DELETE FROM t1 WHERE ts = 1 AND color = 'GREEN';
 SELECT * from t1;
 DROP TABLE t1;
 
+--echo #
+--echo # BUG#11825482: Broken key length calculation for btree index
+--echo #
+CREATE TABLE h1 (f1 VARCHAR(1), f2 INT NOT NULL,
+                 UNIQUE KEY h1i (f1,f2) USING BTREE ) ENGINE=HEAP;
+INSERT INTO h1 VALUES(NULL,0),(NULL,1);
+SELECT 'wrong' as 'result' FROM dual WHERE ('h', 0) NOT IN (SELECT * FROM h1);
+
+CREATE TABLE t1 (  
+  pk int NOT NULL,  
+  col_int_nokey INT,  
+  col_varchar_nokey VARCHAR(1),  
+  PRIMARY KEY (pk)  
+);
+
+INSERT INTO t1 VALUES (19,5,'h'),(20,5,'h');
+  
+CREATE TABLE t2 (col_int_nokey INT);
+
+INSERT INTO t2 VALUES (1),(2);
+
+CREATE VIEW v1 AS
+  SELECT col_varchar_nokey, COUNT( col_varchar_nokey )
+  FROM t1
+  WHERE col_int_nokey <= 141 AND pk <= 4
+;
+
+SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+--echo # shouldn't crash
+EXPLAIN SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+
+DROP TABLE t1,t2,h1;
+DROP VIEW v1;
 # End of 5.1 tests
 
 #

storage/heap/hp_hash.c

@@ -875,9 +875,19 @@ uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old,
     keypart_map>>= 1;
     if (seg->null_bit)
     {
+      /* Convert NULL from MySQL representation into HEAP's. */
       if (!(*key++= (char) 1 - *old++))
+      {
+        /*
+          Skip length part of a variable length field.
+          Length of key-part used with heap_rkey() always 2.
+          See also hp_hashnr().
+        */
+        if (seg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART))
+          old+= 2;
         continue;
       }
+    }
     if (seg->flag & HA_SWAP_KEY)
     {
       uint length= seg->length;