MDEV-27303 Table corruption after insert into a non-InnoDB table with DESC index

optimized prefix search didn't take into account descending indexes

also fixes MDEV-27330

mysql-test/main/myisam.result

@@ -2706,3 +2706,43 @@ DROP TABLE t1;
 #
 # End of 5.5 tests
 #
+#
+# MDEV-27303 Table corruption after insert into a non-InnoDB table with DESC index
+#
+create table t1 (
+a bigint default 0,
+b bigint default 0,
+c binary(128) not null,
+d datetime default '0000-00-00 00:00:00',
+key (c desc,b,d,a)
+) engine=aria;
+insert into t1 (c) values
+('xx'),('bb'),('tt'),('pp'),('mm'),('yy'),('rr'),('bb'),('yy'),('gg'),
+('dd'),('fx'),('wi'),('ix'),('ox'),('mu'),('ux'),('pm'),('mx'),('xu'),
+('ul'),('lp'),('px'),('lp'),('xx'),('pq'),('qs'),('se'),('ee'),('xx'),
+('rv'),('ff'),('vj'),('jy'),('yn'),('nc'),('nx'),('hj'),('ji'),('ik'),
+('kk'),('ww'),('xx'),('yd'),('dw'),('wk'),('kr'),('dd'),('rj'),('jf'),
+('bx'),('fc'),('cp'),('pm'),('mw'),('wy'),('yl'),('li'),('ic'),('he'),
+('ci'),('il'),('lz'),('zd'),('gz'),('xd'),('ze'),('dm'),('ms'),('xd'),
+('sw'),('we'),('nb'),('tx'),('vr'),('xw'),('aa'),('ah'),('hd'),('jl'),
+('lf'),('fw'),('wx'),('xh'),('hr'),('zx'),('vw'),('rm'),('mx'),('xt'),
+('tp'),('ps'),('sh'),('ga'),('df'),('as'),('gz'),('xd'),('yy'),('xr');
+check table t1 extended;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+drop table t1;
+#
+# MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
+#
+create table t (id int, c char(128) not null, key (c desc));
+insert into t values (1,''),(2,'foo'),(3,''),(4,'bar');
+select c from t order by c;
+c
+
+
+bar
+foo
+drop table t;
+#
+# End of 10.8 tests
+#

mysql-test/main/myisam.test

@@ -1830,3 +1830,39 @@ DROP TABLE t1;
 --echo #
 --echo # End of 5.5 tests
 --echo #
+
+--echo #
+--echo # MDEV-27303 Table corruption after insert into a non-InnoDB table with DESC index
+--echo #
+create table t1 (
+  a bigint default 0,
+  b bigint default 0,
+  c binary(128) not null,
+  d datetime default '0000-00-00 00:00:00',
+  key (c desc,b,d,a)
+) engine=aria;
+insert into t1 (c) values
+  ('xx'),('bb'),('tt'),('pp'),('mm'),('yy'),('rr'),('bb'),('yy'),('gg'),
+  ('dd'),('fx'),('wi'),('ix'),('ox'),('mu'),('ux'),('pm'),('mx'),('xu'),
+  ('ul'),('lp'),('px'),('lp'),('xx'),('pq'),('qs'),('se'),('ee'),('xx'),
+  ('rv'),('ff'),('vj'),('jy'),('yn'),('nc'),('nx'),('hj'),('ji'),('ik'),
+  ('kk'),('ww'),('xx'),('yd'),('dw'),('wk'),('kr'),('dd'),('rj'),('jf'),
+  ('bx'),('fc'),('cp'),('pm'),('mw'),('wy'),('yl'),('li'),('ic'),('he'),
+  ('ci'),('il'),('lz'),('zd'),('gz'),('xd'),('ze'),('dm'),('ms'),('xd'),
+  ('sw'),('we'),('nb'),('tx'),('vr'),('xw'),('aa'),('ah'),('hd'),('jl'),
+  ('lf'),('fw'),('wx'),('xh'),('hr'),('zx'),('vw'),('rm'),('mx'),('xt'),
+  ('tp'),('ps'),('sh'),('ga'),('df'),('as'),('gz'),('xd'),('yy'),('xr');
+check table t1 extended;
+drop table t1;
+
+--echo #
+--echo # MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
+--echo #
+create table t (id int, c char(128) not null, key (c desc));
+insert into t values (1,''),(2,'foo'),(3,''),(4,'bar');
+select c from t order by c;
+drop table t;
+
+--echo #
+--echo # End of 10.8 tests
+--echo #

mysql-test/suite/maria/maria2.result

@@ -68,3 +68,46 @@ pk	apk	data
 3	4	NULL
 5	6	NULL
 drop table t1;
+#
+# End of 5.5 tests
+#
+#
+# MDEV-27303 Table corruption after insert into a non-InnoDB table with DESC index
+#
+create table t1 (
+a bigint default 0,
+b bigint default 0,
+c binary(128) not null,
+d datetime default '0000-00-00 00:00:00',
+key (c desc,b,d,a)
+) engine=aria;
+insert into t1 (c) values
+('xx'),('bb'),('tt'),('pp'),('mm'),('yy'),('rr'),('bb'),('yy'),('gg'),
+('dd'),('fx'),('wi'),('ix'),('ox'),('mu'),('ux'),('pm'),('mx'),('xu'),
+('ul'),('lp'),('px'),('lp'),('xx'),('pq'),('qs'),('se'),('ee'),('xx'),
+('rv'),('ff'),('vj'),('jy'),('yn'),('nc'),('nx'),('hj'),('ji'),('ik'),
+('kk'),('ww'),('xx'),('yd'),('dw'),('wk'),('kr'),('dd'),('rj'),('jf'),
+('bx'),('fc'),('cp'),('pm'),('mw'),('wy'),('yl'),('li'),('ic'),('he'),
+('ci'),('il'),('lz'),('zd'),('gz'),('xd'),('ze'),('dm'),('ms'),('xd'),
+('sw'),('we'),('nb'),('tx'),('vr'),('xw'),('aa'),('ah'),('hd'),('jl'),
+('lf'),('fw'),('wx'),('xh'),('hr'),('zx'),('vw'),('rm'),('mx'),('xt'),
+('tp'),('ps'),('sh'),('ga'),('df'),('as'),('gz'),('xd'),('yy'),('xr');
+check table t1 extended;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+drop table t1;
+#
+# MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
+#
+create table t (id int, c char(128) not null, key (c desc)) engine=aria;
+insert into t values (1,''),(2,'foo'),(3,''),(4,'bar');
+select c from t order by c;
+c
+
+
+bar
+foo
+drop table t;
+#
+# End of 10.8 tests
+#

mysql-test/suite/maria/maria2.test

@@ -109,3 +109,43 @@ select * from t1 order by pk;
 load data infile '../../std_data/loaddata5.dat' replace into table t1 fields terminated by '' enclosed by '' ignore 1 lines (pk, apk);
 select * from t1 order by pk;
 drop table t1;
+
+--echo #
+--echo # End of 5.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-27303 Table corruption after insert into a non-InnoDB table with DESC index
+--echo #
+create table t1 (
+  a bigint default 0,
+  b bigint default 0,
+  c binary(128) not null,
+  d datetime default '0000-00-00 00:00:00',
+  key (c desc,b,d,a)
+) engine=aria;
+insert into t1 (c) values
+  ('xx'),('bb'),('tt'),('pp'),('mm'),('yy'),('rr'),('bb'),('yy'),('gg'),
+  ('dd'),('fx'),('wi'),('ix'),('ox'),('mu'),('ux'),('pm'),('mx'),('xu'),
+  ('ul'),('lp'),('px'),('lp'),('xx'),('pq'),('qs'),('se'),('ee'),('xx'),
+  ('rv'),('ff'),('vj'),('jy'),('yn'),('nc'),('nx'),('hj'),('ji'),('ik'),
+  ('kk'),('ww'),('xx'),('yd'),('dw'),('wk'),('kr'),('dd'),('rj'),('jf'),
+  ('bx'),('fc'),('cp'),('pm'),('mw'),('wy'),('yl'),('li'),('ic'),('he'),
+  ('ci'),('il'),('lz'),('zd'),('gz'),('xd'),('ze'),('dm'),('ms'),('xd'),
+  ('sw'),('we'),('nb'),('tx'),('vr'),('xw'),('aa'),('ah'),('hd'),('jl'),
+  ('lf'),('fw'),('wx'),('xh'),('hr'),('zx'),('vw'),('rm'),('mx'),('xt'),
+  ('tp'),('ps'),('sh'),('ga'),('df'),('as'),('gz'),('xd'),('yy'),('xr');
+check table t1 extended;
+drop table t1;
+
+--echo #
+--echo # MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
+--echo #
+create table t (id int, c char(128) not null, key (c desc)) engine=aria;
+insert into t values (1,''),(2,'foo'),(3,''),(4,'bar');
+select c from t order by c;
+drop table t;
+
+--echo #
+--echo # End of 10.8 tests
+--echo #

storage/maria/ma_search.c

@@ -446,6 +446,7 @@ int _ma_prefix_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page,
   MARIA_KEYDEF *keyinfo= key->keyinfo;
   MARIA_SHARE *share= keyinfo->share;
   const uchar *sort_order= keyinfo->seg->charset->sort_order;
+  const int reverse = keyinfo->seg->flag & HA_REVERSE_SORT;
   DBUG_ENTER("_ma_prefix_search");
 
   t_buff[0]=0;                                  /* Avoid bugs */
@@ -579,7 +580,7 @@ int _ma_prefix_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page,
 
       /*
         If prefix_len > cmplen then we are in the end-space comparison
-        phase. Do not try to acces the key any more ==> left= 0.
+        phase. Do not try to access the key any more ==> left= 0.
       */
       left= ((len <= cmplen) ? suffix_len :
              ((prefix_len < cmplen) ? cmplen - prefix_len : 0));
@@ -599,7 +600,7 @@ int _ma_prefix_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page,
             break;
       }
 
-      if (my_flag>0)      /* mismatch */
+      if ((reverse ? -my_flag : my_flag) > 0)      /* mismatch */
         break;
       if (my_flag==0) /* match */
       {
@@ -626,13 +627,10 @@ int _ma_prefix_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page,
 	    for ( ; k < k_end && *k == ' '; k++) ;
 	    if (k == k_end)
 	      goto cmp_rest;		/* should never happen */
-	    if ((uchar) *k < (uchar) ' ')
-	    {
-	      my_flag= 1;		/* Compared string is smaller */
-	      break;
-	    }
-	    my_flag= -1;		/* Continue searching */
+	    my_flag= (uchar)' ' - *k;
 	  }
+          if ((reverse ? -my_flag : my_flag) > 0)
+            break;
         }
         else if (len > cmplen)
         {
@@ -646,13 +644,10 @@ int _ma_prefix_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page,
 	       vseg++, matched++) ;
 	  DBUG_ASSERT(vseg < vseg_end);
 
-	  if ((uchar) *vseg > (uchar) ' ')
-	  {
-	    my_flag= 1;			/* Compared string is smaller */
+          my_flag= *vseg - (uchar)' ';
+          if ((reverse ? -my_flag : my_flag) > 0)
             break;
         }
-	  my_flag= -1;			/* Continue searching */
-        }
         else
 	{
       cmp_rest:
@@ -690,7 +685,7 @@ int _ma_prefix_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page,
     *ret_pos=page;
   }
   if (my_flag)
-    flag=(keyinfo->seg->flag & HA_REVERSE_SORT) ? -my_flag : my_flag;
+    flag= reverse ? -my_flag : my_flag;
   if (flag == 0)
   {
     memcpy(buff,t_buff,saved_length=seg_len_pack+prefix_len);

storage/myisam/mi_search.c

@@ -309,6 +309,7 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
   uchar *UNINIT_VAR(saved_vseg);
   uint  saved_length=0, saved_prefix_len=0;
   uint  length_pack;
+  const int reverse = keyinfo->seg->flag & HA_REVERSE_SORT;
   DBUG_ENTER("_mi_prefix_search");
 
   t_buff[0]=0;                                  /* Avoid bugs */
@@ -452,7 +453,7 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
             break;
       }
 
-      if (my_flag>0)      /* mismatch */
+      if ((reverse ? -my_flag : my_flag) > 0)      /* mismatch */
         break;
       if (my_flag==0) /* match */
       {
@@ -478,13 +479,10 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 	    for ( ; k < k_end && *k == ' '; k++) ;
 	    if (k == k_end)
 	      goto cmp_rest;		/* should never happen */
-	    if (*k < (uchar) ' ')
-	    {
-	      my_flag= 1;		/* Compared string is smaller */
-	      break;
-	    }
-	    my_flag= -1;		/* Continue searching */
+	    my_flag= (uchar)' ' - *k;
 	  }
+          if ((reverse ? -my_flag : my_flag) > 0)
+            break;
         }
         else if (len > cmplen)
         {
@@ -498,13 +496,10 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
 	       vseg++, matched++) ;
 	  DBUG_ASSERT(vseg < vseg_end);
 
-	  if (*vseg > (uchar) ' ')
-	  {
-	    my_flag= 1;			/* Compared string is smaller */
+          my_flag= *vseg - (uchar)' ';
+          if ((reverse ? -my_flag : my_flag) > 0)
 	    break;
 	  }
-	  my_flag= -1;			/* Continue searching */
-        }
         else
 	{
       cmp_rest:
@@ -541,7 +536,7 @@ int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page,
     *ret_pos=page;
   }
   if (my_flag)
-    flag=(keyinfo->seg->flag & HA_REVERSE_SORT) ? -my_flag : my_flag;
+    flag= reverse ? -my_flag : my_flag;
   if (flag == 0)
   {
     memcpy(buff,t_buff,saved_length=seg_len_pack+prefix_len);