Fix for bug #31174: "Repair" command on MyISAM crashes with small

myisam_sort_buffer_size.

An incorrect length of the sort buffer was used when calculating the
maximum number of keys. When myisam_sort_buffer_size is small enough,
this could result in the number of keys < number of
BUFFPEK structures which in turn led to use of uninitialized BUFFPEKs.

Fixed by correcting the buffer length calculation.


myisam/sort.c:
  Use a correct buffer length when calculating the maximum number of keys.
  Assert that for each BUFFPEK structure there is at least one
  corresponding key. Otherwise we would fail earlier and not reach
  merge_buffers().
mysql-test/r/repair.result:
  Added a test case for bug #31174.
mysql-test/t/repair.test:
  Added a test case for bug #31174.

myisam/sort.c

@@ -559,9 +559,10 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
       if (!mergebuf)
       {
         length=param->sort_buffer_length;
-        while (length >= MIN_SORT_MEMORY && !mergebuf)
+        while (length >= MIN_SORT_MEMORY)
         {
-          mergebuf=my_malloc(length, MYF(0));
+          if ((mergebuf= my_malloc(length, MYF(0))))
+              break;
           length=length*3/4;
         }
         if (!mergebuf)
@@ -897,6 +898,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
 
   count=error=0;
   maxcount=keys/((uint) (Tb-Fb) +1);
+  DBUG_ASSERT(maxcount > 0);
   LINT_INIT(to_start_filepos);
   if (to_file)
     to_start_filepos=my_b_tell(to_file);

mysql-test/r/repair.result

@@ -83,3 +83,30 @@ test.t1	repair	status	OK
 SET myisam_repair_threads=@@global.myisam_repair_threads;
 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
 DROP TABLE t1;
+CREATE TABLE t1(a CHAR(255), KEY(a));
+SET myisam_sort_buffer_size=4196;
+INSERT INTO t1 VALUES
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0');
+SET myisam_repair_threads=2;
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	status	OK
+SET myisam_repair_threads=@@global.myisam_repair_threads;
+SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
+DROP TABLE t1;
+End of 4.1 tests

mysql-test/t/repair.test

@@ -83,4 +83,33 @@ SET myisam_repair_threads=@@global.myisam_repair_threads;
 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
 DROP TABLE t1;
 
-# End of 4.1 tests
+#
+# BUG#31174 - "Repair" command on MyISAM crashes with small 
+#              myisam_sort_buffer_size
+#
+CREATE TABLE t1(a CHAR(255), KEY(a));
+SET myisam_sort_buffer_size=4196;
+INSERT INTO t1 VALUES
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0');
+SET myisam_repair_threads=2;
+REPAIR TABLE t1;
+SET myisam_repair_threads=@@global.myisam_repair_threads;
+SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
+DROP TABLE t1;
+
+--echo End of 4.1 tests