Fix of4cb0d43ac63761174a39cea892c176b9cfa6edfc after merge in 10.5

mysql-test/main/myisam.result

@@ -2556,9 +2556,6 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
-Warnings:
-Error	1034	myisam_sort_buffer_size is too small. X
-Warning	1034	Number of rows changed from 0 to 157
 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
 INSERT INTO t1 VALUES('1');
 SELECT * FROM t1, t1 AS a1 WHERE t1.a=1 AND a1.a=1;

mysql-test/main/repair.result

@@ -77,9 +77,6 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
-Warnings:
-Error	1034	myisam_sort_buffer_size is too small. X
-Warning	1034	Number of rows changed from 0 to 157
 SET myisam_repair_threads=2;
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text

mysql-test/suite/maria/maria-64bit.result

@@ -35,9 +35,6 @@ INSERT INTO t1 select seq,'0' from seq_1_to_65536;
 SET aria_sort_buffer_size=16384;
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	repair	error	aria_sort_buffer_size is too small. X
-test.t1	repair	error	Create index by sort failed
-test.t1	repair	info	Retrying repair with keycache
 test.t1	repair	status	OK
 CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
@@ -45,9 +42,6 @@ test.t1	check	status	OK
 SET aria_repair_threads=2;
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	repair	error	aria_sort_buffer_size is too small. X
-test.t1	repair	error	Create index by sort failed
-test.t1	repair	info	Retrying repair with keycache
 test.t1	repair	status	OK
 CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text

storage/maria/ma_check.c

@@ -2483,9 +2483,8 @@ static int initialize_variables_for_repair(HA_CHECK *param,
   tmp= (size_t) MY_MIN(sort_info->filelength,
                        (my_off_t) (SIZE_T_MAX/10/threads));
   tmp= MY_MAX(tmp * 8 * threads, (size_t) 65536);         /* Some margin */
-  param->sort_buffer_length= MY_MIN(param->orig_sort_buffer_length,
-                                    tmp);
-  set_if_smaller(param->sort_buffer_length, tmp);
+  param->sort_buffer_length= MY_MIN(param->orig_sort_buffer_length,                                    tmp);
+  set_if_bigger(param->sort_buffer_length, MARIA_MIN_SORT_MEMORY);
   /* Protect against too big sort buffer length */
 #if SIZEOF_SIZE_T >= 8
   set_if_smaller(param->sort_buffer_length, 16LL*1024LL*1024LL*1024LL);
@@ -3985,16 +3984,6 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info,
     {
       sort_param.key_read=  sort_key_read;
       sort_param.key_write= sort_key_write;
-
-      /*
-        Limit usage of sort memory
-        We assume we don't need more memory than data file length * 2
-        (There is a pointer overhead for each key, but this is hard to
-        estimae as we cannot be sure how many records we have in file to
-        be repaired).
-      */
-      set_if_smaller(param->sort_buffer_length, sort_info.filelength*2);
-      set_if_bigger(param->sort_buffer_length, MARIA_MIN_SORT_MEMORY);
     }
 
     if (sort_info.new_info->s->data_file_type == BLOCK_RECORD)
@@ -4597,16 +4586,6 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info,
     DBUG_PRINT("io_cache_share", ("thread: %u  read_cache: %p",
                                   i, &sort_param[i].read_cache));
 
-    /*
-      Limit usage of sort memory
-      We assume we don't need more memory than data file length * 2
-      (There is a pointer overhead for each key, but this is hard to
-      estimae as we cannot be sure how many records we have in file to
-      be repaired).
-     */
-    set_if_smaller(param->sort_buffer_length, sort_info.filelength*2);
-    set_if_bigger(param->sort_buffer_length, MARIA_MIN_SORT_MEMORY);
-
     /*
       two approaches: the same amount of memory for each thread
       or the memory for the same number of keys for each thread...

storage/maria/ma_sort.c

@@ -177,8 +177,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
 	maxbuffer_org= maxbuffer;
 	if (memavl < sizeof(BUFFPEK) * maxbuffer ||
 	    (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
-             (sort_length+sizeof(char*))) <= 1 ||
-            keys < maxbuffer)
+             (sort_length+sizeof(char*))) <= 1)
 	{
 	  _ma_check_print_error(info->sort_info->param,
                                 "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
@@ -187,6 +186,15 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
           my_errno= ENOMEM;
 	  goto err;
 	}
+        if (keys < maxbuffer)
+        {
+          /*
+            There must be sufficient memory for at least one key per BUFFPEK,
+            otherwise repair by sort/parallel repair cannot operate.
+          */
+          maxbuffer= (uint) keys;
+          break;
+        }
       }
       while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
     }
@@ -231,6 +239,20 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
       == HA_POS_ERROR)
     goto err; /* purecov: tested */
 
+  if (maxbuffer >= keys)
+  {
+    /*
+      merge_many_buff will crash if maxbuffer > keys as then we cannot store in memory
+      the keys for each buffer.
+    */
+    keys= maxbuffer + 1;
+    if (!(sort_keys= ((uchar **)
+                      my_realloc(PSI_INSTRUMENT_ME, sort_keys,
+                                 (size_t) (keys*(sort_length+sizeof(char*))+
+                                           HA_FT_MAXBYTELEN), MYF(MY_FREE_ON_ERROR)))))
+      goto err;
+  }
+
   info->sort_info->param->stage++;                         /* Merge stage */
 
   if (maxbuffer == 0)

storage/myisam/mi_check.c

@@ -2945,6 +2945,8 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
 #else
       param->sort_buffer_length*sort_param[i].key_length/total_key_length;
 #endif
+    set_if_bigger(sort_param[i].sortbuff_size, MIN_SORT_BUFFER);
+
     if ((error= mysql_thread_create(mi_key_thread_find_all_keys,
                                     &sort_param[i].thr, &thr_attr,
                                     thr_find_all_keys,

storage/myisam/sort.c

@@ -175,8 +175,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
 	maxbuffer_org= maxbuffer;
 	if (memavl < sizeof(BUFFPEK) * maxbuffer ||
 	    (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
-             (sort_length+sizeof(char*))) <= 1 ||
-            keys < maxbuffer)
+             (sort_length+sizeof(char*))) <= 1)
 	{
 	  mi_check_print_error(info->sort_info->param,
 			       "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
@@ -185,6 +184,15 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
           my_errno= ENOMEM;
 	  goto err;
 	}
+        if (keys < maxbuffer)
+        {
+          /*
+            There must be sufficient memory for at least one key per BUFFPEK,
+            otherwise repair by sort/parallel repair cannot operate.
+          */
+          keys= maxbuffer;
+          break;
+        }
       }
       while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
     }
@@ -228,6 +236,20 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
                               &tempfile,&tempfile_for_exceptions))
       == HA_POS_ERROR)
     goto err; /* purecov: tested */
+  if (maxbuffer >= keys)
+  {
+    /*
+      merge_many_buff will crash if maxbuffer >= keys as then we cannot store in memory
+      the keys for each buffer.
+    */
+    keys= maxbuffer + 1;
+    if (!(sort_keys= ((uchar **)
+                      my_realloc(PSI_INSTRUMENT_ME, sort_keys,
+                                 (size_t) (keys*(sort_length+sizeof(char*))+
+                                           HA_FT_MAXBYTELEN), MYF(MY_FREE_ON_ERROR)))))
+      goto err;
+  }
+
   if (maxbuffer == 0)
   {
     if (!no_messages)