MDEV-22390: Assertion `m_next_rec_ptr >= m_rawmem' failed in...

MDEV-22390: Assertion `m_next_rec_ptr >= m_rawmem' failed in Filesort_buffer::spaceleft | SIGSEGV in __memmove_avx_unaligned_erms from my_b_write

Make sure that the sort_buffer that is allocated has atleast space for MERGEBUFF2 keys.
The issue here was that the record length is quite high and sort buffer size is very small,
due to which we end up with zero number of keys in the sort buffer. The Sort_param::max_keys_per_buffer
was zero in such a case, due to which we were flushing empty sort_buffer to the disk.

mysql-test/r/order_by.result

@@ -3302,3 +3302,57 @@ a	b	c
 SET @@sort_buffer_size= @save_sort_buffer_size;
 SET @@max_sort_length= @save_max_sort_length;
 DROP TABLE t1;
+#
+# MDEV-22390: Assertion `m_next_rec_ptr >= m_rawmem' failed in Filesort_buffer::spaceleft |
+# SIGSEGV in __memmove_avx_unaligned_erms from my_b_write (on optimized)
+#
+SET @save_max_sort_length= @@max_sort_length;
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_length_for_sort_data= @@max_length_for_sort_data;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+SET max_length_for_sort_data=7000;
+CREATE TABLE t1(a VARCHAR(64), b VARCHAR(2048))DEFAULT CHARSET=utf8;
+INSERT INTO t1 SELECT seq,seq from seq_1_to_100;
+ANALYZE FORMAT=JSON SELECT * FROM t1 ORDER BY a LIMIT 5;
+ANALYZE
+{
+  "query_block": {
+    "select_id": 1,
+    "r_loops": 1,
+    "r_total_time_ms": "REPLACED",
+    "read_sorted_file": {
+      "r_rows": 5,
+      "filesort": {
+        "r_loops": 1,
+        "r_total_time_ms": "REPLACED",
+        "r_limit": 5,
+        "r_used_priority_queue": false,
+        "r_output_rows": 35,
+        "r_sort_passes": 1,
+        "r_buffer_size": "REPLACED",
+        "table": {
+          "table_name": "t1",
+          "access_type": "ALL",
+          "r_loops": 1,
+          "rows": 100,
+          "r_rows": 100,
+          "r_total_time_ms": "REPLACED",
+          "filtered": 100,
+          "r_filtered": 100
+        }
+      }
+    }
+  }
+}
+SELECT * FROM t1 ORDER BY a LIMIT 5;
+a	b
+1	1
+10	10
+100	100
+11	11
+12	12
+SET max_sort_length= @save_max_sort_length;
+SET sort_buffer_size= @save_sort_buffer_size;
+SET max_length_for_sort_data= @save_max_length_for_sort_data;
+DROP TABLE t1;

mysql-test/t/order_by.test

@@ -2174,3 +2174,25 @@ SELECT * FROM t1 ORDER BY a,b;
 SET @@sort_buffer_size= @save_sort_buffer_size;
 SET @@max_sort_length= @save_max_sort_length;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-22390: Assertion `m_next_rec_ptr >= m_rawmem' failed in Filesort_buffer::spaceleft |
+--echo # SIGSEGV in __memmove_avx_unaligned_erms from my_b_write (on optimized)
+--echo #
+
+SET @save_max_sort_length= @@max_sort_length;
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_length_for_sort_data= @@max_length_for_sort_data;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+# needed to make sure we use addon fields
+SET max_length_for_sort_data=7000;
+CREATE TABLE t1(a VARCHAR(64), b VARCHAR(2048))DEFAULT CHARSET=utf8;
+INSERT INTO t1 SELECT seq,seq from seq_1_to_100;
+--source include/analyze-format.inc
+ANALYZE FORMAT=JSON SELECT * FROM t1 ORDER BY a LIMIT 5;
+SELECT * FROM t1 ORDER BY a LIMIT 5;
+SET max_sort_length= @save_max_sort_length;
+SET sort_buffer_size= @save_sort_buffer_size;
+SET max_length_for_sort_data= @save_max_length_for_sort_data;
+DROP TABLE t1;

sql/filesort.cc

@@ -256,7 +256,8 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
     while (memory_available >= min_sort_memory)
     {
       ulonglong keys= memory_available / (param.rec_length + sizeof(char*));
-      param.max_keys_per_buffer= (uint) MY_MIN(num_rows, keys);
+      param.max_keys_per_buffer= MY_MAX(MERGEBUFF2,
+                                        (uint) MY_MIN(num_rows, keys));
       if (table_sort.get_sort_keys())
       {
         // If we have already allocated a buffer, it better have same size!

sql/filesort_utils.cc

@@ -97,7 +97,6 @@ uchar **Filesort_buffer::alloc_sort_buffer(uint num_records, uint record_length)
   if (m_idx_array.is_null())
   {
     sort_buff_sz= ((size_t)num_records) * (record_length + sizeof(uchar*));
-    set_if_bigger(sort_buff_sz, record_length * MERGEBUFF2); 
     uchar **sort_keys=
       (uchar**) my_malloc(sort_buff_sz, MYF(MY_THREAD_SPECIFIC));
     m_idx_array= Idx_array(sort_keys, num_records);