MDEV-24750 Various corruptions caused by Aria subsystem...

The test case was setting aria_sort_buffer_size to MAX_ULONGLONG-1
which was not handled gracefully by my_malloc() or safemalloc().
Fixed by ensuring that the malloc functions returns 0 if the size
is too big.
I also added some protection to Aria repair:
- Limit sort_buffer_size to 16G (after that a bigger sort buffer will
  not help that much anyway)
- Limit sort_buffer_size also according to sort file size. This will
  help by not allocating less memory if someone sets the buffer size too
  high.

mysql-test/suite/maria/aria_sort_buffer.result

+SET SESSION aria_repair_threads=128;
+SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT);
+Warnings:
+Note	1105	Cast to unsigned converted negative integer to it's positive complement
+Note	1105	Cast to unsigned converted negative integer to it's positive complement
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '18446744073709551615'
+SET SESSION tmp_table_size=65535;
+CREATE TABLE t1 (a VARCHAR(255));
+insert into t1 (a) select seq from seq_1_to_1000;
+UPDATE t1 SET a=( (SELECT MAX(a) FROM t1));
+DROP TABLE t1;

mysql-test/suite/maria/aria_sort_buffer.test

+--source include/have_sequence.inc
+
+#
+#
+# MDEV-24750 Various corruptions caused by Aria subsystem asking system call
+#            to overwrite memory that it does not own (InnoDB stacks)
+
+SET SESSION aria_repair_threads=128;
+SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT);
+
+SET SESSION tmp_table_size=65535;
+CREATE TABLE t1 (a VARCHAR(255));
+insert into t1 (a) select seq from seq_1_to_1000;
+UPDATE t1 SET a=( (SELECT MAX(a) FROM t1));
+DROP TABLE t1;

mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff

+--- suite/sys_vars/r/aria_sort_buffer_size_basic.result	2021-02-02 02:58:55.686921205 +0200
++++ suite/sys_vars/r/aria_sort_buffer_size_basic.reject	2021-02-02 11:02:12.361178360 +0200
+@@ -44,5 +44,5 @@
+ set session aria_sort_buffer_size=cast(-1 as unsigned int);
+ select @@session.aria_sort_buffer_size;
+ @@session.aria_sort_buffer_size
+-9223372036854775807
++2147483647
+ SET @@global.aria_sort_buffer_size = @start_global_value;

mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result

@@ -44,5 +44,5 @@ select @@global.aria_sort_buffer_size;
 set session aria_sort_buffer_size=cast(-1 as unsigned int);
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size
-18446744073709551615
+9223372036854775807
 SET @@global.aria_sort_buffer_size = @start_global_value;

mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff

---- sysvars_aria.result	2020-04-29 11:35:30.042243474 +0300
-+++ sysvars_aria,32bit.reject	2020-04-30 11:45:51.128740906 +0300
+--- suite/sys_vars/r/sysvars_aria.result	2021-02-02 02:58:55.686921205 +0200
++++ suite/sys_vars/r/sysvars_aria,32bit.reject	2021-02-02 10:55:53.876791633 +0200
 @@ -5,7 +5,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	8192
@@ -98,8 +98,8 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
  NUMERIC_MIN_VALUE	4096
--NUMERIC_MAX_VALUE	18446744073709551615
-+NUMERIC_MAX_VALUE	4294967295
+-NUMERIC_MAX_VALUE	9223372036854775807
++NUMERIC_MAX_VALUE	2147483647
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO

mysql-test/suite/sys_vars/r/sysvars_aria.result

@@ -212,7 +212,7 @@ VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
 NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MAX_VALUE	9223372036854775807
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO

mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff

---- mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
-+++ mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
+--- suite/sys_vars/r/sysvars_server_embedded.result	2021-02-02 14:07:21.616784062 +0200
++++ suite/sys_vars/r/sysvars_server_embedded,32bit.reject	2021-02-02 18:56:53.727764717 +0200
 @@ -35,7 +35,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	ARIA_BLOCK_SIZE
@@ -98,8 +98,8 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
  NUMERIC_MIN_VALUE	4096
--NUMERIC_MAX_VALUE	18446744073709551615
-+NUMERIC_MAX_VALUE	4294967295
+-NUMERIC_MAX_VALUE	9223372036854775807
++NUMERIC_MAX_VALUE	2147483647
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
@@ -335,15 +335,6 @@
  VARIABLE_COMMENT	Number of best matches to use for query expansion
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	1000
-@@ -1058,7 +1058,7 @@
- VARIABLE_TYPE	BIGINT UNSIGNED
- VARIABLE_COMMENT	The maximum length of the result of function GROUP_CONCAT()
- NUMERIC_MIN_VALUE	4
--NUMERIC_MAX_VALUE	18446744073709551615
-+NUMERIC_MAX_VALUE	4294967295
- NUMERIC_BLOCK_SIZE	1
- ENUM_VALUE_LIST	NULL
- READ_ONLY	NO
 @@ -1185,7 +1185,7 @@
  COMMAND_LINE_ARGUMENT	NULL
  VARIABLE_NAME	HISTOGRAM_SIZE
@@ -554,7 +545,7 @@
 -VARIABLE_TYPE	BIGINT UNSIGNED
 +VARIABLE_TYPE	INT UNSIGNED
  VARIABLE_COMMENT	The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored)
- NUMERIC_MIN_VALUE	4
+ NUMERIC_MIN_VALUE	64
  NUMERIC_MAX_VALUE	8388608
 @@ -1955,7 +1955,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
@@ -713,7 +704,7 @@
  VARIABLE_COMMENT	If this is not 0, then mysqld will use this value to reserve file descriptors to use with setrlimit(). If this value is 0 or autoset then mysqld will reserve max_connections*5 or max_connections + table_cache*2 (whichever is larger) number of file descriptors
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	4294967295
-@@ -2235,10 +2235,10 @@
+@@ -2235,17 +2235,17 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	OPTIMIZER_MAX_SEL_ARG_WEIGHT
  VARIABLE_SCOPE	SESSION
@@ -726,7 +717,6 @@
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -2245,7 +2245,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	OPTIMIZER_PRUNE_LEVEL
  VARIABLE_SCOPE	SESSION
@@ -791,7 +781,7 @@
 +VARIABLE_TYPE	INT
  VARIABLE_COMMENT	Size of the statement digest. Use 0 to disable, -1 for automated sizing.
  NUMERIC_MIN_VALUE	-1
- NUMERIC_MAX_VALUE	200
+ NUMERIC_MAX_VALUE	1048576
 @@ -2345,7 +2345,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	PERFORMANCE_SCHEMA_EVENTS_STAGES_HISTORY_LONG_SIZE
@@ -810,7 +800,7 @@
  VARIABLE_COMMENT	Number of rows per thread in EVENTS_STAGES_HISTORY. Use 0 to disable, -1 for automated sizing.
  NUMERIC_MIN_VALUE	-1
  NUMERIC_MAX_VALUE	1024
-@@ -2355,7 +2355,7 @@
+@@ -2365,7 +2365,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	PERFORMANCE_SCHEMA_EVENTS_STATEMENTS_HISTORY_LONG_SIZE
  VARIABLE_SCOPE	GLOBAL
@@ -828,7 +818,7 @@
  VARIABLE_COMMENT	Number of rows per thread in EVENTS_STATEMENTS_HISTORY. Use 0 to disable, -1 for automated sizing.
  NUMERIC_MIN_VALUE	-1
  NUMERIC_MAX_VALUE	1024
-@@ -2385,7 +2395,7 @@
+@@ -2385,7 +2385,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	PERFORMANCE_SCHEMA_EVENTS_TRANSACTIONS_HISTORY_LONG_SIZE
  VARIABLE_SCOPE	GLOBAL
@@ -1259,7 +1249,7 @@
  VARIABLE_COMMENT	When reading rows in sorted order after a sort, the rows are read through this buffer to avoid a disk seeks
  NUMERIC_MIN_VALUE	1
  NUMERIC_MAX_VALUE	2147483647
-@@ -2915,10 +2915,10 @@
+@@ -3015,10 +3015,10 @@
  COMMAND_LINE_ARGUMENT	OPTIONAL
  VARIABLE_NAME	ROWID_MERGE_BUFF_SIZE
  VARIABLE_SCOPE	SESSION
@@ -1272,7 +1262,7 @@
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -2955,7 +2955,7 @@
+@@ -3055,7 +3055,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	SERVER_ID
  VARIABLE_SCOPE	SESSION
@@ -1281,7 +1271,7 @@
  VARIABLE_COMMENT	Uniquely identifies the server instance in the community of replication partners
  NUMERIC_MIN_VALUE	1
  NUMERIC_MAX_VALUE	4294967295
-@@ -3025,7 +3025,7 @@
+@@ -3125,7 +3125,7 @@
  COMMAND_LINE_ARGUMENT	OPTIONAL
  VARIABLE_NAME	SLAVE_MAX_ALLOWED_PACKET
  VARIABLE_SCOPE	GLOBAL
@@ -1290,7 +1280,7 @@
  VARIABLE_COMMENT	The maximum packet length to sent successfully from the master to slave.
  NUMERIC_MIN_VALUE	1024
  NUMERIC_MAX_VALUE	1073741824
-@@ -3035,7 +3035,7 @@
+@@ -3135,7 +3135,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	SLOW_LAUNCH_TIME
  VARIABLE_SCOPE	GLOBAL
@@ -1299,7 +1289,7 @@
  VARIABLE_COMMENT	If creating the thread takes longer than this value (in seconds), the Slow_launch_threads counter will be incremented
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	31536000
-@@ -3078,7 +3078,7 @@
+@@ -3178,7 +3178,7 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	Each thread that needs to do a sort allocates a buffer of this size
  NUMERIC_MIN_VALUE	1024
@@ -1308,7 +1298,7 @@
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -3285,7 +3285,7 @@
+@@ -3395,7 +3395,7 @@
  COMMAND_LINE_ARGUMENT	NULL
  VARIABLE_NAME	STORED_PROGRAM_CACHE
  VARIABLE_SCOPE	GLOBAL
@@ -1317,7 +1307,7 @@
  VARIABLE_COMMENT	The soft upper limit for number of cached stored routines for one connection.
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	524288
-@@ -3365,7 +3365,7 @@
+@@ -3475,7 +3475,7 @@
  COMMAND_LINE_ARGUMENT	NULL
  VARIABLE_NAME	TABLE_DEFINITION_CACHE
  VARIABLE_SCOPE	GLOBAL
@@ -1326,7 +1316,7 @@
  VARIABLE_COMMENT	The number of cached table definitions
  NUMERIC_MIN_VALUE	400
  NUMERIC_MAX_VALUE	2097152
-@@ -3375,7 +3375,7 @@
+@@ -3485,7 +3485,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	TABLE_OPEN_CACHE
  VARIABLE_SCOPE	GLOBAL
@@ -1335,7 +1325,7 @@
  VARIABLE_COMMENT	The number of cached open tables
  NUMERIC_MIN_VALUE	10
  NUMERIC_MAX_VALUE	1048576
-@@ -3435,7 +3435,7 @@
+@@ -3545,7 +3545,7 @@
  COMMAND_LINE_ARGUMENT	OPTIONAL
  VARIABLE_NAME	THREAD_CACHE_SIZE
  VARIABLE_SCOPE	GLOBAL
@@ -1344,7 +1334,7 @@
  VARIABLE_COMMENT	How many threads we should keep in a cache for reuse. These are freed after 5 minutes of idle time
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	16384
-@@ -3518,7 +3518,7 @@
+@@ -3628,7 +3628,7 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	Max size for data for an internal temporary on-disk MyISAM or Aria table.
  NUMERIC_MIN_VALUE	1024
@@ -1353,7 +1343,7 @@
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -3528,7 +3528,7 @@
+@@ -3638,7 +3638,7 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size.
  NUMERIC_MIN_VALUE	0
@@ -1362,7 +1352,7 @@
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -3538,14 +3538,14 @@
+@@ -3648,14 +3648,14 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table.
  NUMERIC_MIN_VALUE	0
@@ -1379,7 +1369,7 @@
  VARIABLE_COMMENT	Allocation block size for transactions to be stored in binary log
  NUMERIC_MIN_VALUE	1024
  NUMERIC_MAX_VALUE	134217728
-@@ -3555,7 +3555,7 @@
+@@ -3665,7 +3665,7 @@
  COMMAND_LINE_ARGUMENT	REQUIRED
  VARIABLE_NAME	TRANSACTION_PREALLOC_SIZE
  VARIABLE_SCOPE	SESSION
@@ -1388,7 +1378,7 @@
  VARIABLE_COMMENT	Persistent buffer for transactions to be stored in binary log
  NUMERIC_MIN_VALUE	1024
  NUMERIC_MAX_VALUE	134217728
-@@ -3695,7 +3695,7 @@
+@@ -3805,7 +3805,7 @@
  COMMAND_LINE_ARGUMENT	NULL
  VARIABLE_NAME	WAIT_TIMEOUT
  VARIABLE_SCOPE	SESSION
@@ -1397,7 +1387,7 @@
  VARIABLE_COMMENT	The number of seconds the server waits for activity on a connection before closing it
  NUMERIC_MIN_VALUE	1
  NUMERIC_MAX_VALUE	31536000
-@@ -3722,7 +3722,7 @@
+@@ -3832,7 +3832,7 @@
  VARIABLE_NAME	LOG_TC_SIZE
  GLOBAL_VALUE_ORIGIN	AUTO
  VARIABLE_SCOPE	GLOBAL

mysql-test/suite/sys_vars/r/sysvars_server_embedded.result

@@ -208,7 +208,7 @@ VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
 NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MAX_VALUE	9223372036854775807
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO

mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff

@@ -98,8 +98,8 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
  NUMERIC_MIN_VALUE	4096
--NUMERIC_MAX_VALUE	18446744073709551615
-+NUMERIC_MAX_VALUE	4294967295
+-NUMERIC_MAX_VALUE	9223372036854775807
++NUMERIC_MAX_VALUE	2147483647
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -208,7 +208,7 @@ VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
 NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MAX_VALUE	9223372036854775807
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO

mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test

 # ulong session
 --source include/have_maria.inc
+--source include/word_size.inc
 
 SET @start_global_value = @@global.aria_sort_buffer_size;
 

mysys/my_malloc.c

@@ -78,6 +78,8 @@ void *my_malloc(PSI_memory_key key, size_t size, myf my_flags)
   /* Safety */
   if (!size)
     size=1;
+  if (size > SIZE_T_MAX - 1024L*1024L*16L)           /* Wrong call */
+    return 0;
 
   /* We have to align size as we store MY_THREAD_SPECIFIC flag in the LSB */
   size= ALIGN_SIZE(size);

mysys/safemalloc.c

@@ -115,9 +115,10 @@ void *sf_malloc(size_t size, myf my_flags)
     init_done= 1;
   }
 
-  irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4);
+  if (size > SIZE_T_MAX - 1024L*1024L*16L)           /* Wrong call */
+    return 0;
 
-  if (!irem)
+  if (!(irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4)))
     return 0;
 
   /* we guarantee the alignment */

storage/maria/ha_maria.cc

@@ -268,8 +268,9 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
 
 static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
        "The buffer that is allocated when sorting the index when doing a "
-       "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL,
-       SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
+       "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.",
+       NULL, NULL,
+       SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX/2, 1);
 
 static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG,
        "Specifies how Aria index statistics collection code should treat "

storage/maria/ma_check.c

@@ -2359,6 +2359,8 @@ static int initialize_variables_for_repair(HA_CHECK *param,
                                            MARIA_SHARE *org_share)
 {
   MARIA_SHARE *share= info->s;
+  size_t tmp;
+  uint threads;
 
   /*
     We have to clear these variables first, as the cleanup-in-case-of-error
@@ -2419,6 +2421,7 @@ static int initialize_variables_for_repair(HA_CHECK *param,
 
   /* calculate max_records */
   sort_info->filelength= my_seek(info->dfile.file, 0L, MY_SEEK_END, MYF(0));
+
   param->max_progress= sort_info->filelength;
   if ((param->testflag & T_CREATE_MISSING_KEYS) ||
       sort_info->org_data_file_type == COMPRESSED_RECORD)
@@ -2431,6 +2434,19 @@ static int initialize_variables_for_repair(HA_CHECK *param,
     sort_info->max_records= (ha_rows) (sort_info->filelength / rec_length);
   }
 
+  /* We don't need a bigger sort buffer than file_length * 8 */
+  threads= (param->testflag & T_REP_PARALLEL) ? (uint) share->base.keys : 1;
+  tmp= (size_t) MY_MIN(sort_info->filelength,
+                       (my_off_t) (SIZE_T_MAX/10/threads));
+  tmp= MY_MAX(tmp * 8 * threads, (size_t) 65536);         /* Some margin */
+  set_if_smaller(param->sort_buffer_length, tmp);
+  /* Protect against too big sort buffer length */
+#if SIZEOF_SIZE_T >= 8
+  set_if_smaller(param->sort_buffer_length, 16LL*1024LL*1024LL*1024LL);
+#else
+  set_if_smaller(param->sort_buffer_length, 1L*1024L*1024L*1024L);
+#endif
+
   /* Set up transaction handler so that we can see all rows */
   if (param->max_trid == 0)
   {