Added some fixes that should make MyISAM & Aria REPAIR work with more than 4G records

- If one specifies --force twice to myisamchk and aria_chk, then we will try to finnish the repair even if sort_buffer would be too small.
  This was done by dynamically allocate buffer handler objects as long as memory lasts.
- New option for myisamchk and aria_chk: create-missing-keys
- Changed default size of myisam_sort_buffer_size from 8M to 128M.
- Changed default size of sort_buffer_size in aria_chk from 128M to 256M.
- Increased information in error message about 'sort_buffer_size' beeing to small.
- Print also to 'show warnings' if repair was retried.
- Increased size of internal sort-buffer-readers from 16K to 128K
- Changed printing of 'number of records' to use %ll instead of casting to long
- Changed buffer sizes for myisam and aria to use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines.





include/my_global.h:
  Added MY_ALIGN_DOWN() to get previous alignment (for big memory areas)
include/myisam.h:
  Increased size of types to be able to handle more records
include/myisamchk.h:
  Increased size of types to be able to handle more records
  Added T_FORCE_SORT_MEMORY to force repair to work even if sort_buffer would not be big enough
mysql-test/r/myisam.result:
  Updated result
mysql-test/r/mysqld--help.result:
  Updated result
mysql-test/r/repair.result:
  Updated result
mysql-test/suite/maria/maria.result:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria.test:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria3.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test:
  Updated result after sort buffer size increase
mysql-test/t/myisam.test:
  Fixed error messages to not print system specific data
mysql-test/t/repair.test:
  Fixed error messages to not print system specific data
storage/maria/ha_maria.cc:
  Print also to 'show warnings' if repair was retried
  Changed default size of sort_buffer_size from 128M to 256M (same as in mysqld)
storage/maria/ma_check.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/ma_sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug where maria_update_key_parts() was called too early.
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
storage/maria/maria_chk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
storage/maria/maria_def.h:
  Increased size of types to be able to handle more records
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_ftdump.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_read_log.c:
  Use PAGE_BUFFER_INIT for page cache
storage/myisam/ha_myisam.cc:
  Changed default size of myisam_sort_buffer_size from 8M to 128M
storage/myisam/mi_check.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisam_ftdump.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamchk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamdef.h:
  Increased SORT_BUFFER_INIT to 64M
  (speeds up repair a lot and most machines have nowadays a lot of memory)
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.

include/my_global.h

@@ -847,7 +847,8 @@ typedef long		my_ptrdiff_t;
 typedef long long	my_ptrdiff_t;
 #endif
 
-#define MY_ALIGN(A,L)	(((A) + (L) - 1) & ~((L) - 1))
+#define MY_ALIGN(A,L)	   (((A) + (L) - 1) & ~((L) - 1))
+#define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1))
 #define ALIGN_SIZE(A)	MY_ALIGN((A),sizeof(double))
 #define ALIGN_MAX_UNIT  (sizeof(double))
 /* Size to make adressable obj. */

include/myisam.h

@@ -356,8 +356,10 @@ typedef struct st_mi_sort_param
   ulonglong notnull[HA_MAX_KEY_SEG+1];
 
   my_off_t pos,max_pos,filepos,start_recpos;
-  uint key, key_length,real_key_length,sortbuff_size;
-  uint maxbuffers, keys, find_length, sort_keys_length;
+  uint key, key_length,real_key_length;
+  uint maxbuffers, find_length;
+  ulonglong sortbuff_size;
+  ha_rows keys;
   my_bool fix_datafile, master;
   my_bool calc_checksum;                /* calculate table checksum */
 
@@ -366,10 +368,10 @@ typedef struct st_mi_sort_param
   int (*key_write)(struct st_mi_sort_param *, const void *);
   void (*lock_in_memory)(HA_CHECK *);
   int (*write_keys)(struct st_mi_sort_param *, register uchar **,
-                    uint , struct st_buffpek *, IO_CACHE *);
-  uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
+                    ulonglong , struct st_buffpek *, IO_CACHE *);
+  my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
   int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *,
-                   uint, uint);
+                   uint, ulonglong);
 } MI_SORT_PARAM;
 
 /* functions in mi_check */

include/myisamchk.h

@@ -62,9 +62,10 @@
 #define T_ZEROFILL              ((ulonglong) 1L << 32)
 #define T_ZEROFILL_KEEP_LSN     ((ulonglong) 1L << 33)
 /** If repair should not bump create_rename_lsn */
-#define T_NO_CREATE_RENAME_LSN  ((ulonglong) 1L << 33)
-#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 34)
-#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 35)
+#define T_NO_CREATE_RENAME_LSN  ((ulonglong) 1L << 34)
+#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 35)
+#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 36)
+#define T_FORCE_SORT_MEMORY     ((ulonglong) 1L << 37)
 
 #define T_REP_ANY               (T_REP | T_REP_BY_SORT | T_REP_PARALLEL)
 
@@ -180,8 +181,8 @@ typedef struct st_buffpek {
   my_off_t file_pos;                    /* Where we are in the sort file */
   uchar *base, *key;                    /* Key pointers */
   ha_rows count;                        /* Number of rows in table */
-  ulong mem_count;                      /* numbers of keys in memory */
-  ulong max_keys;                       /* Max keys in buffert */
+  ha_rows mem_count;                    /* Numbers of keys in memory */
+  ha_rows max_keys;                     /* Max keys in buffert */
 } BUFFPEK;
 
 #endif /* _myisamchk_h */

mysql-test/r/myisam.result

@@ -2355,7 +2355,7 @@ Warnings:
 Warning	1292	Truncated incorrect myisam_sort_buffer_size value: '4'
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	repair	error	myisam_sort_buffer_size is too small
+test.t1	repair	error	myisam_sort_buffer_size is too small. X
 test.t1	repair	warning	Number of rows changed from 0 to 7168
 test.t1	repair	status	OK
 SET myisam_repair_threads=2;
@@ -2424,7 +2424,7 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
 Warnings:
-Error	1034	myisam_sort_buffer_size is too small
+Error	1034	myisam_sort_buffer_size is too small. X
 Error	1034	Number of rows changed from 0 to 157
 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
 INSERT INTO t1 VALUES('1');

mysql-test/r/mysqld--help.result

@@ -1045,7 +1045,7 @@ myisam-max-sort-file-size 9223372036853727232
 myisam-mmap-size 18446744073709551615
 myisam-recover-options DEFAULT
 myisam-repair-threads 1
-myisam-sort-buffer-size 8388608
+myisam-sort-buffer-size 134216704
 myisam-stats-method nulls_unequal
 myisam-use-mmap FALSE
 net-buffer-length 16384

mysql-test/r/repair.result

@@ -77,12 +77,12 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
 Warnings:
-Error	1034	myisam_sort_buffer_size is too small
+Error	1034	myisam_sort_buffer_size is too small. X
 Error	1034	Number of rows changed from 0 to 157
 SET myisam_repair_threads=2;
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	repair	error	myisam_sort_buffer_size is too small
+test.t1	repair	error	myisam_sort_buffer_size is too small. X
 test.t1	repair	warning	Number of rows changed from 0 to 157
 test.t1	repair	status	OK
 SET myisam_repair_threads=@@global.myisam_repair_threads;

mysql-test/suite/maria/maria.result

@@ -2679,3 +2679,65 @@ select count(*) from t1;
 count(*)
 13
 drop table t1;
+#
+# BUG#47444 - --myisam_repair_threads > 1 can result in all index
+#             cardinalities=1
+#
+SET aria_repair_threads=2;
+SET aria_sort_buffer_size=8192;
+CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
+INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	status	OK
+SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
+CARDINALITY
+14
+14
+14
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+SET aria_repair_threads=@@global.aria_repair_threads;
+#
+# BUG#47073 - valgrind errs, corruption,failed repair of partition,
+#             low myisam_sort_buffer_size
+#
+CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
+INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
+(6,'0'),(7,'0');
+INSERT INTO t1 SELECT a+10,b FROM t1;
+INSERT INTO t1 SELECT a+20,b FROM t1;
+INSERT INTO t1 SELECT a+40,b FROM t1;
+INSERT INTO t1 SELECT a+80,b FROM t1;
+INSERT INTO t1 SELECT a+160,b FROM t1;
+INSERT INTO t1 SELECT a+320,b FROM t1;
+INSERT INTO t1 SELECT a+640,b FROM t1;
+INSERT INTO t1 SELECT a+1280,b FROM t1;
+INSERT INTO t1 SELECT a+2560,b FROM t1;
+INSERT INTO t1 SELECT a+5120,b FROM t1;
+SET aria_sort_buffer_size=4096;
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	error	aria_sort_buffer_size is too small. X
+test.t1	repair	error	Create index by sort failed
+test.t1	repair	info	Retrying repair with keycache
+test.t1	repair	status	OK
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+SET aria_repair_threads=2;
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	error	aria_sort_buffer_size is too small. X
+test.t1	repair	error	Create index by sort failed
+test.t1	repair	info	Retrying repair with keycache
+test.t1	repair	status	OK
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+SET aria_repair_threads=@@global.aria_repair_threads;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+DROP TABLE t1;

mysql-test/suite/maria/maria.test

@@ -1963,6 +1963,52 @@ unlock tables;
 select count(*) from t1;
 drop table t1;
 
+--echo #
+--echo # BUG#47444 - --myisam_repair_threads > 1 can result in all index
+--echo #             cardinalities=1
+--echo #
+SET aria_repair_threads=2;
+SET aria_sort_buffer_size=8192;
+CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
+INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
+--replace_regex /Current aria_sort_buffer_size.*/X/
+REPAIR TABLE t1;
+SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
+CHECK TABLE t1;
+DROP TABLE t1;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+SET aria_repair_threads=@@global.aria_repair_threads;
+
+--echo #
+--echo # BUG#47073 - valgrind errs, corruption,failed repair of partition,
+--echo #             low myisam_sort_buffer_size
+--echo #
+CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
+INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
+                     (6,'0'),(7,'0');
+INSERT INTO t1 SELECT a+10,b FROM t1;
+INSERT INTO t1 SELECT a+20,b FROM t1;
+INSERT INTO t1 SELECT a+40,b FROM t1;
+INSERT INTO t1 SELECT a+80,b FROM t1;
+INSERT INTO t1 SELECT a+160,b FROM t1;
+INSERT INTO t1 SELECT a+320,b FROM t1;
+INSERT INTO t1 SELECT a+640,b FROM t1;
+INSERT INTO t1 SELECT a+1280,b FROM t1;
+INSERT INTO t1 SELECT a+2560,b FROM t1;
+INSERT INTO t1 SELECT a+5120,b FROM t1;
+SET aria_sort_buffer_size=4096;
+--replace_regex /Current aria_sort_buffer_size.*/X/
+REPAIR TABLE t1;
+CHECK TABLE t1;
+SET aria_repair_threads=2;
+# May report different values depending on threads activity.
+--replace_regex /Current aria_sort_buffer_size.*/X/
+REPAIR TABLE t1;
+CHECK TABLE t1;
+SET aria_repair_threads=@@global.aria_repair_threads;
+SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
+DROP TABLE t1;
+
 #
 # End of test
 #

mysql-test/suite/maria/maria3.result

@@ -317,7 +317,7 @@ aria_pagecache_division_limit	100
 aria_page_checksum	OFF
 aria_recover	NORMAL
 aria_repair_threads	1
-aria_sort_buffer_size	134217728
+aria_sort_buffer_size	268434432
 aria_stats_method	nulls_unequal
 aria_sync_log_dir	NEWFILE
 show status like 'aria%';

mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result

 SET @start_global_value = @@global.aria_sort_buffer_size;
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-134217728
+268434432
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size
-134217728
+268434432
 show global variables like 'aria_sort_buffer_size';
 Variable_name	Value
-aria_sort_buffer_size	134217728
+aria_sort_buffer_size	268434432
 show session variables like 'aria_sort_buffer_size';
 Variable_name	Value
-aria_sort_buffer_size	134217728
+aria_sort_buffer_size	268434432
 select * from information_schema.global_variables where variable_name='aria_sort_buffer_size';
 VARIABLE_NAME	VARIABLE_VALUE
-ARIA_SORT_BUFFER_SIZE	134217728
+ARIA_SORT_BUFFER_SIZE	268434432
 select * from information_schema.session_variables where variable_name='aria_sort_buffer_size';
 VARIABLE_NAME	VARIABLE_VALUE
-ARIA_SORT_BUFFER_SIZE	134217728
+ARIA_SORT_BUFFER_SIZE	268434432
 set global aria_sort_buffer_size=10;
+Warnings:
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '10'
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-10
+4096
 set session aria_sort_buffer_size=10;
+Warnings:
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '10'
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size
-10
+4096
 set global aria_sort_buffer_size=1.1;
 ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size'
 set session aria_sort_buffer_size=1e1;
@@ -36,7 +40,7 @@ Warnings:
 Warning	1292	Truncated incorrect aria_sort_buffer_size value: '0'
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-4
+4096
 set session aria_sort_buffer_size=cast(-1 as unsigned int);
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size

mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result

 SET @start_global_value = @@global.myisam_sort_buffer_size ;
 SELECT @start_global_value;
 @start_global_value
-8388608
+134216704
 SET @start_session_value = @@session.myisam_sort_buffer_size ;
 SELECT @start_session_value;
 @start_session_value
-8388608
+134216704
 '#--------------------FN_DYNVARS_005_01-------------------------#'
 SET @@global.myisam_sort_buffer_size   = 100;
 Warnings:
@@ -13,22 +13,22 @@ Warning	1292	Truncated incorrect myisam_sort_buffer_size value: '100'
 SET @@global.myisam_sort_buffer_size   = DEFAULT;
 SELECT @@global.myisam_sort_buffer_size  ;
 @@global.myisam_sort_buffer_size
-8388608
+134216704
 SET @@session.myisam_sort_buffer_size   = 200;
 Warnings:
 Warning	1292	Truncated incorrect myisam_sort_buffer_size value: '200'
 SET @@session.myisam_sort_buffer_size   = DEFAULT;
 SELECT @@session.myisam_sort_buffer_size  ;
 @@session.myisam_sort_buffer_size
-8388608
+134216704
 '#--------------------FN_DYNVARS_005_02-------------------------#'
 SET @@global.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@global.myisam_sort_buffer_size   = 8388608;
-@@global.myisam_sort_buffer_size   = 8388608
+SELECT @@global.myisam_sort_buffer_size   = 134216704;
+@@global.myisam_sort_buffer_size   = 134216704
 1
 SET @@session.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@session.myisam_sort_buffer_size   = 8388608;
-@@session.myisam_sort_buffer_size   = 8388608
+SELECT @@session.myisam_sort_buffer_size   = 134216704;
+@@session.myisam_sort_buffer_size   = 134216704
 1
 '#--------------------FN_DYNVARS_005_03-------------------------#'
 SET @@global.myisam_sort_buffer_size   = 4;
@@ -187,8 +187,8 @@ ERROR 42S22: Unknown column 'myisam_sort_buffer_size' in 'field list'
 SET @@global.myisam_sort_buffer_size   = @start_global_value;
 SELECT @@global.myisam_sort_buffer_size  ;
 @@global.myisam_sort_buffer_size
-8388608
+134216704
 SET @@session.myisam_sort_buffer_size   = @start_session_value;
 SELECT @@session.myisam_sort_buffer_size  ;
 @@session.myisam_sort_buffer_size
-8388608
+134216704

mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test

@@ -60,10 +60,10 @@ SELECT @@session.myisam_sort_buffer_size  ;
 ########################################################################
 
 SET @@global.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@global.myisam_sort_buffer_size   = 8388608;
+SELECT @@global.myisam_sort_buffer_size   = 134216704;
 
 SET @@session.myisam_sort_buffer_size   = DEFAULT;
-SELECT @@session.myisam_sort_buffer_size   = 8388608;
+SELECT @@session.myisam_sort_buffer_size   = 134216704;
 
 
 --echo '#--------------------FN_DYNVARS_005_03-------------------------#'

mysql-test/t/myisam.test

@@ -1600,6 +1600,7 @@ INSERT INTO t1 SELECT a+1280,b FROM t1;
 INSERT INTO t1 SELECT a+2560,b FROM t1;
 INSERT INTO t1 SELECT a+5120,b FROM t1;
 SET myisam_sort_buffer_size=4;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 REPAIR TABLE t1;
 
 SET myisam_repair_threads=2;
@@ -1648,6 +1649,7 @@ DROP TABLE t1, t2, t3;
 CREATE TABLE t1(a CHAR(255), KEY(a));
 SELECT * FROM t1, t1 AS a1;
 SET myisam_sort_buffer_size=4;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),

mysql-test/t/repair.test

@@ -61,6 +61,7 @@ DROP TABLE t1;
 #
 CREATE TABLE t1(a CHAR(255), KEY(a));
 SET myisam_sort_buffer_size=4096;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
@@ -79,6 +80,7 @@ INSERT INTO t1 VALUES
 ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
 ('0'),('0'),('0'),('0'),('0'),('0'),('0');
 SET myisam_repair_threads=2;
+--replace_regex /Current myisam_sort_buffer_size.*/X/
 REPAIR TABLE t1;
 SET myisam_repair_threads=@@global.myisam_repair_threads;
 SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;

storage/maria/ha_maria.cc

@@ -260,10 +260,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
        "disables parallel repair.",
        0, 0, 1, 1, 128, 1);
 
-static MYSQL_THDVAR_ULONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
+static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
        "The buffer that is allocated when sorting the index when doing a "
-       "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.",
-       0, 0, 128L*1024L*1024L, 4, UINT_MAX32, 1);
+       "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL,
+       SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
 
 static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG,
        "Specifies how Aria index statistics collection code should treat "
@@ -1455,6 +1455,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt)
     if ((param.testflag & T_REP_BY_SORT))
     {
       param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP;
+      if (thd->vio_ok())
+        _ma_check_print_info(&param, "Retrying repair with keycache");
       sql_print_information("Retrying repair of: '%s' with keycache",
                             table->s->path.str);
       continue;

storage/maria/ma_check.c

@@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param)
   param->keys_in_use= ~(ulonglong) 0;
   param->search_after_block=HA_OFFSET_ERROR;
   param->auto_increment_value= 0;
-  param->use_buffers=USE_BUFFER_INIT;
+  param->use_buffers= PAGE_BUFFER_INIT;
   param->read_buffer_length=READ_BUFFER_INIT;
   param->write_buffer_length=READ_BUFFER_INIT;
   param->sort_buffer_length=SORT_BUFFER_INIT;

storage/maria/ma_sort.c

@@ -36,8 +36,10 @@
 #define MERGEBUFF2 31
 #define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD)
 #define MYF_RW  MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
-#define DISK_BUFFER_SIZE (IO_SIZE*16)
+#define DISK_BUFFER_SIZE (IO_SIZE*128)
 
+/* How many keys we can keep in memory */
+typedef ulonglong ha_keys;
 
 /*
  Pointers of functions for store and read keys from temp file
@@ -47,41 +49,42 @@ extern void print_error(const char *fmt,...);
 
 /* Functions defined in this file */
 
-static ha_rows find_all_keys(MARIA_SORT_PARAM *info,uint keys,
+static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_keys keys,
                              uchar **sort_keys,
-                             DYNAMIC_ARRAY *buffpek,int *maxbuffer,
+                             DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
                              IO_CACHE *tempfile,
                              IO_CACHE *tempfile_for_exceptions);
-static int write_keys(MARIA_SORT_PARAM *info, uchar **sort_keys,
-                      uint count, BUFFPEK *buffpek,IO_CACHE *tempfile);
+static int write_keys(MARIA_SORT_PARAM *info,uchar **sort_keys,
+                      ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
 static int write_key(MARIA_SORT_PARAM *info, uchar *key,
                      IO_CACHE *tempfile);
 static int write_index(MARIA_SORT_PARAM *info, uchar **sort_keys,
-                       uint count);
-static int merge_many_buff(MARIA_SORT_PARAM *info,uint keys,
+                      ha_keys count);
+static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
                            uchar **sort_keys,
-                           BUFFPEK *buffpek,int *maxbuffer,
+                           BUFFPEK *buffpek, uint *maxbuffer,
                            IO_CACHE *t_file);
-static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
-                           uint sort_length);
-static int merge_buffers(MARIA_SORT_PARAM *info,uint keys,
+static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
+                               uint sort_length);
+static int merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys,
                          IO_CACHE *from_file, IO_CACHE *to_file,
                          uchar **sort_keys, BUFFPEK *lastbuff,
                          BUFFPEK *Fb, BUFFPEK *Tb);
-static int merge_index(MARIA_SORT_PARAM *,uint, uchar **,BUFFPEK *, int,
+static int merge_index(MARIA_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
                        IO_CACHE *);
 static int flush_maria_ft_buf(MARIA_SORT_PARAM *info);
 
-static int write_keys_varlen(MARIA_SORT_PARAM *info, uchar **sort_keys,
-                             uint count, BUFFPEK *buffpek,
+static int write_keys_varlen(MARIA_SORT_PARAM *info,uchar **sort_keys,
+                             ha_keys count, BUFFPEK *buffpek,
                              IO_CACHE *tempfile);
-static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
-                                  uint sort_length);
+static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
+                                      uint sort_length);
 static int write_merge_key(MARIA_SORT_PARAM *info, IO_CACHE *to_file,
-                           uchar *key, uint sort_length, uint count);
+                           uchar *key, uint sort_length, ha_keys count);
 static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
-                                  IO_CACHE *to_file, uchar *key,
-                                  uint sort_length, uint count);
+                                  IO_CACHE *to_file,
+                                  uchar* key, uint sort_length,
+                                  ha_keys count);
 static inline int
 my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
 
@@ -102,11 +105,11 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
 int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
                              size_t sortbuff_size)
 {
-  int error,maxbuffer,skr;
-  size_t memavl,old_memavl;
-  uint keys,sort_length;
+  int error;
+  uint sort_length, maxbuffer;
+  size_t memavl, old_memavl;
   DYNAMIC_ARRAY buffpek;
-  ha_rows records;
+  ha_rows records, keys;
   uchar **sort_keys;
   IO_CACHE tempfile, tempfile_for_exceptions;
   DBUG_ENTER("_ma_create_index_by_sort");
@@ -117,7 +120,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
   if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
   {
     info->write_keys= write_keys_varlen;
-    info->read_to_buffer=read_to_buffer_varlen;
+    info->read_to_buffer= read_to_buffer_varlen;
     info->write_key=write_merge_key_varlen;
   }
   else
@@ -140,31 +143,53 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
 
   while (memavl >= MIN_SORT_MEMORY)
   {
-    if ((records < UINT_MAX32) &&
-       ((my_off_t) (records + 1) *
-        (sort_length + sizeof(char*)) <= (my_off_t) memavl))
-      keys= (uint)records+1;
+    /* Check if we can fit all keys into memory */
+    if (((ulonglong) (records + 1) *
+         (sort_length + sizeof(char*)) <= memavl))
+      keys= records+1;
+    else if ((info->sort_info->param->testflag &
+              (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
+             T_FORCE_SORT_MEMORY)
+    {
+      /*
+        Use all of the given sort buffer for key data.
+        Allocate 1000 buffers at a start for new data. More buffers
+        will be allocated when needed.
+      */
+      keys= memavl / (sort_length+sizeof(char*));
+      maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
+    }
     else
+    {
+      /*
+        All keys can't fit in memory.
+        Calculate how many keys + buffers we can keep in memory
+      */
+      uint maxbuffer_org;
       do
       {
-	skr=maxbuffer;
-	if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
-	    (keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
+	maxbuffer_org= maxbuffer;
+	if (memavl < sizeof(BUFFPEK) * maxbuffer ||
+	    (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
              (sort_length+sizeof(char*))) <= 1 ||
-            keys < (uint) maxbuffer)
+            keys < maxbuffer)
 	{
 	  _ma_check_print_error(info->sort_info->param,
-			       "aria_sort_buffer_size is too small");
+                                "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                                (ulonglong) sortbuff_size, (ulonglong) records,
+                                sort_length);
+          my_errno= ENOMEM;
 	  goto err;
 	}
       }
-      while ((maxbuffer= (int) (records/(keys-1)+1)) != skr);
+      while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
+    }
 
     if ((sort_keys=(uchar**) my_malloc(keys*(sort_length+sizeof(char*))+
                                       HA_FT_MAXBYTELEN, MYF(0))))
     {
       if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
-                                maxbuffer/2, MYF(0)))
+                                min(maxbuffer/2, 1000), MYF(0)))
       {
 	my_free(sort_keys);
         sort_keys= 0;
@@ -178,14 +203,20 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
   }
   if (memavl < MIN_SORT_MEMORY)
   {
-    _ma_check_print_error(info->sort_info->param, "Aria sort buffer"
-                          " too small"); /* purecov: tested */
-    goto err; /* purecov: tested */
+    /* purecov: begin inspected */
+    _ma_check_print_error(info->sort_info->param,
+                          "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                          (ulonglong) sortbuff_size, (ulonglong) records, sort_length);
+    my_errno= ENOMEM;
+    goto err;
+    /* purecov: end inspected */
   }
   (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
 
   if (!no_messages)
-    printf("  - Searching for keys, allocating buffer for %d keys\n",keys);
+    my_fprintf(stdout,
+               "  - Searching for keys, allocating buffer for %llu keys\n",
+               (ulonglong) keys);
 
   if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
                                   &tempfile,&tempfile_for_exceptions))
@@ -197,8 +228,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
   if (maxbuffer == 0)
   {
     if (!no_messages)
-      printf("  - Dumping %lu keys\n", (ulong) records);
-    if (write_index(info,sort_keys, (uint) records))
+      my_fprintf(stdout, "  - Dumping %llu keys\n", (ulonglong) records);
+    if (write_index(info, sort_keys, (ha_keys) records))
       goto err; /* purecov: inspected */
   }
   else
@@ -207,7 +238,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
     if (maxbuffer >= MERGEBUFF2)
     {
       if (!no_messages)
-	printf("  - Merging %lu keys\n", (ulong) records); /* purecov: tested */
+	my_fprintf(stdout, "  - Merging %llu keys\n",
+                   (ulonglong) records); /* purecov: tested */
       if (merge_many_buff(info,keys,sort_keys,
                   dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
 	goto err;				/* purecov: inspected */
@@ -266,13 +298,13 @@ err:
 
 /* Search after all keys and place them in a temp. file */
 
-static ha_rows find_all_keys(MARIA_SORT_PARAM *info, uint keys,
+static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_rows keys,
                              uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
-                             int *maxbuffer, IO_CACHE *tempfile,
+                             uint *maxbuffer, IO_CACHE *tempfile,
                              IO_CACHE *tempfile_for_exceptions)
 {
   int error;
-  uint idx;
+  ha_rows idx;
   DBUG_ENTER("find_all_keys");
 
   idx=error=0;
@@ -328,9 +360,11 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
 {
   MARIA_SORT_PARAM *sort_param= (MARIA_SORT_PARAM*) arg;
   int error;
-  size_t memavl,old_memavl;
+  size_t memavl, old_memavl;
+  longlong sortbuff_size;
+  ha_keys keys, idx;
   uint sort_length;
-  ulong idx, maxbuffer, keys;
+  uint maxbuffer;
   uchar **sort_keys=0;
 
   LINT_INIT(keys);
@@ -364,8 +398,9 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
     bzero((char*) &sort_param->buffpek,sizeof(sort_param->buffpek));
     bzero((char*) &sort_param->unique, sizeof(sort_param->unique));
 
-    memavl=       max(sort_param->sortbuff_size, MIN_SORT_MEMORY);
-    idx=          (uint)sort_param->sort_info->max_records;
+    sortbuff_size= sort_param->sortbuff_size;
+    memavl=       max(sortbuff_size, MIN_SORT_MEMORY);
+    idx=          (ha_keys) sort_param->sort_info->max_records;
     sort_length=  sort_param->key_length;
     maxbuffer=    1;
 
@@ -373,23 +408,36 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
     {
       if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= (my_off_t) memavl)
         keys= idx+1;
+      else if ((sort_param->sort_info->param->testflag &
+                (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
+               T_FORCE_SORT_MEMORY)
+      {
+        /*
+          Use all of the given sort buffer for key data.
+          Allocate 1000 buffers at a start for new data. More buffers
+          will be allocated when needed.
+        */
+        keys= memavl / (sort_length+sizeof(char*));
+        maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
+      }
       else
       {
-        ulong skr;
+        uint maxbuffer_org;
         do
         {
-          skr= maxbuffer;
+          maxbuffer_org= maxbuffer;
           if (memavl < sizeof(BUFFPEK)*maxbuffer ||
               (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
                (sort_length+sizeof(char*))) <= 1 ||
               keys < maxbuffer)
           {
             _ma_check_print_error(sort_param->sort_info->param,
-                                  "aria_sort_buffer_size is too small");
+                                  "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                                  sortbuff_size, (ulonglong) idx, sort_length);
             goto err;
           }
         }
-        while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr);
+        while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
       }
       if ((sort_keys= (uchar **)
            my_malloc(keys*(sort_length+sizeof(char*))+
@@ -397,7 +445,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
                       HA_FT_MAXBYTELEN : 0), MYF(0))))
       {
         if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
-                                  maxbuffer, maxbuffer/2, MYF(0)))
+                                  maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
         {
           my_free(sort_keys);
           sort_keys= (uchar **) NULL;            /* for err: label */
@@ -412,14 +460,19 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
     }
     if (memavl < MIN_SORT_MEMORY)
     {
+      /* purecov: begin inspected */
       _ma_check_print_error(sort_param->sort_info->param,
-                            "Aria sort buffer too small");
-      goto err; /* purecov: tested */
+                            "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                            sortbuff_size, (ulonglong) idx, sort_length);
+      my_errno= ENOMEM;
+      goto err;
+    /* purecov: end inspected */
     }
 
     if (sort_param->sort_info->param->testflag & T_VERBOSE)
-      printf("Key %d - Allocating buffer for %lu keys\n",
-             sort_param->key+1, (ulong) keys);
+      my_fprintf(stdout,
+                 "Key %d - Allocating buffer for %llu keys\n",
+                 sort_param->key + 1, (ulonglong) keys);
     sort_param->sort_keys= sort_keys;
 
     idx= error= 0;
@@ -431,7 +484,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
     {
       if (sort_param->real_key_length > sort_param->key_length)
       {
-        if (write_key(sort_param,sort_keys[idx],
+        if (write_key(sort_param, sort_keys[idx],
                       &sort_param->tempfile_for_exceptions))
           goto err;
         continue;
@@ -449,7 +502,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
                (size_t) sort_param->key_length);
         idx= 1;
       }
-      sort_keys[idx]=sort_keys[idx - 1] + sort_param->key_length;
+      sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
     }
     if (error > 0)
       goto err;
@@ -465,14 +518,13 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
     else
       sort_param->keys= idx;
 
-    sort_param->sort_keys_length= keys;
     goto ok;
 
 err:
     DBUG_PRINT("error", ("got some error"));
     sort_param->sort_info->got_error= 1; /* no need to protect with a mutex */
     my_free(sort_keys);
-    sort_param->sort_keys=0;
+    sort_param->sort_keys= 0;
     delete_dynamic(& sort_param->buffpek);
     close_cached_file(&sort_param->tempfile);
     close_cached_file(&sort_param->tempfile_for_exceptions);
@@ -483,8 +535,12 @@ ok:
       Detach from the share if the writer is involved. Avoid others to
       be blocked. This includes a flush of the write buffer. This will
       also indicate EOF to the readers.
+      That means that a writer always gets here first and readers -
+      only when they see EOF. But if a reader finishes prematurely
+      because of an error it may reach this earlier - don't allow it
+      to detach the writer thread.
     */
-    if (sort_param->sort_info->info->rec_cache.share)
+    if (sort_param->master && sort_param->sort_info->info->rec_cache.share)
       remove_io_thread(&sort_param->sort_info->info->rec_cache);
 
     /* Readers detach from the share if any. Avoid others to be blocked. */
@@ -518,7 +574,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
 
   for (i= 0, sinfo= sort_param ;
        i < sort_info->total_keys ;
-       i++, rec_per_key_part+=sinfo->keyinfo->keysegs, sinfo++)
+       i++, sinfo++)
   {
     if (!sinfo->sort_keys)
     {
@@ -534,19 +590,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
       {
         if (param->testflag & T_VERBOSE)
         {
-          printf("Key %d  - Dumping %u keys\n",sinfo->key+1, sinfo->keys);
+          my_fprintf(stdout,
+                     "Key %d  - Dumping %llu keys\n", sinfo->key+1,
+                     (ulonglong) sinfo->keys);
           fflush(stdout);
         }
         if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
             flush_maria_ft_buf(sinfo) || _ma_flush_pending_blocks(sinfo))
           got_error=1;
       }
-      if (!got_error && param->testflag & T_STATISTICS)
-        maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
-                               param->stats_method ==
-                               MI_STATS_METHOD_IGNORE_NULLS ?
-                               sinfo->notnull : NULL,
-                               (ulonglong) share->state.state.records);
     }
     my_free(sinfo->sort_keys);
     my_free(sinfo->rec_buff);
@@ -559,6 +611,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
 	 delete_dynamic(&sinfo->buffpek),
 	 close_cached_file(&sinfo->tempfile),
 	 close_cached_file(&sinfo->tempfile_for_exceptions),
+         rec_per_key_part+= sinfo->keyinfo->keysegs,
 	 sinfo++)
   {
     if (got_error)
@@ -597,10 +650,12 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
       if (maxbuffer >= MERGEBUFF2)
       {
         if (param->testflag & T_VERBOSE)
-          printf("Key %d  - Merging %u keys\n",sinfo->key+1, sinfo->keys);
-        if (merge_many_buff(sinfo, keys, (uchar **) mergebuf,
+          my_fprintf(stdout,
+                     "Key %d  - Merging %llu keys\n",
+                     sinfo->key+1, (ulonglong) sinfo->keys);
+        if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
 			    dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
-			    (int*) &maxbuffer, &sinfo->tempfile))
+			    &maxbuffer, &sinfo->tempfile))
         {
           got_error=1;
           continue;
@@ -660,6 +715,13 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
         }
       }
     }
+    if (!got_error && (param->testflag & T_STATISTICS))
+      maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
+                             param->stats_method ==
+                             MI_STATS_METHOD_IGNORE_NULLS ?
+                             sinfo->notnull : NULL,
+                             (ulonglong) share->state.state.records);
+
   }
   my_free(mergebuf);
   DBUG_RETURN(got_error);
@@ -669,12 +731,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
 /* Write all keys in memory to file for later merge */
 
 static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
-                      uint count, BUFFPEK *buffpek, IO_CACHE *tempfile)
+                      ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
 {
   uchar **end;
   uint sort_length=info->key_length;
   DBUG_ENTER("write_keys");
 
+  if (!buffpek)
+    DBUG_RETURN(1);                             /* Out of memory */
+
   my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
             info);
   if (!my_b_inited(tempfile) &&
@@ -687,7 +752,7 @@ static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
 
   for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
   {
-    if (my_b_write(tempfile, *sort_keys, (uint) sort_length))
+    if (my_b_write(tempfile, *sort_keys, sort_length))
       DBUG_RETURN(1); /* purecov: inspected */
   }
   DBUG_RETURN(0);
@@ -710,14 +775,17 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
 
 
 static int write_keys_varlen(MARIA_SORT_PARAM *info,
-				    register uchar **sort_keys,
-                                    uint count, BUFFPEK *buffpek,
-				    IO_CACHE *tempfile)
+                             register uchar **sort_keys,
+                             ha_keys count, BUFFPEK *buffpek,
+                             IO_CACHE *tempfile)
 {
   uchar **end;
   int err;
   DBUG_ENTER("write_keys_varlen");
 
+  if (!buffpek)
+    DBUG_RETURN(1);                             /* Out of memory */
+
   my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
             info);
   if (!my_b_inited(tempfile) &&
@@ -756,9 +824,8 @@ static int write_key(MARIA_SORT_PARAM *info, uchar *key,
 
 /* Write index */
 
-static int write_index(MARIA_SORT_PARAM *info,
-                              register uchar **sort_keys,
-                              register uint count)
+static int write_index(MARIA_SORT_PARAM *info, register uchar **sort_keys,
+                       register ha_keys count)
 {
   DBUG_ENTER("write_index");
 
@@ -777,11 +844,11 @@ static int write_index(MARIA_SORT_PARAM *info,
 
         /* Merge buffers to make < MERGEBUFF2 buffers */
 
-static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
-                                  uchar **sort_keys, BUFFPEK *buffpek,
-                                  int *maxbuffer, IO_CACHE *t_file)
+static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
+                           uchar **sort_keys, BUFFPEK *buffpek,
+                           uint *maxbuffer, IO_CACHE *t_file)
 {
-  int tmp, merges, max_merges;
+  uint tmp, merges, max_merges;
   IO_CACHE t_file2, *from_file, *to_file, *temp;
   BUFFPEK *lastbuff;
   DBUG_ENTER("merge_many_buff");
@@ -807,11 +874,11 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
   from_file= t_file ; to_file= &t_file2;
   while (*maxbuffer >= MERGEBUFF2)
   {
-    int i;
+    uint i;
     reinit_io_cache(from_file,READ_CACHE,0L,0,0);
     reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
     lastbuff=buffpek;
-    for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
+    for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
     {
       if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                         buffpek+i,buffpek+i+MERGEBUFF-1))
@@ -825,14 +892,19 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
     if (flush_io_cache(to_file))
       break;                                    /* purecov: inspected */
     temp=from_file; from_file=to_file; to_file=temp;
-    *maxbuffer= (int) (lastbuff-buffpek)-1;
+    *maxbuffer= (uint) (lastbuff-buffpek)-1;
     if (info->sort_info->param->max_stage != 1) /* If not parallel */
       _ma_report_progress(info->sort_info->param, merges++, max_merges);
   }
 cleanup:
   close_cached_file(to_file);                   /* This holds old result */
   if (to_file == t_file)
+  {
+    DBUG_ASSERT(t_file2.type == WRITE_CACHE);
     *t_file=t_file2;                            /* Copy result file */
+    t_file->current_pos= &t_file->write_pos;
+    t_file->current_end= &t_file->write_end;
+  }
 
   DBUG_RETURN(*maxbuffer >= MERGEBUFF2);        /* Return 1 if interrupted */
 } /* merge_many_buff */
@@ -851,33 +923,35 @@ cleanup:
     -1	Error
 */
 
-static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
-                                  uint sort_length)
+static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
+                                uint sort_length)
 {
-  register uint count;
-  uint length;
+  register ha_keys count;
+  my_off_t length;
 
-  if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
+  if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
   {
-    if (mysql_file_pread(fromfile->file, buffpek->base,
-                 (length= sort_length*count),buffpek->file_pos,MYF_RW))
-      return((uint) -1);                        /* purecov: inspected */
+    if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
+                         (length= sort_length * count),
+                         buffpek->file_pos, MYF_RW))
+      return(HA_OFFSET_ERROR);               /* purecov: inspected */
     buffpek->key=buffpek->base;
     buffpek->file_pos+= length;                 /* New filepos */
     buffpek->count-=    count;
     buffpek->mem_count= count;
   }
-  return (count*sort_length);
+  return (((my_off_t) count) * sort_length);
 } /* read_to_buffer */
 
-static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
-                                         uint sort_length)
+
+static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
+                                      uint sort_length)
 {
-  register uint count;
+  register ha_keys count;
   uint idx;
   uchar *buffp;
 
-  if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
+  if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
   {
     buffp= buffpek->base;
 
@@ -886,7 +960,7 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
       uint16 length_of_key;
       if (mysql_file_pread(fromfile->file,(uchar*)&length_of_key,sizeof(length_of_key),
                    buffpek->file_pos,MYF_RW))
-        return((uint) -1);
+        return(HA_OFFSET_ERROR);
       buffpek->file_pos+=sizeof(length_of_key);
       if (mysql_file_pread(fromfile->file, buffp, length_of_key,
                    buffpek->file_pos,MYF_RW))
@@ -898,15 +972,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
     buffpek->count-=    count;
     buffpek->mem_count= count;
   }
-  return (count*sort_length);
+  return (((my_off_t) count) * sort_length);
 } /* read_to_buffer_varlen */
 
 
 static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
                                   IO_CACHE *to_file, uchar* key,
-                                  uint sort_length, uint count)
+                                  uint sort_length, ha_keys count)
 {
-  uint idx;
+  ha_keys idx;
   uchar *bufs = key;
 
   for (idx=1;idx<=count;idx++)
@@ -921,34 +995,40 @@ static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
 
 
 static int write_merge_key(MARIA_SORT_PARAM *info __attribute__((unused)),
-				  IO_CACHE *to_file, uchar *key,
-				  uint sort_length, uint count)
+                           IO_CACHE *to_file, uchar *key,
+                           uint sort_length, ha_keys count)
 {
-  return my_b_write(to_file, key, (size_t) sort_length*count);
+  return my_b_write(to_file, key, ((size_t) sort_length) * count);
 }
 
 /*
   Merge buffers to one buffer
   If to_file == 0 then use info->key_write
+
+  Return:
+  0 ok
+  1 error
 */
 
 static int
-merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
+merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
               IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
               BUFFPEK *Fb, BUFFPEK *Tb)
 {
-  int error;
-  uint sort_length,maxcount;
+  int error= 1;
+  uint sort_length;
+  ha_keys maxcount;
   ha_rows count;
-  my_off_t UNINIT_VAR(to_start_filepos);
+  my_off_t UNINIT_VAR(to_start_filepos), read_length;
   uchar *strpos;
   BUFFPEK *buffpek,**refpek;
   QUEUE queue;
   DBUG_ENTER("merge_buffers");
 
-  count=error=0;
-  maxcount=keys/((uint) (Tb-Fb) +1);
+  count= 0;
+  maxcount= keys/((uint) (Tb-Fb) +1);
   DBUG_ASSERT(maxcount > 0);
+  LINT_INIT(to_start_filepos);
   if (to_file)
     to_start_filepos=my_b_tell(to_file);
   strpos= (uchar*) sort_keys;
@@ -963,10 +1043,10 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
   {
     count+= buffpek->count;
     buffpek->base= strpos;
-    buffpek->max_keys=maxcount;
-    strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,
-                                                      sort_length));
-    if (error == -1)
+    buffpek->max_keys= maxcount;
+    strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
+                                                sort_length));
+    if (read_length == HA_OFFSET_ERROR)
       goto err; /* purecov: inspected */
     queue_insert(&queue,(uchar*) buffpek);
   }
@@ -980,27 +1060,20 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
       {
         if (info->write_key(info,to_file, buffpek->key,
                             (uint) sort_length,1))
-        {
-          error=1; goto err; /* purecov: inspected */
-        }
+          goto err;                           /* purecov: inspected */
       }
       else
       {
         if ((*info->key_write)(info,(void*) buffpek->key))
-        {
-          error=1; goto err; /* purecov: inspected */
-        }
+          goto err;                           /* purecov: inspected */
       }
       buffpek->key+=sort_length;
       if (! --buffpek->mem_count)
       {
         /* It's enough to check for killedptr before a slow operation */
         if (_ma_killed_ptr(info->sort_info->param))
-        {
-          error=1;
           goto err;
-        }
-        if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length)))
+        if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
         {
           uchar *base= buffpek->base;
           uint max_keys=buffpek->max_keys;
@@ -1027,9 +1100,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
           }
           break;                /* One buffer have been removed */
         }
+        else if (read_length == HA_OFFSET_ERROR)
+          goto err;               /* purecov: inspected */
       }
-      else if (error == -1)
-        goto err;               /* purecov: inspected */
       queue_replace_top(&queue);   /* Top element has been replaced */
     }
   }
@@ -1061,8 +1134,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
       }
     }
   }
-  while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) !=
-         -1 && error != 0);
+  while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
+  if (read_length == 0)
+    error= 0;
 
   lastbuff->count=count;
   if (to_file)
@@ -1076,8 +1150,8 @@ err:
         /* Do a merge to output-file (save only positions) */
 
 static int
-merge_index(MARIA_SORT_PARAM *info, uint keys, uchar **sort_keys,
-            BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile)
+merge_index(MARIA_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
+            BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
 {
   DBUG_ENTER("merge_index");
   if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,

storage/maria/maria_chk.c

@@ -191,7 +191,7 @@ end:
 
 enum options_mc {
   OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
-  OPT_CORRECT_CHECKSUM, OPT_PAGE_BUFFER_SIZE,
+  OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_PAGE_BUFFER_SIZE,
   OPT_KEY_CACHE_BLOCK_SIZE, OPT_MARIA_BLOCK_SIZE,
   OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
   OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@@ -229,6 +229,11 @@ static struct my_option my_long_options[] =
   {"correct-checksum", OPT_CORRECT_CHECKSUM,
    "Correct checksum information for table.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"create-missing-keys", OPT_CREATE_MISSING_KEYS,
+   "Create missing keys. This assumes that the data file is correct and that "
+   "the the number of rows stored in the index file is correct. Enables "
+   "--quick",
+   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
 #ifndef DBUG_OFF
   {"debug", '#',
    "Output debug log. Often this is 'd:t:o,filename'.",
@@ -362,8 +367,8 @@ static struct my_option my_long_options[] =
   { "page_buffer_size", OPT_PAGE_BUFFER_SIZE,
     "Size of page buffer. Used by --safe-repair",
     &check_param.use_buffers, &check_param.use_buffers, 0,
-    GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, 1024L*1024L,
-    (long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
+    GET_ULONG, REQUIRED_ARG, PAGE_BUFFER_INIT, 1024L*1024L,
+    SIZE_T_MAX, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
   { "read_buffer_size", OPT_READ_BUFFER_SIZE,
     "Read buffer size for sequential reads during scanning",
     &check_param.read_buffer_length,
@@ -379,9 +384,8 @@ static struct my_option my_long_options[] =
   { "sort_buffer_size", OPT_SORT_BUFFER_SIZE,
     "Size of sort buffer. Used by --recover",
     &check_param.sort_buffer_length,
-    &check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
-    (long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD),
-    (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0},
+    &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG,
+    SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0},
   { "sort_key_blocks", OPT_SORT_KEY_BLOCKS,
     "Internal buffer for sorting keys; Don't touch :)",
     &check_param.sort_key_blocks,
@@ -497,10 +501,18 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\
   --correct-checksum  Correct checksum information for table.\n\
   -D, --data-file-length=#  Max length of data file (when recreating data\n\
                       file when it's full).\n\
+ --create-missing-keys\n\
+                      Create missing keys. This assumes that the data\n\
+                      file is correct and that the the number of rows stored\n\
+                      in the index file is correct. Enables --quick.\n\
   -e, --extend-check  Try to recover every possible row from the data file\n\
 		      Normally this will also find a lot of garbage rows;\n\
 		      Don't use this option if you are not totally desperate.\n\
-  -f, --force         Overwrite old temporary files.\n\
+  -f, --force         Overwrite old temporary files. Add another --force to\n\
+                      avoid 'sort_buffer_size is too small' errors.\n\
+                      In this case we will attempt to do the repair with the\n\
+                      given sort_buffer_size and dynamically allocate\n\
+                      as many management buffers as needed.\n\
   -k, --keys-used=#   Tell Aria to update only some specific keys. # is a\n\
 	              bit mask of which keys to use. This can be used to\n\
 		      get faster inserts.\n\
@@ -664,10 +676,13 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
     {
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
-      check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
+     check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
+                              T_FORCE_SORT_MEMORY);
     }
     else
     {
+     if (check_param.testflag & T_FORCE_CREATE)
+       check_param.testflag= T_FORCE_SORT_MEMORY;
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
       check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
     }
@@ -720,8 +735,26 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
       check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
     else
+    {
+      /*
+        If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
+        then add T_FORCE_UNIQUENESS.
+      */
       check_param.testflag|=
-        (check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
+        ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
+         T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
+    }
+    break;
+  case OPT_CREATE_MISSING_KEYS:
+    if (argument == disabled_my_option)
+      check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
+    else
+    {
+      check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
+      /* Use repair by sort by default */
+      if (!(check_param.testflag & T_REP_ANY))
+        check_param.testflag|= T_REP_BY_SORT;
+    }
     break;
   case 'u':
     if (argument == disabled_my_option)

storage/maria/maria_def.h

@@ -94,9 +94,10 @@ typedef struct st_maria_sort_param
   */
   ulonglong unique[HA_MAX_KEY_SEG+1];
   ulonglong notnull[HA_MAX_KEY_SEG+1];
+  ulonglong sortbuff_size;
 
   MARIA_RECORD_POS pos,max_pos,filepos,start_recpos, current_filepos;
-  uint key, key_length,real_key_length,sortbuff_size;
+  uint key, key_length,real_key_length;
   uint maxbuffers, keys, find_length, sort_keys_length;
   my_bool fix_datafile, master;
   my_bool calc_checksum;                /* calculate table checksum */
@@ -107,10 +108,10 @@ typedef struct st_maria_sort_param
   int (*key_write)(struct st_maria_sort_param *, const uchar *);
   void (*lock_in_memory)(HA_CHECK *);
   int (*write_keys)(struct st_maria_sort_param *, register uchar **,
-                         uint , struct st_buffpek *, IO_CACHE *);
-  uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
+                         ulonglong , struct st_buffpek *, IO_CACHE *);
+  my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
   int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,uchar *,
-                   uint, uint);
+                   uint, ulonglong);
 } MARIA_SORT_PARAM;
 
 int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile);
@@ -1223,10 +1224,11 @@ typedef struct st_maria_block_info
 #define UPDATE_AUTO_INC		8
 #define UPDATE_OPEN_COUNT	16
 
-#define USE_BUFFER_INIT		(((1024L*1024L*128-MALLOC_OVERHEAD)/8192)*8192)
-#define READ_BUFFER_INIT	(1024L*256L-MALLOC_OVERHEAD)
-#define SORT_BUFFER_INIT	(1024L*1024L*256-MALLOC_OVERHEAD)
-#define MIN_SORT_BUFFER		(4096-MALLOC_OVERHEAD)
+/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
+#define PAGE_BUFFER_INIT	MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 8192)
+#define READ_BUFFER_INIT	MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
+#define SORT_BUFFER_INIT	MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 1024)
+#define MIN_SORT_BUFFER		4096
 
 #define fast_ma_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _ma_writeinfo((INFO),0)
 #define fast_ma_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _ma_readinfo((INFO),F_RDLCK,1)

storage/maria/maria_ftdump.c

@@ -84,7 +84,7 @@ int main(int argc,char *argv[])
       usage();
   }
 
-  init_pagecache(maria_pagecache, USE_BUFFER_INIT, 0, 0,
+  init_pagecache(maria_pagecache, PAGE_BUFFER_INIT, 0, 0,
                  MARIA_KEY_BLOCK_LENGTH, MY_WME);
 
   if (!(info=maria_open(argv[0], O_RDONLY,

storage/maria/maria_read_log.c

@@ -32,7 +32,8 @@ const char *default_dbug_option= "d:t:o,/tmp/aria_read_log.trace";
 static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent;
 static my_bool opt_check;
 static const char *opt_tmpdir;
-static ulong opt_page_buffer_size, opt_translog_buffer_size;
+static ulong opt_translog_buffer_size;
+static ulonglong opt_page_buffer_size;
 static ulonglong opt_start_from_lsn, opt_end_lsn, opt_start_from_checkpoint;
 static MY_TMPDIR maria_chk_tmpdir;
 
@@ -204,9 +205,8 @@ static struct my_option my_long_options[] =
   { "page-buffer-size", 'P',
     "The size of the buffer used for index blocks for Aria tables",
     &opt_page_buffer_size, &opt_page_buffer_size, 0,
-    GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT,
-    1024L*1024L, (long) ~(ulong) 0, (long) MALLOC_OVERHEAD,
-    (long) IO_SIZE, 0},
+    GET_ULL, REQUIRED_ARG, PAGE_BUFFER_INIT,
+    PAGE_BUFFER_INIT, SIZE_T_MAX, MALLOC_OVERHEAD, (long) IO_SIZE, 0},
   { "start-from-lsn", 'o', "Start reading log from this lsn",
     &opt_start_from_lsn, &opt_start_from_lsn,
     0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 },

storage/myisam/ha_myisam.cc

@@ -78,7 +78,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
 static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
   "The buffer that is allocated when sorting the index when doing "
   "a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL,
-  8192 * 1024, MIN_SORT_BUFFER + MALLOC_OVERHEAD, SIZE_T_MAX, 1);
+  SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
 
 static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG,
   "Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE);

storage/myisam/mi_check.c

@@ -91,7 +91,7 @@ void myisamchk_init(HA_CHECK *param)
   param->opt_follow_links=1;
   param->keys_in_use= ~(ulonglong) 0;
   param->search_after_block=HA_OFFSET_ERROR;
-  param->use_buffers=USE_BUFFER_INIT;
+  param->use_buffers= KEY_BUFFER_INIT;
   param->read_buffer_length=READ_BUFFER_INIT;
   param->write_buffer_length=READ_BUFFER_INIT;
   param->sort_buffer_length=SORT_BUFFER_INIT;

storage/myisam/myisam_ftdump.c

@@ -84,7 +84,7 @@ int main(int argc,char *argv[])
       usage();
   }
 
-  init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0, 0);
+  init_key_cache(dflt_key_cache, MI_KEY_BLOCK_LENGTH, KEY_BUFFER_INIT, 0, 0, 0);
 
   if (!(info=mi_open(argv[0], O_RDONLY,
                      HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER)))

storage/myisam/myisamchk.c

@@ -134,7 +134,7 @@ int main(int argc, char **argv)
 
 enum options_mc {
   OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
-  OPT_CORRECT_CHECKSUM, OPT_KEY_BUFFER_SIZE,
+  OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_KEY_BUFFER_SIZE,
   OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE,
   OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
   OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@@ -165,6 +165,11 @@ static struct my_option my_long_options[] =
   {"correct-checksum", OPT_CORRECT_CHECKSUM,
    "Correct checksum information for table.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"create-missing-keys", OPT_CREATE_MISSING_KEYS,
+   "Create missing keys. This assumes that the data file is correct and that "
+   "the the number of rows stored in the index file is correct. Enables "
+   "--quick",
+   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
 #ifndef DBUG_OFF
   {"debug", '#',
    "Output debug log. Often this is 'd:t:o,filename'.",
@@ -270,7 +275,7 @@ static struct my_option my_long_options[] =
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   { "key_buffer_size", OPT_KEY_BUFFER_SIZE, "",
     &check_param.use_buffers, &check_param.use_buffers, 0,
-    GET_ULL, REQUIRED_ARG, USE_BUFFER_INIT, MALLOC_OVERHEAD,
+    GET_ULL, REQUIRED_ARG, KEY_BUFFER_INIT, MALLOC_OVERHEAD,
     SIZE_T_MAX, MALLOC_OVERHEAD,  IO_SIZE, 0},
   { "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE,  "",
     &opt_key_cache_block_size,
@@ -395,10 +400,18 @@ static void usage(void)
   -e, --extend-check  Try to recover every possible row from the data file\n\
 		      Normally this will also find a lot of garbage rows;\n\
 		      Don't use this option if you are not totally desperate.\n\
-  -f, --force         Overwrite old temporary files.\n\
+  -f, --force         Overwrite old temporary files. Add another --force to\n\
+                      avoid 'myisam_sort_buffer_size is too small' errors.\n\
+                      In this case we will attempt to do the repair with the\n\
+                      given myisam_sort_buffer_size and dynamically allocate\n\
+                      as many management buffers as needed.\n\
   -k, --keys-used=#   Tell MyISAM to update only some specific keys. # is a\n\
 	              bit mask of which keys to use. This can be used to\n\
 		      get faster inserts.\n\
+  --create-missing-keys\n\
+                      Create missing keys. This assumes that the data\n\
+                      file is correct and that the the number of rows stored\n\
+                      in the index file is correct. Enables --quick\n\
   --max-record-length=#\n\
                       Skip rows bigger than this if myisamchk can't allocate\n\
 		      memory to hold it.\n\
@@ -541,10 +554,13 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
     {
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
-      check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
+      check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
+                               T_FORCE_SORT_MEMORY);
     }
     else
     {
+      if (check_param.testflag & T_FORCE_CREATE)
+        check_param.testflag= T_FORCE_SORT_MEMORY;
       check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
       check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
     }
@@ -597,8 +613,26 @@ get_one_option(int optid,
     if (argument == disabled_my_option)
       check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
     else
+    {
+      /*
+        If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
+        then add T_FORCE_UNIQUENESS.
+      */
       check_param.testflag|=
-        (check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
+        ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
+         T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
+    }
+    break;
+  case OPT_CREATE_MISSING_KEYS:
+    if (argument == disabled_my_option)
+      check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
+    else
+    {
+      check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
+      /* Use repair by sort by default */
+      if (!(check_param.testflag & T_REP_ANY))
+        check_param.testflag|= T_REP_BY_SORT;
+    }
     break;
   case 'u':
     if (argument == disabled_my_option)

storage/myisam/myisamdef.h

@@ -629,10 +629,11 @@ typedef struct st_mi_block_info         /* Parameter to _mi_get_block_info */
 #define UPDATE_AUTO_INC         8
 #define UPDATE_OPEN_COUNT       16
 
-#define USE_BUFFER_INIT         (((1024L*512L-MALLOC_OVERHEAD)/IO_SIZE)*IO_SIZE)
-#define READ_BUFFER_INIT        (1024L*256L-MALLOC_OVERHEAD)
-#define SORT_BUFFER_INIT        (2048L*1024L-MALLOC_OVERHEAD)
-#define MIN_SORT_BUFFER         (4096-MALLOC_OVERHEAD)
+/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
+#define KEY_BUFFER_INIT	        MY_ALIGN_DOWN(1024L*1024L-MALLOC_OVERHEAD, IO_SIZE)
+#define READ_BUFFER_INIT	MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
+#define SORT_BUFFER_INIT	MY_ALIGN_DOWN(1024L*1024L*128L-MALLOC_OVERHEAD, 1024)
+#define MIN_SORT_BUFFER		4096
 
 enum myisam_log_commands
 {

storage/myisam/sort.c

@@ -34,8 +34,10 @@
 #define MERGEBUFF 15
 #define MERGEBUFF2 31
 #define MYF_RW  MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
-#define DISK_BUFFER_SIZE (IO_SIZE*16)
+#define DISK_BUFFER_SIZE (IO_SIZE*128)
 
+/* How many keys we can keep in memory */
+typedef ulonglong ha_keys;
 
 /*
  Pointers of functions for store and read keys from temp file
@@ -45,42 +47,42 @@ extern void print_error(const char *fmt,...);
 
 /* Functions defined in this file */
 
-static ha_rows find_all_keys(MI_SORT_PARAM *info,uint keys,
+static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_keys keys,
                              uchar **sort_keys,
-                             DYNAMIC_ARRAY *buffpek,int *maxbuffer,
+                             DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
                              IO_CACHE *tempfile,
                              IO_CACHE *tempfile_for_exceptions);
 static int write_keys(MI_SORT_PARAM *info,uchar **sort_keys,
-                      uint count, BUFFPEK *buffpek,IO_CACHE *tempfile);
+                      ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
 static int write_key(MI_SORT_PARAM *info, uchar *key,
                      IO_CACHE *tempfile);
 static int write_index(MI_SORT_PARAM *info,uchar * *sort_keys,
-                       uint count);
-static int merge_many_buff(MI_SORT_PARAM *info,uint keys,
+                       ha_keys count);
+static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
                            uchar * *sort_keys,
-                           BUFFPEK *buffpek,int *maxbuffer,
+                           BUFFPEK *buffpek, uint *maxbuffer,
                            IO_CACHE *t_file);
-static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
-                           uint sort_length);
-static int merge_buffers(MI_SORT_PARAM *info,uint keys,
+static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
+                               uint sort_length);
+static int merge_buffers(MI_SORT_PARAM *info, ha_keys keys,
                          IO_CACHE *from_file, IO_CACHE *to_file,
                          uchar * *sort_keys, BUFFPEK *lastbuff,
                          BUFFPEK *Fb, BUFFPEK *Tb);
-static int merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int,
+static int merge_index(MI_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
                        IO_CACHE *);
 static int flush_ft_buf(MI_SORT_PARAM *info);
 
 static int write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys,
-                             uint count, BUFFPEK *buffpek,
+                             ha_keys count, BUFFPEK *buffpek,
                              IO_CACHE *tempfile);
-static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
+static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
                                   uint sort_length);
 static int write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,
-                           uchar *key, uint sort_length, uint count);
+                           uchar *key, uint sort_length, ha_keys count);
 static int write_merge_key_varlen(MI_SORT_PARAM *info,
                                   IO_CACHE *to_file,
                                   uchar* key, uint sort_length,
-                                  uint count);
+                                  ha_keys count);
 static inline int
 my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
 
@@ -101,15 +103,15 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
 int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
 			  ulonglong sortbuff_size)
 {
-  int error,maxbuffer,skr;
-  uint sort_length, keys;
+  int error;
+  uint sort_length, maxbuffer;
   ulonglong memavl, old_memavl;
   DYNAMIC_ARRAY buffpek;
-  ha_rows records;
+  ha_rows records, keys;
   uchar **sort_keys;
   IO_CACHE tempfile, tempfile_for_exceptions;
   DBUG_ENTER("_create_index_by_sort");
-  DBUG_PRINT("enter",("sort_length: %d", info->key_length));
+  DBUG_PRINT("enter",("sort_length: %u", info->key_length));
 
   if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
   {
@@ -135,37 +137,55 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
   sort_length=	info->key_length;
   LINT_INIT(keys);
 
-  if ((memavl - sizeof(BUFFPEK)) / (sort_length + sizeof(char *)) > UINT_MAX32)
-    memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
-
   while (memavl >= MIN_SORT_BUFFER)
   {
-    if ((records < UINT_MAX32) && 
-       ((my_off_t) (records + 1) * 
-        (sort_length + sizeof(char*)) <= (my_off_t) memavl))
-      keys= (uint)records+1;
+    /* Check if we can fit all keys into memory */
+    if (((ulonglong) (records + 1) *
+         (sort_length + sizeof(char*)) <= memavl))
+      keys= records+1;
+    else if ((info->sort_info->param->testflag &
+              (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
+             T_FORCE_SORT_MEMORY)
+    {
+      /*
+        Use all of the given sort buffer for key data.
+        Allocate 1000 buffers at a start for new data. More buffers
+        will be allocated when needed.
+      */
+      keys= memavl / (sort_length+sizeof(char*));
+      maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
+    }
     else
+    {
+      /*
+        All keys can't fit in memory.
+        Calculate how many keys + buffers we can keep in memory
+      */
+      uint maxbuffer_org;
       do
       {
-	skr=maxbuffer;
-	if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
-	    (keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
+	maxbuffer_org= maxbuffer;
+	if (memavl < sizeof(BUFFPEK) * maxbuffer ||
+	    (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
              (sort_length+sizeof(char*))) <= 1 ||
-            keys < (uint) maxbuffer)
+            keys < maxbuffer)
 	{
 	  mi_check_print_error(info->sort_info->param,
-			       "myisam_sort_buffer_size is too small");
+			       "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                               sortbuff_size, (ulonglong) records,
+                               sort_length);
           my_errno= ENOMEM;
 	  goto err;
 	}
       }
-      while ((maxbuffer= (int) (records/(keys-1)+1)) != skr);
+      while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
+    }
 
     if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+
 				       HA_FT_MAXBYTELEN, MYF(0))))
     {
       if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
-                                maxbuffer/2, MYF(0)))
+                                min(maxbuffer/2, 1000), MYF(0)))
       {
 	my_free(sort_keys);
         sort_keys= 0;
@@ -179,24 +199,30 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
   }
   if (memavl < MIN_SORT_BUFFER)
   {
-    mi_check_print_error(info->sort_info->param,"MyISAM sort buffer too small"); /* purecov: tested */
-    my_errno= ENOMEM;                           /* purecov: tested */
-    goto err;                                   /* purecov: tested */
+    /* purecov: begin inspected */
+    mi_check_print_error(info->sort_info->param,
+                         "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                         sortbuff_size, (ulonglong) records, sort_length);
+    my_errno= ENOMEM;
+    goto err;
+    /* purecov: end inspected */
   }
   (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
 
   if (!no_messages)
-    printf("  - Searching for keys, allocating buffer for %d keys\n",keys);
+    my_fprintf(stdout,
+               "  - Searching for keys, allocating buffer for %llu keys\n",
+               (ulonglong) keys);
 
-  if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
-                                  &tempfile,&tempfile_for_exceptions))
+  if ((records= find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
+                              &tempfile,&tempfile_for_exceptions))
       == HA_POS_ERROR)
     goto err; /* purecov: tested */
   if (maxbuffer == 0)
   {
     if (!no_messages)
-      printf("  - Dumping %lu keys\n", (ulong) records);
-    if (write_index(info,sort_keys, (uint) records))
+      my_fprintf(stdout, "  - Dumping %llu keys\n", (ulonglong) records);
+    if (write_index(info, sort_keys, (ha_keys) records))
       goto err; /* purecov: inspected */
   }
   else
@@ -205,7 +231,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
     if (maxbuffer >= MERGEBUFF2)
     {
       if (!no_messages)
-	printf("  - Merging %lu keys\n", (ulong) records); /* purecov: tested */
+	my_fprintf(stdout, "  - Merging %llu keys\n",
+                   (ulonglong) records); /* purecov: tested */
       if (merge_many_buff(info,keys,sort_keys,
                   dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
 	goto err;				/* purecov: inspected */
@@ -259,13 +286,13 @@ err:
 
 /* Search after all keys and place them in a temp. file */
 
-static ha_rows find_all_keys(MI_SORT_PARAM *info, uint keys,
+static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_rows keys,
                              uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
-                             int *maxbuffer, IO_CACHE *tempfile,
+                             uint *maxbuffer, IO_CACHE *tempfile,
                              IO_CACHE *tempfile_for_exceptions)
 {
   int error;
-  uint idx;
+  ha_rows idx;
   DBUG_ENTER("find_all_keys");
 
   idx=error=0;
@@ -314,9 +341,10 @@ pthread_handler_t thr_find_all_keys(void *arg)
 {
   MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg;
   int error;
-  ulonglong memavl, old_memavl;
-  uint keys, sort_length;
-  uint idx, maxbuffer;
+  ulonglong memavl, old_memavl, sortbuff_size;
+  ha_keys keys, idx;
+  uint sort_length;
+  uint maxbuffer;
   uchar **sort_keys=0;
 
   LINT_INIT(keys);
@@ -351,37 +379,47 @@ pthread_handler_t thr_find_all_keys(void *arg)
     bzero((char*) &sort_param->unique,  sizeof(sort_param->unique));
     sort_keys= (uchar **) NULL;
 
-    memavl=       max(sort_param->sortbuff_size, MIN_SORT_BUFFER);
-    idx=          (uint)sort_param->sort_info->max_records;
+    sortbuff_size= sort_param->sortbuff_size;
+    memavl=       max(sortbuff_size, MIN_SORT_BUFFER);
+    idx=          (ha_keys) sort_param->sort_info->max_records;
     sort_length=  sort_param->key_length;
     maxbuffer=    1;
 
-    if ((memavl - sizeof(BUFFPEK)) / (sort_length +
-                                      sizeof(char *)) > UINT_MAX32)
-      memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));    
-
     while (memavl >= MIN_SORT_BUFFER)
     {
       if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <=
           (my_off_t) memavl)
         keys= idx+1;
+      else if ((sort_param->sort_info->param->testflag &
+                (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
+               T_FORCE_SORT_MEMORY)
+      {
+        /*
+          Use all of the given sort buffer for key data.
+          Allocate 1000 buffers at a start for new data. More buffers
+          will be allocated when needed.
+        */
+        keys= memavl / (sort_length+sizeof(char*));
+        maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
+      }
       else
       {
-        uint skr;
+        uint maxbuffer_org;
         do
         {
-          skr= maxbuffer;
+          maxbuffer_org= maxbuffer;
           if (memavl < sizeof(BUFFPEK)*maxbuffer ||
               (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
                (sort_length+sizeof(char*))) <= 1 ||
               keys < (uint) maxbuffer)
           {
             mi_check_print_error(sort_param->sort_info->param,
-                                 "myisam_sort_buffer_size is too small");
+                                 "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                                 sortbuff_size, (ulonglong) idx, sort_length);
             goto err;
           }
         }
-        while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr);
+        while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
       }
       if ((sort_keys= (uchar**)
            my_malloc(keys*(sort_length+sizeof(char*))+
@@ -389,7 +427,7 @@ pthread_handler_t thr_find_all_keys(void *arg)
                       HA_FT_MAXBYTELEN : 0), MYF(0))))
       {
         if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
-                                  maxbuffer, maxbuffer/2, MYF(0)))
+                                  maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
         {
           my_free(sort_keys);
           sort_keys= (uchar **) NULL; /* for err: label */
@@ -404,14 +442,19 @@ pthread_handler_t thr_find_all_keys(void *arg)
     }
     if (memavl < MIN_SORT_BUFFER)
     {
+      /* purecov: begin inspected */
       mi_check_print_error(sort_param->sort_info->param,
-                           "MyISAM sort buffer too small");
-      goto err; /* purecov: tested */
+                           "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu  rows: %llu  sort_length: %u",
+                           sortbuff_size, (ulonglong) idx, sort_length);
+      my_errno= ENOMEM;
+      goto err;
+    /* purecov: end inspected */
     }
 
     if (sort_param->sort_info->param->testflag & T_VERBOSE)
-      printf("Key %d - Allocating buffer for %d keys\n",
-             sort_param->key + 1, keys);
+      my_fprintf(stdout,
+                 "Key %d - Allocating buffer for %llu keys\n",
+                 sort_param->key + 1, (ulonglong) keys);
     sort_param->sort_keys= sort_keys;
 
     idx= error= 0;
@@ -436,7 +479,8 @@ pthread_handler_t thr_find_all_keys(void *arg)
                                    &sort_param->tempfile))
           goto err;
         sort_keys[0]= (uchar*) (sort_keys+keys);
-        memcpy(sort_keys[0], sort_keys[idx - 1], (size_t) sort_param->key_length);
+        memcpy(sort_keys[0], sort_keys[idx - 1],
+               (size_t) sort_param->key_length);
         idx= 1;
       }
       sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
@@ -454,7 +498,6 @@ pthread_handler_t thr_find_all_keys(void *arg)
     else
       sort_param->keys= idx;
 
-    sort_param->sort_keys_length= keys;
     goto ok;
 
 err:
@@ -527,7 +570,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
       {
         if (param->testflag & T_VERBOSE)
         {
-          printf("Key %d  - Dumping %u keys\n",sinfo->key+1, sinfo->keys);
+          my_fprintf(stdout,
+                     "Key %d  - Dumping %llu keys\n", sinfo->key+1,
+                     (ulonglong) sinfo->keys);
           fflush(stdout);
         }
         if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
@@ -584,10 +629,12 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
       if (maxbuffer >= MERGEBUFF2)
       {
         if (param->testflag & T_VERBOSE)
-          printf("Key %d  - Merging %u keys\n",sinfo->key+1, sinfo->keys);
+          my_fprintf(stdout,
+                     "Key %d  - Merging %llu keys\n",
+                     sinfo->key+1, (ulonglong) sinfo->keys);
         if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
 			    dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
-			    (int*) &maxbuffer, &sinfo->tempfile))
+			    &maxbuffer, &sinfo->tempfile))
         {
           got_error=1;
           continue;
@@ -651,12 +698,15 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
         /* Write all keys in memory to file for later merge */
 
 static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
-                      uint count, BUFFPEK *buffpek, IO_CACHE *tempfile)
+                      ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
 {
   uchar **end;
   uint sort_length=info->key_length;
   DBUG_ENTER("write_keys");
 
+  if (!buffpek)
+    DBUG_RETURN(1);                             /* Out of memory */
+
   my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
             info);
   if (!my_b_inited(tempfile) &&
@@ -669,7 +719,7 @@ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
 
   for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
   {
-    if (my_b_write(tempfile,(uchar*) *sort_keys,(uint) sort_length))
+    if (my_b_write(tempfile,(uchar*) *sort_keys, sort_length))
       DBUG_RETURN(1); /* purecov: inspected */
   }
   DBUG_RETURN(0);
@@ -693,13 +743,16 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
 
 static int write_keys_varlen(MI_SORT_PARAM *info,
                              register uchar **sort_keys,
-                             uint count, BUFFPEK *buffpek,
+                             ha_keys count, BUFFPEK *buffpek,
                              IO_CACHE *tempfile)
 {
   uchar **end;
   int err;
   DBUG_ENTER("write_keys_varlen");
 
+  if (!buffpek)
+    DBUG_RETURN(1);                             /* Out of memory */
+
   my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
             info);
   if (!my_b_inited(tempfile) &&
@@ -738,7 +791,7 @@ static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile)
 /* Write index */
 
 static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
-                       register uint count)
+                       register ha_keys count)
 {
   DBUG_ENTER("write_index");
 
@@ -755,11 +808,11 @@ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
 
         /* Merge buffers to make < MERGEBUFF2 buffers */
 
-static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
+static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
                            uchar **sort_keys, BUFFPEK *buffpek,
-                           int *maxbuffer, IO_CACHE *t_file)
+                           uint *maxbuffer, IO_CACHE *t_file)
 {
-  register int i;
+  register uint i;
   IO_CACHE t_file2, *from_file, *to_file, *temp;
   BUFFPEK *lastbuff;
   DBUG_ENTER("merge_many_buff");
@@ -777,7 +830,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
     reinit_io_cache(from_file,READ_CACHE,0L,0,0);
     reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
     lastbuff=buffpek;
-    for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
+    for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
     {
       if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                         buffpek+i,buffpek+i+MERGEBUFF-1))
@@ -789,7 +842,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
     if (flush_io_cache(to_file))
       break;                                    /* purecov: inspected */
     temp=from_file; from_file=to_file; to_file=temp;
-    *maxbuffer= (int) (lastbuff-buffpek)-1;
+    *maxbuffer= (uint) (lastbuff-buffpek)-1;
   }
 cleanup:
   close_cached_file(to_file);                   /* This holds old result */
@@ -818,35 +871,36 @@ cleanup:
     -1	Error
 */
 
-static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
-                           uint sort_length)
+static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
+                                uint sort_length)
 {
-  register uint count;
-  uint length;
+  register ha_keys count;
+  my_off_t length;
 
-  if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
+  if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
   {
     if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
-                         (length= sort_length*count),
+                         (length= sort_length * count),
                          buffpek->file_pos, MYF_RW))
-      return((uint) -1);                        /* purecov: inspected */
+      return(HA_OFFSET_ERROR);               /* purecov: inspected */
     buffpek->key=buffpek->base;
-    buffpek->file_pos+= length;                 /* New filepos */
+    buffpek->file_pos+= length;               /* New filepos */
     buffpek->count-=    count;
     buffpek->mem_count= count;
   }
-  return (count*sort_length);
+  return (((my_off_t) count) * sort_length);
 } /* read_to_buffer */
 
-static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
-                                  uint sort_length)
+
+static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
+                                      uint sort_length)
 {
-  register uint count;
+  register ha_keys count;
   uint16 length_of_key = 0;
   uint idx;
   uchar *buffp;
 
-  if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
+  if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
   {
     buffp = buffpek->base;
 
@@ -854,11 +908,11 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
     {
       if (mysql_file_pread(fromfile->file, (uchar*)&length_of_key,
                            sizeof(length_of_key), buffpek->file_pos, MYF_RW))
-        return((uint) -1);
+        return(HA_OFFSET_ERROR);
       buffpek->file_pos+=sizeof(length_of_key);
       if (mysql_file_pread(fromfile->file, (uchar*) buffp,
                            length_of_key, buffpek->file_pos, MYF_RW))
-        return((uint) -1);
+        return(HA_OFFSET_ERROR);
       buffpek->file_pos+=length_of_key;
       buffp = buffp + sort_length;
     }
@@ -866,15 +920,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
     buffpek->count-=    count;
     buffpek->mem_count= count;
   }
-  return (count*sort_length);
+  return (((my_off_t) count) * sort_length);
 } /* read_to_buffer_varlen */
 
 
 static int write_merge_key_varlen(MI_SORT_PARAM *info,
                                   IO_CACHE *to_file, uchar* key,
-                                  uint sort_length, uint count)
+                                  uint sort_length, ha_keys count)
 {
-  uint idx;
+  ha_keys idx;
   uchar *bufs = key;
 
   for (idx=1;idx<=count;idx++)
@@ -890,32 +944,37 @@ static int write_merge_key_varlen(MI_SORT_PARAM *info,
 
 static int write_merge_key(MI_SORT_PARAM *info __attribute__((unused)),
                            IO_CACHE *to_file, uchar *key,
-                           uint sort_length, uint count)
+                           uint sort_length, ha_keys count)
 {
-  return my_b_write(to_file, key, (size_t) sort_length*count);
+  return my_b_write(to_file, key, ((size_t) sort_length) * count);
 }
 
 /*
   Merge buffers to one buffer
   If to_file == 0 then use info->key_write
+
+  Return:
+  0 ok
+  1 error
 */
 
 static int
-merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
+merge_buffers(MI_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
               IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
               BUFFPEK *Fb, BUFFPEK *Tb)
 {
-  int error;
-  uint sort_length,maxcount;
+  int error= 1;
+  uint sort_length;
+  ha_keys maxcount;
   ha_rows count;
-  my_off_t UNINIT_VAR(to_start_filepos);
+  my_off_t UNINIT_VAR(to_start_filepos), read_length;
   uchar *strpos;
   BUFFPEK *buffpek,**refpek;
   QUEUE queue;
   DBUG_ENTER("merge_buffers");
 
   count=error=0;
-  maxcount=keys/((uint) (Tb-Fb) +1);
+  maxcount= keys/((uint) (Tb-Fb) +1);
   DBUG_ASSERT(maxcount > 0);
   LINT_INIT(to_start_filepos);
   if (to_file)
@@ -932,10 +991,10 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
   {
     count+= buffpek->count;
     buffpek->base= (uchar*) strpos;
-    buffpek->max_keys=maxcount;
-    strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,
-                                                      sort_length));
-    if (error == -1)
+    buffpek->max_keys= maxcount;
+    strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
+                                                sort_length));
+    if (read_length == HA_OFFSET_ERROR)
       goto err; /* purecov: inspected */
     queue_insert(&queue,(uchar*) buffpek);
   }
@@ -966,10 +1025,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
         /* It's enough to check for killedptr before a slow operation */
         if (killed_ptr(info->sort_info->param))
         {
-          error=1;
           goto err;
         }
-        if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length)))
+        if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
         {
           uchar *base= buffpek->base;
           uint max_keys=buffpek->max_keys;
@@ -996,9 +1054,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
           }
           break;                /* One buffer have been removed */
         }
+        else if (read_length == HA_OFFSET_ERROR)
+          goto err;               /* purecov: inspected */
       }
-      else if (error == -1)
-        goto err;               /* purecov: inspected */
       queue_replace_top(&queue);   /* Top element has been replaced */
     }
   }
@@ -1030,8 +1088,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
       }
     }
   }
-  while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != -1 &&
-         error != 0);
+  while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
+  if (read_length == 0)
+    error= 0;
 
   lastbuff->count=count;
   if (to_file)
@@ -1045,8 +1104,8 @@ err:
         /* Do a merge to output-file (save only positions) */
 
 static int
-merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
-            BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile)
+merge_index(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
+            BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
 {
   DBUG_ENTER("merge_index");
   if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,
@@ -1055,6 +1114,7 @@ merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
   DBUG_RETURN(0);
 } /* merge_index */
 
+
 static int
 flush_ft_buf(MI_SORT_PARAM *info)
 {
@@ -1067,4 +1127,3 @@ flush_ft_buf(MI_SORT_PARAM *info)
   }
   return err;
 }
-