Skip to content

M
mariadb
Commit 53d44ad1 authored by Michael Widenius's avatar Michael Widenius
Browse files
Options

Added some fixes that should make MyISAM & Aria REPAIR work with more than 4G records 

- If one specifies --force twice to myisamchk and aria_chk, then we will try to finnish the repair even if sort_buffer would be too small.
  This was done by dynamically allocate buffer handler objects as long as memory lasts.
- New option for myisamchk and aria_chk: create-missing-keys
- Changed default size of myisam_sort_buffer_size from 8M to 128M.
- Changed default size of sort_buffer_size in aria_chk from 128M to 256M.
- Increased information in error message about 'sort_buffer_size' beeing to small.
- Print also to 'show warnings' if repair was retried.
- Increased size of internal sort-buffer-readers from 16K to 128K
- Changed printing of 'number of records' to use %ll instead of casting to long
- Changed buffer sizes for myisam and aria to use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines.





include/my_global.h:
  Added MY_ALIGN_DOWN() to get previous alignment (for big memory areas)
include/myisam.h:
  Increased size of types to be able to handle more records
include/myisamchk.h:
  Increased size of types to be able to handle more records
  Added T_FORCE_SORT_MEMORY to force repair to work even if sort_buffer would not be big enough
mysql-test/r/myisam.result:
  Updated result
mysql-test/r/mysqld--help.result:
  Updated result
mysql-test/r/repair.result:
  Updated result
mysql-test/suite/maria/maria.result:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria.test:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria3.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test:
  Updated result after sort buffer size increase
mysql-test/t/myisam.test:
  Fixed error messages to not print system specific data
mysql-test/t/repair.test:
  Fixed error messages to not print system specific data
storage/maria/ha_maria.cc:
  Print also to 'show warnings' if repair was retried
  Changed default size of sort_buffer_size from 128M to 256M (same as in mysqld)
storage/maria/ma_check.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/ma_sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug where maria_update_key_parts() was called too early.
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
storage/maria/maria_chk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
storage/maria/maria_def.h:
  Increased size of types to be able to handle more records
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_ftdump.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_read_log.c:
  Use PAGE_BUFFER_INIT for page cache
storage/myisam/ha_myisam.cc:
  Changed default size of myisam_sort_buffer_size from 8M to 128M
storage/myisam/mi_check.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisam_ftdump.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamchk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamdef.h:
  Increased SORT_BUFFER_INIT to 64M
  (speeds up repair a lot and most machines have nowadays a lot of memory)
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
parent e5a0daae
Hide whitespace changes
Inline Side-by-side
Showing with 651 additions and 326 deletions
include/my_global.h
View file @ 53d44ad1
...@@ -847,7 +847,8 @@ typedef long my_ptrdiff_t; ...@@ -847,7 +847,8 @@ typedef long my_ptrdiff_t;
typedef long long my_ptrdiff_t; typedef long long my_ptrdiff_t;
#endif #endif
#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1)) #define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1))
#define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1))
#define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) #define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double))
#define ALIGN_MAX_UNIT (sizeof(double)) #define ALIGN_MAX_UNIT (sizeof(double))
/* Size to make adressable obj. */ /* Size to make adressable obj. */
......
include/myisam.h
View file @ 53d44ad1
...@@ -356,8 +356,10 @@ typedef struct st_mi_sort_param ...@@ -356,8 +356,10 @@ typedef struct st_mi_sort_param
ulonglong notnull[HA_MAX_KEY_SEG+1]; ulonglong notnull[HA_MAX_KEY_SEG+1];
my_off_t pos,max_pos,filepos,start_recpos; my_off_t pos,max_pos,filepos,start_recpos;
uint key, key_length,real_key_length,sortbuff_size; uint key, key_length,real_key_length;
uint maxbuffers, keys, find_length, sort_keys_length; uint maxbuffers, find_length;
ulonglong sortbuff_size;
ha_rows keys;
my_bool fix_datafile, master; my_bool fix_datafile, master;
my_bool calc_checksum; /* calculate table checksum */ my_bool calc_checksum; /* calculate table checksum */
...@@ -366,10 +368,10 @@ typedef struct st_mi_sort_param ...@@ -366,10 +368,10 @@ typedef struct st_mi_sort_param
int (*key_write)(struct st_mi_sort_param *, const void *); int (*key_write)(struct st_mi_sort_param *, const void *);
void (*lock_in_memory)(HA_CHECK *); void (*lock_in_memory)(HA_CHECK *);
int (*write_keys)(struct st_mi_sort_param *, register uchar **, int (*write_keys)(struct st_mi_sort_param *, register uchar **,
uint , struct st_buffpek *, IO_CACHE *); ulonglong , struct st_buffpek *, IO_CACHE *);
uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *, int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *,
uint, uint); uint, ulonglong);
} MI_SORT_PARAM; } MI_SORT_PARAM;
/* functions in mi_check */ /* functions in mi_check */
......
include/myisamchk.h
View file @ 53d44ad1
...@@ -62,9 +62,10 @@ ...@@ -62,9 +62,10 @@
#define T_ZEROFILL ((ulonglong) 1L << 32) #define T_ZEROFILL ((ulonglong) 1L << 32)
#define T_ZEROFILL_KEEP_LSN ((ulonglong) 1L << 33) #define T_ZEROFILL_KEEP_LSN ((ulonglong) 1L << 33)
/** If repair should not bump create_rename_lsn */ /** If repair should not bump create_rename_lsn */
#define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 33) #define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 34)
#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 34) #define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 35)
#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 35) #define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 36)
#define T_FORCE_SORT_MEMORY ((ulonglong) 1L << 37)
#define T_REP_ANY (T_REP | T_REP_BY_SORT | T_REP_PARALLEL) #define T_REP_ANY (T_REP | T_REP_BY_SORT | T_REP_PARALLEL)
...@@ -180,8 +181,8 @@ typedef struct st_buffpek { ...@@ -180,8 +181,8 @@ typedef struct st_buffpek {
my_off_t file_pos; /* Where we are in the sort file */ my_off_t file_pos; /* Where we are in the sort file */
uchar *base, *key; /* Key pointers */ uchar *base, *key; /* Key pointers */
ha_rows count; /* Number of rows in table */ ha_rows count; /* Number of rows in table */
ulong mem_count; /* numbers of keys in memory */ ha_rows mem_count; /* Numbers of keys in memory */
ulong max_keys; /* Max keys in buffert */ ha_rows max_keys; /* Max keys in buffert */
} BUFFPEK; } BUFFPEK;
#endif /* _myisamchk_h */ #endif /* _myisamchk_h */
mysql-test/r/myisam.result
View file @ 53d44ad1
...@@ -2355,7 +2355,7 @@ Warnings: ...@@ -2355,7 +2355,7 @@ Warnings:
Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '4' Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '4'
REPAIR TABLE t1; REPAIR TABLE t1;
Table Op Msg_type Msg_text Table Op Msg_type Msg_text
test.t1 repair error myisam_sort_buffer_size is too small test.t1 repair error myisam_sort_buffer_size is too small. X
test.t1 repair warning Number of rows changed from 0 to 7168 test.t1 repair warning Number of rows changed from 0 to 7168
test.t1 repair status OK test.t1 repair status OK
SET myisam_repair_threads=2; SET myisam_repair_threads=2;
...@@ -2424,7 +2424,7 @@ INSERT INTO t1 VALUES ...@@ -2424,7 +2424,7 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'); ('0'),('0'),('0'),('0'),('0'),('0'),('0');
Warnings: Warnings:
Error 1034 myisam_sort_buffer_size is too small Error 1034 myisam_sort_buffer_size is too small. X
Error 1034 Number of rows changed from 0 to 157 Error 1034 Number of rows changed from 0 to 157
SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size; SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
INSERT INTO t1 VALUES('1'); INSERT INTO t1 VALUES('1');
......
mysql-test/r/mysqld--help.result
View file @ 53d44ad1
...@@ -1045,7 +1045,7 @@ myisam-max-sort-file-size 9223372036853727232 ...@@ -1045,7 +1045,7 @@ myisam-max-sort-file-size 9223372036853727232
myisam-mmap-size 18446744073709551615 myisam-mmap-size 18446744073709551615
myisam-recover-options DEFAULT myisam-recover-options DEFAULT
myisam-repair-threads 1 myisam-repair-threads 1
myisam-sort-buffer-size 8388608 myisam-sort-buffer-size 134216704
myisam-stats-method nulls_unequal myisam-stats-method nulls_unequal
myisam-use-mmap FALSE myisam-use-mmap FALSE
net-buffer-length 16384 net-buffer-length 16384
......
mysql-test/r/repair.result
View file @ 53d44ad1
...@@ -77,12 +77,12 @@ INSERT INTO t1 VALUES ...@@ -77,12 +77,12 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'); ('0'),('0'),('0'),('0'),('0'),('0'),('0');
Warnings: Warnings:
Error 1034 myisam_sort_buffer_size is too small Error 1034 myisam_sort_buffer_size is too small. X
Error 1034 Number of rows changed from 0 to 157 Error 1034 Number of rows changed from 0 to 157
SET myisam_repair_threads=2; SET myisam_repair_threads=2;
REPAIR TABLE t1; REPAIR TABLE t1;
Table Op Msg_type Msg_text Table Op Msg_type Msg_text
test.t1 repair error myisam_sort_buffer_size is too small test.t1 repair error myisam_sort_buffer_size is too small. X
test.t1 repair warning Number of rows changed from 0 to 157 test.t1 repair warning Number of rows changed from 0 to 157
test.t1 repair status OK test.t1 repair status OK
SET myisam_repair_threads=@@global.myisam_repair_threads; SET myisam_repair_threads=@@global.myisam_repair_threads;
......
mysql-test/suite/maria/maria.result
View file @ 53d44ad1
...@@ -2679,3 +2679,65 @@ select count(*) from t1; ...@@ -2679,3 +2679,65 @@ select count(*) from t1;
count(*) count(*)
13 13
drop table t1; drop table t1;
#
# BUG#47444 - --myisam_repair_threads > 1 can result in all index
# cardinalities=1
#
SET aria_repair_threads=2;
SET aria_sort_buffer_size=8192;
CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair status OK
SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
CARDINALITY
14
14
14
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
DROP TABLE t1;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
SET aria_repair_threads=@@global.aria_repair_threads;
#
# BUG#47073 - valgrind errs, corruption,failed repair of partition,
# low myisam_sort_buffer_size
#
CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
(6,'0'),(7,'0');
INSERT INTO t1 SELECT a+10,b FROM t1;
INSERT INTO t1 SELECT a+20,b FROM t1;
INSERT INTO t1 SELECT a+40,b FROM t1;
INSERT INTO t1 SELECT a+80,b FROM t1;
INSERT INTO t1 SELECT a+160,b FROM t1;
INSERT INTO t1 SELECT a+320,b FROM t1;
INSERT INTO t1 SELECT a+640,b FROM t1;
INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET aria_sort_buffer_size=4096;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error aria_sort_buffer_size is too small. X
test.t1 repair error Create index by sort failed
test.t1 repair info Retrying repair with keycache
test.t1 repair status OK
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SET aria_repair_threads=2;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error aria_sort_buffer_size is too small. X
test.t1 repair error Create index by sort failed
test.t1 repair info Retrying repair with keycache
test.t1 repair status OK
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SET aria_repair_threads=@@global.aria_repair_threads;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
DROP TABLE t1;
mysql-test/suite/maria/maria.test
View file @ 53d44ad1
...@@ -1963,6 +1963,52 @@ unlock tables; ...@@ -1963,6 +1963,52 @@ unlock tables;
select count(*) from t1; select count(*) from t1;
drop table t1; drop table t1;
--echo #
--echo # BUG#47444 - --myisam_repair_threads > 1 can result in all index
--echo # cardinalities=1
--echo #
SET aria_repair_threads=2;
SET aria_sort_buffer_size=8192;
CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
CHECK TABLE t1;
DROP TABLE t1;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
SET aria_repair_threads=@@global.aria_repair_threads;
--echo #
--echo # BUG#47073 - valgrind errs, corruption,failed repair of partition,
--echo # low myisam_sort_buffer_size
--echo #
CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
(6,'0'),(7,'0');
INSERT INTO t1 SELECT a+10,b FROM t1;
INSERT INTO t1 SELECT a+20,b FROM t1;
INSERT INTO t1 SELECT a+40,b FROM t1;
INSERT INTO t1 SELECT a+80,b FROM t1;
INSERT INTO t1 SELECT a+160,b FROM t1;
INSERT INTO t1 SELECT a+320,b FROM t1;
INSERT INTO t1 SELECT a+640,b FROM t1;
INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET aria_sort_buffer_size=4096;
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
CHECK TABLE t1;
SET aria_repair_threads=2;
# May report different values depending on threads activity.
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
CHECK TABLE t1;
SET aria_repair_threads=@@global.aria_repair_threads;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
DROP TABLE t1;
# #
# End of test # End of test
# #
......
mysql-test/suite/maria/maria3.result
View file @ 53d44ad1
...@@ -317,7 +317,7 @@ aria_pagecache_division_limit 100 ...@@ -317,7 +317,7 @@ aria_pagecache_division_limit 100
aria_page_checksum OFF aria_page_checksum OFF
aria_recover NORMAL aria_recover NORMAL
aria_repair_threads 1 aria_repair_threads 1
aria_sort_buffer_size 134217728 aria_sort_buffer_size 268434432
aria_stats_method nulls_unequal aria_stats_method nulls_unequal
aria_sync_log_dir NEWFILE aria_sync_log_dir NEWFILE
show status like 'aria%'; show status like 'aria%';
......
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result
View file @ 53d44ad1
SET @start_global_value = @@global.aria_sort_buffer_size; SET @start_global_value = @@global.aria_sort_buffer_size;
select @@global.aria_sort_buffer_size; select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size @@global.aria_sort_buffer_size
134217728 268434432
select @@session.aria_sort_buffer_size; select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size @@session.aria_sort_buffer_size
134217728 268434432
show global variables like 'aria_sort_buffer_size'; show global variables like 'aria_sort_buffer_size';
Variable_name Value Variable_name Value
aria_sort_buffer_size 134217728 aria_sort_buffer_size 268434432
show session variables like 'aria_sort_buffer_size'; show session variables like 'aria_sort_buffer_size';
Variable_name Value Variable_name Value
aria_sort_buffer_size 134217728 aria_sort_buffer_size 268434432
select * from information_schema.global_variables where variable_name='aria_sort_buffer_size'; select * from information_schema.global_variables where variable_name='aria_sort_buffer_size';
VARIABLE_NAME VARIABLE_VALUE VARIABLE_NAME VARIABLE_VALUE
ARIA_SORT_BUFFER_SIZE 134217728 ARIA_SORT_BUFFER_SIZE 268434432
select * from information_schema.session_variables where variable_name='aria_sort_buffer_size'; select * from information_schema.session_variables where variable_name='aria_sort_buffer_size';
VARIABLE_NAME VARIABLE_VALUE VARIABLE_NAME VARIABLE_VALUE
ARIA_SORT_BUFFER_SIZE 134217728 ARIA_SORT_BUFFER_SIZE 268434432
set global aria_sort_buffer_size=10; set global aria_sort_buffer_size=10;
Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10'
select @@global.aria_sort_buffer_size; select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size @@global.aria_sort_buffer_size
10 4096
set session aria_sort_buffer_size=10; set session aria_sort_buffer_size=10;
Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10'
select @@session.aria_sort_buffer_size; select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size @@session.aria_sort_buffer_size
10 4096
set global aria_sort_buffer_size=1.1; set global aria_sort_buffer_size=1.1;
ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size' ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size'
set session aria_sort_buffer_size=1e1; set session aria_sort_buffer_size=1e1;
...@@ -36,7 +40,7 @@ Warnings: ...@@ -36,7 +40,7 @@ Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '0' Warning 1292 Truncated incorrect aria_sort_buffer_size value: '0'
select @@global.aria_sort_buffer_size; select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size @@global.aria_sort_buffer_size
4 4096
set session aria_sort_buffer_size=cast(-1 as unsigned int); set session aria_sort_buffer_size=cast(-1 as unsigned int);
select @@session.aria_sort_buffer_size; select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size @@session.aria_sort_buffer_size
......
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result
View file @ 53d44ad1
SET @start_global_value = @@global.myisam_sort_buffer_size ; SET @start_global_value = @@global.myisam_sort_buffer_size ;
SELECT @start_global_value; SELECT @start_global_value;
@start_global_value @start_global_value
8388608 134216704
SET @start_session_value = @@session.myisam_sort_buffer_size ; SET @start_session_value = @@session.myisam_sort_buffer_size ;
SELECT @start_session_value; SELECT @start_session_value;
@start_session_value @start_session_value
8388608 134216704
'#--------------------FN_DYNVARS_005_01-------------------------#' '#--------------------FN_DYNVARS_005_01-------------------------#'
SET @@global.myisam_sort_buffer_size = 100; SET @@global.myisam_sort_buffer_size = 100;
Warnings: Warnings:
...@@ -13,22 +13,22 @@ Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '100' ...@@ -13,22 +13,22 @@ Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '100'
SET @@global.myisam_sort_buffer_size = DEFAULT; SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size ; SELECT @@global.myisam_sort_buffer_size ;
@@global.myisam_sort_buffer_size @@global.myisam_sort_buffer_size
8388608 134216704
SET @@session.myisam_sort_buffer_size = 200; SET @@session.myisam_sort_buffer_size = 200;
Warnings: Warnings:
Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '200' Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '200'
SET @@session.myisam_sort_buffer_size = DEFAULT; SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size ; SELECT @@session.myisam_sort_buffer_size ;
@@session.myisam_sort_buffer_size @@session.myisam_sort_buffer_size
8388608 134216704
'#--------------------FN_DYNVARS_005_02-------------------------#' '#--------------------FN_DYNVARS_005_02-------------------------#'
SET @@global.myisam_sort_buffer_size = DEFAULT; SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size = 8388608; SELECT @@global.myisam_sort_buffer_size = 134216704;
@@global.myisam_sort_buffer_size = 8388608 @@global.myisam_sort_buffer_size = 134216704
1 1
SET @@session.myisam_sort_buffer_size = DEFAULT; SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size = 8388608; SELECT @@session.myisam_sort_buffer_size = 134216704;
@@session.myisam_sort_buffer_size = 8388608 @@session.myisam_sort_buffer_size = 134216704
1 1
'#--------------------FN_DYNVARS_005_03-------------------------#' '#--------------------FN_DYNVARS_005_03-------------------------#'
SET @@global.myisam_sort_buffer_size = 4; SET @@global.myisam_sort_buffer_size = 4;
...@@ -187,8 +187,8 @@ ERROR 42S22: Unknown column 'myisam_sort_buffer_size' in 'field list' ...@@ -187,8 +187,8 @@ ERROR 42S22: Unknown column 'myisam_sort_buffer_size' in 'field list'
SET @@global.myisam_sort_buffer_size = @start_global_value; SET @@global.myisam_sort_buffer_size = @start_global_value;
SELECT @@global.myisam_sort_buffer_size ; SELECT @@global.myisam_sort_buffer_size ;
@@global.myisam_sort_buffer_size @@global.myisam_sort_buffer_size
8388608 134216704
SET @@session.myisam_sort_buffer_size = @start_session_value; SET @@session.myisam_sort_buffer_size = @start_session_value;
SELECT @@session.myisam_sort_buffer_size ; SELECT @@session.myisam_sort_buffer_size ;
@@session.myisam_sort_buffer_size @@session.myisam_sort_buffer_size
8388608 134216704
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test
View file @ 53d44ad1
...@@ -60,10 +60,10 @@ SELECT @@session.myisam_sort_buffer_size ; ...@@ -60,10 +60,10 @@ SELECT @@session.myisam_sort_buffer_size ;
######################################################################## ########################################################################
SET @@global.myisam_sort_buffer_size = DEFAULT; SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size = 8388608; SELECT @@global.myisam_sort_buffer_size = 134216704;
SET @@session.myisam_sort_buffer_size = DEFAULT; SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size = 8388608; SELECT @@session.myisam_sort_buffer_size = 134216704;
--echo '#--------------------FN_DYNVARS_005_03-------------------------#' --echo '#--------------------FN_DYNVARS_005_03-------------------------#'
......
mysql-test/t/myisam.test
View file @ 53d44ad1
...@@ -1600,6 +1600,7 @@ INSERT INTO t1 SELECT a+1280,b FROM t1; ...@@ -1600,6 +1600,7 @@ INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1; INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1; INSERT INTO t1 SELECT a+5120,b FROM t1;
SET myisam_sort_buffer_size=4; SET myisam_sort_buffer_size=4;
--replace_regex /Current myisam_sort_buffer_size.*/X/
REPAIR TABLE t1; REPAIR TABLE t1;
SET myisam_repair_threads=2; SET myisam_repair_threads=2;
...@@ -1648,6 +1649,7 @@ DROP TABLE t1, t2, t3; ...@@ -1648,6 +1649,7 @@ DROP TABLE t1, t2, t3;
CREATE TABLE t1(a CHAR(255), KEY(a)); CREATE TABLE t1(a CHAR(255), KEY(a));
SELECT * FROM t1, t1 AS a1; SELECT * FROM t1, t1 AS a1;
SET myisam_sort_buffer_size=4; SET myisam_sort_buffer_size=4;
--replace_regex /Current myisam_sort_buffer_size.*/X/
INSERT INTO t1 VALUES INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
......
mysql-test/t/repair.test
View file @ 53d44ad1
...@@ -61,6 +61,7 @@ DROP TABLE t1; ...@@ -61,6 +61,7 @@ DROP TABLE t1;
# #
CREATE TABLE t1(a CHAR(255), KEY(a)); CREATE TABLE t1(a CHAR(255), KEY(a));
SET myisam_sort_buffer_size=4096; SET myisam_sort_buffer_size=4096;
--replace_regex /Current myisam_sort_buffer_size.*/X/
INSERT INTO t1 VALUES INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
...@@ -79,6 +80,7 @@ INSERT INTO t1 VALUES ...@@ -79,6 +80,7 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'), ('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'); ('0'),('0'),('0'),('0'),('0'),('0'),('0');
SET myisam_repair_threads=2; SET myisam_repair_threads=2;
--replace_regex /Current myisam_sort_buffer_size.*/X/
REPAIR TABLE t1; REPAIR TABLE t1;
SET myisam_repair_threads=@@global.myisam_repair_threads; SET myisam_repair_threads=@@global.myisam_repair_threads;
SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size; SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
......
storage/maria/ha_maria.cc
View file @ 53d44ad1
...@@ -260,10 +260,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, ...@@ -260,10 +260,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
"disables parallel repair.", "disables parallel repair.",
0, 0, 1, 1, 128, 1); 0, 0, 1, 1, 128, 1);
static MYSQL_THDVAR_ULONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
"The buffer that is allocated when sorting the index when doing a " "The buffer that is allocated when sorting the index when doing a "
"REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL,
0, 0, 128L*1024L*1024L, 4, UINT_MAX32, 1); SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG,
"Specifies how Aria index statistics collection code should treat " "Specifies how Aria index statistics collection code should treat "
...@@ -1455,6 +1455,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) ...@@ -1455,6 +1455,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt)
if ((param.testflag & T_REP_BY_SORT)) if ((param.testflag & T_REP_BY_SORT))
{ {
param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP; param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP;
if (thd->vio_ok())
_ma_check_print_info(&param, "Retrying repair with keycache");
sql_print_information("Retrying repair of: '%s' with keycache", sql_print_information("Retrying repair of: '%s' with keycache",
table->s->path.str); table->s->path.str);
continue; continue;
......
storage/maria/ma_check.c
View file @ 53d44ad1
...@@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param) ...@@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param)
param->keys_in_use= ~(ulonglong) 0; param->keys_in_use= ~(ulonglong) 0;
param->search_after_block=HA_OFFSET_ERROR; param->search_after_block=HA_OFFSET_ERROR;
param->auto_increment_value= 0; param->auto_increment_value= 0;
param->use_buffers=USE_BUFFER_INIT; param->use_buffers= PAGE_BUFFER_INIT;
param->read_buffer_length=READ_BUFFER_INIT; param->read_buffer_length=READ_BUFFER_INIT;
param->write_buffer_length=READ_BUFFER_INIT; param->write_buffer_length=READ_BUFFER_INIT;
param->sort_buffer_length=SORT_BUFFER_INIT; param->sort_buffer_length=SORT_BUFFER_INIT;
......
storage/maria/ma_sort.c
View file @ 53d44ad1
...@@ -36,8 +36,10 @@ ...@@ -36,8 +36,10 @@
#define MERGEBUFF2 31 #define MERGEBUFF2 31
#define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD) #define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD)
#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) #define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
#define DISK_BUFFER_SIZE (IO_SIZE*16) #define DISK_BUFFER_SIZE (IO_SIZE*128)
/* How many keys we can keep in memory */
typedef ulonglong ha_keys;
/* /*
Pointers of functions for store and read keys from temp file Pointers of functions for store and read keys from temp file
...@@ -47,41 +49,42 @@ extern void print_error(const char *fmt,...); ...@@ -47,41 +49,42 @@ extern void print_error(const char *fmt,...);
/* Functions defined in this file */ /* Functions defined in this file */
static ha_rows find_all_keys(MARIA_SORT_PARAM *info,uint keys, static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, uchar **sort_keys,
DYNAMIC_ARRAY *buffpek,int *maxbuffer, DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
IO_CACHE *tempfile, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions); IO_CACHE *tempfile_for_exceptions);
static int write_keys(MARIA_SORT_PARAM *info, uchar **sort_keys, static int write_keys(MARIA_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_key(MARIA_SORT_PARAM *info, uchar *key, static int write_key(MARIA_SORT_PARAM *info, uchar *key,
IO_CACHE *tempfile); IO_CACHE *tempfile);
static int write_index(MARIA_SORT_PARAM *info, uchar **sort_keys, static int write_index(MARIA_SORT_PARAM *info, uchar **sort_keys,
uint count); ha_keys count);
static int merge_many_buff(MARIA_SORT_PARAM *info,uint keys, static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, uchar **sort_keys,
BUFFPEK *buffpek,int *maxbuffer, BUFFPEK *buffpek, uint *maxbuffer,
IO_CACHE *t_file); IO_CACHE *t_file);
static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length); uint sort_length);
static int merge_buffers(MARIA_SORT_PARAM *info,uint keys, static int merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys,
IO_CACHE *from_file, IO_CACHE *to_file, IO_CACHE *from_file, IO_CACHE *to_file,
uchar **sort_keys, BUFFPEK *lastbuff, uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb); BUFFPEK *Fb, BUFFPEK *Tb);
static int merge_index(MARIA_SORT_PARAM *,uint, uchar **,BUFFPEK *, int, static int merge_index(MARIA_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
IO_CACHE *); IO_CACHE *);
static int flush_maria_ft_buf(MARIA_SORT_PARAM *info); static int flush_maria_ft_buf(MARIA_SORT_PARAM *info);
static int write_keys_varlen(MARIA_SORT_PARAM *info, uchar **sort_keys, static int write_keys_varlen(MARIA_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek, ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile); IO_CACHE *tempfile);
static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length); uint sort_length);
static int write_merge_key(MARIA_SORT_PARAM *info, IO_CACHE *to_file, static int write_merge_key(MARIA_SORT_PARAM *info, IO_CACHE *to_file,
uchar *key, uint sort_length, uint count); uchar *key, uint sort_length, ha_keys count);
static int write_merge_key_varlen(MARIA_SORT_PARAM *info, static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
IO_CACHE *to_file, uchar *key, IO_CACHE *to_file,
uint sort_length, uint count); uchar* key, uint sort_length,
ha_keys count);
static inline int static inline int
my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
...@@ -102,11 +105,11 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); ...@@ -102,11 +105,11 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
size_t sortbuff_size) size_t sortbuff_size)
{ {
int error,maxbuffer,skr; int error;
size_t memavl,old_memavl; uint sort_length, maxbuffer;
uint keys,sort_length; size_t memavl, old_memavl;
DYNAMIC_ARRAY buffpek; DYNAMIC_ARRAY buffpek;
ha_rows records; ha_rows records, keys;
uchar **sort_keys; uchar **sort_keys;
IO_CACHE tempfile, tempfile_for_exceptions; IO_CACHE tempfile, tempfile_for_exceptions;
DBUG_ENTER("_ma_create_index_by_sort"); DBUG_ENTER("_ma_create_index_by_sort");
...@@ -117,7 +120,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, ...@@ -117,7 +120,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (info->keyinfo->flag & HA_VAR_LENGTH_KEY) if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
{ {
info->write_keys= write_keys_varlen; info->write_keys= write_keys_varlen;
info->read_to_buffer=read_to_buffer_varlen; info->read_to_buffer= read_to_buffer_varlen;
info->write_key=write_merge_key_varlen; info->write_key=write_merge_key_varlen;
} }
else else
...@@ -140,31 +143,53 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, ...@@ -140,31 +143,53 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
while (memavl >= MIN_SORT_MEMORY) while (memavl >= MIN_SORT_MEMORY)
{ {
if ((records < UINT_MAX32) && /* Check if we can fit all keys into memory */
((my_off_t) (records + 1) * if (((ulonglong) (records + 1) *
(sort_length + sizeof(char*)) <= (my_off_t) memavl)) (sort_length + sizeof(char*)) <= memavl))
keys= (uint)records+1; keys= records+1;
else if ((info->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
}
else else
{
/*
All keys can't fit in memory.
Calculate how many keys + buffers we can keep in memory
*/
uint maxbuffer_org;
do do
{ {
skr=maxbuffer; maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer || if (memavl < sizeof(BUFFPEK) * maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/ (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 || (sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer) keys < maxbuffer)
{ {
_ma_check_print_error(info->sort_info->param, _ma_check_print_error(info->sort_info->param,
"aria_sort_buffer_size is too small"); "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
(ulonglong) sortbuff_size, (ulonglong) records,
sort_length);
my_errno= ENOMEM;
goto err; goto err;
} }
} }
while ((maxbuffer= (int) (records/(keys-1)+1)) != skr); while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys=(uchar**) my_malloc(keys*(sort_length+sizeof(char*))+ if ((sort_keys=(uchar**) my_malloc(keys*(sort_length+sizeof(char*))+
HA_FT_MAXBYTELEN, MYF(0)))) HA_FT_MAXBYTELEN, MYF(0))))
{ {
if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer, if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
maxbuffer/2, MYF(0))) min(maxbuffer/2, 1000), MYF(0)))
{ {
my_free(sort_keys); my_free(sort_keys);
sort_keys= 0; sort_keys= 0;
...@@ -178,14 +203,20 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, ...@@ -178,14 +203,20 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
} }
if (memavl < MIN_SORT_MEMORY) if (memavl < MIN_SORT_MEMORY)
{ {
_ma_check_print_error(info->sort_info->param, "Aria sort buffer" /* purecov: begin inspected */
" too small"); /* purecov: tested */ _ma_check_print_error(info->sort_info->param,
goto err; /* purecov: tested */ "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
(ulonglong) sortbuff_size, (ulonglong) records, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
} }
(*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */ (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
if (!no_messages) if (!no_messages)
printf(" - Searching for keys, allocating buffer for %d keys\n",keys); my_fprintf(stdout,
" - Searching for keys, allocating buffer for %llu keys\n",
(ulonglong) keys);
if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions)) &tempfile,&tempfile_for_exceptions))
...@@ -197,8 +228,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, ...@@ -197,8 +228,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (maxbuffer == 0) if (maxbuffer == 0)
{ {
if (!no_messages) if (!no_messages)
printf(" - Dumping %lu keys\n", (ulong) records); my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records);
if (write_index(info,sort_keys, (uint) records)) if (write_index(info, sort_keys, (ha_keys) records))
goto err; /* purecov: inspected */ goto err; /* purecov: inspected */
} }
else else
...@@ -207,7 +238,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, ...@@ -207,7 +238,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (maxbuffer >= MERGEBUFF2) if (maxbuffer >= MERGEBUFF2)
{ {
if (!no_messages) if (!no_messages)
printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */ my_fprintf(stdout, " - Merging %llu keys\n",
(ulonglong) records); /* purecov: tested */
if (merge_many_buff(info,keys,sort_keys, if (merge_many_buff(info,keys,sort_keys,
dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
goto err; /* purecov: inspected */ goto err; /* purecov: inspected */
...@@ -266,13 +298,13 @@ err: ...@@ -266,13 +298,13 @@ err:
/* Search after all keys and place them in a temp. file */ /* Search after all keys and place them in a temp. file */
static ha_rows find_all_keys(MARIA_SORT_PARAM *info, uint keys, static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_rows keys,
uchar **sort_keys, DYNAMIC_ARRAY *buffpek, uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
int *maxbuffer, IO_CACHE *tempfile, uint *maxbuffer, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions) IO_CACHE *tempfile_for_exceptions)
{ {
int error; int error;
uint idx; ha_rows idx;
DBUG_ENTER("find_all_keys"); DBUG_ENTER("find_all_keys");
idx=error=0; idx=error=0;
...@@ -328,9 +360,11 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -328,9 +360,11 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{ {
MARIA_SORT_PARAM *sort_param= (MARIA_SORT_PARAM*) arg; MARIA_SORT_PARAM *sort_param= (MARIA_SORT_PARAM*) arg;
int error; int error;
size_t memavl,old_memavl; size_t memavl, old_memavl;
longlong sortbuff_size;
ha_keys keys, idx;
uint sort_length; uint sort_length;
ulong idx, maxbuffer, keys; uint maxbuffer;
uchar **sort_keys=0; uchar **sort_keys=0;
LINT_INIT(keys); LINT_INIT(keys);
...@@ -364,8 +398,9 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -364,8 +398,9 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
bzero((char*) &sort_param->buffpek,sizeof(sort_param->buffpek)); bzero((char*) &sort_param->buffpek,sizeof(sort_param->buffpek));
bzero((char*) &sort_param->unique, sizeof(sort_param->unique)); bzero((char*) &sort_param->unique, sizeof(sort_param->unique));
memavl= max(sort_param->sortbuff_size, MIN_SORT_MEMORY); sortbuff_size= sort_param->sortbuff_size;
idx= (uint)sort_param->sort_info->max_records; memavl= max(sortbuff_size, MIN_SORT_MEMORY);
idx= (ha_keys) sort_param->sort_info->max_records;
sort_length= sort_param->key_length; sort_length= sort_param->key_length;
maxbuffer= 1; maxbuffer= 1;
...@@ -373,23 +408,36 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -373,23 +408,36 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{ {
if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= (my_off_t) memavl) if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= (my_off_t) memavl)
keys= idx+1; keys= idx+1;
else if ((sort_param->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
}
else else
{ {
ulong skr; uint maxbuffer_org;
do do
{ {
skr= maxbuffer; maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*maxbuffer || if (memavl < sizeof(BUFFPEK)*maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/ (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 || (sort_length+sizeof(char*))) <= 1 ||
keys < maxbuffer) keys < maxbuffer)
{ {
_ma_check_print_error(sort_param->sort_info->param, _ma_check_print_error(sort_param->sort_info->param,
"aria_sort_buffer_size is too small"); "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
goto err; goto err;
} }
} }
while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr); while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
} }
if ((sort_keys= (uchar **) if ((sort_keys= (uchar **)
my_malloc(keys*(sort_length+sizeof(char*))+ my_malloc(keys*(sort_length+sizeof(char*))+
...@@ -397,7 +445,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -397,7 +445,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
HA_FT_MAXBYTELEN : 0), MYF(0)))) HA_FT_MAXBYTELEN : 0), MYF(0))))
{ {
if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK), if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
maxbuffer, maxbuffer/2, MYF(0))) maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
{ {
my_free(sort_keys); my_free(sort_keys);
sort_keys= (uchar **) NULL; /* for err: label */ sort_keys= (uchar **) NULL; /* for err: label */
...@@ -412,14 +460,19 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -412,14 +460,19 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
} }
if (memavl < MIN_SORT_MEMORY) if (memavl < MIN_SORT_MEMORY)
{ {
/* purecov: begin inspected */
_ma_check_print_error(sort_param->sort_info->param, _ma_check_print_error(sort_param->sort_info->param,
"Aria sort buffer too small"); "aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
goto err; /* purecov: tested */ sortbuff_size, (ulonglong) idx, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
} }
if (sort_param->sort_info->param->testflag & T_VERBOSE) if (sort_param->sort_info->param->testflag & T_VERBOSE)
printf("Key %d - Allocating buffer for %lu keys\n", my_fprintf(stdout,
sort_param->key+1, (ulong) keys); "Key %d - Allocating buffer for %llu keys\n",
sort_param->key + 1, (ulonglong) keys);
sort_param->sort_keys= sort_keys; sort_param->sort_keys= sort_keys;
idx= error= 0; idx= error= 0;
...@@ -431,7 +484,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -431,7 +484,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{ {
if (sort_param->real_key_length > sort_param->key_length) if (sort_param->real_key_length > sort_param->key_length)
{ {
if (write_key(sort_param,sort_keys[idx], if (write_key(sort_param, sort_keys[idx],
&sort_param->tempfile_for_exceptions)) &sort_param->tempfile_for_exceptions))
goto err; goto err;
continue; continue;
...@@ -449,7 +502,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -449,7 +502,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
(size_t) sort_param->key_length); (size_t) sort_param->key_length);
idx= 1; idx= 1;
} }
sort_keys[idx]=sort_keys[idx - 1] + sort_param->key_length; sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
} }
if (error > 0) if (error > 0)
goto err; goto err;
...@@ -465,14 +518,13 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg) ...@@ -465,14 +518,13 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
else else
sort_param->keys= idx; sort_param->keys= idx;
sort_param->sort_keys_length= keys;
goto ok; goto ok;
err: err:
DBUG_PRINT("error", ("got some error")); DBUG_PRINT("error", ("got some error"));
sort_param->sort_info->got_error= 1; /* no need to protect with a mutex */ sort_param->sort_info->got_error= 1; /* no need to protect with a mutex */
my_free(sort_keys); my_free(sort_keys);
sort_param->sort_keys=0; sort_param->sort_keys= 0;
delete_dynamic(& sort_param->buffpek); delete_dynamic(& sort_param->buffpek);
close_cached_file(&sort_param->tempfile); close_cached_file(&sort_param->tempfile);
close_cached_file(&sort_param->tempfile_for_exceptions); close_cached_file(&sort_param->tempfile_for_exceptions);
...@@ -483,8 +535,12 @@ ok: ...@@ -483,8 +535,12 @@ ok:
Detach from the share if the writer is involved. Avoid others to Detach from the share if the writer is involved. Avoid others to
be blocked. This includes a flush of the write buffer. This will be blocked. This includes a flush of the write buffer. This will
also indicate EOF to the readers. also indicate EOF to the readers.
That means that a writer always gets here first and readers -
only when they see EOF. But if a reader finishes prematurely
because of an error it may reach this earlier - don't allow it
to detach the writer thread.
*/ */
if (sort_param->sort_info->info->rec_cache.share) if (sort_param->master && sort_param->sort_info->info->rec_cache.share)
remove_io_thread(&sort_param->sort_info->info->rec_cache); remove_io_thread(&sort_param->sort_info->info->rec_cache);
/* Readers detach from the share if any. Avoid others to be blocked. */ /* Readers detach from the share if any. Avoid others to be blocked. */
...@@ -518,7 +574,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) ...@@ -518,7 +574,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
for (i= 0, sinfo= sort_param ; for (i= 0, sinfo= sort_param ;
i < sort_info->total_keys ; i < sort_info->total_keys ;
i++, rec_per_key_part+=sinfo->keyinfo->keysegs, sinfo++) i++, sinfo++)
{ {
if (!sinfo->sort_keys) if (!sinfo->sort_keys)
{ {
...@@ -534,19 +590,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) ...@@ -534,19 +590,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
{ {
if (param->testflag & T_VERBOSE) if (param->testflag & T_VERBOSE)
{ {
printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys); my_fprintf(stdout,
"Key %d - Dumping %llu keys\n", sinfo->key+1,
(ulonglong) sinfo->keys);
fflush(stdout); fflush(stdout);
} }
if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) || if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
flush_maria_ft_buf(sinfo) || _ma_flush_pending_blocks(sinfo)) flush_maria_ft_buf(sinfo) || _ma_flush_pending_blocks(sinfo))
got_error=1; got_error=1;
} }
if (!got_error && param->testflag & T_STATISTICS)
maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
param->stats_method ==
MI_STATS_METHOD_IGNORE_NULLS ?
sinfo->notnull : NULL,
(ulonglong) share->state.state.records);
} }
my_free(sinfo->sort_keys); my_free(sinfo->sort_keys);
my_free(sinfo->rec_buff); my_free(sinfo->rec_buff);
...@@ -559,6 +611,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) ...@@ -559,6 +611,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
delete_dynamic(&sinfo->buffpek), delete_dynamic(&sinfo->buffpek),
close_cached_file(&sinfo->tempfile), close_cached_file(&sinfo->tempfile),
close_cached_file(&sinfo->tempfile_for_exceptions), close_cached_file(&sinfo->tempfile_for_exceptions),
rec_per_key_part+= sinfo->keyinfo->keysegs,
sinfo++) sinfo++)
{ {
if (got_error) if (got_error)
...@@ -597,10 +650,12 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) ...@@ -597,10 +650,12 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
if (maxbuffer >= MERGEBUFF2) if (maxbuffer >= MERGEBUFF2)
{ {
if (param->testflag & T_VERBOSE) if (param->testflag & T_VERBOSE)
printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys); my_fprintf(stdout,
if (merge_many_buff(sinfo, keys, (uchar **) mergebuf, "Key %d - Merging %llu keys\n",
sinfo->key+1, (ulonglong) sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
(int*) &maxbuffer, &sinfo->tempfile)) &maxbuffer, &sinfo->tempfile))
{ {
got_error=1; got_error=1;
continue; continue;
...@@ -660,6 +715,13 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) ...@@ -660,6 +715,13 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
} }
} }
} }
if (!got_error && (param->testflag & T_STATISTICS))
maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
param->stats_method ==
MI_STATS_METHOD_IGNORE_NULLS ?
sinfo->notnull : NULL,
(ulonglong) share->state.state.records);
} }
my_free(mergebuf); my_free(mergebuf);
DBUG_RETURN(got_error); DBUG_RETURN(got_error);
...@@ -669,12 +731,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param) ...@@ -669,12 +731,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
/* Write all keys in memory to file for later merge */ /* Write all keys in memory to file for later merge */
static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys, static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
{ {
uchar **end; uchar **end;
uint sort_length=info->key_length; uint sort_length=info->key_length;
DBUG_ENTER("write_keys"); DBUG_ENTER("write_keys");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info); info);
if (!my_b_inited(tempfile) && if (!my_b_inited(tempfile) &&
...@@ -687,7 +752,7 @@ static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys, ...@@ -687,7 +752,7 @@ static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
for (end=sort_keys+count ; sort_keys != end ; sort_keys++) for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
{ {
if (my_b_write(tempfile, *sort_keys, (uint) sort_length)) if (my_b_write(tempfile, *sort_keys, sort_length))
DBUG_RETURN(1); /* purecov: inspected */ DBUG_RETURN(1); /* purecov: inspected */
} }
DBUG_RETURN(0); DBUG_RETURN(0);
...@@ -710,14 +775,17 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs) ...@@ -710,14 +775,17 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
static int write_keys_varlen(MARIA_SORT_PARAM *info, static int write_keys_varlen(MARIA_SORT_PARAM *info,
register uchar **sort_keys, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile) IO_CACHE *tempfile)
{ {
uchar **end; uchar **end;
int err; int err;
DBUG_ENTER("write_keys_varlen"); DBUG_ENTER("write_keys_varlen");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info); info);
if (!my_b_inited(tempfile) && if (!my_b_inited(tempfile) &&
...@@ -756,9 +824,8 @@ static int write_key(MARIA_SORT_PARAM *info, uchar *key, ...@@ -756,9 +824,8 @@ static int write_key(MARIA_SORT_PARAM *info, uchar *key,
/* Write index */ /* Write index */
static int write_index(MARIA_SORT_PARAM *info, static int write_index(MARIA_SORT_PARAM *info, register uchar **sort_keys,
register uchar **sort_keys, register ha_keys count)
register uint count)
{ {
DBUG_ENTER("write_index"); DBUG_ENTER("write_index");
...@@ -777,11 +844,11 @@ static int write_index(MARIA_SORT_PARAM *info, ...@@ -777,11 +844,11 @@ static int write_index(MARIA_SORT_PARAM *info,
/* Merge buffers to make < MERGEBUFF2 buffers */ /* Merge buffers to make < MERGEBUFF2 buffers */
static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, BUFFPEK *buffpek, uchar **sort_keys, BUFFPEK *buffpek,
int *maxbuffer, IO_CACHE *t_file) uint *maxbuffer, IO_CACHE *t_file)
{ {
int tmp, merges, max_merges; uint tmp, merges, max_merges;
IO_CACHE t_file2, *from_file, *to_file, *temp; IO_CACHE t_file2, *from_file, *to_file, *temp;
BUFFPEK *lastbuff; BUFFPEK *lastbuff;
DBUG_ENTER("merge_many_buff"); DBUG_ENTER("merge_many_buff");
...@@ -807,11 +874,11 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, ...@@ -807,11 +874,11 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
from_file= t_file ; to_file= &t_file2; from_file= t_file ; to_file= &t_file2;
while (*maxbuffer >= MERGEBUFF2) while (*maxbuffer >= MERGEBUFF2)
{ {
int i; uint i;
reinit_io_cache(from_file,READ_CACHE,0L,0,0); reinit_io_cache(from_file,READ_CACHE,0L,0,0);
reinit_io_cache(to_file,WRITE_CACHE,0L,0,0); reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
lastbuff=buffpek; lastbuff=buffpek;
for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
{ {
if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
buffpek+i,buffpek+i+MERGEBUFF-1)) buffpek+i,buffpek+i+MERGEBUFF-1))
...@@ -825,14 +892,19 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, ...@@ -825,14 +892,19 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
if (flush_io_cache(to_file)) if (flush_io_cache(to_file))
break; /* purecov: inspected */ break; /* purecov: inspected */
temp=from_file; from_file=to_file; to_file=temp; temp=from_file; from_file=to_file; to_file=temp;
*maxbuffer= (int) (lastbuff-buffpek)-1; *maxbuffer= (uint) (lastbuff-buffpek)-1;
if (info->sort_info->param->max_stage != 1) /* If not parallel */ if (info->sort_info->param->max_stage != 1) /* If not parallel */
_ma_report_progress(info->sort_info->param, merges++, max_merges); _ma_report_progress(info->sort_info->param, merges++, max_merges);
} }
cleanup: cleanup:
close_cached_file(to_file); /* This holds old result */ close_cached_file(to_file); /* This holds old result */
if (to_file == t_file) if (to_file == t_file)
{
DBUG_ASSERT(t_file2.type == WRITE_CACHE);
*t_file=t_file2; /* Copy result file */ *t_file=t_file2; /* Copy result file */
t_file->current_pos= &t_file->write_pos;
t_file->current_end= &t_file->write_end;
}
DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */ DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */
} /* merge_many_buff */ } /* merge_many_buff */
...@@ -851,33 +923,35 @@ cleanup: ...@@ -851,33 +923,35 @@ cleanup:
-1 Error -1 Error
*/ */
static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length) uint sort_length)
{ {
register uint count; register ha_keys count;
uint length; my_off_t length;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{ {
if (mysql_file_pread(fromfile->file, buffpek->base, if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
(length= sort_length*count),buffpek->file_pos,MYF_RW)) (length= sort_length * count),
return((uint) -1); /* purecov: inspected */ buffpek->file_pos, MYF_RW))
return(HA_OFFSET_ERROR); /* purecov: inspected */
buffpek->key=buffpek->base; buffpek->key=buffpek->base;
buffpek->file_pos+= length; /* New filepos */ buffpek->file_pos+= length; /* New filepos */
buffpek->count-= count; buffpek->count-= count;
buffpek->mem_count= count; buffpek->mem_count= count;
} }
return (count*sort_length); return (((my_off_t) count) * sort_length);
} /* read_to_buffer */ } /* read_to_buffer */
static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length) static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{ {
register uint count; register ha_keys count;
uint idx; uint idx;
uchar *buffp; uchar *buffp;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{ {
buffp= buffpek->base; buffp= buffpek->base;
...@@ -886,7 +960,7 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, ...@@ -886,7 +960,7 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint16 length_of_key; uint16 length_of_key;
if (mysql_file_pread(fromfile->file,(uchar*)&length_of_key,sizeof(length_of_key), if (mysql_file_pread(fromfile->file,(uchar*)&length_of_key,sizeof(length_of_key),
buffpek->file_pos,MYF_RW)) buffpek->file_pos,MYF_RW))
return((uint) -1); return(HA_OFFSET_ERROR);
buffpek->file_pos+=sizeof(length_of_key); buffpek->file_pos+=sizeof(length_of_key);
if (mysql_file_pread(fromfile->file, buffp, length_of_key, if (mysql_file_pread(fromfile->file, buffp, length_of_key,
buffpek->file_pos,MYF_RW)) buffpek->file_pos,MYF_RW))
...@@ -898,15 +972,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, ...@@ -898,15 +972,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
buffpek->count-= count; buffpek->count-= count;
buffpek->mem_count= count; buffpek->mem_count= count;
} }
return (count*sort_length); return (((my_off_t) count) * sort_length);
} /* read_to_buffer_varlen */ } /* read_to_buffer_varlen */
static int write_merge_key_varlen(MARIA_SORT_PARAM *info, static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
IO_CACHE *to_file, uchar* key, IO_CACHE *to_file, uchar* key,
uint sort_length, uint count) uint sort_length, ha_keys count)
{ {
uint idx; ha_keys idx;
uchar *bufs = key; uchar *bufs = key;
for (idx=1;idx<=count;idx++) for (idx=1;idx<=count;idx++)
...@@ -921,34 +995,40 @@ static int write_merge_key_varlen(MARIA_SORT_PARAM *info, ...@@ -921,34 +995,40 @@ static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
static int write_merge_key(MARIA_SORT_PARAM *info __attribute__((unused)), static int write_merge_key(MARIA_SORT_PARAM *info __attribute__((unused)),
IO_CACHE *to_file, uchar *key, IO_CACHE *to_file, uchar *key,
uint sort_length, uint count) uint sort_length, ha_keys count)
{ {
return my_b_write(to_file, key, (size_t) sort_length*count); return my_b_write(to_file, key, ((size_t) sort_length) * count);
} }
/* /*
Merge buffers to one buffer Merge buffers to one buffer
If to_file == 0 then use info->key_write If to_file == 0 then use info->key_write
Return:
0 ok
1 error
*/ */
static int static int
merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb) BUFFPEK *Fb, BUFFPEK *Tb)
{ {
int error; int error= 1;
uint sort_length,maxcount; uint sort_length;
ha_keys maxcount;
ha_rows count; ha_rows count;
my_off_t UNINIT_VAR(to_start_filepos); my_off_t UNINIT_VAR(to_start_filepos), read_length;
uchar *strpos; uchar *strpos;
BUFFPEK *buffpek,**refpek; BUFFPEK *buffpek,**refpek;
QUEUE queue; QUEUE queue;
DBUG_ENTER("merge_buffers"); DBUG_ENTER("merge_buffers");
count=error=0; count= 0;
maxcount=keys/((uint) (Tb-Fb) +1); maxcount= keys/((uint) (Tb-Fb) +1);
DBUG_ASSERT(maxcount > 0); DBUG_ASSERT(maxcount > 0);
LINT_INIT(to_start_filepos);
if (to_file) if (to_file)
to_start_filepos=my_b_tell(to_file); to_start_filepos=my_b_tell(to_file);
strpos= (uchar*) sort_keys; strpos= (uchar*) sort_keys;
...@@ -963,10 +1043,10 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -963,10 +1043,10 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{ {
count+= buffpek->count; count+= buffpek->count;
buffpek->base= strpos; buffpek->base= strpos;
buffpek->max_keys=maxcount; buffpek->max_keys= maxcount;
strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek, strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
sort_length)); sort_length));
if (error == -1) if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */ goto err; /* purecov: inspected */
queue_insert(&queue,(uchar*) buffpek); queue_insert(&queue,(uchar*) buffpek);
} }
...@@ -980,27 +1060,20 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -980,27 +1060,20 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{ {
if (info->write_key(info,to_file, buffpek->key, if (info->write_key(info,to_file, buffpek->key,
(uint) sort_length,1)) (uint) sort_length,1))
{ goto err; /* purecov: inspected */
error=1; goto err; /* purecov: inspected */
}
} }
else else
{ {
if ((*info->key_write)(info,(void*) buffpek->key)) if ((*info->key_write)(info,(void*) buffpek->key))
{ goto err; /* purecov: inspected */
error=1; goto err; /* purecov: inspected */
}
} }
buffpek->key+=sort_length; buffpek->key+=sort_length;
if (! --buffpek->mem_count) if (! --buffpek->mem_count)
{ {
/* It's enough to check for killedptr before a slow operation */ /* It's enough to check for killedptr before a slow operation */
if (_ma_killed_ptr(info->sort_info->param)) if (_ma_killed_ptr(info->sort_info->param))
{
error=1;
goto err; goto err;
} if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length)))
{ {
uchar *base= buffpek->base; uchar *base= buffpek->base;
uint max_keys=buffpek->max_keys; uint max_keys=buffpek->max_keys;
...@@ -1027,9 +1100,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -1027,9 +1100,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
} }
break; /* One buffer have been removed */ break; /* One buffer have been removed */
} }
else if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
} }
else if (error == -1)
goto err; /* purecov: inspected */
queue_replace_top(&queue); /* Top element has been replaced */ queue_replace_top(&queue); /* Top element has been replaced */
} }
} }
...@@ -1061,8 +1134,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -1061,8 +1134,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
} }
} }
} }
while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
-1 && error != 0); if (read_length == 0)
error= 0;
lastbuff->count=count; lastbuff->count=count;
if (to_file) if (to_file)
...@@ -1076,8 +1150,8 @@ err: ...@@ -1076,8 +1150,8 @@ err:
/* Do a merge to output-file (save only positions) */ /* Do a merge to output-file (save only positions) */
static int static int
merge_index(MARIA_SORT_PARAM *info, uint keys, uchar **sort_keys, merge_index(MARIA_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
{ {
DBUG_ENTER("merge_index"); DBUG_ENTER("merge_index");
if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,
......
storage/maria/maria_chk.c
View file @ 53d44ad1
...@@ -191,7 +191,7 @@ end: ...@@ -191,7 +191,7 @@ end:
enum options_mc { enum options_mc {
OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS, OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
OPT_CORRECT_CHECKSUM, OPT_PAGE_BUFFER_SIZE, OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_PAGE_BUFFER_SIZE,
OPT_KEY_CACHE_BLOCK_SIZE, OPT_MARIA_BLOCK_SIZE, OPT_KEY_CACHE_BLOCK_SIZE, OPT_MARIA_BLOCK_SIZE,
OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE, OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN, OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
...@@ -229,6 +229,11 @@ static struct my_option my_long_options[] = ...@@ -229,6 +229,11 @@ static struct my_option my_long_options[] =
{"correct-checksum", OPT_CORRECT_CHECKSUM, {"correct-checksum", OPT_CORRECT_CHECKSUM,
"Correct checksum information for table.", "Correct checksum information for table.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"create-missing-keys", OPT_CREATE_MISSING_KEYS,
"Create missing keys. This assumes that the data file is correct and that "
"the the number of rows stored in the index file is correct. Enables "
"--quick",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF #ifndef DBUG_OFF
{"debug", '#', {"debug", '#',
"Output debug log. Often this is 'd:t:o,filename'.", "Output debug log. Often this is 'd:t:o,filename'.",
...@@ -362,8 +367,8 @@ static struct my_option my_long_options[] = ...@@ -362,8 +367,8 @@ static struct my_option my_long_options[] =
{ "page_buffer_size", OPT_PAGE_BUFFER_SIZE, { "page_buffer_size", OPT_PAGE_BUFFER_SIZE,
"Size of page buffer. Used by --safe-repair", "Size of page buffer. Used by --safe-repair",
&check_param.use_buffers, &check_param.use_buffers, 0, &check_param.use_buffers, &check_param.use_buffers, 0,
GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, 1024L*1024L, GET_ULONG, REQUIRED_ARG, PAGE_BUFFER_INIT, 1024L*1024L,
(long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0}, SIZE_T_MAX, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
{ "read_buffer_size", OPT_READ_BUFFER_SIZE, { "read_buffer_size", OPT_READ_BUFFER_SIZE,
"Read buffer size for sequential reads during scanning", "Read buffer size for sequential reads during scanning",
&check_param.read_buffer_length, &check_param.read_buffer_length,
...@@ -379,9 +384,8 @@ static struct my_option my_long_options[] = ...@@ -379,9 +384,8 @@ static struct my_option my_long_options[] =
{ "sort_buffer_size", OPT_SORT_BUFFER_SIZE, { "sort_buffer_size", OPT_SORT_BUFFER_SIZE,
"Size of sort buffer. Used by --recover", "Size of sort buffer. Used by --recover",
&check_param.sort_buffer_length, &check_param.sort_buffer_length,
&check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG, &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG,
(long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD), SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0},
(long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0},
{ "sort_key_blocks", OPT_SORT_KEY_BLOCKS, { "sort_key_blocks", OPT_SORT_KEY_BLOCKS,
"Internal buffer for sorting keys; Don't touch :)", "Internal buffer for sorting keys; Don't touch :)",
&check_param.sort_key_blocks, &check_param.sort_key_blocks,
...@@ -497,10 +501,18 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\ ...@@ -497,10 +501,18 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\
--correct-checksum Correct checksum information for table.\n\ --correct-checksum Correct checksum information for table.\n\
-D, --data-file-length=# Max length of data file (when recreating data\n\ -D, --data-file-length=# Max length of data file (when recreating data\n\
file when it's full).\n\ file when it's full).\n\
--create-missing-keys\n\
Create missing keys. This assumes that the data\n\
file is correct and that the the number of rows stored\n\
in the index file is correct. Enables --quick.\n\
-e, --extend-check Try to recover every possible row from the data file\n\ -e, --extend-check Try to recover every possible row from the data file\n\
Normally this will also find a lot of garbage rows;\n\ Normally this will also find a lot of garbage rows;\n\
Don't use this option if you are not totally desperate.\n\ Don't use this option if you are not totally desperate.\n\
-f, --force Overwrite old temporary files.\n\ -f, --force Overwrite old temporary files. Add another --force to\n\
avoid 'sort_buffer_size is too small' errors.\n\
In this case we will attempt to do the repair with the\n\
given sort_buffer_size and dynamically allocate\n\
as many management buffers as needed.\n\
-k, --keys-used=# Tell Aria to update only some specific keys. # is a\n\ -k, --keys-used=# Tell Aria to update only some specific keys. # is a\n\
bit mask of which keys to use. This can be used to\n\ bit mask of which keys to use. This can be used to\n\
get faster inserts.\n\ get faster inserts.\n\
...@@ -664,10 +676,13 @@ get_one_option(int optid, ...@@ -664,10 +676,13 @@ get_one_option(int optid,
if (argument == disabled_my_option) if (argument == disabled_my_option)
{ {
check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL; check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE); check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
T_FORCE_SORT_MEMORY);
} }
else else
{ {
if (check_param.testflag & T_FORCE_CREATE)
check_param.testflag= T_FORCE_SORT_MEMORY;
check_param.tmpfile_createflag= O_RDWR | O_TRUNC; check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE; check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
} }
...@@ -720,8 +735,26 @@ get_one_option(int optid, ...@@ -720,8 +735,26 @@ get_one_option(int optid,
if (argument == disabled_my_option) if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS); check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
else else
{
/*
If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
then add T_FORCE_UNIQUENESS.
*/
check_param.testflag|= check_param.testflag|=
(check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK; ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
}
break;
case OPT_CREATE_MISSING_KEYS:
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
else
{
check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
/* Use repair by sort by default */
if (!(check_param.testflag & T_REP_ANY))
check_param.testflag|= T_REP_BY_SORT;
}
break; break;
case 'u': case 'u':
if (argument == disabled_my_option) if (argument == disabled_my_option)
......
storage/maria/maria_def.h
View file @ 53d44ad1
...@@ -94,9 +94,10 @@ typedef struct st_maria_sort_param ...@@ -94,9 +94,10 @@ typedef struct st_maria_sort_param
*/ */
ulonglong unique[HA_MAX_KEY_SEG+1]; ulonglong unique[HA_MAX_KEY_SEG+1];
ulonglong notnull[HA_MAX_KEY_SEG+1]; ulonglong notnull[HA_MAX_KEY_SEG+1];
ulonglong sortbuff_size;
MARIA_RECORD_POS pos,max_pos,filepos,start_recpos, current_filepos; MARIA_RECORD_POS pos,max_pos,filepos,start_recpos, current_filepos;
uint key, key_length,real_key_length,sortbuff_size; uint key, key_length,real_key_length;
uint maxbuffers, keys, find_length, sort_keys_length; uint maxbuffers, keys, find_length, sort_keys_length;
my_bool fix_datafile, master; my_bool fix_datafile, master;
my_bool calc_checksum; /* calculate table checksum */ my_bool calc_checksum; /* calculate table checksum */
...@@ -107,10 +108,10 @@ typedef struct st_maria_sort_param ...@@ -107,10 +108,10 @@ typedef struct st_maria_sort_param
int (*key_write)(struct st_maria_sort_param *, const uchar *); int (*key_write)(struct st_maria_sort_param *, const uchar *);
void (*lock_in_memory)(HA_CHECK *); void (*lock_in_memory)(HA_CHECK *);
int (*write_keys)(struct st_maria_sort_param *, register uchar **, int (*write_keys)(struct st_maria_sort_param *, register uchar **,
uint , struct st_buffpek *, IO_CACHE *); ulonglong , struct st_buffpek *, IO_CACHE *);
uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,uchar *, int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,uchar *,
uint, uint); uint, ulonglong);
} MARIA_SORT_PARAM; } MARIA_SORT_PARAM;
int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile); int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile);
...@@ -1223,10 +1224,11 @@ typedef struct st_maria_block_info ...@@ -1223,10 +1224,11 @@ typedef struct st_maria_block_info
#define UPDATE_AUTO_INC 8 #define UPDATE_AUTO_INC 8
#define UPDATE_OPEN_COUNT 16 #define UPDATE_OPEN_COUNT 16
#define USE_BUFFER_INIT (((1024L*1024L*128-MALLOC_OVERHEAD)/8192)*8192) /* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD) #define PAGE_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 8192)
#define SORT_BUFFER_INIT (1024L*1024L*256-MALLOC_OVERHEAD) #define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD) #define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER 4096
#define fast_ma_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _ma_writeinfo((INFO),0) #define fast_ma_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _ma_writeinfo((INFO),0)
#define fast_ma_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _ma_readinfo((INFO),F_RDLCK,1) #define fast_ma_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _ma_readinfo((INFO),F_RDLCK,1)
......
storage/maria/maria_ftdump.c
View file @ 53d44ad1
...@@ -84,7 +84,7 @@ int main(int argc,char *argv[]) ...@@ -84,7 +84,7 @@ int main(int argc,char *argv[])
usage(); usage();
} }
init_pagecache(maria_pagecache, USE_BUFFER_INIT, 0, 0, init_pagecache(maria_pagecache, PAGE_BUFFER_INIT, 0, 0,
MARIA_KEY_BLOCK_LENGTH, MY_WME); MARIA_KEY_BLOCK_LENGTH, MY_WME);
if (!(info=maria_open(argv[0], O_RDONLY, if (!(info=maria_open(argv[0], O_RDONLY,
......
storage/maria/maria_read_log.c
View file @ 53d44ad1
...@@ -32,7 +32,8 @@ const char *default_dbug_option= "d:t:o,/tmp/aria_read_log.trace"; ...@@ -32,7 +32,8 @@ const char *default_dbug_option= "d:t:o,/tmp/aria_read_log.trace";
static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent; static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent;
static my_bool opt_check; static my_bool opt_check;
static const char *opt_tmpdir; static const char *opt_tmpdir;
static ulong opt_page_buffer_size, opt_translog_buffer_size; static ulong opt_translog_buffer_size;
static ulonglong opt_page_buffer_size;
static ulonglong opt_start_from_lsn, opt_end_lsn, opt_start_from_checkpoint; static ulonglong opt_start_from_lsn, opt_end_lsn, opt_start_from_checkpoint;
static MY_TMPDIR maria_chk_tmpdir; static MY_TMPDIR maria_chk_tmpdir;
...@@ -204,9 +205,8 @@ static struct my_option my_long_options[] = ...@@ -204,9 +205,8 @@ static struct my_option my_long_options[] =
{ "page-buffer-size", 'P', { "page-buffer-size", 'P',
"The size of the buffer used for index blocks for Aria tables", "The size of the buffer used for index blocks for Aria tables",
&opt_page_buffer_size, &opt_page_buffer_size, 0, &opt_page_buffer_size, &opt_page_buffer_size, 0,
GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, GET_ULL, REQUIRED_ARG, PAGE_BUFFER_INIT,
1024L*1024L, (long) ~(ulong) 0, (long) MALLOC_OVERHEAD, PAGE_BUFFER_INIT, SIZE_T_MAX, MALLOC_OVERHEAD, (long) IO_SIZE, 0},
(long) IO_SIZE, 0},
{ "start-from-lsn", 'o', "Start reading log from this lsn", { "start-from-lsn", 'o', "Start reading log from this lsn",
&opt_start_from_lsn, &opt_start_from_lsn, &opt_start_from_lsn, &opt_start_from_lsn,
0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 }, 0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 },
......
storage/myisam/ha_myisam.cc
View file @ 53d44ad1
...@@ -78,7 +78,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, ...@@ -78,7 +78,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
"The buffer that is allocated when sorting the index when doing " "The buffer that is allocated when sorting the index when doing "
"a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL, "a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL,
8192 * 1024, MIN_SORT_BUFFER + MALLOC_OVERHEAD, SIZE_T_MAX, 1); SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG, static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG,
"Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE); "Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE);
......
storage/myisam/mi_check.c
View file @ 53d44ad1
...@@ -91,7 +91,7 @@ void myisamchk_init(HA_CHECK *param) ...@@ -91,7 +91,7 @@ void myisamchk_init(HA_CHECK *param)
param->opt_follow_links=1; param->opt_follow_links=1;
param->keys_in_use= ~(ulonglong) 0; param->keys_in_use= ~(ulonglong) 0;
param->search_after_block=HA_OFFSET_ERROR; param->search_after_block=HA_OFFSET_ERROR;
param->use_buffers=USE_BUFFER_INIT; param->use_buffers= KEY_BUFFER_INIT;
param->read_buffer_length=READ_BUFFER_INIT; param->read_buffer_length=READ_BUFFER_INIT;
param->write_buffer_length=READ_BUFFER_INIT; param->write_buffer_length=READ_BUFFER_INIT;
param->sort_buffer_length=SORT_BUFFER_INIT; param->sort_buffer_length=SORT_BUFFER_INIT;
......
storage/myisam/myisam_ftdump.c
View file @ 53d44ad1
...@@ -84,7 +84,7 @@ int main(int argc,char *argv[]) ...@@ -84,7 +84,7 @@ int main(int argc,char *argv[])
usage(); usage();
} }
init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0, 0); init_key_cache(dflt_key_cache, MI_KEY_BLOCK_LENGTH, KEY_BUFFER_INIT, 0, 0, 0);
if (!(info=mi_open(argv[0], O_RDONLY, if (!(info=mi_open(argv[0], O_RDONLY,
HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER))) HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER)))
......
storage/myisam/myisamchk.c
View file @ 53d44ad1
...@@ -134,7 +134,7 @@ int main(int argc, char **argv) ...@@ -134,7 +134,7 @@ int main(int argc, char **argv)
enum options_mc { enum options_mc {
OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS, OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
OPT_CORRECT_CHECKSUM, OPT_KEY_BUFFER_SIZE, OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_KEY_BUFFER_SIZE,
OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE, OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE,
OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE, OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN, OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
...@@ -165,6 +165,11 @@ static struct my_option my_long_options[] = ...@@ -165,6 +165,11 @@ static struct my_option my_long_options[] =
{"correct-checksum", OPT_CORRECT_CHECKSUM, {"correct-checksum", OPT_CORRECT_CHECKSUM,
"Correct checksum information for table.", "Correct checksum information for table.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"create-missing-keys", OPT_CREATE_MISSING_KEYS,
"Create missing keys. This assumes that the data file is correct and that "
"the the number of rows stored in the index file is correct. Enables "
"--quick",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF #ifndef DBUG_OFF
{"debug", '#', {"debug", '#',
"Output debug log. Often this is 'd:t:o,filename'.", "Output debug log. Often this is 'd:t:o,filename'.",
...@@ -270,7 +275,7 @@ static struct my_option my_long_options[] = ...@@ -270,7 +275,7 @@ static struct my_option my_long_options[] =
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{ "key_buffer_size", OPT_KEY_BUFFER_SIZE, "", { "key_buffer_size", OPT_KEY_BUFFER_SIZE, "",
&check_param.use_buffers, &check_param.use_buffers, 0, &check_param.use_buffers, &check_param.use_buffers, 0,
GET_ULL, REQUIRED_ARG, USE_BUFFER_INIT, MALLOC_OVERHEAD, GET_ULL, REQUIRED_ARG, KEY_BUFFER_INIT, MALLOC_OVERHEAD,
SIZE_T_MAX, MALLOC_OVERHEAD, IO_SIZE, 0}, SIZE_T_MAX, MALLOC_OVERHEAD, IO_SIZE, 0},
{ "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE, "", { "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE, "",
&opt_key_cache_block_size, &opt_key_cache_block_size,
...@@ -395,10 +400,18 @@ static void usage(void) ...@@ -395,10 +400,18 @@ static void usage(void)
-e, --extend-check Try to recover every possible row from the data file\n\ -e, --extend-check Try to recover every possible row from the data file\n\
Normally this will also find a lot of garbage rows;\n\ Normally this will also find a lot of garbage rows;\n\
Don't use this option if you are not totally desperate.\n\ Don't use this option if you are not totally desperate.\n\
-f, --force Overwrite old temporary files.\n\ -f, --force Overwrite old temporary files. Add another --force to\n\
avoid 'myisam_sort_buffer_size is too small' errors.\n\
In this case we will attempt to do the repair with the\n\
given myisam_sort_buffer_size and dynamically allocate\n\
as many management buffers as needed.\n\
-k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\ -k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\
bit mask of which keys to use. This can be used to\n\ bit mask of which keys to use. This can be used to\n\
get faster inserts.\n\ get faster inserts.\n\
--create-missing-keys\n\
Create missing keys. This assumes that the data\n\
file is correct and that the the number of rows stored\n\
in the index file is correct. Enables --quick\n\
--max-record-length=#\n\ --max-record-length=#\n\
Skip rows bigger than this if myisamchk can't allocate\n\ Skip rows bigger than this if myisamchk can't allocate\n\
memory to hold it.\n\ memory to hold it.\n\
...@@ -541,10 +554,13 @@ get_one_option(int optid, ...@@ -541,10 +554,13 @@ get_one_option(int optid,
if (argument == disabled_my_option) if (argument == disabled_my_option)
{ {
check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL; check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE); check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
T_FORCE_SORT_MEMORY);
} }
else else
{ {
if (check_param.testflag & T_FORCE_CREATE)
check_param.testflag= T_FORCE_SORT_MEMORY;
check_param.tmpfile_createflag= O_RDWR | O_TRUNC; check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE; check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
} }
...@@ -597,8 +613,26 @@ get_one_option(int optid, ...@@ -597,8 +613,26 @@ get_one_option(int optid,
if (argument == disabled_my_option) if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS); check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
else else
{
/*
If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
then add T_FORCE_UNIQUENESS.
*/
check_param.testflag|= check_param.testflag|=
(check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK; ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
}
break;
case OPT_CREATE_MISSING_KEYS:
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
else
{
check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
/* Use repair by sort by default */
if (!(check_param.testflag & T_REP_ANY))
check_param.testflag|= T_REP_BY_SORT;
}
break; break;
case 'u': case 'u':
if (argument == disabled_my_option) if (argument == disabled_my_option)
......
storage/myisam/myisamdef.h
View file @ 53d44ad1
...@@ -629,10 +629,11 @@ typedef struct st_mi_block_info /* Parameter to _mi_get_block_info */ ...@@ -629,10 +629,11 @@ typedef struct st_mi_block_info /* Parameter to _mi_get_block_info */
#define UPDATE_AUTO_INC 8 #define UPDATE_AUTO_INC 8
#define UPDATE_OPEN_COUNT 16 #define UPDATE_OPEN_COUNT 16
#define USE_BUFFER_INIT (((1024L*512L-MALLOC_OVERHEAD)/IO_SIZE)*IO_SIZE) /* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD) #define KEY_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L-MALLOC_OVERHEAD, IO_SIZE)
#define SORT_BUFFER_INIT (2048L*1024L-MALLOC_OVERHEAD) #define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD) #define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*128L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER 4096
enum myisam_log_commands enum myisam_log_commands
{ {
......
storage/myisam/sort.c
View file @ 53d44ad1
...@@ -34,8 +34,10 @@ ...@@ -34,8 +34,10 @@
#define MERGEBUFF 15 #define MERGEBUFF 15
#define MERGEBUFF2 31 #define MERGEBUFF2 31
#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) #define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
#define DISK_BUFFER_SIZE (IO_SIZE*16) #define DISK_BUFFER_SIZE (IO_SIZE*128)
/* How many keys we can keep in memory */
typedef ulonglong ha_keys;
/* /*
Pointers of functions for store and read keys from temp file Pointers of functions for store and read keys from temp file
...@@ -45,42 +47,42 @@ extern void print_error(const char *fmt,...); ...@@ -45,42 +47,42 @@ extern void print_error(const char *fmt,...);
/* Functions defined in this file */ /* Functions defined in this file */
static ha_rows find_all_keys(MI_SORT_PARAM *info,uint keys, static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, uchar **sort_keys,
DYNAMIC_ARRAY *buffpek,int *maxbuffer, DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
IO_CACHE *tempfile, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions); IO_CACHE *tempfile_for_exceptions);
static int write_keys(MI_SORT_PARAM *info,uchar **sort_keys, static int write_keys(MI_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_key(MI_SORT_PARAM *info, uchar *key, static int write_key(MI_SORT_PARAM *info, uchar *key,
IO_CACHE *tempfile); IO_CACHE *tempfile);
static int write_index(MI_SORT_PARAM *info,uchar * *sort_keys, static int write_index(MI_SORT_PARAM *info,uchar * *sort_keys,
uint count); ha_keys count);
static int merge_many_buff(MI_SORT_PARAM *info,uint keys, static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
uchar * *sort_keys, uchar * *sort_keys,
BUFFPEK *buffpek,int *maxbuffer, BUFFPEK *buffpek, uint *maxbuffer,
IO_CACHE *t_file); IO_CACHE *t_file);
static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length); uint sort_length);
static int merge_buffers(MI_SORT_PARAM *info,uint keys, static int merge_buffers(MI_SORT_PARAM *info, ha_keys keys,
IO_CACHE *from_file, IO_CACHE *to_file, IO_CACHE *from_file, IO_CACHE *to_file,
uchar * *sort_keys, BUFFPEK *lastbuff, uchar * *sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb); BUFFPEK *Fb, BUFFPEK *Tb);
static int merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, static int merge_index(MI_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
IO_CACHE *); IO_CACHE *);
static int flush_ft_buf(MI_SORT_PARAM *info); static int flush_ft_buf(MI_SORT_PARAM *info);
static int write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys, static int write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek, ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile); IO_CACHE *tempfile);
static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length); uint sort_length);
static int write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file, static int write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,
uchar *key, uint sort_length, uint count); uchar *key, uint sort_length, ha_keys count);
static int write_merge_key_varlen(MI_SORT_PARAM *info, static int write_merge_key_varlen(MI_SORT_PARAM *info,
IO_CACHE *to_file, IO_CACHE *to_file,
uchar* key, uint sort_length, uchar* key, uint sort_length,
uint count); ha_keys count);
static inline int static inline int
my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
...@@ -101,15 +103,15 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); ...@@ -101,15 +103,15 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
ulonglong sortbuff_size) ulonglong sortbuff_size)
{ {
int error,maxbuffer,skr; int error;
uint sort_length, keys; uint sort_length, maxbuffer;
ulonglong memavl, old_memavl; ulonglong memavl, old_memavl;
DYNAMIC_ARRAY buffpek; DYNAMIC_ARRAY buffpek;
ha_rows records; ha_rows records, keys;
uchar **sort_keys; uchar **sort_keys;
IO_CACHE tempfile, tempfile_for_exceptions; IO_CACHE tempfile, tempfile_for_exceptions;
DBUG_ENTER("_create_index_by_sort"); DBUG_ENTER("_create_index_by_sort");
DBUG_PRINT("enter",("sort_length: %d", info->key_length)); DBUG_PRINT("enter",("sort_length: %u", info->key_length));
if (info->keyinfo->flag & HA_VAR_LENGTH_KEY) if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
{ {
...@@ -135,37 +137,55 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ...@@ -135,37 +137,55 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
sort_length= info->key_length; sort_length= info->key_length;
LINT_INIT(keys); LINT_INIT(keys);
if ((memavl - sizeof(BUFFPEK)) / (sort_length + sizeof(char *)) > UINT_MAX32)
memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
while (memavl >= MIN_SORT_BUFFER) while (memavl >= MIN_SORT_BUFFER)
{ {
if ((records < UINT_MAX32) && /* Check if we can fit all keys into memory */
((my_off_t) (records + 1) * if (((ulonglong) (records + 1) *
(sort_length + sizeof(char*)) <= (my_off_t) memavl)) (sort_length + sizeof(char*)) <= memavl))
keys= (uint)records+1; keys= records+1;
else if ((info->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
}
else else
{
/*
All keys can't fit in memory.
Calculate how many keys + buffers we can keep in memory
*/
uint maxbuffer_org;
do do
{ {
skr=maxbuffer; maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer || if (memavl < sizeof(BUFFPEK) * maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/ (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 || (sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer) keys < maxbuffer)
{ {
mi_check_print_error(info->sort_info->param, mi_check_print_error(info->sort_info->param,
"myisam_sort_buffer_size is too small"); "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) records,
sort_length);
my_errno= ENOMEM; my_errno= ENOMEM;
goto err; goto err;
} }
} }
while ((maxbuffer= (int) (records/(keys-1)+1)) != skr); while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+ if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+
HA_FT_MAXBYTELEN, MYF(0)))) HA_FT_MAXBYTELEN, MYF(0))))
{ {
if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer, if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
maxbuffer/2, MYF(0))) min(maxbuffer/2, 1000), MYF(0)))
{ {
my_free(sort_keys); my_free(sort_keys);
sort_keys= 0; sort_keys= 0;
...@@ -179,24 +199,30 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ...@@ -179,24 +199,30 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
} }
if (memavl < MIN_SORT_BUFFER) if (memavl < MIN_SORT_BUFFER)
{ {
mi_check_print_error(info->sort_info->param,"MyISAM sort buffer too small"); /* purecov: tested */ /* purecov: begin inspected */
my_errno= ENOMEM; /* purecov: tested */ mi_check_print_error(info->sort_info->param,
goto err; /* purecov: tested */ "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) records, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
} }
(*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */ (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
if (!no_messages) if (!no_messages)
printf(" - Searching for keys, allocating buffer for %d keys\n",keys); my_fprintf(stdout,
" - Searching for keys, allocating buffer for %llu keys\n",
(ulonglong) keys);
if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, if ((records= find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions)) &tempfile,&tempfile_for_exceptions))
== HA_POS_ERROR) == HA_POS_ERROR)
goto err; /* purecov: tested */ goto err; /* purecov: tested */
if (maxbuffer == 0) if (maxbuffer == 0)
{ {
if (!no_messages) if (!no_messages)
printf(" - Dumping %lu keys\n", (ulong) records); my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records);
if (write_index(info,sort_keys, (uint) records)) if (write_index(info, sort_keys, (ha_keys) records))
goto err; /* purecov: inspected */ goto err; /* purecov: inspected */
} }
else else
...@@ -205,7 +231,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ...@@ -205,7 +231,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
if (maxbuffer >= MERGEBUFF2) if (maxbuffer >= MERGEBUFF2)
{ {
if (!no_messages) if (!no_messages)
printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */ my_fprintf(stdout, " - Merging %llu keys\n",
(ulonglong) records); /* purecov: tested */
if (merge_many_buff(info,keys,sort_keys, if (merge_many_buff(info,keys,sort_keys,
dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
goto err; /* purecov: inspected */ goto err; /* purecov: inspected */
...@@ -259,13 +286,13 @@ err: ...@@ -259,13 +286,13 @@ err:
/* Search after all keys and place them in a temp. file */ /* Search after all keys and place them in a temp. file */
static ha_rows find_all_keys(MI_SORT_PARAM *info, uint keys, static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_rows keys,
uchar **sort_keys, DYNAMIC_ARRAY *buffpek, uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
int *maxbuffer, IO_CACHE *tempfile, uint *maxbuffer, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions) IO_CACHE *tempfile_for_exceptions)
{ {
int error; int error;
uint idx; ha_rows idx;
DBUG_ENTER("find_all_keys"); DBUG_ENTER("find_all_keys");
idx=error=0; idx=error=0;
...@@ -314,9 +341,10 @@ pthread_handler_t thr_find_all_keys(void *arg) ...@@ -314,9 +341,10 @@ pthread_handler_t thr_find_all_keys(void *arg)
{ {
MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg; MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg;
int error; int error;
ulonglong memavl, old_memavl; ulonglong memavl, old_memavl, sortbuff_size;
uint keys, sort_length; ha_keys keys, idx;
uint idx, maxbuffer; uint sort_length;
uint maxbuffer;
uchar **sort_keys=0; uchar **sort_keys=0;
LINT_INIT(keys); LINT_INIT(keys);
...@@ -351,37 +379,47 @@ pthread_handler_t thr_find_all_keys(void *arg) ...@@ -351,37 +379,47 @@ pthread_handler_t thr_find_all_keys(void *arg)
bzero((char*) &sort_param->unique, sizeof(sort_param->unique)); bzero((char*) &sort_param->unique, sizeof(sort_param->unique));
sort_keys= (uchar **) NULL; sort_keys= (uchar **) NULL;
memavl= max(sort_param->sortbuff_size, MIN_SORT_BUFFER); sortbuff_size= sort_param->sortbuff_size;
idx= (uint)sort_param->sort_info->max_records; memavl= max(sortbuff_size, MIN_SORT_BUFFER);
idx= (ha_keys) sort_param->sort_info->max_records;
sort_length= sort_param->key_length; sort_length= sort_param->key_length;
maxbuffer= 1; maxbuffer= 1;
if ((memavl - sizeof(BUFFPEK)) / (sort_length +
sizeof(char *)) > UINT_MAX32)
memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
while (memavl >= MIN_SORT_BUFFER) while (memavl >= MIN_SORT_BUFFER)
{ {
if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <=
(my_off_t) memavl) (my_off_t) memavl)
keys= idx+1; keys= idx+1;
else if ((sort_param->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
}
else else
{ {
uint skr; uint maxbuffer_org;
do do
{ {
skr= maxbuffer; maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*maxbuffer || if (memavl < sizeof(BUFFPEK)*maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/ (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 || (sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer) keys < (uint) maxbuffer)
{ {
mi_check_print_error(sort_param->sort_info->param, mi_check_print_error(sort_param->sort_info->param,
"myisam_sort_buffer_size is too small"); "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
goto err; goto err;
} }
} }
while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr); while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
} }
if ((sort_keys= (uchar**) if ((sort_keys= (uchar**)
my_malloc(keys*(sort_length+sizeof(char*))+ my_malloc(keys*(sort_length+sizeof(char*))+
...@@ -389,7 +427,7 @@ pthread_handler_t thr_find_all_keys(void *arg) ...@@ -389,7 +427,7 @@ pthread_handler_t thr_find_all_keys(void *arg)
HA_FT_MAXBYTELEN : 0), MYF(0)))) HA_FT_MAXBYTELEN : 0), MYF(0))))
{ {
if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK), if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
maxbuffer, maxbuffer/2, MYF(0))) maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
{ {
my_free(sort_keys); my_free(sort_keys);
sort_keys= (uchar **) NULL; /* for err: label */ sort_keys= (uchar **) NULL; /* for err: label */
...@@ -404,14 +442,19 @@ pthread_handler_t thr_find_all_keys(void *arg) ...@@ -404,14 +442,19 @@ pthread_handler_t thr_find_all_keys(void *arg)
} }
if (memavl < MIN_SORT_BUFFER) if (memavl < MIN_SORT_BUFFER)
{ {
/* purecov: begin inspected */
mi_check_print_error(sort_param->sort_info->param, mi_check_print_error(sort_param->sort_info->param,
"MyISAM sort buffer too small"); "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
goto err; /* purecov: tested */ sortbuff_size, (ulonglong) idx, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
} }
if (sort_param->sort_info->param->testflag & T_VERBOSE) if (sort_param->sort_info->param->testflag & T_VERBOSE)
printf("Key %d - Allocating buffer for %d keys\n", my_fprintf(stdout,
sort_param->key + 1, keys); "Key %d - Allocating buffer for %llu keys\n",
sort_param->key + 1, (ulonglong) keys);
sort_param->sort_keys= sort_keys; sort_param->sort_keys= sort_keys;
idx= error= 0; idx= error= 0;
...@@ -436,7 +479,8 @@ pthread_handler_t thr_find_all_keys(void *arg) ...@@ -436,7 +479,8 @@ pthread_handler_t thr_find_all_keys(void *arg)
&sort_param->tempfile)) &sort_param->tempfile))
goto err; goto err;
sort_keys[0]= (uchar*) (sort_keys+keys); sort_keys[0]= (uchar*) (sort_keys+keys);
memcpy(sort_keys[0], sort_keys[idx - 1], (size_t) sort_param->key_length); memcpy(sort_keys[0], sort_keys[idx - 1],
(size_t) sort_param->key_length);
idx= 1; idx= 1;
} }
sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length; sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
...@@ -454,7 +498,6 @@ pthread_handler_t thr_find_all_keys(void *arg) ...@@ -454,7 +498,6 @@ pthread_handler_t thr_find_all_keys(void *arg)
else else
sort_param->keys= idx; sort_param->keys= idx;
sort_param->sort_keys_length= keys;
goto ok; goto ok;
err: err:
...@@ -527,7 +570,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) ...@@ -527,7 +570,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
{ {
if (param->testflag & T_VERBOSE) if (param->testflag & T_VERBOSE)
{ {
printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys); my_fprintf(stdout,
"Key %d - Dumping %llu keys\n", sinfo->key+1,
(ulonglong) sinfo->keys);
fflush(stdout); fflush(stdout);
} }
if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) || if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
...@@ -584,10 +629,12 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) ...@@ -584,10 +629,12 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
if (maxbuffer >= MERGEBUFF2) if (maxbuffer >= MERGEBUFF2)
{ {
if (param->testflag & T_VERBOSE) if (param->testflag & T_VERBOSE)
printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys); my_fprintf(stdout,
"Key %d - Merging %llu keys\n",
sinfo->key+1, (ulonglong) sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **)mergebuf, if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
(int*) &maxbuffer, &sinfo->tempfile)) &maxbuffer, &sinfo->tempfile))
{ {
got_error=1; got_error=1;
continue; continue;
...@@ -651,12 +698,15 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) ...@@ -651,12 +698,15 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
/* Write all keys in memory to file for later merge */ /* Write all keys in memory to file for later merge */
static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
{ {
uchar **end; uchar **end;
uint sort_length=info->key_length; uint sort_length=info->key_length;
DBUG_ENTER("write_keys"); DBUG_ENTER("write_keys");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info); info);
if (!my_b_inited(tempfile) && if (!my_b_inited(tempfile) &&
...@@ -669,7 +719,7 @@ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, ...@@ -669,7 +719,7 @@ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
for (end=sort_keys+count ; sort_keys != end ; sort_keys++) for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
{ {
if (my_b_write(tempfile,(uchar*) *sort_keys,(uint) sort_length)) if (my_b_write(tempfile,(uchar*) *sort_keys, sort_length))
DBUG_RETURN(1); /* purecov: inspected */ DBUG_RETURN(1); /* purecov: inspected */
} }
DBUG_RETURN(0); DBUG_RETURN(0);
...@@ -693,13 +743,16 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs) ...@@ -693,13 +743,16 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
static int write_keys_varlen(MI_SORT_PARAM *info, static int write_keys_varlen(MI_SORT_PARAM *info,
register uchar **sort_keys, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile) IO_CACHE *tempfile)
{ {
uchar **end; uchar **end;
int err; int err;
DBUG_ENTER("write_keys_varlen"); DBUG_ENTER("write_keys_varlen");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp, my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info); info);
if (!my_b_inited(tempfile) && if (!my_b_inited(tempfile) &&
...@@ -738,7 +791,7 @@ static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile) ...@@ -738,7 +791,7 @@ static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile)
/* Write index */ /* Write index */
static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys, static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
register uint count) register ha_keys count)
{ {
DBUG_ENTER("write_index"); DBUG_ENTER("write_index");
...@@ -755,11 +808,11 @@ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys, ...@@ -755,11 +808,11 @@ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
/* Merge buffers to make < MERGEBUFF2 buffers */ /* Merge buffers to make < MERGEBUFF2 buffers */
static int merge_many_buff(MI_SORT_PARAM *info, uint keys, static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, BUFFPEK *buffpek, uchar **sort_keys, BUFFPEK *buffpek,
int *maxbuffer, IO_CACHE *t_file) uint *maxbuffer, IO_CACHE *t_file)
{ {
register int i; register uint i;
IO_CACHE t_file2, *from_file, *to_file, *temp; IO_CACHE t_file2, *from_file, *to_file, *temp;
BUFFPEK *lastbuff; BUFFPEK *lastbuff;
DBUG_ENTER("merge_many_buff"); DBUG_ENTER("merge_many_buff");
...@@ -777,7 +830,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys, ...@@ -777,7 +830,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
reinit_io_cache(from_file,READ_CACHE,0L,0,0); reinit_io_cache(from_file,READ_CACHE,0L,0,0);
reinit_io_cache(to_file,WRITE_CACHE,0L,0,0); reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
lastbuff=buffpek; lastbuff=buffpek;
for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
{ {
if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
buffpek+i,buffpek+i+MERGEBUFF-1)) buffpek+i,buffpek+i+MERGEBUFF-1))
...@@ -789,7 +842,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys, ...@@ -789,7 +842,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
if (flush_io_cache(to_file)) if (flush_io_cache(to_file))
break; /* purecov: inspected */ break; /* purecov: inspected */
temp=from_file; from_file=to_file; to_file=temp; temp=from_file; from_file=to_file; to_file=temp;
*maxbuffer= (int) (lastbuff-buffpek)-1; *maxbuffer= (uint) (lastbuff-buffpek)-1;
} }
cleanup: cleanup:
close_cached_file(to_file); /* This holds old result */ close_cached_file(to_file); /* This holds old result */
...@@ -818,35 +871,36 @@ cleanup: ...@@ -818,35 +871,36 @@ cleanup:
-1 Error -1 Error
*/ */
static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length) uint sort_length)
{ {
register uint count; register ha_keys count;
uint length; my_off_t length;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{ {
if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base, if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
(length= sort_length*count), (length= sort_length * count),
buffpek->file_pos, MYF_RW)) buffpek->file_pos, MYF_RW))
return((uint) -1); /* purecov: inspected */ return(HA_OFFSET_ERROR); /* purecov: inspected */
buffpek->key=buffpek->base; buffpek->key=buffpek->base;
buffpek->file_pos+= length; /* New filepos */ buffpek->file_pos+= length; /* New filepos */
buffpek->count-= count; buffpek->count-= count;
buffpek->mem_count= count; buffpek->mem_count= count;
} }
return (count*sort_length); return (((my_off_t) count) * sort_length);
} /* read_to_buffer */ } /* read_to_buffer */
static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length) static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{ {
register uint count; register ha_keys count;
uint16 length_of_key = 0; uint16 length_of_key = 0;
uint idx; uint idx;
uchar *buffp; uchar *buffp;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{ {
buffp = buffpek->base; buffp = buffpek->base;
...@@ -854,11 +908,11 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, ...@@ -854,11 +908,11 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
{ {
if (mysql_file_pread(fromfile->file, (uchar*)&length_of_key, if (mysql_file_pread(fromfile->file, (uchar*)&length_of_key,
sizeof(length_of_key), buffpek->file_pos, MYF_RW)) sizeof(length_of_key), buffpek->file_pos, MYF_RW))
return((uint) -1); return(HA_OFFSET_ERROR);
buffpek->file_pos+=sizeof(length_of_key); buffpek->file_pos+=sizeof(length_of_key);
if (mysql_file_pread(fromfile->file, (uchar*) buffp, if (mysql_file_pread(fromfile->file, (uchar*) buffp,
length_of_key, buffpek->file_pos, MYF_RW)) length_of_key, buffpek->file_pos, MYF_RW))
return((uint) -1); return(HA_OFFSET_ERROR);
buffpek->file_pos+=length_of_key; buffpek->file_pos+=length_of_key;
buffp = buffp + sort_length; buffp = buffp + sort_length;
} }
...@@ -866,15 +920,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, ...@@ -866,15 +920,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
buffpek->count-= count; buffpek->count-= count;
buffpek->mem_count= count; buffpek->mem_count= count;
} }
return (count*sort_length); return (((my_off_t) count) * sort_length);
} /* read_to_buffer_varlen */ } /* read_to_buffer_varlen */
static int write_merge_key_varlen(MI_SORT_PARAM *info, static int write_merge_key_varlen(MI_SORT_PARAM *info,
IO_CACHE *to_file, uchar* key, IO_CACHE *to_file, uchar* key,
uint sort_length, uint count) uint sort_length, ha_keys count)
{ {
uint idx; ha_keys idx;
uchar *bufs = key; uchar *bufs = key;
for (idx=1;idx<=count;idx++) for (idx=1;idx<=count;idx++)
...@@ -890,32 +944,37 @@ static int write_merge_key_varlen(MI_SORT_PARAM *info, ...@@ -890,32 +944,37 @@ static int write_merge_key_varlen(MI_SORT_PARAM *info,
static int write_merge_key(MI_SORT_PARAM *info __attribute__((unused)), static int write_merge_key(MI_SORT_PARAM *info __attribute__((unused)),
IO_CACHE *to_file, uchar *key, IO_CACHE *to_file, uchar *key,
uint sort_length, uint count) uint sort_length, ha_keys count)
{ {
return my_b_write(to_file, key, (size_t) sort_length*count); return my_b_write(to_file, key, ((size_t) sort_length) * count);
} }
/* /*
Merge buffers to one buffer Merge buffers to one buffer
If to_file == 0 then use info->key_write If to_file == 0 then use info->key_write
Return:
0 ok
1 error
*/ */
static int static int
merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, merge_buffers(MI_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb) BUFFPEK *Fb, BUFFPEK *Tb)
{ {
int error; int error= 1;
uint sort_length,maxcount; uint sort_length;
ha_keys maxcount;
ha_rows count; ha_rows count;
my_off_t UNINIT_VAR(to_start_filepos); my_off_t UNINIT_VAR(to_start_filepos), read_length;
uchar *strpos; uchar *strpos;
BUFFPEK *buffpek,**refpek; BUFFPEK *buffpek,**refpek;
QUEUE queue; QUEUE queue;
DBUG_ENTER("merge_buffers"); DBUG_ENTER("merge_buffers");
count=error=0; count=error=0;
maxcount=keys/((uint) (Tb-Fb) +1); maxcount= keys/((uint) (Tb-Fb) +1);
DBUG_ASSERT(maxcount > 0); DBUG_ASSERT(maxcount > 0);
LINT_INIT(to_start_filepos); LINT_INIT(to_start_filepos);
if (to_file) if (to_file)
...@@ -932,10 +991,10 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -932,10 +991,10 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{ {
count+= buffpek->count; count+= buffpek->count;
buffpek->base= (uchar*) strpos; buffpek->base= (uchar*) strpos;
buffpek->max_keys=maxcount; buffpek->max_keys= maxcount;
strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek, strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
sort_length)); sort_length));
if (error == -1) if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */ goto err; /* purecov: inspected */
queue_insert(&queue,(uchar*) buffpek); queue_insert(&queue,(uchar*) buffpek);
} }
...@@ -966,10 +1025,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -966,10 +1025,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
/* It's enough to check for killedptr before a slow operation */ /* It's enough to check for killedptr before a slow operation */
if (killed_ptr(info->sort_info->param)) if (killed_ptr(info->sort_info->param))
{ {
error=1;
goto err; goto err;
} }
if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length))) if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
{ {
uchar *base= buffpek->base; uchar *base= buffpek->base;
uint max_keys=buffpek->max_keys; uint max_keys=buffpek->max_keys;
...@@ -996,9 +1054,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -996,9 +1054,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
} }
break; /* One buffer have been removed */ break; /* One buffer have been removed */
} }
else if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
} }
else if (error == -1)
goto err; /* purecov: inspected */
queue_replace_top(&queue); /* Top element has been replaced */ queue_replace_top(&queue); /* Top element has been replaced */
} }
} }
...@@ -1030,8 +1088,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, ...@@ -1030,8 +1088,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
} }
} }
} }
while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != -1 && while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
error != 0); if (read_length == 0)
error= 0;
lastbuff->count=count; lastbuff->count=count;
if (to_file) if (to_file)
...@@ -1045,8 +1104,8 @@ err: ...@@ -1045,8 +1104,8 @@ err:
/* Do a merge to output-file (save only positions) */ /* Do a merge to output-file (save only positions) */
static int static int
merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, merge_index(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
{ {
DBUG_ENTER("merge_index"); DBUG_ENTER("merge_index");
if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,
...@@ -1055,6 +1114,7 @@ merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, ...@@ -1055,6 +1114,7 @@ merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
DBUG_RETURN(0); DBUG_RETURN(0);
} /* merge_index */ } /* merge_index */
static int static int
flush_ft_buf(MI_SORT_PARAM *info) flush_ft_buf(MI_SORT_PARAM *info)
{ {
...@@ -1067,4 +1127,3 @@ flush_ft_buf(MI_SORT_PARAM *info) ...@@ -1067,4 +1127,3 @@ flush_ft_buf(MI_SORT_PARAM *info)
} }
return err; return err;
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7