Skip to content

M
MariaDB
Commit 6edcb18a authored by Rich Prohaska's avatar Rich Prohaska Committed by Yoni Fogel
Browse files
Options

closes[t:2639] recover from translation table realloc failures in the allocate... 

closes[t:2639] recover from translation table realloc failures in the allocate block function of the brtloader. merge -r 20450:head from tokudb.2639

git-svn-id: file:///svn/toku/tokudb@20466 c7de825b-a66e-492c-adef-691d508d4ae1
parent 73c323a8
Hide whitespace changes
Inline Side-by-side
Showing with 149 additions and 79 deletions
newbrt/brt-serialize.c
View file @ 6edcb18a
......@@ -473,6 +473,7 @@ serialize_uncompressed_block_to_memory(char * uncompressed_buf,
int
toku_serialize_brtnode_to_memory (BRTNODE node, int UU(n_workitems), int UU(n_threads), /*out*/ size_t *n_bytes_to_write, /*out*/ char **bytes_to_write) {
int result = 0;
// get the size of the serialized node
size_t calculated_size = toku_serialize_brtnode_size(node);
......@@ -492,18 +493,22 @@ toku_serialize_brtnode_to_memory (BRTNODE node, int UU(n_workitems), int UU(n_th
// allocate space for the serialized node
char *MALLOC_N(calculated_size, buf);
//toku_verify_counts(node);
//assert(size>0);
//printf("%s:%d serializing %lld w height=%d p0=%p\n", __FILE__, __LINE__, off, node->height, node->mdicts[0]);
if (buf == NULL)
result = errno;
else {
//toku_verify_counts(node);
//assert(size>0);
//printf("%s:%d serializing %lld w height=%d p0=%p\n", __FILE__, __LINE__, off, node->height, node->mdicts[0]);
// serialize the node into buf
serialize_node(node, buf, calculated_size, n_sub_blocks, sub_block);
// serialize the node into buf
serialize_node(node, buf, calculated_size, n_sub_blocks, sub_block);
//Compress and malloc buffer to write
serialize_uncompressed_block_to_memory(buf, n_sub_blocks, sub_block,
n_bytes_to_write, bytes_to_write);
toku_free(buf);
return 0;
//Compress and malloc buffer to write
serialize_uncompressed_block_to_memory(buf, n_sub_blocks, sub_block,
n_bytes_to_write, bytes_to_write);
toku_free(buf);
}
return result;
}
int
......
newbrt/brtloader.c
View file @ 6edcb18a
......@@ -261,17 +261,22 @@ int brtloader_open_temp_file (BRTLOADER bl, FIDX *file_idx)
*/
{
int result = 0;
char *fname = toku_strdup(bl->temp_file_template);
FILE *f = NULL;
int fd = mkstemp(fname);
if (fd < 0) {
int fd = -1;
char *fname = toku_strdup(bl->temp_file_template);
if (fname == NULL)
result = errno;
} else {
f = toku_os_fdopen(fd, "r+");
if (f == NULL)
else {
fd = mkstemp(fname);
if (fd < 0) {
result = errno;
else
result = open_file_add(&bl->file_infos, f, fname, file_idx);
} else {
f = toku_os_fdopen(fd, "r+");
if (f == NULL)
result = errno;
else
result = open_file_add(&bl->file_infos, f, fname, file_idx);
}
}
if (result != 0) {
if (fd >= 0) {
......@@ -1865,7 +1870,10 @@ static inline void dbout_init(struct dbout *out) {
}
static inline void dbout_destroy(struct dbout *out) {
invariant(out->fd == -1);
if (out->fd >= 0) {
toku_os_close(out->fd);
out->fd = -1;
}
toku_free(out->translation);
out->translation = NULL;
}
......@@ -1918,21 +1926,31 @@ static void dbuf_destroy (struct dbuf *dbuf) {
toku_free(dbuf->buf); dbuf->buf = NULL;
}
static int64_t allocate_block (struct dbout *out)
static int allocate_block (struct dbout *out, int64_t *ret_block_number)
// Return the new block number
{
int result = 0;
dbout_lock(out);
int64_t result = out->n_translations;
if (result >= out->n_translations_limit) {
int64_t block_number = out->n_translations;
if (block_number >= out->n_translations_limit) {
int64_t old_n_translations_limit = out->n_translations_limit;
struct translation *old_translation = out->translation;
if (out->n_translations_limit==0) {
out->n_translations_limit = 1;
} else {
out->n_translations_limit *= 2;
}
REALLOC_N(out->n_translations_limit, out->translation);
lazy_assert(out->translation);
if (out->translation == NULL) {
result = errno;
out->n_translations_limit = old_n_translations_limit;
out->translation = old_translation;
}
}
if (result == 0) {
out->n_translations++;
*ret_block_number = block_number;
}
out->n_translations++;
dbout_unlock(out);
return result;
}
......@@ -2130,7 +2148,9 @@ static int toku_loader_write_brt_from_q (BRTLOADER bl,
out.translation[1].off = -1; // block 1 is the block translation, filled in later
out.translation[2].off = -1; // block 2 is the descriptor
seek_align(&out);
int64_t lblock = allocate_block(&out);
int64_t lblock;
result = allocate_block(&out, &lblock);
lazy_assert(result == 0); // can not fail since translations reserved above
struct leaf_buf *lbuf = start_leaf(&out, descriptor, lblock);
struct subtree_estimates est = zero_estimates;
est.exact = TRUE;
......@@ -2153,6 +2173,7 @@ static int toku_loader_write_brt_from_q (BRTLOADER bl,
}
struct rowset *output_rowset = (struct rowset *)item;
if (result == 0)
for (unsigned int i = 0; i < output_rowset->n_rows; i++) {
DBT key = make_dbt(output_rowset->data+output_rowset->rows[i].off, output_rowset->rows[i].klen);
DBT val = make_dbt(output_rowset->data+output_rowset->rows[i].off + output_rowset->rows[i].klen, output_rowset->rows[i].vlen);
......@@ -2183,12 +2204,19 @@ static int toku_loader_write_brt_from_q (BRTLOADER bl,
if ((r = bl_write_dbt(&key, pivots_stream, NULL, bl))) {
brt_loader_set_panic(bl, r); // error after cilk sync
if (result == 0) result = r;
break;
}
cilk_spawn finish_leafnode(&out, lbuf, progress_this_node, bl);
lbuf = NULL;
lblock = allocate_block(&out);
r = allocate_block(&out, &lblock);
if (r != 0) {
brt_loader_set_panic(bl, r);
if (result == 0) result = r;
break;
}
lbuf = start_leaf(&out, descriptor, lblock);
}
......@@ -2203,11 +2231,13 @@ static int toku_loader_write_brt_from_q (BRTLOADER bl,
toku_free(output_rowset);
}
allocate_node(&sts, lblock, est, lbuf->local_fingerprint);
{
int p = progress_allocation/2;
finish_leafnode(&out, lbuf, p, bl);
progress_allocation -= p;
if (lbuf) {
allocate_node(&sts, lblock, est, lbuf->local_fingerprint);
{
int p = progress_allocation/2;
finish_leafnode(&out, lbuf, p, bl);
progress_allocation -= p;
}
}
cilk_sync;
......@@ -2704,8 +2734,11 @@ static int write_translation_table (struct dbout *out, long long *off_of_transla
}
unsigned int checksum = x1764_memory(ttable.buf, ttable.off);
putbuf_int32(&ttable, checksum);
invariant(bt_size_on_disk==ttable.off);
int result = toku_os_pwrite(out->fd, ttable.buf, ttable.off, off_of_translation);
int result = ttable.error;
if (result == 0) {
invariant(bt_size_on_disk==ttable.off);
result = toku_os_pwrite(out->fd, ttable.buf, ttable.off, off_of_translation);
}
dbuf_destroy(&ttable);
*off_of_translation_p = off_of_translation;
return result;
......@@ -2794,19 +2827,15 @@ static int setup_nonleaf_block (int n_children,
if (result == 0) {
int r = read_some_pivots(pivots_file, n_children, bl, pivots);
if (r) {
delete_pivots(pivots, n_children);
if (r)
result = r;
}
}
if (result == 0) {
FILE *next_pivots_stream = toku_bl_fidx2file(bl, next_pivots_file);
int r = bl_write_dbt(&pivots[n_children-1], next_pivots_stream, NULL, bl);
if (r) {
delete_pivots(pivots, n_children);
if (r)
result = r;
}
}
if (result == 0) {
......@@ -2827,11 +2856,22 @@ static int setup_nonleaf_block (int n_children,
fingerprint += subtrees->subtrees[from_blocknum].fingerprint;
}
*blocknum = allocate_block(out);
allocate_node(next_subtrees, *blocknum, new_subtree_estimates, fingerprint);
int r = allocate_block(out, blocknum);
if (r) {
toku_free(subtrees_array);
result = r;
} else {
allocate_node(next_subtrees, *blocknum, new_subtree_estimates, fingerprint);
*pivots_p = pivots;
*subtrees_info_p = subtrees_array;
}
}
*pivots_p = pivots;
*subtrees_info_p = subtrees_array;
if (result != 0) {
if (pivots) {
delete_pivots(pivots, n_children); pivots = NULL;
}
}
return result;
......@@ -2860,10 +2900,15 @@ static void write_nonleaf_node (BRTLOADER bl, struct dbout *out, int64_t blocknu
node->rand4fingerprint = loader_random();
XMALLOC_N(n_children-1, node->u.n.childkeys);
for (int i=0; i<n_children-1; i++)
node->u.n.childkeys[i] = NULL;
unsigned int totalchildkeylens = 0;
for (int i=0; i<n_children-1; i++) {
struct kv_pair *childkey = kv_pair_malloc(pivots[i].data, pivots[i].size, NULL, 0);
lazy_assert(childkey);
if (childkey == NULL) {
result = errno;
break;
}
node->u.n.childkeys[i] = childkey;
totalchildkeylens += kv_pair_keylen(childkey);
}
......@@ -2882,27 +2927,29 @@ static void write_nonleaf_node (BRTLOADER bl, struct dbout *out, int64_t blocknu
ci->n_bytes_in_buffer = 0;
}
size_t n_bytes;
char *bytes;
int r;
r = toku_serialize_brtnode_to_memory(node, 1, 1, &n_bytes, &bytes);
if (r) {
result = r;
} else {
dbout_lock(out);
out->translation[blocknum_of_new_node].off = out->current_off;
out->translation[blocknum_of_new_node].size = n_bytes;
//fprintf(stderr, "Wrote internal node at %ld (%ld bytes)\n", out->current_off, n_bytes);
//for (uint32_t i=0; i<n_bytes; i++) { unsigned char b = bytes[i]; printf("%d:%02x (%d) ('%c')\n", i, b, b, (b>=' ' && b<128) ? b : '*'); }
r = write_literal(out, bytes, n_bytes);
if (r)
if (result == 0) {
size_t n_bytes;
char *bytes;
int r;
r = toku_serialize_brtnode_to_memory(node, 1, 1, &n_bytes, &bytes);
if (r) {
result = r;
else
seek_align_locked(out);
dbout_unlock(out);
} else {
dbout_lock(out);
out->translation[blocknum_of_new_node].off = out->current_off;
out->translation[blocknum_of_new_node].size = n_bytes;
//fprintf(stderr, "Wrote internal node at %ld (%ld bytes)\n", out->current_off, n_bytes);
//for (uint32_t i=0; i<n_bytes; i++) { unsigned char b = bytes[i]; printf("%d:%02x (%d) ('%c')\n", i, b, b, (b>=' ' && b<128) ? b : '*'); }
r = write_literal(out, bytes, n_bytes);
if (r)
result = r;
else
seek_align_locked(out);
dbout_unlock(out);
toku_free(bytes);
}
}
toku_free(bytes);
for (int i=0; i<n_children-1; i++) {
toku_free(pivots[i].data);
toku_free(node->u.n.childkeys[i]);
......
newbrt/tests/Makefile
View file @ 6edcb18a
......@@ -101,7 +101,7 @@ check_brtloader-test-bad-generate$(BINSUF): EXTRA_ARGS=dir.$@
check_brtloader-test-extractor-errors$(BINSUF): $(patsubst %,check_brtloader-test-extractor-errors-%, 1 2)
true $(SUMMARIZE_CMD)
check_brtloader-test-extractor-errors-1: brtloader-test-extractor-errors$(BINSUF)
$(VGRIND) ./$< -q -f -m -u -r 1 dir.$@ $(SUMMARIZE_CMD)
$(VGRIND) ./$< -q -w -m -u -r 1 dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-extractor-errors-2: brtloader-test-extractor-errors$(BINSUF)
$(VGRIND) ./$< -q -m -r 10000 dir.$@ $(SUMMARIZE_CMD)
......@@ -114,12 +114,14 @@ check_brtloader-test-writer-1: brtloader-test-writer$(BINSUF)
check_brtloader-test-writer-2: brtloader-test-writer$(BINSUF)
$(VGRIND) ./$< -q -r 10000000 dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-writer-errors$(BINSUF): $(patsubst %,check_brtloader-test-writer-errors-%, 1 2)
check_brtloader-test-writer-errors$(BINSUF): $(patsubst %,check_brtloader-test-writer-errors-%, 1 2 3)
true $(SUMMARIZE_CMD)
check_brtloader-test-writer-errors-1: brtloader-test-writer-errors$(BINSUF)
$(VGRIND) ./$< -q -f -m -u -r 1000000 dir.$@ $(SUMMARIZE_CMD)
$(VGRIND) ./$< -q -w -m -u -r 1000000 dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-writer-errors-2: brtloader-test-writer-errors$(BINSUF)
$(VGRIND) ./$< -q -s -f -m -u -r 10000 dir.$@ $(SUMMARIZE_CMD)
$(VGRIND) ./$< -q -s -w -m -u -r 10000 dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-writer-errors-3: brtloader-test-writer-errors$(BINSUF)
$(VGRIND) ./$< -q -s -r 10000 --malloc_limit 0 --realloc_errors dir.$@ $(SUMMARIZE_CMD)
brtloader-%$(BINSUF): brtloader-%.$(OEXT)
ifeq ($(BRTLOADER),cilk)
......
newbrt/tests/brtloader-test-extractor-errors.c
View file @ 6edcb18a
......@@ -82,6 +82,7 @@ static ssize_t bad_pwrite(int fd, const void * bp, size_t len, toku_off_t off) {
static int do_malloc_errors = 0;
static int my_malloc_count = 0, my_big_malloc_count = 0;
static int my_realloc_count = 0, my_big_realloc_count = 0;
static size_t my_big_malloc_limit = 64*1024;
static void reset_my_malloc_counts(void) {
my_malloc_count = my_big_malloc_count = 0;
......@@ -93,7 +94,7 @@ static void *my_malloc(size_t n) {
if (!((void*)toku_malloc <= caller && caller <= (void*)toku_free))
goto skip;
my_malloc_count++;
if (n >= 64*1024) {
if (n >= my_big_malloc_limit) {
my_big_malloc_count++;
if (do_malloc_errors) {
caller = __builtin_return_address(1);
......@@ -116,7 +117,7 @@ static void *my_realloc(void *p, size_t n) {
if (!((void*)toku_realloc <= caller && caller <= (void*)toku_free))
goto skip;
my_realloc_count++;
if (n >= 64*1024) {
if (n >= my_big_malloc_limit) {
my_big_realloc_count++;
if (do_malloc_errors) {
caller = __builtin_return_address(1);
......@@ -254,7 +255,8 @@ static int usage(const char *progname) {
fprintf(stderr, "[--rowsets %d] set the number of rowsets\n", nrowsets);
fprintf(stderr, "[-s] set the small loader size factor\n");
fprintf(stderr, "[-m] inject big malloc and realloc errors\n");
fprintf(stderr, "[-f] inject write errors\n");
fprintf(stderr, "[--malloc_limit %u] set the threshold for failing malloc and realloc\n", (unsigned) my_big_malloc_limit);
fprintf(stderr, "[-w] inject write errors\n");
fprintf(stderr, "[-u] inject user errors\n");
return 1;
}
......@@ -278,12 +280,15 @@ int test_main (int argc, const char *argv[]) {
nrowsets = atoi(argv[0]);
} else if (strcmp(argv[0],"-s") == 0) {
toku_brtloader_set_size_factor(1);
} else if (strcmp(argv[0],"-f") == 0) {
} else if (strcmp(argv[0],"-w") == 0) {
do_write_errors = 1;
} else if (strcmp(argv[0],"-m") == 0) {
do_malloc_errors = 1;
} else if (strcmp(argv[0],"-u") == 0) {
do_user_errors = 1;
} else if (strcmp(argv[0],"--malloc_limit") == 0 && argc > 1) {
argc--; argv++;
my_big_malloc_limit = atoi(argv[0]);
} else if (strcmp(argv[0],"--max_error_limit") == 0 && argc >= 1) {
argc--; argv++;
max_error_limit = atoi(argv[0]);
......
newbrt/tests/brtloader-test-writer-errors.c
View file @ 6edcb18a
......@@ -82,6 +82,7 @@ static ssize_t bad_pwrite(int fd, const void * bp, size_t len, toku_off_t off) {
static int do_malloc_errors = 0;
static int my_malloc_count = 0, my_big_malloc_count = 0;
static int my_realloc_count = 0, my_big_realloc_count = 0;
static size_t my_big_malloc_limit = 64*1024;
static void reset_my_malloc_counts(void) {
my_malloc_count = my_big_malloc_count = 0;
......@@ -93,7 +94,7 @@ static void *my_malloc(size_t n) {
if (!((void*)toku_malloc <= caller && caller <= (void*)toku_free))
goto skip;
my_malloc_count++;
if (n >= 64*1024) {
if (n >= my_big_malloc_limit) {
my_big_malloc_count++;
if (do_malloc_errors) {
caller = __builtin_return_address(1);
......@@ -111,14 +112,16 @@ static void *my_malloc(size_t n) {
return malloc(n);
}
static int do_realloc_errors = 0;
static void *my_realloc(void *p, size_t n) {
void *caller = __builtin_return_address(0);
if (!((void*)toku_realloc <= caller && caller <= (void*)toku_free))
goto skip;
my_realloc_count++;
if (n >= 64*1024) {
if (n >= my_big_malloc_limit) {
my_big_realloc_count++;
if (do_malloc_errors) {
if (do_realloc_errors) {
caller = __builtin_return_address(1);
if ((void*)toku_xrealloc <= caller && caller <= (void*)toku_malloc_report)
goto skip;
......@@ -155,8 +158,8 @@ static void err_cb(DB *db UU(), int dbn UU(), int err UU(), DBT *key UU(), DBT *
abort();
}
static void write_dbfile (char *template, int n, char *output_name, BOOL expect_error) {
if (verbose) printf("test start %d %d\n", n, expect_error);
static void write_dbfile (char *template, int n, char *output_name, BOOL expect_error, int testno) {
if (verbose) printf("test start %d %d testno=%d\n", n, expect_error, testno);
DB *dest_db = NULL;
struct brtloader_s bl = {
......@@ -271,7 +274,8 @@ static int usage(const char *progname, int n) {
fprintf(stderr, "[-r %d] set the number of rows\n", n);
fprintf(stderr, "[-s] set the small loader size factor\n");
fprintf(stderr, "[-m] inject big malloc and realloc errors\n");
fprintf(stderr, "[-f] inject write errors\n");
fprintf(stderr, "[--malloc_limit %u] set the threshold for failing malloc and realloc\n", (unsigned) my_big_malloc_limit);
fprintf(stderr, "[-w] inject write errors\n");
fprintf(stderr, "[-u] inject user errors\n");
return 1;
}
......@@ -292,12 +296,18 @@ int test_main (int argc, const char *argv[]) {
n = atoi(argv[0]);
} else if (strcmp(argv[0],"-s") == 0) {
toku_brtloader_set_size_factor(1);
} else if (strcmp(argv[0],"-f") == 0) {
} else if (strcmp(argv[0],"-w") == 0) {
do_write_errors = 1;
} else if (strcmp(argv[0],"-m") == 0) {
do_malloc_errors = 1;
do_realloc_errors = 1;
} else if (strcmp(argv[0],"-u") == 0) {
do_user_errors = 1;
} else if (strcmp(argv[0],"--realloc_errors") == 0) {
do_realloc_errors = 1;
} else if (strcmp(argv[0],"--malloc_limit") == 0 && argc > 1) {
argc--; argv++;
my_big_malloc_limit = atoi(argv[0]);
} else if (argc!=1) {
return usage(progname, n);
}
......@@ -324,7 +334,7 @@ int test_main (int argc, const char *argv[]) {
int r;
r = system(unlink_all); CKERR(r);
r = toku_os_mkdir(directory, 0755); CKERR(r);
write_dbfile(template, n, output_name, FALSE);
write_dbfile(template, n, output_name, FALSE, 0);
if (verbose) printf("my_malloc_count=%d big_count=%d\n", my_malloc_count, my_big_malloc_count);
if (verbose) printf("my_realloc_count=%d big_count=%d\n", my_realloc_count, my_big_realloc_count);
......@@ -333,12 +343,13 @@ int test_main (int argc, const char *argv[]) {
if (verbose) printf("event_limit=%d\n", event_limit);
for (int i = 1; i <= event_limit; i++) {
// if (i == 62 || i == 63 || i == 64 || i == 65 || i == 66 || i == 67) continue;
reset_event_counts();
reset_my_malloc_counts();
event_count_trigger = i;
r = system(unlink_all); CKERR(r);
r = toku_os_mkdir(directory, 0755); CKERR(r);
write_dbfile(template, n, output_name, TRUE);
write_dbfile(template, n, output_name, TRUE, i);
}
return 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7