From 3444202c8e739ef296a39f08fe9445b0f1ecfc6b Mon Sep 17 00:00:00 2001 From: Barry Perlman <barry@tokutek.com> Date: Wed, 19 May 2010 00:13:54 +0000 Subject: [PATCH] Closes [t:2624] Added toku_os_close() which retries on EINTR. Closes [t:2621] Added toku_os_open(), toku_os_fopen(), toku_os_fdopen(), toku_os_fclose() and related override hooks db_env_set_func_open(), db_env_set_func_fopen(), db_env_set_func_fdopen(), db_env_set_func_fclose(). Modified test program loader-cleanup-test to use new hooks to inject synthetic errors. Marked some error cases in brtloader.c. with ###. git-svn-id: file:///svn/toku/tokudb@20360 c7de825b-a66e-492c-adef-691d508d4ae1 --- src/tests/loader-cleanup-test.c | 241 +++++++++++++++++++++++++------- 1 file changed, 191 insertions(+), 50 deletions(-) diff --git a/src/tests/loader-cleanup-test.c b/src/tests/loader-cleanup-test.c index 1298815cdde..c97991c2096 100644 --- a/src/tests/loader-cleanup-test.c +++ b/src/tests/loader-cleanup-test.c @@ -5,7 +5,7 @@ /* TODO: * - * When ready, add simulated errors on calls to pwrite() and malloc() + * When ready, add simulated errors on calls to malloc() * */ @@ -18,10 +18,7 @@ * - user calls loader->abort() * - user aborts transaction * - disk full (ENOSPC) - * - crash - * - * In the event of a crash, the verification of no temp files and - * no loader-generated iname file is done after recovery. + * - crash (not tested in this test program) * * Mechanism: * This test is derived from the loader-stress-test. @@ -29,7 +26,7 @@ * The outline of the test is as follows: * - use loader to create table * - verify presence of temp files - * - commit / abort / enospc / crash + * - commit / abort / inject error (simulated error from system call) * - verify absence of temp files * - verify absence of unwanted iname files (old inames if committed, new inames if aborted) * @@ -53,7 +50,11 @@ enum test_type {commit, // close loader, commit txn abort_via_poll, // close loader, but poll function returns non-zero, abort txn enospc_w, // close loader, but close fails due to enospc return from toku_os_write enospc_f, // either loader->put() or loader->close() fails due to enospc return from do_fwrite() - enospc_p}; // loader->close() fails due to enospc return from toku_os_pwrite() + enospc_p, // loader->close() fails due to enospc return from toku_os_pwrite() + einval_fdo, // return einval from fdopen() + einval_fo, // return einval from fopen() + einval_o, // return einval from open() + enospc_fc}; // return enospc from fclose() int abort_on_poll = 0; // set when test_loader() called with test_type of abort_via_poll @@ -81,38 +82,70 @@ static void run_all_tests(void); static void free_inames(DBT* inames); -#define NUM_ENOSPC_TYPES 3 +// how many different system calls are intercepted with error injection +#define NUM_ERR_TYPES 7 int fwrite_count = 0; -int fwrite_enospc = 0; int fwrite_count_nominal = 0; // number of fwrite calls for normal operation, initially zero int fwrite_count_trigger = 0; // sequence number of fwrite call that will fail (zero disables induced failure) int write_count = 0; -int write_enospc = 0; int write_count_nominal = 0; // number of write calls for normal operation, initially zero int write_count_trigger = 0; // sequence number of write call that will fail (zero disables induced failure) int pwrite_count = 0; -int pwrite_enospc = 0; int pwrite_count_nominal = 0; // number of pwrite calls for normal operation, initially zero int pwrite_count_trigger = 0; // sequence number of pwrite call that will fail (zero disables induced failure) +int fdopen_count = 0; +int fdopen_count_nominal = 0; // number of fdopen calls for normal operation, initially zero +int fdopen_count_trigger = 0; // sequence number of fdopen call that will fail (zero disables induced failure) + +int fopen_count = 0; +int fopen_count_nominal = 0; // number of fopen calls for normal operation, initially zero +int fopen_count_trigger = 0; // sequence number of fopen call that will fail (zero disables induced failure) + +int open_count = 0; +int open_count_nominal = 0; // number of open calls for normal operation, initially zero +int open_count_trigger = 0; // sequence number of open call that will fail (zero disables induced failure) + +int fclose_count = 0; +int fclose_count_nominal = 0; // number of fclose calls for normal operation, initially zero +int fclose_count_trigger = 0; // sequence number of fclose call that will fail (zero disables induced failure) + + + + const char * fwrite_str = "fwrite"; const char * write_str = "write"; const char * pwrite_str = "pwrite"; +const char * fdopen_str = "fdopen"; +const char * fopen_str = "fopen"; +const char * open_str = "open"; +const char * fclose_str = "fclose"; + static const char * -enospc_type_str (enum test_type t){ +err_type_str (enum test_type t) { const char * rval; - if (t == enospc_f) - rval = fwrite_str; - else if (t == enospc_w) - rval = write_str; - else if (t == enospc_p) - rval = pwrite_str; - else + switch(t) { + case enospc_f: + rval = fwrite_str; break; + case enospc_w: + rval = write_str; break; + case enospc_p: + rval = pwrite_str; break; + case einval_fdo: + rval = fdopen_str; break; + case einval_fo: + rval = fopen_str; break; + case einval_o: + rval = open_str; break; + case enospc_fc: + rval = fclose_str; break; + default: assert(0); + } return rval; } @@ -120,7 +153,6 @@ static size_t bad_fwrite (const void *ptr, size_t size, size_t nmemb, FILE *stre fwrite_count++; size_t r; if (fwrite_count_trigger == fwrite_count) { - fwrite_enospc++; errno = ENOSPC; r = -1; } else { @@ -138,7 +170,6 @@ bad_write(int fd, const void * bp, size_t len) { ssize_t r; write_count++; if (write_count_trigger == write_count) { - write_enospc++; errno = ENOSPC; r = -1; } else { @@ -152,7 +183,6 @@ bad_pwrite (int fd, const void *buf, size_t len, toku_off_t off) { int r; pwrite_count++; if (pwrite_count_trigger == pwrite_count) { - pwrite_enospc++; errno = ENOSPC; r = -1; } else { @@ -163,6 +193,66 @@ bad_pwrite (int fd, const void *buf, size_t len, toku_off_t off) { +static FILE * +bad_fdopen(int fd, const char * mode) { + FILE * rval; + fdopen_count++; + if (fdopen_count_trigger == fdopen_count) { + errno = EINVAL; + rval = NULL; + } else { + rval = fdopen(fd, mode); + } + return rval; +} + +static FILE * +bad_fopen(const char *filename, const char *mode) { + FILE * rval; + fopen_count++; + if (fopen_count_trigger == fopen_count) { + errno = EINVAL; + rval = NULL; + } else { + rval = fopen(filename, mode); + } + return rval; +} + + +static int +bad_open(const char *path, int oflag, int mode) { + int rval; + open_count++; + if (open_count_trigger == open_count) { + errno = EINVAL; + rval = -1; + } else { + rval = open(path, oflag, mode); + } + return rval; +} + + + +static int +bad_fclose(FILE * stream) { + int rval; + fclose_count++; + if (fclose_count_trigger == fclose_count) { + errno = ENOSPC; + rval = -1; + } else { + rval = fclose(stream); + } + return rval; +} + + + +/////////////// + + // return number of temp files static int count_temp(char * dirname) { @@ -449,12 +539,23 @@ static int poll_function (void *extra, float progress) { static void test_loader(enum test_type t, DB **dbs) { int failed_put = 0; + int error_injection; // are we expecting simulated errors from system calls? + + if (t == commit || + t == abort_txn || + t == abort_loader || + t == abort_via_poll) + error_injection = 0; + else + error_injection = 1; + if (t == abort_via_poll) abort_on_poll = 1; else abort_on_poll = 0; + int r; DB_TXN *txn; DB_LOADER *loader; @@ -494,7 +595,7 @@ static void test_loader(enum test_type t, DB **dbs) dbt_init(&key, &k, sizeof(unsigned int)); dbt_init(&val, &v, sizeof(unsigned int)); r = loader->put(loader, &key, &val); - if (t == enospc_f || t == enospc_w || t == enospc_p) + if (error_injection) failed_put = r; else CKERR(r); @@ -527,10 +628,9 @@ static void test_loader(enum test_type t, DB **dbs) r = loader->close(loader); assert(r); // not defined what close() returns when poll function returns non-zero } - else if ((t == enospc_f || t == enospc_w || t == enospc_p) - && !failed_put) { - const char * type = enospc_type_str(t); - printf("closing, but expecting failure from enospc %s\n", type); + else if (error_injection && !failed_put) { + const char * type = err_type_str(t); + printf("closing, but expecting failure from simulated error (enospc or einval)%s\n", type); r = loader->close(loader); if (!USE_PUTS) assert(r); @@ -553,8 +653,22 @@ static void test_loader(enum test_type t, DB **dbs) fwrite_count_nominal = fwrite_count; // capture how many fwrites were required for normal operation write_count_nominal = write_count; // capture how many writes were required for normal operation pwrite_count_nominal = pwrite_count; // capture how many pwrites were required for normal operation - if (verbose) printf("Calls to fwrite nominal: %d, calls to write nominal: %d, calls to pwrite nominal: %d\n", - fwrite_count_nominal, write_count_nominal, pwrite_count_nominal); + fdopen_count_nominal = fdopen_count; // capture how many fdopens were required for normal operation + fopen_count_nominal = fopen_count; // capture how many fopens were required for normal operation + open_count_nominal = open_count; // capture how many opens were required for normal operation + fclose_count_nominal = fclose_count; // capture how many fcloses were required for normal operation + + if (verbose) { + printf("Nominal calls: function calls (number of calls for normal operation)\n"); + printf(" fwrite %d\n", fwrite_count_nominal); + printf(" write %d\n", write_count_nominal); + printf(" pwrite %d\n", pwrite_count_nominal); + printf(" fdopen %d\n", fdopen_count_nominal); + printf(" fopen %d\n", fopen_count_nominal); + printf(" open %d\n", open_count_nominal); + printf(" fclose %d\n", fclose_count_nominal); + } + r = txn->commit(txn, 0); CKERR(r); if (!USE_PUTS) { @@ -589,7 +703,7 @@ static void run_test(enum test_type t, int trigger) r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r); r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r); -// int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE; + int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE; r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); env->set_errfile(env, stderr); @@ -614,23 +728,46 @@ static void run_test(enum test_type t, int trigger) generate_permute_tables(); - fwrite_count_trigger = fwrite_count = fwrite_enospc = 0; - write_count_trigger = write_count = write_enospc = 0; - pwrite_count_trigger = pwrite_count = pwrite_enospc = 0; - - if (t == enospc_f) { - fwrite_count_trigger = trigger; - } - else if (t == enospc_w) { - write_count_trigger = trigger; - } - else if (t == enospc_p) { - pwrite_count_trigger = trigger; + fwrite_count_trigger = fwrite_count = 0; + write_count_trigger = write_count = 0; + pwrite_count_trigger = pwrite_count = 0; + fdopen_count_trigger = fdopen_count = 0; + fopen_count_trigger = fopen_count = 0; + open_count_trigger = open_count = 0; + fclose_count_trigger = fclose_count = 0; + + switch(t) { + case commit: + case abort_txn: + case abort_loader: + case abort_via_poll: + break; + case enospc_f: + fwrite_count_trigger = trigger; break; + case enospc_w: + write_count_trigger = trigger; break; + case enospc_p: + pwrite_count_trigger = trigger; break; + case einval_fdo: + fdopen_count_trigger = trigger; break; + case einval_fo: + fopen_count_trigger = trigger; break; + case einval_o: + open_count_trigger = trigger; break; + case enospc_fc: + fclose_count_trigger = trigger; break; + default: + assert(0); } + db_env_set_func_loader_fwrite(bad_fwrite); db_env_set_func_write(bad_write); db_env_set_func_pwrite(bad_pwrite); + db_env_set_func_fdopen(bad_fdopen); + db_env_set_func_fopen(bad_fopen); + db_env_set_func_open(bad_open); + db_env_set_func_fclose(bad_fclose); test_loader(t, dbs); @@ -638,6 +775,9 @@ static void run_test(enum test_type t, int trigger) dbs[i]->close(dbs[i], 0); CKERR(r); dbs[i] = NULL; } + if (verbose >= 3) + print_engine_status(env); + r = env->close(env, 0); CKERR(r); toku_free(dbs); } @@ -661,34 +801,35 @@ static void run_all_tests(void) { if (verbose) printf("\n\nTesting loader with loader close and txn abort\n"); run_test(abort_txn, 0); - enum test_type et[NUM_ENOSPC_TYPES] = {enospc_f, enospc_w, enospc_p}; - int * nomp[NUM_ENOSPC_TYPES] = {&fwrite_count_nominal, &write_count_nominal, &pwrite_count_nominal}; + enum test_type et[NUM_ERR_TYPES] = {enospc_f, enospc_w, enospc_p, einval_fdo, einval_fo, einval_o, enospc_fc}; + int * nomp[NUM_ERR_TYPES] = {&fwrite_count_nominal, &write_count_nominal, &pwrite_count_nominal, + &fdopen_count_nominal, &fopen_count_nominal, &open_count_nominal, &fclose_count_nominal}; int limit = NUM_DBS * 5; int j; - for (j = 0; j<NUM_ENOSPC_TYPES; j++) { + for (j = 0; j<NUM_ERR_TYPES; j++) { enum test_type t = et[j]; - const char * write_type = enospc_type_str(t); + const char * write_type = err_type_str(t); int nominal = *(nomp[j]); - printf("\nNow test with induced ENOSPC errors returned from %s, nominal = %d\n", write_type, nominal); + printf("\nNow test with induced ENOSPC/EINVAL errors returned from %s, nominal = %d\n", write_type, nominal); int i; // induce write error at beginning of process for (i = 1; i < limit && i < nominal+1; i++) { trigger = i; - if (verbose) printf("\n\nTesting loader with enospc induced at %s count %d (of %d)\n", write_type, trigger, nominal); + if (verbose) printf("\n\nTesting loader with enospc/einval induced at %s count %d (of %d)\n", write_type, trigger, nominal); run_test(t, trigger); } if (nominal > limit) { // if we didn't already test every possible case // induce write error sprinkled through process for (i = 2; i < 5; i++) { trigger = nominal / i; - if (verbose) printf("\n\nTesting loader with enospc induced at %s count %d (of %d)\n", write_type, trigger, nominal); + if (verbose) printf("\n\nTesting loader with enospc/einval induced at %s count %d (of %d)\n", write_type, trigger, nominal); run_test(t, trigger); } // induce write error at end of process for (i = 0; i < limit; i++) { trigger = nominal - i; assert(trigger > 0); - if (verbose) printf("\n\nTesting loader with enospc induced at %s count %d (of %d)\n", write_type, trigger, nominal); + if (verbose) printf("\n\nTesting loader with enospc/einval induced at %s count %d (of %d)\n", write_type, trigger, nominal); run_test(t, trigger); } } -- 2.30.9