[t:2542] Modify all loader tests to support -p (USE_PUTS) and logging

All now support the behavior of the put loader

git-svn-id: file:///svn/toku/tokudb@19683 c7de825b-a66e-492c-adef-691d508d4ae1

src/tests/Makefile

@@ -32,6 +32,8 @@ endif
 
 SRCS = $(sort $(wildcard *.c))
 RECOVER_SRCS = $(wildcard recover-*.c)
+LOADER_SRCS = $(wildcard loader-*.c)
+NONSTANDARD_SRCS=$(RECOVER_SRCS) $(LOADER_SRCS)
 
 #Tests that don't compile in windows.  SHould
 WINDOWS_NOT_PORTED_TESTS = \
@@ -64,10 +66,10 @@ WINDOWS_BDB_DONTRUN_TESTS += \
 
 ifeq ($(OS_CHOICE),windows)
     TDB_BINS = $(patsubst %.c,%.tdb$(BINSUF),$(filter-out $(patsubst %,%.c,$(WINDOWS_DONTRUN_TESTS)),$(SRCS)))
-    TDB_TESTS = $(patsubst %.c,%.tdb$(BINSUF),$(filter-out $(patsubst %,%.c,$(WINDOWS_DONTRUN_TESTS)),$(filter-out $(RECOVER_SRCS),$(SRCS))))
+    TDB_TESTS = $(patsubst %.c,%.tdb$(BINSUF),$(filter-out $(patsubst %,%.c,$(WINDOWS_DONTRUN_TESTS)),$(filter-out $(NONSTANDARD_SRCS),$(SRCS))))
 else
     TDB_BINS = $(patsubst %.c,%.tdb$(BINSUF),$(SRCS))
-    TDB_TESTS = $(patsubst %.c,%.tdb$(BINSUF),$(filter-out $(RECOVER_SRCS),$(SRCS)))
+    TDB_TESTS = $(patsubst %.c,%.tdb$(BINSUF),$(filter-out $(NONSTANDARD_SRCS),$(SRCS)))
 endif
 
 # For diskfull.bdb: db-4.6 seems OK, but db-4.3 segfaults
@@ -238,11 +240,14 @@ EXTRA_TDB_TESTS = \
 endif
 
 RECOVER_TESTS = $(patsubst %.c,%.abortrecover,$(RECOVER_SRCS))
+LOADER_TESTS= $(patsubst %.c,%.loader,$(LOADER_SRCS))
 ifeq ($(OS_CHOICE),windows)
 RECOVER_TESTS = $(patsubst %.c,%.abortrecover,$(filter-out $(patsubst %,%.c,$(WINDOWS_DONTRUN_TESTS)),$(RECOVER_SRCS)))
+LOADER_TESTS = $(patsubst %.c,%.loader,$(filter-out $(patsubst %,%.c,$(WINDOWS_DONTRUN_TESTS)),$(LOADER_SRCS)))
 endif
 
-RUN_TDB_TESTS = $(patsubst %.tdb$(BINSUF),%.tdbrun,$(TDB_TESTS))  $(EXTRA_TDB_TESTS) $(RECOVER_TESTS)
+
+RUN_TDB_TESTS = $(patsubst %.tdb$(BINSUF),%.tdbrun,$(TDB_TESTS))  $(EXTRA_TDB_TESTS) $(RECOVER_TESTS) $(LOADER_TESTS)
 RUN_BDB_TESTS  = $(patsubst %.bdb$(BINSUF),%.bdbrun,$(BDB_TESTS))
 RUN_ALL_TESTS  = $(RUN_BDB_TESTS) $(RUN_TDB_TESTS)
 
@@ -317,8 +322,6 @@ endif
 endif
 #empty on purpose
 
-
-
 # Use -s on the command line to make things quiet.
 # Use -s on the command line to make things quiet.
 %.bdbrun: %.bdb$(BINSUF) $(DEPEND_COMPILE) $(DEPEND_LINK) $(WINDOWS_BDB_LIB_NAME)
@@ -370,6 +373,13 @@ ABORTCODE=137
 	fi \
 	$(MAYBEINVERTER) $(SUMMARIZE_CMD)
 
+.PHONY: %.loader
+%.loader: %.tdb$(BINSUF) $(PTHREAD_LOCAL)
+	echo doing ./$< fastloader && \
+	$(TDBVGRIND) ./$< $(VERBVERBOSE) && \
+	echo doing ./$< putloader && \
+	$(TDBVGRIND) ./$< -p $(VERBVERBOSE) $(MAYBEINVERTER) $(SUMMARIZE_CMD)
+
 .PHONY: %.recover
 all.recover: $(patsubst %,test_log%.recover,$(TLRECOVER)) $(PTHREAD_LOCAL) ;
 
@@ -633,3 +643,4 @@ dumpit: $(TDBBRTDUMP)
 test_thread_stack.%run: test_thread_stack.%$(BINSUF) $(PTHREAD_LOCAL)
 	./$< -a -thread_stack 16384         && \
 	./$< -a -thread_stack 16384 -resume $(SUMMARIZE_CMD)
+

src/tests/loader-dup-test.c

@@ -240,7 +240,13 @@ static void test_loader(DB **dbs)
         dbt_init(&key, &k, sizeof(unsigned int));
         dbt_init(&val, &v, sizeof(unsigned int));
         r = loader->put(loader, &key, &val);
-        CKERR(r);
+        if (USE_PUTS) {
+            //PUT loader can return -1 if it finds an error during the puts.
+            CKERR2s(r, 0,-1);
+        }
+        else {
+            CKERR(r);
+        }
         if ( CHECK_RESULTS || verbose) { if((i%10000) == 0){printf("."); fflush(stdout);} }
     }
     if( CHECK_RESULTS || verbose ) {printf("\n"); fflush(stdout);}        
@@ -273,7 +279,7 @@ static void run_test(void)
     r = env->set_default_dup_compare(env, uint_dbt_cmp);                                                      CKERR(r);
     r = env->set_generate_row_callback_for_put(env, put_multiple_generate);
     CKERR(r);
-    int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
+    int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOG | DB_CREATE | DB_PRIVATE;
     r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO);                                            CKERR(r);
     env->set_errfile(env, stderr);
     //Disable auto-checkpointing
@@ -322,6 +328,8 @@ int test_main(int argc, char * const *argv) {
 	run_test();
     else {
 	int sizes[]={1,4000000,-1};
+        //Make PUT loader take about the same amount of time:
+        if (USE_PUTS) sizes[1] /= 25;
 	for (int i=0; sizes[i]>=0; i++) {
 	    if (verbose) printf("Doing %d\n", sizes[i]);
 	    NUM_ROWS = sizes[i];

src/tests/loader-no-puts.c

@@ -8,6 +8,7 @@
 #include <sys/stat.h>
 
 DB_ENV *env;
+int USE_PUTS=0;
 enum {MAX_NAME=128};
 enum {NUM_DBS=1};
 enum {NUM_KV_PAIRS=3};
@@ -47,7 +48,7 @@ static void test_loader(DB **dbs)
         db_flags[i] = DB_NOOVERWRITE; 
         dbt_flags[i] = 0;
     }
-    uint32_t loader_flags = 0;
+    uint32_t loader_flags = USE_PUTS; // set with -p option
 
     // create and initialize loader
     r = env->txn_begin(env, NULL, &txn, 0);                                                               
@@ -172,6 +173,8 @@ static void do_args(int argc, char * const argv[]) {
 	} else if (strcmp(argv[0],"-q")==0) {
 	    verbose--;
 	    if (verbose<0) verbose=0;
+        } else if (strcmp(argv[0], "-p")==0) {
+            USE_PUTS = 1;
 	} else {
 	    fprintf(stderr, "Unknown arg: %s\n", argv[0]);
 	    resultcode=1;

src/tests/loader-reference-test.c

@@ -8,6 +8,7 @@
 #include <sys/stat.h>
 
 DB_ENV *env;
+int USE_PUTS=0;
 enum {MAX_NAME=128};
 enum {NUM_DBS=1};
 enum {NUM_KV_PAIRS=3};
@@ -47,7 +48,7 @@ static void test_loader(DB **dbs)
         db_flags[i] = DB_NOOVERWRITE; 
         dbt_flags[i] = 0;
     }
-    uint32_t loader_flags = 0;
+    uint32_t loader_flags = USE_PUTS; // set with -p option
 
     // create and initialize loader
     r = env->txn_begin(env, NULL, &txn, 0);                                                               
@@ -165,11 +166,8 @@ static void do_args(int argc, char * const argv[]) {
 	do_usage:
 	    fprintf(stderr, "Usage:\n%s\n", cmd);
 	    exit(resultcode);
-        } else if (strcmp(argv[0], "-v")==0) {
-	    verbose++;
-	} else if (strcmp(argv[0],"-q")==0) {
-	    verbose--;
-	    if (verbose<0) verbose=0;
+        } else if (strcmp(argv[0], "-p")==0) {
+            USE_PUTS = 1;
 	} else {
 	    fprintf(stderr, "Unknown arg: %s\n", argv[0]);
 	    resultcode=1;

src/tests/loader-stress-test.c

@@ -242,7 +242,8 @@ static void test_loader(DB **dbs)
     printf(" done\n");
     CKERR(r);
 
-    assert(poll_count>0);
+    if (!USE_PUTS)
+        assert(poll_count>0);
 
     r = txn->commit(txn, 0);
     CKERR(r);