[t:2935], merge to main

git-svn-id: file:///svn/toku/tokudb@24885 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -494,6 +494,7 @@ void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__v
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/db.h_4_3

@@ -510,6 +510,7 @@ void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__v
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/db.h_4_4

@@ -516,6 +516,7 @@ void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__v
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/db.h_4_5

@@ -516,6 +516,7 @@ void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__v
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/db.h_4_6

@@ -520,6 +520,7 @@ void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__v
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/make_db_h.c

@@ -701,6 +701,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("void db_env_set_recover_callback (void (*)(void*), void*) %s;\n", VISIBLE);
     printf("void db_env_set_recover_callback2 (void (*)(void*), void*) %s;\n", VISIBLE);
     printf("void db_env_set_loader_size_factor (uint32_t) %s;\n", VISIBLE);
+    printf("void db_env_set_mvcc_garbage_collection_verification(u_int32_t) %s;\n", VISIBLE);
     printf("#if defined(__cplusplus)\n}\n#endif\n");
     printf("#endif\n");
     return 0;

buildheader/tdb.h

@@ -453,6 +453,7 @@ void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__v
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

include/db.h

@@ -453,6 +453,7 @@ void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__v
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

newbrt/brt.h

@@ -241,6 +241,8 @@ int toku_brt_strerror_r(int error, char *buf, size_t buflen);
 // If error>=0 then the result is to do strerror_r(error, buf, buflen), that is fill buf with a descriptive error message.
 // If error<0 then return a TokuDB-specific error code.  For unknown cases, we return -1 and set errno=EINVAL, even for cases that *should* be known.  (Not all DB errors are known by this function which is a bug.)
 
+extern BOOL garbage_collection_debug;
+
 C_END
 
 #endif

newbrt/txn.c

@@ -9,9 +9,10 @@
 #include "checkpoint.h"
 #include "ule.h"
 
-#if GARBAGE_COLLECTION_DEBUG
+
+BOOL garbage_collection_debug = FALSE;
+
 static void verify_snapshot_system(TOKULOGGER logger);
-#endif
 
 // accountability
 static TXN_STATUS_S status = {.begin  = 0, 
@@ -127,9 +128,9 @@ int toku_txn_begin_with_xid (
     ) 
 {
     if (logger->is_panicked) return EINVAL;
-#if GARBAGE_COLLECTION_DEBUG
-    verify_snapshot_system(logger);
-#endif
+    if (garbage_collection_debug) {
+        verify_snapshot_system(logger);
+    }
     assert(logger->rollback_cachefile);
     TOKUTXN MALLOC(result);
     if (result==0) 
@@ -222,9 +223,9 @@ int toku_txn_begin_with_xid (
     toku_list_init(&result->checkpoint_before_commit);
     *tokutxn = result;
     status.begin++;
-#if GARBAGE_COLLECTION_DEBUG
-    verify_snapshot_system(logger);
-#endif
+    if (garbage_collection_debug) {
+        verify_snapshot_system(logger);
+    }
     return 0;
 
 died:
@@ -315,9 +316,9 @@ local_checkpoints_and_log_xcommit(void *thunk) {
 
 int toku_txn_commit_with_lsn(TOKUTXN txn, int nosync, YIELDF yield, void *yieldv, LSN oplsn,
                              TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra) {
-#if GARBAGE_COLLECTION_DEBUG
-    verify_snapshot_system(txn->logger);
-#endif
+    if (garbage_collection_debug) {
+        verify_snapshot_system(txn->logger);
+    }
     int r;
     // panic handled in log_commit
 
@@ -351,9 +352,9 @@ int toku_txn_abort_txn(TOKUTXN txn, YIELDF yield, void *yieldv,
 
 int toku_txn_abort_with_lsn(TOKUTXN txn, YIELDF yield, void *yieldv, LSN oplsn,
                             TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra) {
-#if GARBAGE_COLLECTION_DEBUG
-    verify_snapshot_system(txn->logger);
-#endif
+    if (garbage_collection_debug) {
+        verify_snapshot_system(txn->logger);
+    }
     //printf("%s:%d aborting\n", __FILE__, __LINE__);
     // Must undo everything.  Must undo it all in reverse order.
     // Build the reverse list
@@ -371,14 +372,11 @@ int toku_txn_abort_with_lsn(TOKUTXN txn, YIELDF yield, void *yieldv, LSN oplsn,
 }
 
 void toku_txn_close_txn(TOKUTXN txn) {
-#if GARBAGE_COLLECTION_DEBUG
-    TOKULOGGER logger = txn->logger;
-#endif
-    
+    TOKULOGGER logger = txn->logger;    
     toku_rollback_txn_close(txn);
-#if GARBAGE_COLLECTION_DEBUG
-    verify_snapshot_system(logger);
-#endif
+    if (garbage_collection_debug) {
+        verify_snapshot_system(logger);
+    }
     status.close++;
     return;
 }
@@ -442,7 +440,6 @@ BOOL toku_is_txn_in_live_root_txn_list(TOKUTXN txn, TXNID xid) {
     return retval;
 }
 
-#if GARBAGE_COLLECTION_DEBUG
 static void
 verify_snapshot_system(TOKULOGGER logger) {
     int     num_snapshot_txnids = toku_omt_size(logger->snapshot_txnids);
@@ -551,5 +548,4 @@ verify_snapshot_system(TOKULOGGER logger) {
         }
     }
 }
-#endif
 

newbrt/ule.c

@@ -202,14 +202,13 @@ garbage_collection(ULE ule, OMT snapshot_xids, OMT live_list_reverse) {
                 break;
             }
         }
-#if GARBAGE_COLLECTION_DEBUG
+        if (garbage_collection_debug)
         {
             u_int32_t idx;
             OMTVALUE txnagain;
             int r = toku_omt_find_zero(snapshot_xids, toku_find_xid_by_xid, &tl1, &txnagain, &idx, NULL);
             invariant(r==0); //make sure that the txn you are claiming is live is actually live
         }
-#endif
         //
         // tl1 should now be set
         //

newbrt/ule.h

@@ -18,7 +18,6 @@ extern "C" {
 
 //1 does much slower debugging
 #define ULE_DEBUG 0
-#define GARBAGE_COLLECTION_DEBUG 0
 
 /////////////////////////////////////////////////////////////////////////////////
 // Following data structures are the unpacked format of a leafentry. 

src/export.map

@@ -23,6 +23,7 @@
    db_env_set_recover_callback;
    db_env_set_recover_callback2;
    db_env_set_loader_size_factor;
+   db_env_set_mvcc_garbage_collection_verification;
 
    toku_os_get_max_rss;
 

src/tests/stress-gc.c

@@ -2,11 +2,18 @@
 // In particular, check to see what happens if a subtransaction has different isolation level from its parent.
 
 #include "test.h"
-
 const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE;
 
 int test_main (int argc, char * const argv[]) {
     parse_args(argc, argv);
+    int useseed;
+
+    {
+      struct timeval tv;
+      gettimeofday(&tv, 0);
+      useseed = tv.tv_sec+tv.tv_usec*997;  // magic:  997 is a prime, and a million (microseconds/second) times 997 is still 32 bits.
+    }
+    
     int r;
     r = system("rm -rf " ENVDIR);
     CKERR(r);
@@ -17,20 +24,26 @@ int test_main (int argc, char * const argv[]) {
     env->set_errfile(env, stderr);
     r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO);
     CKERR(r);
-
-    int max_txns = 2000;
-    int num_runs = 10000;
+    db_env_set_mvcc_garbage_collection_verification(1);
+    int max_txns = 400;
+    int num_runs = 2000;
     DB_TXN* txns[max_txns];
     memset(txns, 0, sizeof(txns));
     int num_txns = 0;
     int i;
 
+    if (verbose) printf("seed=%d\n", useseed);
+    srandom(useseed);
+
     for (i = 0; i < num_runs; i++) {
         int rand_num = random()%max_txns;
-        //if (i%50 == 0) {
-        //    printf("num_txns %d\n", num_txns);
-        //    printf("iteration %d\n", i);
-        //}
+        /*
+            if (i%50 == 0) {
+                printf("rand_num %d\n", rand_num);
+                printf("num_txns %d\n", num_txns);
+                printf("iteration %d\n", i);
+            }
+            */
         if (rand_num >= num_txns) {
             // add a txn
             assert(txns[num_txns] == NULL);

src/ydb.c

@@ -4992,6 +4992,9 @@ void db_env_set_loader_size_factor (uint32_t factor) {
 }
 
 
+void db_env_set_mvcc_garbage_collection_verification(u_int32_t verification_mode) {
+    garbage_collection_debug = (verification_mode != 0);
+}