[t:5067] Merge #5067 to main. New GC algorithm that calculates...

[t:5067] Merge #5067 to main.  New GC algorithm that calculates "live_list_reverse" on the fly when needed

git-svn-id: file:///svn/toku/tokudb@44749 c7de825b-a66e-492c-adef-691d508d4ae1

ft/ft-ops.c

@@ -2100,7 +2100,7 @@ ft_basement_node_gc_once(BASEMENTNODE bn,
                           u_int32_t index,
                           LEAFENTRY leaf_entry,
                           OMT snapshot_xids,
-                          OMT live_list_reverse,
+                          OMT referenced_xids,
                           OMT live_root_txns,
                           STAT64INFO_S * delta)
 {
@@ -2129,7 +2129,7 @@ ft_basement_node_gc_once(BASEMENTNODE bn,
                               &bn->buffer_mempool,
                               &maybe_free,
                               snapshot_xids,
-                              live_list_reverse,
+                              referenced_xids,
                               live_root_txns);
 
     // These will represent the number of bytes and rows changed as
@@ -2171,7 +2171,7 @@ ft_basement_node_gc_once(BASEMENTNODE bn,
 static void
 basement_node_gc_all_les(BASEMENTNODE bn,
                          OMT snapshot_xids,
-                         OMT live_list_reverse,
+                         OMT referenced_xids,
                          OMT live_root_txns,
                          STAT64INFO_S * delta)
 {
@@ -2184,7 +2184,7 @@ basement_node_gc_all_les(BASEMENTNODE bn,
         r = toku_omt_fetch(bn->buffer, index, &storedatav);
         assert(r == 0);
         leaf_entry = storedatav;
-        ft_basement_node_gc_once(bn, index, leaf_entry, snapshot_xids, live_list_reverse, live_root_txns, delta);
+        ft_basement_node_gc_once(bn, index, leaf_entry, snapshot_xids, referenced_xids, live_root_txns, delta);
         // Check if the leaf entry was deleted or not.
         if (num_leafentries_before == toku_omt_size(bn->buffer)) {
             ++index;
@@ -2197,7 +2197,7 @@ static void
 ft_leaf_gc_all_les(FTNODE node,
                     FT h,
                     OMT snapshot_xids,
-                    OMT live_list_reverse,
+                    OMT referenced_xids,
                     OMT live_root_txns)
 {
     toku_assert_entire_node_in_memory(node);
@@ -2209,7 +2209,7 @@ ft_leaf_gc_all_les(FTNODE node,
         STAT64INFO_S delta;
         delta.numrows = 0;
         delta.numbytes = 0;
-        basement_node_gc_all_les(bn, snapshot_xids, live_list_reverse, live_root_txns, &delta);
+        basement_node_gc_all_les(bn, snapshot_xids, referenced_xids, live_root_txns, &delta);
         toku_ft_update_stats(&h->in_memory_stats, delta);
     }
 }
@@ -2247,12 +2247,12 @@ toku_bnc_flush_to_child(
     TOKULOGGER logger = toku_cachefile_logger(h->cf);
     if (child->height == 0 && logger) {
         OMT snapshot_txnids = NULL;
-        OMT live_list_reverse = NULL;
+        OMT referenced_xids = NULL;
         OMT live_root_txns = NULL;
         toku_txn_manager_clone_state_for_gc(
             logger->txn_manager,
             &snapshot_txnids,
-            &live_list_reverse,
+            &referenced_xids,
             &live_root_txns
             );
         size_t buffsize = toku_fifo_buffer_size_in_use(bnc->buffer);
@@ -2260,13 +2260,13 @@ toku_bnc_flush_to_child(
         // may be misleading if there's a broadcast message in there
         STATUS_VALUE(FT_MSG_BYTES_CURR) -= buffsize;
         // Perform the garbage collection.
-        ft_leaf_gc_all_les(child, h, snapshot_txnids, live_list_reverse, live_root_txns);
+        ft_leaf_gc_all_les(child, h, snapshot_txnids, referenced_xids, live_root_txns);
 
         // Free the OMT's we used for garbage collecting.
         toku_omt_destroy(&snapshot_txnids);
         toku_omt_destroy(&live_root_txns);
-        toku_omt_free_items_pool(live_list_reverse);
-        toku_omt_destroy(&live_list_reverse);
+        toku_omt_free_items_pool(referenced_xids);
+        toku_omt_destroy(&referenced_xids);
     }
 
     return 0;

ft/rollback.c

@@ -35,15 +35,6 @@ toku_find_xid_by_xid (OMTVALUE v, void *xidv) {
     return 0;
 }
 
-int
-toku_find_pair_by_xid (OMTVALUE v, void *xidv) {
-    XID_PAIR pair = v;
-    TXNID xidfind = (TXNID)xidv;
-    if (pair->xid1<xidfind) return -1;
-    if (pair->xid1>xidfind) return +1;
-    return 0;
-}
-
 void *toku_malloc_in_rollback(ROLLBACK_LOG_NODE log, size_t size) {
     return malloc_in_memarena(log->rollentry_arena, size);
 }

ft/rollback.h

@@ -53,7 +53,6 @@ void toku_maybe_spill_rollbacks(TOKUTXN txn, ROLLBACK_LOG_NODE log);
 void toku_txn_maybe_note_ft (TOKUTXN txn, FT h);
 int toku_logger_txn_rollback_raw_count(TOKUTXN txn, u_int64_t *raw_count);
 
-int toku_find_pair_by_xid (OMTVALUE v, void *txnv);
 int toku_find_xid_by_xid (OMTVALUE v, void *xidv);
 
 PAIR_ATTR rollback_memory_size(ROLLBACK_LOG_NODE log);

ft/txn.h

@@ -101,11 +101,6 @@ BOOL toku_is_txn_in_live_root_txn_list(OMT live_root_txn_list, TXNID xid);
 
 TXNID toku_get_oldest_in_live_root_txn_list(TOKUTXN txn);
 
-typedef struct {
-    TXNID xid1;
-    TXNID xid2;
-} XID_PAIR_S, *XID_PAIR;
-
 #include "txn_state.h"
 
 TOKUTXN_STATE toku_txn_get_state(TOKUTXN txn);

ft/txn_manager.h

@@ -47,9 +47,9 @@ int toku_txn_manager_start_txn(
 void toku_txn_manager_finish_txn(TXN_MANAGER txn_manager, TOKUTXN txn);
 
 void toku_txn_manager_clone_state_for_gc(
-    TXN_MANAGER txn_manager, 
-    OMT* snapshot_xids, 
-    OMT* live_list_reverse, 
+    TXN_MANAGER txn_manager,
+    OMT* snapshot_xids,
+    OMT* referenced_xids,
     OMT* live_root_txns
     );
 
@@ -61,8 +61,8 @@ int toku_txn_manager_get_txn_from_xid (TXN_MANAGER txn_manager, TOKU_XA_XID *xid
 u_int32_t toku_txn_manager_num_live_txns(TXN_MANAGER txn_manager);
 
 int toku_txn_manager_iter_over_live_txns(
-    TXN_MANAGER txn_manager, 
-    int (*f)(OMTVALUE, u_int32_t, void*), 
+    TXN_MANAGER txn_manager,
+    int (*f)(OMTVALUE, u_int32_t, void*),
     void* v
     );
 
@@ -71,10 +71,10 @@ void toku_txn_manager_note_abort_txn(TXN_MANAGER txn_manager, TOKUTXN txn);
 void toku_txn_manager_note_commit_txn(TXN_MANAGER txn_manager, TOKUTXN txn);
 
 int toku_txn_manager_recover_txn(
-    TXN_MANAGER txn_manager, 
-    struct tokulogger_preplist preplist[/*count*/], 
-    long count, 
-    long *retp, /*out*/ 
+    TXN_MANAGER txn_manager,
+    struct tokulogger_preplist preplist[/*count*/],
+    long count,
+    long *retp, /*out*/
     u_int32_t flags
     );
 
@@ -91,6 +91,7 @@ TXNID toku_txn_manager_get_last_xid(TXN_MANAGER mgr);
 // Test-only function
 void toku_txn_manager_increase_last_xid(TXN_MANAGER mgr, uint64_t increment);
 
+TXNID toku_get_youngest_live_list_txnid_for(TXNID xc, OMT snapshot_txnids, OMT referenced_xids);
 #if defined(__cplusplus) || defined(__cilkplusplus)
 }
 #endif

ft/ule.c

@@ -27,6 +27,7 @@
 #include "xids.h"
 #include "ft_msg.h"
 #include "ule.h"
+#include "txn_manager.h"
 #include "ule-internal.h"
 
 
@@ -188,36 +189,16 @@ get_next_older_txnid(TXNID xc, OMT omt) {
     return xid;
 }
 
-TXNID
-toku_get_youngest_live_list_txnid_for(TXNID xc, OMT live_list_reverse) {
-    OMTVALUE pairv;
-    XID_PAIR pair;
-    uint32_t idx;
-    TXNID rval;
-    int r;
-    r = toku_omt_find_zero(live_list_reverse, toku_find_pair_by_xid, (void *)xc, &pairv, &idx);
-    if (r==0) {
-        pair = pairv;
-        invariant(pair->xid1 == xc); //sanity check
-        rval = pair->xid2;
-    }
-    else {
-        invariant(r==DB_NOTFOUND);
-        rval = TXNID_NONE;
-    }
-    return rval;
-}
-
 //
 // This function returns TRUE if live transaction TL1 is allowed to read a value committed by
 // transaction xc, false otherwise.
 //
 static BOOL
-xid_reads_committed_xid(TXNID tl1, TXNID xc, OMT live_list_reverse) {
+xid_reads_committed_xid(TXNID tl1, TXNID xc, OMT snapshot_txnids, OMT referenced_xids) {
     BOOL rval;
     if (tl1 < xc) rval = FALSE; //cannot read a newer txn
     else {
-        TXNID x = toku_get_youngest_live_list_txnid_for(xc, live_list_reverse);
+        TXNID x = toku_get_youngest_live_list_txnid_for(xc, snapshot_txnids, referenced_xids);
         if (x == TXNID_NONE) rval = TRUE; //Not in ANY live list, tl1 can read it.
         else rval = tl1 > x;              //Newer than the 'newest one that has it in live list'
         // we know tl1 > xc
@@ -230,7 +211,7 @@ xid_reads_committed_xid(TXNID tl1, TXNID xc, OMT live_list_reverse) {
 }
 
 static void
-garbage_collection(ULE ule, OMT snapshot_xids, OMT live_list_reverse, OMT live_root_txns) {
+garbage_collection(ULE ule, OMT snapshot_xids, OMT referenced_xids, OMT live_root_txns) {
     if (ule->num_cuxrs == 1) goto done;
     // will fail if too many num_cuxrs
     BOOL necessary_static[MAX_TRANSACTION_RECORDS];
@@ -266,7 +247,7 @@ garbage_collection(ULE ule, OMT snapshot_xids, OMT live_list_reverse, OMT live_r
             continue;            
         }
 
-        tl1 = toku_get_youngest_live_list_txnid_for(xc, live_list_reverse);
+        tl1 = toku_get_youngest_live_list_txnid_for(xc, snapshot_xids, referenced_xids);
         if (tl1 == xc) {
             // if tl1 == xc, that means xc should be live and show up in 
             // live_root_txns, which we check above. So, if we get
@@ -294,7 +275,7 @@ garbage_collection(ULE ule, OMT snapshot_xids, OMT live_list_reverse, OMT live_r
         curr_committed_entry--;
         while (curr_committed_entry > 0) {
             xc = ule->uxrs[curr_committed_entry].xid;
-            if (xid_reads_committed_xid(tl1, xc, live_list_reverse)) {
+            if (xid_reads_committed_xid(tl1, xc, snapshot_xids, referenced_xids)) {
                 break;
             }
             curr_committed_entry--;
@@ -387,7 +368,7 @@ apply_msg_to_leafentry(FT_MSG   msg,		// message to apply to leafentry
 // the memory completely, if we removed the leaf entry.
 // -- snapshot_xids : we use these in memory transaction ids to
 // determine what to garbage collect.
-// -- live_list_reverse : list of in memory active transactions.
+// -- referenced_xids : list of in memory active transactions.
 // NOTE: it is not a good idea to garbage collect a leaf
 // entry with only one committed value.
 int
@@ -398,15 +379,15 @@ garbage_collect_leafentry(LEAFENTRY old_leaf_entry,
                           struct mempool *mp,
                           void **maybe_free,
                           OMT snapshot_xids,
-                          OMT live_list_reverse,
+                          OMT referenced_xids,
                           OMT live_root_txns) {
     int r = 0;
     ULE_S ule;
     le_unpack(&ule, old_leaf_entry);
     assert(snapshot_xids);
-    assert(live_list_reverse);
+    assert(referenced_xids);
     assert(live_root_txns);
-    garbage_collection(&ule, snapshot_xids, live_list_reverse, live_root_txns);
+    garbage_collection(&ule, snapshot_xids, referenced_xids, live_root_txns);
     r = le_pack(&ule,
                 new_leaf_entry_memory_size,
                 new_leaf_entry,

ft/ule.h

@@ -69,11 +69,9 @@ int garbage_collect_leafentry(LEAFENTRY old_leaf_entry,
                               struct mempool *mp,
                               void **maybe_free,
                               OMT snapshot_xids,
-                              OMT live_list_reverse,
+                              OMT referenced_xids,
                               OMT live_root_txns);
 
-TXNID toku_get_youngest_live_list_txnid_for(TXNID xc, OMT live_list_reverse);
-
 #if defined(__cplusplus) || defined(__cilkplusplus)
 }
 #endif