Merge tag 'tokudb-ft-index/tokudb-7.5.7' into 5.5

storage/tokudb/ft-index/buildheader/make_tdb.cc

@@ -587,6 +587,7 @@ static void print_db_txn_struct (void) {
         "uint64_t (*get_client_id)(DB_TXN *)",
         "bool (*is_prepared)(DB_TXN *)",
         "DB_TXN *(*get_child)(DB_TXN *)",
+        "uint64_t (*get_start_time)(DB_TXN *)",
 	NULL};
     sort_and_dump_fields("db_txn", false, extra);
 }
@@ -786,7 +787,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
 
     printf("typedef void (*lock_timeout_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid);\n");
     printf("typedef int (*iterate_row_locks_callback)(DB **db, DBT *left_key, DBT *right_key, void *extra);\n");
-    printf("typedef int (*iterate_transactions_callback)(uint64_t txnid, uint64_t client_id, iterate_row_locks_callback cb, void *locks_extra, void *extra);\n");
+    printf("typedef int (*iterate_transactions_callback)(DB_TXN *dbtxn, iterate_row_locks_callback cb, void *locks_extra, void *extra);\n");
     printf("typedef int (*iterate_requests_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid, uint64_t start_time, void *extra);\n");
     print_db_env_struct();
     print_db_key_range_struct();

storage/tokudb/ft-index/ft/ft-verify.cc

@@ -160,10 +160,14 @@ get_ith_key_dbt (BASEMENTNODE bn, int i) {
 
 #define VERIFY_ASSERTION(predicate, i, string) ({                                                                              \
     if(!(predicate)) {                                                                                                         \
-        (void) verbose; \
-        if (true) {                                                                                                         \
         fprintf(stderr, "%s:%d: Looking at child %d of block %" PRId64 ": %s\n", __FILE__, __LINE__, i, blocknum.b, string);   \
-        }                                                                                                                      \
+        result = TOKUDB_NEEDS_REPAIR;                                                                                          \
+        if (!keep_going_on_failure) goto done;                                                                                 \
+    }})
+
+#define VERIFY_ASSERTION_BASEMENT(predicate, bn, entry, string) ({           \
+    if(!(predicate)) {                                                                                                         \
+        fprintf(stderr, "%s:%d: Looking at block %" PRId64 " bn %d entry %d: %s\n", __FILE__, __LINE__, blocknum.b, bn, entry, string); \
         result = TOKUDB_NEEDS_REPAIR;                                                                                          \
         if (!keep_going_on_failure) goto done;                                                                                 \
     }})
@@ -201,7 +205,6 @@ struct verify_message_tree_extra {
 int verify_message_tree(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) __attribute__((nonnull(3)));
 int verify_message_tree(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e)
 {
-    int verbose = e->verbose;
     BLOCKNUM blocknum = e->blocknum;
     int keep_going_on_failure = e->keep_going_on_failure;
     int result = 0;
@@ -236,7 +239,6 @@ int error_on_iter(const int32_t &UU(offset), const uint32_t UU(idx), void *UU(e)
 int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) __attribute__((nonnull(3)));
 int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e)
 {
-    int verbose = e->verbose;
     BLOCKNUM blocknum = e->blocknum;
     int keep_going_on_failure = e->keep_going_on_failure;
     int result = 0;
@@ -462,16 +464,16 @@ toku_verify_ftnode_internal(FT_HANDLE ft_handle,
                 DBT kdbt = get_ith_key_dbt(bn, j);
                 if (curr_less_pivot) {
                     int compare = compare_pairs(ft_handle, curr_less_pivot, &kdbt);
-                    VERIFY_ASSERTION(compare < 0, j, "The leafentry is >= the lower-bound pivot");
+                    VERIFY_ASSERTION_BASEMENT(compare < 0, i, j, "The leafentry is >= the lower-bound pivot");
                 }
                 if (curr_geq_pivot) {
                     int compare = compare_pairs(ft_handle, curr_geq_pivot, &kdbt);
-                    VERIFY_ASSERTION(compare >= 0, j, "The leafentry is < the upper-bound pivot");
+                    VERIFY_ASSERTION_BASEMENT(compare >= 0, i, j, "The leafentry is < the upper-bound pivot");
                 }
                 if (0 < j) {
                     DBT prev_key_dbt = get_ith_key_dbt(bn, j-1);
                     int compare = compare_pairs(ft_handle, &prev_key_dbt, &kdbt);
-                    VERIFY_ASSERTION(compare < 0, j, "Adjacent leafentries are out of order");
+                    VERIFY_ASSERTION_BASEMENT(compare < 0, i, j, "Adjacent leafentries are out of order");
                 }
             }
         }

storage/tokudb/ft-index/ft/logger/recover.cc

@@ -111,7 +111,8 @@ int tokuft_recovery_trace = 0;                    // turn on recovery tracing, d
 #endif
 
 // time in seconds between recovery progress reports
-#define TOKUDB_RECOVERY_PROGRESS_TIME 15
+#define TOKUFT_RECOVERY_PROGRESS_TIME 15
+time_t tokuft_recovery_progress_time = TOKUFT_RECOVERY_PROGRESS_TIME;
 
 enum ss {
     BACKWARD_NEWER_CHECKPOINT_END = 1,
@@ -325,14 +326,12 @@ static int recover_env_init (RECOVER_ENV renv,
 }
 
 static void recover_env_cleanup (RECOVER_ENV renv) {
-    int r;
-
     invariant_zero(renv->fmap.filenums->size());
     file_map_destroy(&renv->fmap);
 
     if (renv->destroy_logger_at_end) {
         toku_logger_close_rollback(renv->logger);
-        r = toku_logger_close(&renv->logger);
+        int r = toku_logger_close(&renv->logger);
         assert(r == 0);
     } else {
         toku_logger_write_log_files(renv->logger, true);
@@ -749,6 +748,36 @@ static int toku_recover_backward_xbegin (struct logtype_xbegin *UU(l), RECOVER_E
     return 0;
 }
 
+struct toku_txn_progress_extra {
+    time_t tlast;
+    LSN lsn;
+    const char *type;
+    TXNID_PAIR xid;
+    uint64_t last_total;
+};
+
+static void toku_recover_txn_progress(TOKU_TXN_PROGRESS txn_progress, void *extra) {
+    toku_txn_progress_extra *txn_progress_extra = static_cast<toku_txn_progress_extra *>(extra);
+    if (txn_progress_extra->last_total == 0)
+        txn_progress_extra->last_total = txn_progress->entries_total;
+    else
+        assert(txn_progress_extra->last_total == txn_progress->entries_total);
+    time_t tnow = time(NULL);
+    if (tnow - txn_progress_extra->tlast >= tokuft_recovery_progress_time) {
+        txn_progress_extra->tlast = tnow;
+        fprintf(stderr, "%.24s TokuFT ", ctime(&tnow));
+        if (txn_progress_extra->lsn.lsn != 0)
+            fprintf(stderr, "lsn %" PRIu64 " ", txn_progress_extra->lsn.lsn);
+        fprintf(stderr, "%s xid %" PRIu64 ":%" PRIu64 " ",
+                txn_progress_extra->type, txn_progress_extra->xid.parent_id64, txn_progress_extra->xid.child_id64);
+        fprintf(stderr, "%" PRIu64 "/%" PRIu64 " ",
+                txn_progress->entries_processed, txn_progress->entries_total);
+        if (txn_progress->entries_total > 0)
+            fprintf(stderr, "%.0f%% ", ((double) txn_progress->entries_processed / (double) txn_progress->entries_total) * 100.0);
+        fprintf(stderr, "\n");
+    }
+}
+
 static int toku_recover_xcommit (struct logtype_xcommit *l, RECOVER_ENV renv) {
     // find the transaction by transaction id
     TOKUTXN txn = NULL;
@@ -756,8 +785,8 @@ static int toku_recover_xcommit (struct logtype_xcommit *l, RECOVER_ENV renv) {
     assert(txn!=NULL);
 
     // commit the transaction
-    int r = toku_txn_commit_with_lsn(txn, true, l->lsn,
-                                 NULL, NULL);
+    toku_txn_progress_extra extra = { time(NULL), l->lsn, "commit", l->xid };
+    int r = toku_txn_commit_with_lsn(txn, true, l->lsn, toku_recover_txn_progress, &extra);
     assert(r == 0);
 
     // close the transaction
@@ -799,7 +828,8 @@ static int toku_recover_xabort (struct logtype_xabort *l, RECOVER_ENV renv) {
     assert(txn!=NULL);
 
     // abort the transaction
-    r = toku_txn_abort_with_lsn(txn, l->lsn, NULL, NULL);
+    toku_txn_progress_extra extra = { time(NULL), l->lsn, "abort", l->xid };
+    r = toku_txn_abort_with_lsn(txn, l->lsn, toku_recover_txn_progress, &extra);
     assert(r == 0);
 
     // close the transaction
@@ -1301,7 +1331,6 @@ static int is_txn_unprepared(TOKUTXN txn, void* extra) {
     return 0;
 }
 
-
 static int find_an_unprepared_txn (RECOVER_ENV renv, TOKUTXN *txnp) {
     TOKUTXN txn = nullptr;
     int r = toku_txn_manager_iter_over_live_root_txns(
@@ -1326,6 +1355,7 @@ static int call_prepare_txn_callback_iter(TOKUTXN txn, void* extra) {
 }
 
 static void recover_abort_live_txn(TOKUTXN txn) {
+    fprintf(stderr, "%s %" PRIu64 "\n", __FUNCTION__, txn->txnid.parent_id64);
     // recursively abort all children first
     if (txn->child != NULL) {
         recover_abort_live_txn(txn->child);
@@ -1333,7 +1363,8 @@ static void recover_abort_live_txn(TOKUTXN txn) {
     // sanity check that the recursive call successfully NULLs out txn->child
     invariant(txn->child == NULL);
     // abort the transaction
-    int r = toku_txn_abort_txn(txn, NULL, NULL);
+    toku_txn_progress_extra extra = { time(NULL), ZERO_LSN, "abort live", txn->txnid };
+    int r = toku_txn_abort_txn(txn, toku_recover_txn_progress, &extra);
     assert(r == 0);
     
     // close the transaction
@@ -1451,9 +1482,10 @@ static int do_recovery(RECOVER_ENV renv, const char *env_dir, const char *log_di
         // trace progress
         if ((i % 1000) == 0) {
             tnow = time(NULL);
-            if (tnow - tlast >= TOKUDB_RECOVERY_PROGRESS_TIME) {
+            if (tnow - tlast >= tokuft_recovery_progress_time) {
                 thislsn = toku_log_entry_get_lsn(le);
-                fprintf(stderr, "%.24s TokuFT recovery scanning backward from %" PRIu64 " at %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, recover_state(renv));
+                fprintf(stderr, "%.24s TokuFT recovery scanning backward from %" PRIu64 " at %" PRIu64 " (%s)\n",
+                        ctime(&tnow), lastlsn.lsn, thislsn.lsn, recover_state(renv));
                 tlast = tnow;
             }
         }
@@ -1482,16 +1514,18 @@ static int do_recovery(RECOVER_ENV renv, const char *env_dir, const char *log_di
     assert(le);
     thislsn = toku_log_entry_get_lsn(le);
     tnow = time(NULL);
-    fprintf(stderr, "%.24s TokuFT recovery starts scanning forward to %" PRIu64 " from %" PRIu64 " left %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
+    fprintf(stderr, "%.24s TokuFT recovery starts scanning forward to %" PRIu64 " from %" PRIu64 " left %" PRIu64 " (%s)\n",
+            ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
 
     for (unsigned i=0; 1; i++) {
 
         // trace progress
         if ((i % 1000) == 0) {
             tnow = time(NULL);
-            if (tnow - tlast >= TOKUDB_RECOVERY_PROGRESS_TIME) {
+            if (tnow - tlast >= tokuft_recovery_progress_time) {
                 thislsn = toku_log_entry_get_lsn(le);
-                fprintf(stderr, "%.24s TokuFT recovery scanning forward to %" PRIu64 " at %" PRIu64 " left %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
+                fprintf(stderr, "%.24s TokuFT recovery scanning forward to %" PRIu64 " at %" PRIu64 " left %" PRIu64 " (%s)\n",
+                        ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
                 tlast = tnow;
             }
         }

storage/tokudb/ft-index/ft/txn/txn.cc

@@ -344,6 +344,7 @@ static txn_child_manager tcm;
         .state = TOKUTXN_LIVE,
         .num_pin = 0,
         .client_id = 0,
+        .start_time = time(NULL),
     };
 
     TOKUTXN result = NULL;
@@ -787,6 +788,10 @@ void toku_txn_set_client_id(TOKUTXN txn, uint64_t client_id) {
     txn->client_id = client_id;
 }
 
+time_t toku_txn_get_start_time(struct tokutxn *txn) {
+    return txn->start_time;
+}
+
 int toku_txn_reads_txnid(TXNID txnid, TOKUTXN txn) {
     int r = 0;
     TXNID oldest_live_in_snapshot = toku_get_oldest_in_live_root_txn_list(txn);

storage/tokudb/ft-index/ft/txn/txn.h

@@ -253,6 +253,7 @@ struct tokutxn {
     uint32_t num_pin; // number of threads (all hot indexes) that want this
                       // txn to not transition to commit or abort
     uint64_t client_id;
+    time_t start_time;
 };
 typedef struct tokutxn *TOKUTXN;
 
@@ -368,6 +369,8 @@ bool toku_txn_has_spilled_rollback(struct tokutxn *txn);
 uint64_t toku_txn_get_client_id(struct tokutxn *txn);
 void toku_txn_set_client_id(struct tokutxn *txn, uint64_t client_id);
 
+time_t toku_txn_get_start_time(struct tokutxn *txn);
+
 //
 // This function is used by the leafentry iterators.
 // returns TOKUDB_ACCEPT if live transaction context is allowed to read a value

storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc

@@ -104,9 +104,11 @@ struct iterate_extra {
     bool visited_txn[3];
 };
 
-static int iterate_callback(uint64_t txnid, uint64_t client_id,
+static int iterate_callback(DB_TXN *txn,
                             iterate_row_locks_callback iterate_locks,
                             void *locks_extra, void *extra) {
+    uint64_t txnid = txn->id64(txn);
+    uint64_t client_id = txn->get_client_id(txn);
     iterate_extra *info = reinterpret_cast<iterate_extra *>(extra);
     DB *db;
     DBT left_key, right_key;

storage/tokudb/ft-index/src/tests/test_stress0.cc

@@ -140,9 +140,11 @@ static int UU() iterate_pending_lock_requests_op(DB_TXN *UU(txn), ARG arg, void
     return r;
 }
 
-static int iterate_txns(uint64_t txnid, uint64_t client_id,
+static int iterate_txns(DB_TXN *txn,
                         iterate_row_locks_callback iterate_locks,
                         void *locks_extra, void *extra) {
+    uint64_t txnid = txn->id64(txn);
+    uint64_t client_id = txn->get_client_id(txn);
     invariant_null(extra);
     invariant(txnid > 0);
     invariant(client_id == 0);

storage/tokudb/ft-index/src/ydb.cc

@@ -2492,24 +2492,21 @@ struct iter_txns_callback_extra {
 };
 
 static int iter_txns_callback(TOKUTXN txn, void *extra) {
+    int r = 0;
     iter_txns_callback_extra *info =
         reinterpret_cast<iter_txns_callback_extra *>(extra);
-
     DB_TXN *dbtxn = toku_txn_get_container_db_txn(txn);
     invariant_notnull(dbtxn);
-
+    if (db_txn_struct_i(dbtxn)->tokutxn == txn) { // make sure that the dbtxn is fully initialized
         toku_mutex_lock(&db_txn_struct_i(dbtxn)->txn_mutex);
         toku_pthread_rwlock_rdlock(&info->env->i->open_dbs_rwlock);
 
         iter_txn_row_locks_callback_extra e(info->env, &db_txn_struct_i(dbtxn)->lt_map);
-    const int r = info->callback(toku_txn_get_txnid(txn).parent_id64,
-                                 toku_txn_get_client_id(txn),
-                                 iter_txn_row_locks_callback,
-                                 &e,
-                                 info->extra);
+        r = info->callback(dbtxn, iter_txn_row_locks_callback, &e, info->extra);
 
         toku_pthread_rwlock_rdunlock(&info->env->i->open_dbs_rwlock);
         toku_mutex_unlock(&db_txn_struct_i(dbtxn)->txn_mutex);
+    }
 
     return r;
 }

storage/tokudb/ft-index/src/ydb_txn.cc

@@ -433,6 +433,11 @@ static DB_TXN *toku_txn_get_child(DB_TXN *txn) {
     return db_txn_struct_i(txn)->child;
 }
 
+static uint64_t toku_txn_get_start_time(DB_TXN *txn) {
+    TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
+    return toku_txn_get_start_time(ttxn);
+}
+
 static inline void txn_func_init(DB_TXN *txn) {
 #define STXN(name) txn->name = locked_txn_ ## name
     STXN(abort);
@@ -451,6 +456,7 @@ static inline void txn_func_init(DB_TXN *txn) {
     txn->id64 = toku_txn_id64;
     txn->is_prepared = toku_txn_is_prepared;
     txn->get_child = toku_txn_get_child;
+    txn->get_start_time = toku_txn_get_start_time;
 }
 
 //

storage/tokudb/ft-index/tools/CMakeLists.txt

 set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE DONT_DEPRECATE_ERRNO)
 
-set(tools tokudb_dump tokuftdump tdb_logprint tdb-recover ftverify ba_replay)
+set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify ba_replay)
 foreach(tool ${tools})
   add_executable(${tool} ${tool}.cc)
   add_dependencies(${tool} install_tdb_h)
@@ -12,9 +12,6 @@ endforeach(tool)
 # link in math.h library just for this tool.
 target_link_libraries(ftverify m)
 
-install(
-  TARGETS tokuftdump
-  DESTINATION ${INSTALL_BINDIR}
-  COMPONENT Server
-  )
+install(TARGETS tokuftdump      DESTINATION ${INSTALL_BINDIR} COMPONENT Server)
+install(TARGETS tokuft_logprint DESTINATION ${INSTALL_BINDIR} COMPONENT Server)
 

storage/tokudb/ft-index/tools/tdb_logprint.cc → storage/tokudb/ft-index/tools/tokuft_logprint.cc

@@ -91,8 +91,6 @@ PATENT RIGHTS GRANT:
 
 /* Dump the log from stdin to stdout. */
 
-#include <config.h>
-
 #include "ft/log_header.h"
 #include "ft/logger/logger.h"
 

storage/tokudb/ft-index/tools/tokuftdump.cc

@@ -250,6 +250,8 @@ static int64_t getRootNode(FT ft) {
 }
 
 static int print_le(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t idx UU(), void *const ai UU()) {
+    unsigned int *le_index = (unsigned int *) ai;
+    printf("%u: ", *le_index); *le_index += 1;
     print_klpair(stdout, key, keylen, le);
     printf("\n");
     return 0;
@@ -539,7 +541,8 @@ static void dump_node(int fd, BLOCKNUM blocknum, FT ft) {
             printf(" n_bytes_in_buffer= %" PRIu64 "", BLB_DATA(n, i)->get_disk_size());
             printf(" items_in_buffer=%u\n", BLB_DATA(n, i)->num_klpairs());
             if (do_dump_data) {
-                BLB_DATA(n, i)->iterate<void, print_le>(NULL);
+                unsigned int le_index = 0;
+                BLB_DATA(n, i)->iterate<void, print_le>(&le_index);
             }
         }
     }
@@ -938,6 +941,7 @@ static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) {
         } else if (strcmp(fields[0], "header") == 0) {
             toku_ft_free(ft);
             open_header(fd, &ft, cf);
+            dump_header(ft);
         } else if (strcmp(fields[0], "rn") == 0||strcmp(fields[0], "rootNode")==0||strcmp(fields[0], "rootnode") == 0) {
             printf("Root node :%d\n",root);
         } else if (strcmp(fields[0], "block") == 0 && nfields == 2) {

storage/tokudb/ft-index/util/scoped_malloc.cc

@@ -145,6 +145,9 @@ namespace toku {
         }
 
         void destroy() {
+#if TOKU_SCOPED_MALLOC_DEBUG
+            printf("%s %p %p\n", __FUNCTION__, this, m_stack);
+#endif
             if (m_stack != NULL) {
                 toku_free(m_stack);
                 m_stack = NULL;
@@ -167,13 +170,17 @@ namespace toku {
         static void destroy_and_deregister(void *key) {
             invariant_notnull(key);
             tl_stack *st = reinterpret_cast<tl_stack *>(key);
-            st->destroy();
 
+            size_t n = 0;
             toku_mutex_lock(&global_stack_set_mutex);
-            invariant_notnull(global_stack_set);
-            size_t n = global_stack_set->erase(st);
-            invariant(n == 1);
+            if (global_stack_set) {
+                n = global_stack_set->erase(st);
+            }
             toku_mutex_unlock(&global_stack_set_mutex);
+
+            if (n == 1) {
+                st->destroy(); // destroy the stack if this function erased it from the set.  otherwise, somebody else destroyed it.
+            }
         }
 
         // Allocate 'size' bytes and return a pointer to the first byte
@@ -244,6 +251,11 @@ void toku_scoped_malloc_init(void) {
 }
 
 void toku_scoped_malloc_destroy(void) {
+    toku_scoped_malloc_destroy_key();
+    toku_scoped_malloc_destroy_set();
+}
+
+void toku_scoped_malloc_destroy_set(void) {
     toku_mutex_lock(&toku::global_stack_set_mutex);
     invariant_notnull(toku::global_stack_set);
     // Destroy any tl_stacks that were registered as thread locals but did not
@@ -254,10 +266,11 @@ void toku_scoped_malloc_destroy(void) {
         (*i)->destroy();
     }
     delete toku::global_stack_set;
+    toku::global_stack_set = nullptr;
     toku_mutex_unlock(&toku::global_stack_set_mutex);
+}
 
-    // We're deregistering the destructor key here. When this thread exits,
-    // the tl_stack destructor won't get called, so we need to do that first.
+void toku_scoped_malloc_destroy_key(void) {
     int r = pthread_key_delete(toku::tl_stack_destroy_pthread_key);
     invariant_zero(r);
 }

storage/tokudb/ft-index/util/scoped_malloc.h

@@ -151,3 +151,7 @@ void toku_scoped_malloc_init(void);
 
 void toku_scoped_malloc_destroy(void);
 
+void toku_scoped_malloc_destroy_set(void);
+
+void toku_scoped_malloc_destroy_key(void);
+

util/tests/sm-basic.cc

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuFT, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2007-2013 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+// test that basic scoped malloc works with a thread
+
+#ident "Copyright (c) 2015 Tokutek Inc.  All rights reserved."
+#include <toku_portability.h>
+#include <toku_assert.h>
+#include <toku_pthread.h>
+#include <util/scoped_malloc.h>
+
+static void sm_test(void) {
+    toku::scoped_malloc a(1);
+    {
+        toku::scoped_malloc b(2);
+        {
+            toku::scoped_malloc c(3);
+        }
+    }
+}
+
+static void *sm_test_f(void *arg) {
+    sm_test();
+    return arg;
+}
+
+int main(void) {
+    toku_scoped_malloc_init();
+
+    // run the test
+    toku_pthread_t tid;
+    int r;
+    r = toku_pthread_create(&tid, NULL, sm_test_f, NULL);
+    assert_zero(r);
+    void *ret;
+    r = toku_pthread_join(tid, &ret);
+    assert_zero(r);
+
+    toku_scoped_malloc_destroy();
+
+    return 0;
+}

util/tests/sm-crash-double-free.cc

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuFT, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2007-2013 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+// force a race between the scoped malloc global destructor and a thread variable destructor
+
+#ident "Copyright (c) 2015 Tokutek Inc.  All rights reserved."
+#define TOKU_SCOPED_MALLOC_DEBUG 1
+#include <toku_portability.h>
+#include <toku_assert.h>
+#include <toku_pthread.h>
+#include <toku_race_tools.h>
+#include <util/scoped_malloc.h>
+
+volatile int state = 0;
+
+static void sm_test(void) {
+    toku::scoped_malloc a(1);
+}
+
+static void *sm_test_f(void *arg) {
+    sm_test();
+    state = 1;
+    while (state != 2) sleep(1);
+    return arg;
+}
+
+int main(void) {
+    TOKU_VALGRIND_HG_DISABLE_CHECKING(&state, sizeof state);
+    state = 0;
+    toku_scoped_malloc_init();
+    toku_pthread_t tid;
+    int r;
+    r = toku_pthread_create(&tid, NULL, sm_test_f, NULL);
+    assert_zero(r);
+    void *ret;
+    while (state != 1) sleep(1);
+    toku_scoped_malloc_destroy_set();
+    state = 2;
+    r = toku_pthread_join(tid, &ret);
+    assert_zero(r);
+    toku_scoped_malloc_destroy_key();
+    return 0;
+}