[t:4357], [t:4364], [t:4365], merge fixes to main

git-svn-id: file:///svn/toku/tokudb@38595 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt-internal.h

@@ -756,11 +756,21 @@ void *mempool_malloc_from_omt(OMT omt, struct mempool *mp, size_t size, void **m
 //  from the OMT (which items refer to items in the old mempool) into the new mempool.
 //  If MAYBE_FREE is NULL then free the old mempool's space.
 //  Otherwise, store the old mempool's space in maybe_free.
+void
+toku_get_node_for_verify(
+    BLOCKNUM blocknum,
+    BRT brt,
+    BRTNODE* nodep
+    );
 
-int toku_verify_brtnode (BRT brt, MSN rootmsn, MSN parentmsn,
-                         BLOCKNUM blocknum, int height, struct kv_pair *lesser_pivot, struct kv_pair *greatereq_pivot, 
-                         int (*progress_callback)(void *extra, float progress), void *extra,
-                         int recurse, int verbose, int keep_on_going)
+int
+toku_verify_brtnode (BRT brt,
+                     MSN rootmsn, MSN parentmsn,
+                     BRTNODE node, int height,
+                     struct kv_pair *lesser_pivot,               // Everything in the subtree should be > lesser_pivot.  (lesser_pivot==NULL if there is no lesser pivot.)
+                     struct kv_pair *greatereq_pivot,            // Everything in the subtree should be <= lesser_pivot.  (lesser_pivot==NULL if there is no lesser pivot.)
+                     int (*progress_callback)(void *extra, float progress), void *progress_extra,
+                     int recurse, int verbose, int keep_going_on_failure)
     __attribute__ ((warn_unused_result));
 
 void toku_brtheader_free (struct brt_header *h);

newbrt/brt-verify.c

@@ -13,6 +13,7 @@
 
 #include "includes.h"
 #include <brt-flusher.h>
+#include "brt-cachetable-wrappers.h"
 
 static int 
 compare_pairs (BRT brt, struct kv_pair *a, struct kv_pair *b) {
@@ -197,44 +198,42 @@ count_eq_key_msn(BRT brt, FIFO fifo, OMT mt, const void *key, size_t keylen, MSN
     return count;
 }
 
+void
+toku_get_node_for_verify(
+    BLOCKNUM blocknum,
+    BRT brt,
+    BRTNODE* nodep
+    )
+{
+    u_int32_t fullhash = toku_cachetable_hash(brt->cf, blocknum);
+    struct brtnode_fetch_extra bfe;
+    fill_bfe_for_full_read(&bfe, brt->h);
+    toku_pin_brtnode_off_client_thread(
+        brt->h,
+        blocknum,
+        fullhash,
+        &bfe,
+        0,
+        NULL,
+        nodep
+        );
+}
+
+// input is a pinned node, on exit, node is unpinned
 int
 toku_verify_brtnode (BRT brt,
                      MSN rootmsn, MSN parentmsn,
-                     BLOCKNUM blocknum, int height,
+                     BRTNODE node, int height,
                      struct kv_pair *lesser_pivot,               // Everything in the subtree should be > lesser_pivot.  (lesser_pivot==NULL if there is no lesser pivot.)
                      struct kv_pair *greatereq_pivot,            // Everything in the subtree should be <= lesser_pivot.  (lesser_pivot==NULL if there is no lesser pivot.)
                      int (*progress_callback)(void *extra, float progress), void *progress_extra,
                      int recurse, int verbose, int keep_going_on_failure)
 {
     int result=0;
-    BRTNODE node;
-    void *node_v;
     MSN   this_msn;
+    BLOCKNUM blocknum = node->thisnodename;
 
-    u_int32_t fullhash = toku_cachetable_hash(brt->cf, blocknum);
-    {
-        struct brtnode_fetch_extra bfe;
-        fill_bfe_for_full_read(&bfe, brt->h);
-        int r = toku_cachetable_get_and_pin(
-            brt->cf, 
-            blocknum, 
-            fullhash, 
-            &node_v, 
-            NULL,
-            toku_brtnode_flush_callback, 
-            toku_brtnode_fetch_callback, 
-            toku_brtnode_pe_est_callback,
-            toku_brtnode_pe_callback, 
-            toku_brtnode_pf_req_callback,
-            toku_brtnode_pf_callback,
-            toku_brtnode_cleaner_callback,
-            &bfe, 
-            brt->h
-            );
-        assert_zero(r); // this is a bad failure if it happens.
-    }
     //printf("%s:%d pin %p\n", __FILE__, __LINE__, node_v);
-    node = node_v;
     toku_assert_entire_node_in_memory(node);
     this_msn = node->max_msn_applied_to_node_on_disk;
     if (rootmsn.msn == ZERO_MSN.msn) {
@@ -243,7 +242,6 @@ toku_verify_brtnode (BRT brt,
         parentmsn = this_msn;
     }
 
-    invariant(node->fullhash == fullhash);   // this is a bad failure if wrong
     if (height >= 0) {
         invariant(height == node->height);   // this is a bad failure if wrong
     }
@@ -326,7 +324,7 @@ toku_verify_brtnode (BRT brt,
                              }
                              last_msn = msn;
                          }));
-            struct verify_message_tree_extra extra = { .fifo = bnc->buffer, .broadcast = false, .is_fresh = true, .i = i, .verbose = verbose, .blocknum = blocknum, .keep_going_on_failure = keep_going_on_failure };
+            struct verify_message_tree_extra extra = { .fifo = bnc->buffer, .broadcast = false, .is_fresh = true, .i = i, .verbose = verbose, .blocknum = node->thisnodename, .keep_going_on_failure = keep_going_on_failure };
             int r = toku_omt_iterate(bnc->fresh_message_tree, verify_message_tree, &extra);
             if (r != 0) { result = r; goto done; }
             extra.is_fresh = false;
@@ -361,8 +359,10 @@ toku_verify_brtnode (BRT brt,
     // Verify that the subtrees have the right properties.
     if (recurse && node->height > 0) {
         for (int i = 0; i < node->n_children; i++) {
+            BRTNODE child_node;
+            toku_get_node_for_verify(BP_BLOCKNUM(node, i), brt, &child_node);
             int r = toku_verify_brtnode(brt, rootmsn, this_msn,
-                                        BP_BLOCKNUM(node, i), node->height-1,
+                                        child_node, node->height-1,
                                         (i==0)                  ? lesser_pivot        : node->childkeys[i-1],
                                         (i==node->n_children-1) ? greatereq_pivot     : node->childkeys[i],
                                         progress_callback, progress_extra,
@@ -375,7 +375,13 @@ toku_verify_brtnode (BRT brt,
     }
 done:
     {
-    int r = toku_cachetable_unpin(brt->cf, blocknum, fullhash, CACHETABLE_CLEAN, make_brtnode_pair_attr(node));
+    int r = toku_cachetable_unpin(
+        brt->cf, 
+        node->thisnodename, 
+        toku_cachetable_hash(brt->cf, node->thisnodename), 
+        CACHETABLE_CLEAN, 
+        make_brtnode_pair_attr(node)
+        );
     assert_zero(r); // this is a bad failure if it happens.
     }
     
@@ -388,9 +394,13 @@ toku_verify_brtnode (BRT brt,
 int 
 toku_verify_brt_with_progress (BRT brt, int (*progress_callback)(void *extra, float progress), void *progress_extra, int verbose, int keep_on_going) {
     assert(brt->h);
+    toku_cachetable_call_ydb_lock(brt->cf);
     u_int32_t root_hash;
     CACHEKEY *rootp = toku_calculate_root_offset_pointer(brt->h, &root_hash);
-    int r = toku_verify_brtnode(brt, ZERO_MSN, ZERO_MSN, *rootp, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
+    BRTNODE root_node;
+    toku_get_node_for_verify(*rootp, brt, &root_node);
+    toku_cachetable_call_ydb_unlock(brt->cf);
+    int r = toku_verify_brtnode(brt, ZERO_MSN, ZERO_MSN, root_node, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
     if (r == 0) {
         toku_brtheader_lock(brt->h);
         brt->h->time_of_last_verification = time(NULL);

newbrt/brt.c

@@ -5729,9 +5729,10 @@ static int
 toku_dump_brtnode (FILE *file, BRT brt, BLOCKNUM blocknum, int depth, struct kv_pair *lorange, struct kv_pair *hirange) {
     int result=0;
     BRTNODE node;
-    void *node_v;
+    void* node_v;
+    toku_get_node_for_verify(blocknum, brt, &node);
+    result=toku_verify_brtnode(brt, ZERO_MSN, ZERO_MSN, node, -1, lorange, hirange, NULL, NULL, 0, 1, 0);
     u_int32_t fullhash = toku_cachetable_hash(brt->cf, blocknum);
-    result=toku_verify_brtnode(brt, ZERO_MSN, ZERO_MSN, blocknum, -1, lorange, hirange, NULL, NULL, 0, 1, 0);
     struct brtnode_fetch_extra bfe;
     fill_bfe_for_full_read(&bfe, brt->h);
     int r = toku_cachetable_get_and_pin(

newbrt/tests/cachetable-4357.c

+#ident "$Id: cachetable-simple-verify.c 36689 2011-11-07 22:08:05Z zardosht $"
+#ident "Copyright (c) 2007-2011 Tokutek Inc.  All rights reserved."
+#include "includes.h"
+#include "test.h"
+
+static void
+flush (CACHEFILE f __attribute__((__unused__)),
+       int UU(fd),
+       CACHEKEY k  __attribute__((__unused__)),
+       void *v     __attribute__((__unused__)),
+       void *e     __attribute__((__unused__)),
+       PAIR_ATTR s      __attribute__((__unused__)),
+       PAIR_ATTR* new_size      __attribute__((__unused__)),
+       BOOL w      __attribute__((__unused__)),
+       BOOL keep   __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
+       ) {
+  /* Do nothing */
+  if (verbose) { printf("FLUSH: %d\n", (int)k.b); }
+  //usleep (5*1024*1024);
+}
+
+static int
+fetch (CACHEFILE f        __attribute__((__unused__)),
+       int UU(fd),
+       CACHEKEY k         __attribute__((__unused__)),
+       u_int32_t fullhash __attribute__((__unused__)),
+       void **value       __attribute__((__unused__)),
+       PAIR_ATTR *sizep        __attribute__((__unused__)),
+       int  *dirtyp,
+       void *extraargs    __attribute__((__unused__))
+       ) {
+  *dirtyp = 0;
+  *value = NULL;
+  *sizep = make_pair_attr(8);
+  return 0;
+}
+
+CACHEFILE f1;
+
+static void *pin_nonblocking(void *arg) {    
+    void* v1;
+    long s1;
+    int r = toku_cachetable_get_and_pin_nonblocking(
+        f1, 
+        make_blocknum(1), 
+        toku_cachetable_hash(f1, make_blocknum(1)), 
+        &v1, 
+        &s1, 
+        flush, fetch, def_pe_est_callback, def_pe_callback, def_pf_req_callback, def_pf_callback, def_cleaner_callback, 
+        NULL, 
+        NULL, 
+        NULL
+        );
+    assert(r==TOKUDB_TRY_AGAIN);
+    return arg;
+}
+
+
+static void
+cachetable_test (void) {
+  const int test_limit = 12;
+  int r;
+  CACHETABLE ct;
+  r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
+  char fname1[] = __FILE__ "test1.dat";
+  unlink(fname1);
+  r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
+
+  void* v1;
+  long s1;
+  r = toku_cachetable_get_and_pin(
+      f1, 
+      make_blocknum(1), 
+      toku_cachetable_hash(f1, make_blocknum(1)), 
+      &v1, 
+      &s1, 
+      flush, fetch, def_pe_est_callback, def_pe_callback, def_pf_req_callback, def_pf_callback, def_cleaner_callback, 
+      NULL, 
+      NULL
+      );
+  toku_pthread_t pin_nonblocking_tid;
+  r = toku_pthread_create(&pin_nonblocking_tid, NULL, pin_nonblocking, NULL); 
+  assert_zero(r);    
+  // sleep 3 seconds
+  usleep(3*1024*1024);
+  r = toku_cachetable_unpin_and_remove(f1, make_blocknum(1), NULL, NULL);
+  assert_zero(r);
+  
+  void *ret;
+  r = toku_pthread_join(pin_nonblocking_tid, &ret); 
+  assert_zero(r);
+  
+  toku_cachetable_verify(ct);
+  r = toku_cachefile_close(&f1, 0, FALSE, ZERO_LSN); assert(r == 0 && f1 == 0);
+  r = toku_cachetable_close(&ct); lazy_assert_zero(r);
+
+
+}
+
+int
+test_main(int argc, const char *argv[]) {
+  default_parse_args(argc, argv);
+  cachetable_test();
+  return 0;
+}

newbrt/tests/cachetable-4365.c

+#ident "$Id: cachetable-simple-verify.c 36689 2011-11-07 22:08:05Z zardosht $"
+#ident "Copyright (c) 2007-2011 Tokutek Inc.  All rights reserved."
+#include "includes.h"
+#include "test.h"
+
+static void
+flush (CACHEFILE f __attribute__((__unused__)),
+       int UU(fd),
+       CACHEKEY k  __attribute__((__unused__)),
+       void *v     __attribute__((__unused__)),
+       void *e     __attribute__((__unused__)),
+       PAIR_ATTR s      __attribute__((__unused__)),
+       PAIR_ATTR* new_size      __attribute__((__unused__)),
+       BOOL w      __attribute__((__unused__)),
+       BOOL keep   __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
+       ) {
+  /* Do nothing */
+  if (verbose) { printf("FLUSH: %d\n", (int)k.b); }
+  //usleep (5*1024*1024);
+}
+
+static int
+fetch (CACHEFILE f        __attribute__((__unused__)),
+       int UU(fd),
+       CACHEKEY k         __attribute__((__unused__)),
+       u_int32_t fullhash __attribute__((__unused__)),
+       void **value       __attribute__((__unused__)),
+       PAIR_ATTR *sizep        __attribute__((__unused__)),
+       int  *dirtyp,
+       void *extraargs    __attribute__((__unused__))
+       ) {
+  *dirtyp = 0;
+  *value = NULL;
+  *sizep = make_pair_attr(8);
+  return 0;
+}
+
+CACHEFILE f1;
+
+static void *pin_nonblocking(void *arg) {    
+    void* v1;
+    long s1;
+    int r = toku_cachetable_get_and_pin_nonblocking(
+        f1, 
+        make_blocknum(1), 
+        toku_cachetable_hash(f1, make_blocknum(1)), 
+        &v1, 
+        &s1, 
+        flush, fetch, def_pe_est_callback, def_pe_callback, def_pf_req_callback, def_pf_callback, def_cleaner_callback, 
+        NULL, 
+        NULL, 
+        NULL
+        );
+    assert(r==TOKUDB_TRY_AGAIN);
+    return arg;
+}
+
+static void *put_same_key(void *arg) {
+    int r = toku_cachetable_put(
+        f1, 
+        make_blocknum(1),
+        toku_cachetable_hash(f1,make_blocknum(1)),
+        NULL, 
+        make_pair_attr(4),
+        flush, def_pe_est_callback, def_pe_callback, def_cleaner_callback, 
+        NULL
+        );
+    assert(r==0);
+    return arg;
+}
+
+
+toku_pthread_t put_tid;
+
+static void test_remove_key(CACHEKEY* UU(cachekey), BOOL UU(for_checkpoint), void* UU(extra)) {
+    int r = toku_pthread_create(&put_tid, NULL, put_same_key, NULL); 
+    assert_zero(r);    
+}
+
+
+static void
+cachetable_test (void) {
+  const int test_limit = 12;
+  int r;
+  CACHETABLE ct;
+  r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
+  char fname1[] = __FILE__ "test1.dat";
+  unlink(fname1);
+  r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
+
+  void* v1;
+  long s1;
+  r = toku_cachetable_get_and_pin(
+      f1, 
+      make_blocknum(1), 
+      toku_cachetable_hash(f1, make_blocknum(1)), 
+      &v1, 
+      &s1, 
+      flush, fetch, def_pe_est_callback, def_pe_callback, def_pf_req_callback, def_pf_callback, def_cleaner_callback, 
+      NULL, 
+      NULL
+      );
+  toku_pthread_t pin_nonblocking_tid;
+  r = toku_pthread_create(&pin_nonblocking_tid, NULL, pin_nonblocking, NULL); 
+  assert_zero(r);    
+  // sleep 3 seconds
+  usleep(3*1024*1024);
+  r = toku_cachetable_unpin_and_remove(f1, make_blocknum(1), test_remove_key, NULL);
+  assert_zero(r);
+  
+  void *ret;
+  r = toku_pthread_join(pin_nonblocking_tid, &ret); 
+  assert_zero(r);
+  r = toku_pthread_join(put_tid, &ret); 
+  assert_zero(r);
+
+  r = toku_cachetable_unpin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), CACHETABLE_CLEAN, make_pair_attr(2));
+  
+  toku_cachetable_verify(ct);
+  r = toku_cachefile_close(&f1, 0, FALSE, ZERO_LSN); assert(r == 0 && f1 == 0);
+  r = toku_cachetable_close(&ct); lazy_assert_zero(r);
+
+
+}
+
+int
+test_main(int argc, const char *argv[]) {
+  default_parse_args(argc, argv);
+  for (int i = 0; i < 20; i++) {
+      cachetable_test();
+  }
+  return 0;
+}

src/ydb.c

@@ -6476,20 +6476,17 @@ struct ydb_verify_context {
 static int
 ydb_verify_progress_callback(void *extra, float progress) {
     struct ydb_verify_context *context = (struct ydb_verify_context *) extra;
-    toku_ydb_unlock_and_yield(1000);
     int r = 0;
-    if (context->progress_callback)
+    if (context->progress_callback) {
         r = context->progress_callback(context->progress_extra, progress);
-    toku_ydb_lock();
+    }
     return r;
 }
 
 static int
 locked_db_verify_with_progress(DB *db, int (*progress_callback)(void *extra, float progress), void *progress_extra, int verbose, int keep_going) {
     struct ydb_verify_context context = { progress_callback, progress_extra };
-    toku_ydb_lock();
     int r = toku_verify_brt_with_progress(db->i->brt, ydb_verify_progress_callback, &context, verbose, keep_going);
-    toku_ydb_unlock();
     return r;
 }