FT-260 Move ftnode related structures and declarations to ft/node.h and ft/node.cc

ft/CMakeLists.txt

@@ -53,6 +53,7 @@ set(FT_SOURCES
   logger
   log_upgrade
   msg_buffer
+  node
   quicklz
   recover
   rollback

ft/ft-cachetable-wrappers.cc

@@ -89,12 +89,13 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <ft-cachetable-wrappers.h>
+#include "ft/fttypes.h"
+#include "ft/ft-cachetable-wrappers.h"
+#include "ft/ft-flusher.h"
+#include "ft/ft-internal.h"
+#include "ft/ft.h"
+#include "ft/node.h"
 
-#include <fttypes.h>
-#include <ft-flusher.h>
-#include <ft-internal.h>
-#include <ft.h>
 #include <util/context.h>
 
 static void

ft/ft-cachetable-wrappers.h

@@ -92,8 +92,9 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "ft/ft-internal.h"
 #include "ft/cachetable.h"
+#include "ft/ft-internal.h"
+#include "ft/node.h"
 
 /**
  * Put an empty node (that is, no fields filled) into the cachetable. 

ft/ft-flusher.h

@@ -151,11 +151,14 @@ toku_flusher_thread_set_callback(
  * Puts a workitem on the flusher thread queue, scheduling the node to be
  * flushed by toku_ft_flush_some_child.
  */
-void
-toku_ft_flush_node_on_background_thread(
-    FT ft,
-    FTNODE parent
-    );
+void toku_ft_flush_node_on_background_thread(FT ft, FTNODE parent);
+
+enum split_mode {
+    SPLIT_EVENLY,
+    SPLIT_LEFT_HEAVY,
+    SPLIT_RIGHT_HEAVY
+};
+
 
 // Given pinned node and pinned child, split child into two
 // and update node with information about its new child.
@@ -166,6 +169,7 @@ void toku_ft_split_child(
     FTNODE child,
     enum split_mode split_mode
     );
+
 // Given pinned node, merge childnum with a neighbor and update node with
 // information about the change
 void toku_ft_merge_child(
@@ -217,8 +221,6 @@ ft_nonleaf_split(
     FTNODE* dependent_nodes
     );
 
-
-
 /************************************************************************
  * HOT optimize, should perhaps be factored out to its own header file  *
  ************************************************************************
@@ -249,5 +251,5 @@ void toku_ft_hot_get_status(FT_HOT_STATUS);
  */
 int
 toku_ft_hot_optimize(FT_HANDLE ft_h, DBT* left, DBT* right,
-                      int (*progress_callback)(void *extra, float progress),
-                      void *progress_extra, uint64_t* loops_run);
+                     int (*progress_callback)(void *extra, float progress),
+                     void *progress_extra, uint64_t* loops_run);

ft/ft-hot-flusher.cc

@@ -89,14 +89,15 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <ft-flusher.h>
-#include <ft-flusher-internal.h>
-#include <ft-cachetable-wrappers.h>
-#include <ft-internal.h>
-#include <ft.h>
-#include <portability/toku_atomic.h>
-#include <util/status.h>
-#include <util/context.h>
+#include "ft/ft.h"
+#include "ft/ft-cachetable-wrappers.h"
+#include "ft/ft-flusher.h"
+#include "ft/ft-flusher-internal.h"
+#include "ft/ft-internal.h"
+#include "ft/node.h"
+#include "portability/toku_atomic.h"
+#include "util/context.h"
+#include "util/status.h"
 
 // Member Descirption:
 // 1. highest_pivot_key - this is the key that corresponds to the 
@@ -339,7 +340,7 @@ toku_ft_hot_optimize(FT_HANDLE ft_handle, DBT* left, DBT* right,
                             PL_WRITE_EXPENSIVE, 
                             &root,
                             true);
-            toku_assert_entire_node_in_memory(root);
+            toku_ftnode_assert_fully_in_memory(root);
         }
 
         // Prepare HOT diagnostics.

ft/ft-node-deserialize.cc

@@ -89,8 +89,8 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <ft-internal.h>
-#include <db.h>
+#include "ft/node.h"
+#include "ft-internal.h"
 
 /*
  * ft-node-deserialize.c -

ft/ft-test-helpers.cc

@@ -93,6 +93,7 @@ PATENT RIGHTS GRANT:
 #include "ft-flusher.h"
 #include "ft-internal.h"
 #include "ft.h"
+#include "node.h"
 #include "fttypes.h"
 #include "ule.h"
 
@@ -222,7 +223,7 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const
 
     static size_t zero_flow_deltas[] = { 0, 0 };
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
-    toku_ft_node_put_msg (
+    toku_ftnode_put_msg(
         ft_handle->ft->compare_fun,
         ft_handle->ft->update_fun,
         &ft_handle->ft->cmp_descriptor,

ft/ft-verify.cc

@@ -100,6 +100,7 @@ PATENT RIGHTS GRANT:
 #include "ft-cachetable-wrappers.h"
 #include "ft-internal.h"
 #include "ft.h"
+#include "node.h"
 
 static int 
 compare_pairs (FT_HANDLE ft_handle, const DBT *a, const DBT *b) {
@@ -399,7 +400,7 @@ toku_verify_ftnode_internal(FT_HANDLE ft_handle,
     BLOCKNUM blocknum = node->thisnodename;
 
     //printf("%s:%d pin %p\n", __FILE__, __LINE__, node_v);
-    toku_assert_entire_node_in_memory(node);
+    toku_ftnode_assert_fully_in_memory(node);
     this_msn = node->max_msn_applied_to_node_on_disk;
 
     if (height >= 0) {
@@ -501,7 +502,7 @@ toku_verify_ftnode (FT_HANDLE ft_handle,
     MSN   this_msn;
 
     //printf("%s:%d pin %p\n", __FILE__, __LINE__, node_v);
-    toku_assert_entire_node_in_memory(node);
+    toku_ftnode_assert_fully_in_memory(node);
     this_msn = node->max_msn_applied_to_node_on_disk;
 
     int result = 0;

ft/ft.cc

@@ -89,12 +89,12 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "ft.h"
-#include "ft-internal.h"
-#include "ft-cachetable-wrappers.h"
-#include "log-internal.h"
-
-#include <ft/log_header.h>
+#include "ft/ft.h"
+#include "ft/ft-cachetable-wrappers.h"
+#include "ft/ft-internal.h"
+#include "ft/log-internal.h"
+#include "ft/log_header.h"
+#include "ft/node.h"
 
 #include <memory.h>
 #include <toku_assert.h>

ft/ftverify.cc

@@ -96,6 +96,7 @@ PATENT RIGHTS GRANT:
 
 #include "fttypes.h"
 #include "ft-internal.h"
+#include "node.h"
 #include "ft_layout_version.h"
 #include "block_table.h"
 #include "rbuf.h"

ft/loader/loader.cc

@@ -111,6 +111,7 @@ PATENT RIGHTS GRANT:
 #include "leafentry.h"
 #include "log-internal.h"
 #include "ft.h"
+#include "node.h"
 
 static size_t (*os_fwrite_fun)(const void *,size_t,size_t,FILE*)=NULL;
 void ft_loader_set_os_fwrite (size_t (*fwrite_fun)(const void*,size_t,size_t,FILE*)) {

ft/msg_buffer.cc

@@ -129,23 +129,19 @@ struct message_buffer::buffer_entry *message_buffer::get_buffer_entry(int32_t of
 }
 
 void message_buffer::enqueue(FT_MSG msg, bool is_fresh, int32_t *offset) {
-    ITEMLEN keylen = ft_msg_get_keylen(msg);
-    ITEMLEN datalen = ft_msg_get_vallen(msg);
-    XIDS xids = ft_msg_get_xids(msg);
-    int need_space_here = sizeof(struct buffer_entry)
-                          + keylen + datalen
-                          + xids_get_size(xids)
-                          - sizeof(XIDS_S); //Prevent double counting
+    int need_space_here = msg_memsize_in_buffer(msg);
     int need_space_total = _memory_used + need_space_here;
     if (_memory == nullptr || need_space_total > _memory_size) {
         // resize the buffer to the next power of 2 greater than the needed space
         int next_2 = next_power_of_two(need_space_total);
         resize(next_2);
     }
+    ITEMLEN keylen = ft_msg_get_keylen(msg);
+    ITEMLEN datalen = ft_msg_get_vallen(msg);
     struct buffer_entry *entry = get_buffer_entry(_memory_used);
     entry->type = (unsigned char) ft_msg_get_type(msg);
     entry->msn = msg->msn;
-    xids_cpy(&entry->xids_s, xids);
+    xids_cpy(&entry->xids_s, ft_msg_get_xids(msg));
     entry->is_fresh = is_fresh;
     unsigned char *e_key = xids_get_end_of_array(&entry->xids_s);
     entry->keylen = keylen;
@@ -217,8 +213,8 @@ bool message_buffer::equals(message_buffer *other) const {
 }
 
 size_t message_buffer::msg_memsize_in_buffer(FT_MSG msg) {
-    return sizeof(struct buffer_entry)
-        + msg->u.id.key->size + msg->u.id.val->size
-        + xids_get_size(msg->xids)
-        - sizeof(XIDS_S);
+    const uint32_t keylen = ft_msg_get_keylen(msg);
+    const uint32_t datalen = ft_msg_get_vallen(msg);
+    const size_t xidslen = xids_get_size(msg->xids);
+    return sizeof(struct buffer_entry) + keylen + datalen + xidslen - sizeof(XIDS_S);
 }

ft/tests/test-pick-child-to-flush.cc

@@ -245,7 +245,7 @@ doit (void) {
     // what we say and flushes the child we pick
     FTNODE node = NULL;
     toku_pin_node_with_min_bfe(&node, node_internal, t);
-    toku_assert_entire_node_in_memory(node);
+    toku_ftnode_assert_fully_in_memory(node);
     assert(node->n_children == 2);
     assert(!node->dirty);
     assert(toku_bnc_n_entries(node->bp[0].ptr.u.nonleaf) > 0);
@@ -268,7 +268,7 @@ doit (void) {
     assert(num_flushes_called == 1);
 
     toku_pin_node_with_min_bfe(&node, node_internal, t);
-    toku_assert_entire_node_in_memory(node);
+    toku_ftnode_assert_fully_in_memory(node);
     assert(node->dirty);
     assert(node->n_children == 2);
     // child 0 should have empty buffer because it flushed
@@ -287,7 +287,7 @@ doit (void) {
     
     toku_pin_node_with_min_bfe(&node, node_internal, t);
     assert(node->dirty);
-    toku_assert_entire_node_in_memory(node);
+    toku_ftnode_assert_fully_in_memory(node);
     assert(node->n_children == 2);
     // both buffers should be empty now
     assert(toku_bnc_n_entries(node->bp[0].ptr.u.nonleaf) == 0);
@@ -305,7 +305,7 @@ doit (void) {
 
     toku_pin_node_with_min_bfe(&node, node_internal, t);
     assert(node->dirty); // nothing was flushed, but since we were trying to flush to a leaf, both become dirty
-    toku_assert_entire_node_in_memory(node);
+    toku_ftnode_assert_fully_in_memory(node);
     assert(node->n_children == 2);
     // both buffers should be empty now
     assert(toku_bnc_n_entries(node->bp[0].ptr.u.nonleaf) == 0);
@@ -326,7 +326,7 @@ doit (void) {
     // use a for loop so to get us down both paths
     for (int i = 0; i < 2; i++) {
         toku_pin_node_with_min_bfe(&node, node_root, t);
-        toku_assert_entire_node_in_memory(node); // entire root is in memory
+        toku_ftnode_assert_fully_in_memory(node); // entire root is in memory
         curr_child_to_flush = i;
         num_flushes_called = 0;
         toku_ft_flush_some_child(t->ft, node, &fa);
@@ -376,7 +376,7 @@ doit (void) {
 
     //now let's do the same test as above
     toku_pin_node_with_min_bfe(&node, node_root, t);
-    toku_assert_entire_node_in_memory(node); // entire root is in memory
+    toku_ftnode_assert_fully_in_memory(node); // entire root is in memory
     curr_child_to_flush = 0;
     num_flushes_called = 0;
     toku_ft_flush_some_child(t->ft, node, &fa);

ft/tests/test.h

@@ -102,6 +102,7 @@ PATENT RIGHTS GRANT:
 #include <portability/toku_path.h>
 
 #include "ft.h"
+#include "node.h"
 #include "block_table.h"
 #include "log-internal.h"
 #include "logger.h"

ft/tests/test_rightmost_leaf_split_merge.cc

@@ -180,7 +180,7 @@ static void test_split_merge(void) {
     toku_pin_ftnode(ft, rightmost_blocknum_before_merge,
                    toku_cachetable_hash(ft->cf, rightmost_blocknum_before_merge),
                    &bfe, PL_WRITE_EXPENSIVE, &rightmost_leaf, true);
-    invariant(get_node_reactivity(ft, rightmost_leaf) == RE_FUSIBLE);
+    invariant(toku_ftnode_get_reactivity(ft, rightmost_leaf) == RE_FUSIBLE);
     toku_unpin_ftnode(ft, rightmost_leaf);
 
     // - merge the rightmost child now that it's fusible

ft/tokuftdump.cc

@@ -95,6 +95,7 @@ PATENT RIGHTS GRANT:
 #include "ft.h"
 #include "fttypes.h"
 #include "ft-internal.h"
+#include "ft/node.h"
 #include <ctype.h>
 #include <stdint.h>
 #include <stdio.h>