Skip to content

M
MariaDB
Commit f165ee02 authored by Leif Walsh's avatar Leif Walsh Committed by Yoni Fogel
Browse files
Options

[t:4002] Commiting HOT to main. 

git-svn-id: file:///svn/toku/tokudb@38549 c7de825b-a66e-492c-adef-691d508d4ae1
parent e44c7d7a
Expand all Show whitespace changes
Inline Side-by-side
Showing with 2570 additions and 483 deletions
buildheader/db.h_4_1
View file @ f165ee02
......@@ -197,7 +197,7 @@ typedef struct __toku_engine_status {
uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */
uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */
uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the "flush from root" process to merge a leaf node */
uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */
uint64_t flush_in_memory; /* number of in memory flushes */
uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */
......@@ -223,6 +223,10 @@ typedef struct __toku_engine_status {
uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */
uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */
uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */
uint64_t hot_num_started; /* number of HOT operations that have begun */
uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */
uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */
uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */
uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/
uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/
uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/
......@@ -541,6 +545,7 @@ struct __toku_db {
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra);
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*get_readpagesize)(DB*,u_int32_t*);
int (*set_readpagesize)(DB*,u_int32_t);
......@@ -549,7 +554,7 @@ struct __toku_db {
int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going);
int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, u_int32_t flags);
int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, u_int32_t flags);
void* __toku_dummy0[11];
void* __toku_dummy0[10];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=236 size=4, 64=bit offset=376 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_3
View file @ f165ee02
......@@ -197,7 +197,7 @@ typedef struct __toku_engine_status {
uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */
uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */
uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the "flush from root" process to merge a leaf node */
uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */
uint64_t flush_in_memory; /* number of in memory flushes */
uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */
......@@ -223,6 +223,10 @@ typedef struct __toku_engine_status {
uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */
uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */
uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */
uint64_t hot_num_started; /* number of HOT operations that have begun */
uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */
uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */
uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */
uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/
uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/
uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/
......@@ -551,6 +555,7 @@ struct __toku_db {
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra);
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*get_readpagesize)(DB*,u_int32_t*);
int (*set_readpagesize)(DB*,u_int32_t);
......@@ -559,7 +564,7 @@ struct __toku_db {
int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going);
int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, u_int32_t flags);
int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, u_int32_t flags);
void* __toku_dummy0[14];
void* __toku_dummy0[13];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=248 size=4, 64=bit offset=400 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_4
View file @ f165ee02
......@@ -197,7 +197,7 @@ typedef struct __toku_engine_status {
uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */
uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */
uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the "flush from root" process to merge a leaf node */
uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */
uint64_t flush_in_memory; /* number of in memory flushes */
uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */
......@@ -223,6 +223,10 @@ typedef struct __toku_engine_status {
uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */
uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */
uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */
uint64_t hot_num_started; /* number of HOT operations that have begun */
uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */
uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */
uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */
uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/
uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/
uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/
......@@ -553,6 +557,7 @@ struct __toku_db {
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra);
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*get_readpagesize)(DB*,u_int32_t*);
int (*set_readpagesize)(DB*,u_int32_t);
......@@ -561,7 +566,7 @@ struct __toku_db {
int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going);
int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, u_int32_t flags);
int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, u_int32_t flags);
void* __toku_dummy0[16];
void* __toku_dummy0[15];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=256 size=4, 64=bit offset=416 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_5
View file @ f165ee02
......@@ -197,7 +197,7 @@ typedef struct __toku_engine_status {
uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */
uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */
uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the "flush from root" process to merge a leaf node */
uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */
uint64_t flush_in_memory; /* number of in memory flushes */
uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */
......@@ -223,6 +223,10 @@ typedef struct __toku_engine_status {
uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */
uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */
uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */
uint64_t hot_num_started; /* number of HOT operations that have begun */
uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */
uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */
uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */
uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/
uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/
uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/
......@@ -553,6 +557,7 @@ struct __toku_db {
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra);
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*get_readpagesize)(DB*,u_int32_t*);
int (*set_readpagesize)(DB*,u_int32_t);
......@@ -561,7 +566,7 @@ struct __toku_db {
int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going);
int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, u_int32_t flags);
int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, u_int32_t flags);
void* __toku_dummy0[19];
void* __toku_dummy0[18];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=268 size=4, 64=bit offset=440 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_6
View file @ f165ee02
......@@ -197,7 +197,7 @@ typedef struct __toku_engine_status {
uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */
uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */
uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the "flush from root" process to merge a leaf node */
uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */
uint64_t flush_in_memory; /* number of in memory flushes */
uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */
......@@ -223,6 +223,10 @@ typedef struct __toku_engine_status {
uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */
uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */
uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */
uint64_t hot_num_started; /* number of HOT operations that have begun */
uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */
uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */
uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */
uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/
uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/
uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/
......@@ -556,6 +560,7 @@ struct __toku_db {
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra);
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*get_readpagesize)(DB*,u_int32_t*);
int (*set_readpagesize)(DB*,u_int32_t);
......@@ -564,7 +569,7 @@ struct __toku_db {
int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going);
int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, u_int32_t flags);
int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, u_int32_t flags);
void* __toku_dummy1[23];
void* __toku_dummy1[22];
char __toku_dummy2[80];
void *api_internal; /* 32-bit offset=276 size=4, 64=bit offset=464 size=8 */
void* __toku_dummy3[5];
......
buildheader/make_db_h.c
View file @ f165ee02
......@@ -590,7 +590,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
printf(" uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */\n");
printf(" uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */\n");
printf(" uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */\n");
printf(" uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */\n");
printf(" uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the \"flush from root\" process to merge a leaf node */\n");
printf(" uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */\n");
printf(" uint64_t flush_in_memory; /* number of in memory flushes */\n");
printf(" uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */\n");
......@@ -616,6 +616,10 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
printf(" uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */\n");
printf(" uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */\n");
printf(" uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */\n");
printf(" uint64_t hot_num_started; /* number of HOT operations that have begun */\n");
printf(" uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */\n");
printf(" uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */\n");
printf(" uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */\n");
printf(" uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/\n");
printf(" uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/\n");
printf(" uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/\n");
......@@ -804,6 +808,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
"int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */",
"int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */",
"int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */",
"int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra)",
"int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION)",
"int (*get_readpagesize)(DB*,u_int32_t*)",
"int (*set_readpagesize)(DB*,u_int32_t)",
......
buildheader/tdb.h
View file @ f165ee02
......@@ -197,7 +197,7 @@ typedef struct __toku_engine_status {
uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */
uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */
uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the "flush from root" process to merge a leaf node */
uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */
uint64_t flush_in_memory; /* number of in memory flushes */
uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */
......@@ -223,6 +223,10 @@ typedef struct __toku_engine_status {
uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */
uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */
uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */
uint64_t hot_num_started; /* number of HOT operations that have begun */
uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */
uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */
uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */
uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/
uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/
uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/
......@@ -525,6 +529,7 @@ struct __toku_db {
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra);
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*get_readpagesize)(DB*,u_int32_t*);
int (*set_readpagesize)(DB*,u_int32_t);
......
include/db.h
View file @ f165ee02
......@@ -197,7 +197,7 @@ typedef struct __toku_engine_status {
uint64_t cleaner_max_buffer_workdone; /* max workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_min_buffer_workdone; /* min workdone value of any message buffer flushed by cleaner thread */
uint64_t cleaner_total_buffer_workdone; /* total workdone value of message buffers flushed by cleaner thread */
uint64_t cleaner_num_leaves_unmerged; /* number of leaves left unmerged by the cleaner thread */
uint64_t cleaner_num_dirtied_for_leaf_merge; /* nodes dirtied by the "flush from root" process to merge a leaf node */
uint64_t flush_total; /* total number of flushes done by flusher threads or cleaner threads */
uint64_t flush_in_memory; /* number of in memory flushes */
uint64_t flush_needed_io; /* number of flushes that had to read a child (or part) off disk */
......@@ -223,6 +223,10 @@ typedef struct __toku_engine_status {
uint64_t dirty_leaf; /* number of times leaf nodes are dirtied when previously clean */
uint64_t dirty_nonleaf; /* number of times nonleaf nodes are dirtied when previously clean */
uint64_t balance_leaf; /* number of times a leaf node is balanced inside brt */
uint64_t hot_num_started; /* number of HOT operations that have begun */
uint64_t hot_num_completed; /* number of HOT operations that have successfully completed */
uint64_t hot_num_aborted; /* number of HOT operations that have been aborted */
uint64_t hot_max_root_flush_count; /* max number of flushes from root ever required to optimize a tree */
uint64_t msg_bytes_in; /* how many bytes of messages injected at root (for all trees)*/
uint64_t msg_bytes_out; /* how many bytes of messages flushed from h1 nodes to leaves*/
uint64_t msg_bytes_curr; /* how many bytes of messages currently in trees (estimate)*/
......@@ -525,6 +529,7 @@ struct __toku_db {
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*hot_optimize)(DB*, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra);
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*get_readpagesize)(DB*,u_int32_t*);
int (*set_readpagesize)(DB*,u_int32_t);
......
newbrt/Makefile
View file @ f165ee02
......@@ -51,6 +51,7 @@ BRT_SOURCES = \
brt \
brt-cachetable-wrappers \
brt-flusher \
brt-hot-flusher \
brt_msg \
brt-test-helpers \
cachetable \
......
newbrt/brt-XY.c
View file @ f165ee02
......@@ -8,7 +8,7 @@ static int brt_root_put_cmd_XY (BRT brt, BRT_MSG *md, TOKUTXN txn) {
if (0) { died0: toku_unpin_brt_header(brt); }
return r;
}
CACHEKEY *rootp = toku_calculate_root_offset_pointer(brt);
CACHEKEY *rootp = toku_calculate_root_offset_pointer(brt->h);
if ((r=cachetable_get_and_pin(brt->cf, *rootp, &node_v, NULL,
toku_brtnode_flush_callback, toku_brtnode_fetch_callback, (void*)(long)brt->h->nodesize))) {
goto died0;
......
newbrt/brt-cachetable-wrappers.c
View file @ f165ee02
......@@ -6,6 +6,7 @@
#include <brt-cachetable-wrappers.h>
#include <brttypes.h>
#include <brt-flusher.h>
#include <brt-internal.h>
#include <cachetable.h>
......
newbrt/brt-flusher-internal.h 0 → 100644
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ifndef BRT_FLUSHER_INTERNAL
#define BRT_FLUSHER_INTERNAL
#ident "$Id$"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#include <brttypes.h>
#include <c_dialects.h>
C_BEGIN
typedef struct flusher_advice FLUSHER_ADVICE;
/**
* Choose a child to flush to. Returns a childnum, or -1 if we should
* go no further.
*
* Flusher threads: pick the heaviest child buffer
* Cleaner threads: pick the heaviest child buffer
* Cleaner thread merging leaf nodes: follow down to a key
* Hot optimize table: follow down to the right of a key
*/
typedef int (*FA_PICK_CHILD)(struct brt_header *h, BRTNODE parent, void* extra);
/**
* Decide whether to call `flush_some_child` on the child if it is
* stable and a nonleaf node.
*
* Flusher threads: yes if child is gorged
* Cleaner threads: yes if child is gorged
* Cleaner thread merging leaf nodes: always yes
* Hot optimize table: always yes
*/
typedef bool (*FA_SHOULD_RECURSIVELY_FLUSH)(BRTNODE child, void* extra);
/**
* Called if the child needs merging. Should do something to get the
* child out of a fusible state. Must unpin parent and child.
*
* Flusher threads: just do the merge
* Cleaner threads: if nonleaf, just merge, otherwise start a "cleaner
* thread merge"
* Cleaner thread merging leaf nodes: just do the merge
* Hot optimize table: just do the merge
*/
typedef void (*FA_MAYBE_MERGE_CHILD)(struct flusher_advice *fa,
struct brt_header *h,
BRTNODE parent,
int childnum,
BRTNODE child,
void* extra);
/**
* Cleaner threads may need to destroy basement nodes which have been
* brought more up to date than the height 1 node flushing to them.
* This function is used to determine if we need to check for basement
* nodes that are too up to date, and then destroy them if we find
* them.
*
* Flusher threads: no
* Cleaner threads: yes
* Cleaner thread merging leaf nodes: no
* Hot optimize table: no
*/
typedef bool (*FA_SHOULD_DESTROY_BN)(void* extra);
/**
* Update `brt_flusher_status` in whatever way necessary. Called once
* by `flush_some_child` right before choosing what to do next (split,
* merge, recurse), with the number of nodes that were dirtied by this
* execution of `flush_some_child`.
*/
typedef void (*FA_UPDATE_STATUS)(BRTNODE child, int dirtied, void* extra);
/**
* Choose whether to go to the left or right child after a split. Called
* by `brt_split_child`. If -1 is returned, `brt_split_child` defaults to
* the old behavior.
*/
typedef int (*FA_PICK_CHILD_AFTER_SPLIT)(struct brt_header* h,
BRTNODE node,
int childnuma,
int childnumb,
void* extra);
/**
* A collection of callbacks used by the flushing machinery to make
* various decisions. There are implementations of each of these
* functions for flusher threads (ft_*), cleaner threads (ct_*), , and hot
* optimize table (hot_*).
*/
struct flusher_advice {
FA_PICK_CHILD pick_child;
FA_SHOULD_RECURSIVELY_FLUSH should_recursively_flush;
FA_MAYBE_MERGE_CHILD maybe_merge_child;
FA_SHOULD_DESTROY_BN should_destroy_basement_nodes;
FA_UPDATE_STATUS update_status;
FA_PICK_CHILD_AFTER_SPLIT pick_child_after_split;
void* extra; // parameter passed into callbacks
};
void
flusher_advice_init(
struct flusher_advice *fa,
FA_PICK_CHILD pick_child,
FA_SHOULD_DESTROY_BN should_destroy_basement_nodes,
FA_SHOULD_RECURSIVELY_FLUSH should_recursively_flush,
FA_MAYBE_MERGE_CHILD maybe_merge_child,
FA_UPDATE_STATUS update_status,
FA_PICK_CHILD_AFTER_SPLIT pick_child_after_split,
void* extra
);
void
flush_some_child(
struct brt_header* h,
BRTNODE parent,
struct flusher_advice *fa);
bool
always_recursively_flush(BRTNODE child, void* extra);
bool
dont_destroy_basement_nodes(void* extra);
void
default_merge_child(struct flusher_advice *fa,
struct brt_header *h,
BRTNODE parent,
int childnum,
BRTNODE child,
void* extra);
int
default_pick_child_after_split(struct brt_header *h,
BRTNODE parent,
int childnuma,
int childnumb,
void *extra);
C_END
#endif // End of header guardian.
newbrt/brt-flusher.c
View file @ f165ee02
This diff is collapsed.
newbrt/brt-flusher.h
View file @ f165ee02
......@@ -11,6 +11,39 @@
C_BEGIN
typedef struct brt_flusher_status {
uint64_t cleaner_total_nodes; // total number of nodes whose buffers are potentially flushed by cleaner thread
uint64_t cleaner_h1_nodes; // number of nodes of height one whose message buffers are flushed by cleaner thread
uint64_t cleaner_hgt1_nodes; // number of nodes of height > 1 whose message buffers are flushed by cleaner thread
uint64_t cleaner_empty_nodes; // number of nodes that are selected by cleaner, but whose buffers are empty
uint64_t cleaner_nodes_dirtied; // number of nodes that are made dirty by the cleaner thread
uint64_t cleaner_max_buffer_size; // max number of bytes in message buffer flushed by cleaner thread
uint64_t cleaner_min_buffer_size;
uint64_t cleaner_total_buffer_size;
uint64_t cleaner_max_buffer_workdone; // max workdone value of any message buffer flushed by cleaner thread
uint64_t cleaner_min_buffer_workdone;
uint64_t cleaner_total_buffer_workdone;
uint64_t cleaner_num_dirtied_for_leaf_merge; // nodes dirtied by the "flush from root" process to merge a leaf node
uint64_t flush_total; // total number of flushes done by flusher threads or cleaner threads
uint64_t flush_in_memory; // number of in memory flushes
uint64_t flush_needed_io; // number of flushes that had to read a child (or part) off disk
uint64_t flush_cascades; // number of flushes that triggered another flush in the child
uint64_t flush_cascades_1; // number of flushes that triggered 1 cascading flush
uint64_t flush_cascades_2; // number of flushes that triggered 2 cascading flushes
uint64_t flush_cascades_3; // number of flushes that triggered 3 cascading flushes
uint64_t flush_cascades_4; // number of flushes that triggered 4 cascading flushes
uint64_t flush_cascades_5; // number of flushes that triggered 5 cascading flushes
uint64_t flush_cascades_gt_5; // number of flushes that triggered more than 5 cascading flushes
uint64_t split_leaf; // number of leaf nodes split
uint64_t split_nonleaf; // number of nonleaf nodes split
uint64_t merge_leaf; // number of times leaf nodes are merged
uint64_t merge_nonleaf; // number of times nonleaf nodes are merged
uint64_t balance_leaf; // number of times a leaf node is balanced inside brt
} BRT_FLUSHER_STATUS_S, *BRT_FLUSHER_STATUS;
void toku_brt_flusher_status_init(void);
void toku_brt_flusher_get_status(BRT_FLUSHER_STATUS);
/**
* Only for testing, not for production.
*
......@@ -32,12 +65,11 @@ toku_flusher_thread_set_callback(
* brt_status which currently just lives in brt.c.
*/
int
toku_brtnode_cleaner_callback_internal(
toku_brtnode_cleaner_callback(
void *brtnode_pv,
BLOCKNUM blocknum,
u_int32_t fullhash,
void *extraargs,
BRT_STATUS brt_status
void *extraargs
);
/**
......@@ -47,8 +79,7 @@ toku_brtnode_cleaner_callback_internal(
void
flush_node_on_background_thread(
BRT brt,
BRTNODE parent,
BRT_STATUS brt_status
BRTNODE parent
);
/**
......@@ -68,8 +99,7 @@ brtleaf_split(
DBT *splitk,
BOOL create_new_node,
u_int32_t num_dependent_nodes,
BRTNODE* dependent_nodes,
BRT_STATUS brt_status
BRTNODE* dependent_nodes
);
/**
......@@ -89,10 +119,33 @@ brt_nonleaf_split(
BRTNODE *nodeb,
DBT *splitk,
u_int32_t num_dependent_nodes,
BRTNODE* dependent_nodes,
BRT_STATUS brt_status
BRTNODE* dependent_nodes
);
/************************************************************************
* HOT optimize, should perhaps be factored out to its own header file *
************************************************************************
*/
typedef struct brt_hot_status {
uint64_t num_started; // number of HOT operations that have begun
uint64_t num_completed; // number of HOT operations that have successfully completed
uint64_t num_aborted; // number of HOT operations that have been aborted
uint64_t max_root_flush_count; // max number of flushes from root ever required to optimize a tree
} BRT_HOT_STATUS_S, *BRT_HOT_STATUS;
void toku_brt_hot_get_status(BRT_HOT_STATUS);
/**
* Takes given BRT and pushes all pending messages to the leaf nodes.
*/
int
toku_brt_hot_optimize(BRT brt,
int (*progress_callback)(void *extra, float progress),
void *progress_extra);
C_END
#endif // End of header guardian.
newbrt/brt-hot-flusher.c 0 → 100644
View file @ f165ee02
This diff is collapsed.
newbrt/brt-internal.h
View file @ f165ee02
......@@ -341,13 +341,6 @@ enum {
u_int32_t compute_child_fullhash (CACHEFILE cf, BRTNODE node, int childnum);
struct remembered_hash {
BOOL valid; // set to FALSE if the fullhash is invalid
FILENUM fnum;
BLOCKNUM root;
u_int32_t fullhash; // fullhash is the hashed value of fnum and root.
};
// The brt_header is not managed by the cachetable. Instead, it hangs off the cachefile as userdata.
enum brtheader_type {BRTHEADER_CURRENT=1, BRTHEADER_CHECKPOINT_INPROGRESS};
......@@ -380,7 +373,6 @@ struct brt_header {
unsigned int nodesize;
unsigned int basementnodesize;
BLOCKNUM root; // roots of the dictionary
struct remembered_hash root_hash; // hash of the root offset.
unsigned int flags;
DESCRIPTOR_S descriptor;
......@@ -404,6 +396,11 @@ struct brt_header {
STAT64INFO_S in_memory_stats;
STAT64INFO_S on_disk_stats;
STAT64INFO_S checkpoint_staging_stats;
uint64_t time_of_last_optimize_begin; // last time that a hot optimize operation was begun
uint64_t time_of_last_optimize_end; // last time that a hot optimize operation was successfully completed
uint32_t count_of_optimize_in_progress; // the number of hot optimize operations currently in progress on this tree
uint32_t count_of_optimize_in_progress_read_from_disk; // the number of hot optimize operations in progress on this tree at the time of the last crash (this field is in-memory only)
MSN msn_at_start_of_last_completed_optimize; // all messages before this msn have been applied to leaf nodes
};
struct brt {
......@@ -526,10 +523,9 @@ extern void toku_brtnode_pe_est_callback(void* brtnode_pv, long* bytes_freed_est
extern int toku_brtnode_pe_callback (void *brtnode_pv, PAIR_ATTR old_attr, PAIR_ATTR* new_attr, void *extraargs);
extern BOOL toku_brtnode_pf_req_callback(void* brtnode_pv, void* read_extraargs);
int toku_brtnode_pf_callback(void* brtnode_pv, void* read_extraargs, int fd, PAIR_ATTR* sizep);
extern int toku_brtnode_cleaner_callback (void* brtnode_pv, BLOCKNUM blocknum, u_int32_t fullhash, void* extraargs);
extern int toku_brt_alloc_init_header(BRT t, TOKUTXN txn);
extern int toku_read_brt_header_and_store_in_cachefile (BRT brt, CACHEFILE cf, LSN max_acceptable_lsn, struct brt_header **header, BOOL* was_open);
extern CACHEKEY* toku_calculate_root_offset_pointer (BRT brt, u_int32_t *root_hash);
extern CACHEKEY* toku_calculate_root_offset_pointer (struct brt_header* h, u_int32_t *root_hash);
static const BRTNODE null_brtnode=0;
......@@ -716,15 +712,31 @@ unsigned int toku_brtnode_which_child(BRTNODE node, const DBT *k,
DESCRIPTOR desc, brt_compare_func cmp)
__attribute__((__warn_unused_result__));
/**
* Finds the next child for HOT to flush to, given that everything up to
* and including k has been flattened.
*
* If k falls between pivots in node, then we return the childnum where k
* lies.
*
* If k is equal to some pivot, then we return the next (to the right)
* childnum.
*/
unsigned int toku_brtnode_hot_next_child(BRTNODE node,
const DBT *k,
DESCRIPTOR desc,
brt_compare_func cmp);
/* Stuff for testing */
// toku_testsetup_initialize() must be called before any other test_setup_xxx() functions are called.
void toku_testsetup_initialize(void);
int toku_testsetup_leaf(BRT brt, BLOCKNUM *);
int toku_testsetup_leaf(BRT brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens);
int toku_testsetup_nonleaf (BRT brt, int height, BLOCKNUM *diskoff, int n_children, BLOCKNUM *children, char **keys, int *keylens);
int toku_testsetup_root(BRT brt, BLOCKNUM);
int toku_testsetup_get_sersize(BRT brt, BLOCKNUM); // Return the size on disk.
int toku_testsetup_insert_to_leaf (BRT brt, BLOCKNUM, char *key, int keylen, char *val, int vallen);
int toku_testsetup_insert_to_nonleaf (BRT brt, BLOCKNUM, enum brt_msg_type, char *key, int keylen, char *val, int vallen);
void toku_pin_node_with_min_bfe(BRTNODE* node, BLOCKNUM b, BRT t);
// These two go together to do lookups in a brtnode using the keys in a command.
struct cmd_leafval_heaviside_extra {
......@@ -799,28 +811,6 @@ struct brt_status {
uint64_t search_root_retries; // number of searches that required the root node to be fetched more than once
uint64_t search_tries_gt_height; // number of searches that required more tries than the height of the tree
uint64_t search_tries_gt_heightplus3; // number of searches that required more tries than the height of the tree plus three
uint64_t cleaner_total_nodes; // total number of nodes whose buffers are potentially flushed by cleaner thread
uint64_t cleaner_h1_nodes; // number of nodes of height one whose message buffers are flushed by cleaner thread
uint64_t cleaner_hgt1_nodes; // number of nodes of height > 1 whose message buffers are flushed by cleaner thread
uint64_t cleaner_empty_nodes; // number of nodes that are selected by cleaner, but whose buffers are empty
uint64_t cleaner_nodes_dirtied; // number of nodes that are made dirty by the cleaner thread
uint64_t cleaner_max_buffer_size; // max number of bytes in message buffer flushed by cleaner thread
uint64_t cleaner_min_buffer_size;
uint64_t cleaner_total_buffer_size;
uint64_t cleaner_max_buffer_workdone; // max workdone value of any message buffer flushed by cleaner thread
uint64_t cleaner_min_buffer_workdone;
uint64_t cleaner_total_buffer_workdone;
uint64_t cleaner_num_leaves_unmerged; // number of leaves left unmerged by the cleaner thread
uint64_t flush_total; // total number of flushes done by flusher threads or cleaner threads
uint64_t flush_in_memory; // number of in memory flushes
uint64_t flush_needed_io; // number of flushes that had to read a child (or part) off disk
uint64_t flush_cascades; // number of flushes that triggered another flush in the child
uint64_t flush_cascades_1; // number of flushes that triggered 1 cascading flush
uint64_t flush_cascades_2; // number of flushes that triggered 2 cascading flushes
uint64_t flush_cascades_3; // number of flushes that triggered 3 cascading flushes
uint64_t flush_cascades_4; // number of flushes that triggered 4 cascading flushes
uint64_t flush_cascades_5; // number of flushes that triggered 5 cascading flushes
uint64_t flush_cascades_gt_5; // number of flushes that triggered more than 5 cascading flushes
uint64_t disk_flush_leaf; // number of leaf nodes flushed to disk, not for checkpoint
uint64_t disk_flush_nonleaf; // number of nonleaf nodes flushed to disk, not for checkpoint
uint64_t disk_flush_leaf_for_checkpoint; // number of leaf nodes flushed to disk for checkpoint
......@@ -829,13 +819,8 @@ struct brt_status {
uint64_t create_nonleaf; // number of nonleaf nodes created
uint64_t destroy_leaf; // number of leaf nodes destroyed
uint64_t destroy_nonleaf; // number of nonleaf nodes destroyed
uint64_t split_leaf; // number of leaf nodes split
uint64_t split_nonleaf; // number of nonleaf nodes split
uint64_t merge_leaf; // number of times leaf nodes are merged
uint64_t merge_nonleaf; // number of times nonleaf nodes are merged
uint64_t dirty_leaf; // number of times leaf nodes are dirtied when previously clean
uint64_t dirty_nonleaf; // number of times nonleaf nodes are dirtied when previously clean
uint64_t balance_leaf; // number of times a leaf node is balanced inside brt
uint64_t msg_bytes_in; // how many bytes of messages injected at root (for all trees)
uint64_t msg_bytes_out; // how many bytes of messages flushed from h1 nodes to leaves
uint64_t msg_bytes_curr; // how many bytes of messages currently in trees (estimate)
......@@ -865,9 +850,6 @@ struct brt_status {
void toku_brt_get_status(BRT_STATUS);
void
brtleaf_split (struct brt_header* h, BRTNODE node, BRTNODE *nodea, BRTNODE *nodeb, DBT *splitk, BOOL create_new_node, u_int32_t num_dependent_nodes, BRTNODE* dependent_nodes, BRT_STATUS brt_status);
void
brt_leaf_apply_cmd_once (
BRTNODE leafnode,
......@@ -906,6 +888,18 @@ void toku_apply_cmd_to_leaf(
OMT live_list_reverse
);
void brtnode_put_cmd (
brt_compare_func compare_fun,
brt_update_func update_fun,
DESCRIPTOR desc,
BRTNODE node,
BRT_MSG cmd,
bool is_fresh,
OMT snapshot_txnids,
OMT live_list_reverse
);
void toku_reset_root_xid_that_created(BRT brt, TXNID new_root_xid_that_created);
// Reset the root_xid_that_created field to the given value.
// This redefines which xid created the dictionary.
......@@ -913,6 +907,10 @@ void toku_reset_root_xid_that_created(BRT brt, TXNID new_root_xid_that_created);
void toku_flusher_thread_set_callback(void (*callback_f)(int, void*), void* extra);
void toku_brt_header_note_hot_begin(BRT brt);
void toku_brt_header_note_hot_complete(BRT brt, BOOL success, MSN msn_at_start_of_hot);
C_END
#endif
newbrt/brt-serialize.c
View file @ f165ee02
......@@ -1815,6 +1815,11 @@ serialize_brt_header_min_size (u_int32_t version) {
switch(version) {
case BRT_LAYOUT_VERSION_18:
size += sizeof(uint64_t); // time_of_last_optimize_begin
size += sizeof(uint64_t); // time_of_last_optimize_end
size += sizeof(uint32_t); // count_of_optimize_in_progress
size += sizeof(MSN); // msn_at_start_of_last_completed_optimize
case BRT_LAYOUT_VERSION_17:
size += 16;
invariant(sizeof(STAT64INFO_S) == 16);
......@@ -1891,6 +1896,10 @@ int toku_serialize_brt_header_to_wbuf (struct wbuf *wbuf, struct brt_header *h,
wbuf_ulonglong(wbuf, h->time_of_last_verification);
wbuf_ulonglong(wbuf, h->checkpoint_staging_stats.numrows);
wbuf_ulonglong(wbuf, h->checkpoint_staging_stats.numbytes);
wbuf_ulonglong(wbuf, h->time_of_last_optimize_begin);
wbuf_ulonglong(wbuf, h->time_of_last_optimize_end);
wbuf_int(wbuf, h->count_of_optimize_in_progress);
wbuf_MSN(wbuf, h->msn_at_start_of_last_completed_optimize);
u_int32_t checksum = x1764_finish(&wbuf->checksum);
wbuf_int(wbuf, checksum);
lazy_assert(wbuf->ndone == wbuf->size);
......@@ -2143,7 +2152,6 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
}
h->root = rbuf_blocknum(&rc);
h->root_hash.valid = FALSE;
h->flags = rbuf_int(&rc);
h->layout_version_original = rbuf_int(&rc);
h->build_id_original = rbuf_int(&rc);
......@@ -2161,10 +2169,15 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
h->basementnodesize = rbuf_int(&rc);
h->time_of_last_verification = rbuf_ulonglong(&rc);
}
if (h->layout_version >= BRT_LAYOUT_VERSION_17) {
if (h->layout_version >= BRT_LAYOUT_VERSION_18) {
h->on_disk_stats.numrows = rbuf_ulonglong(&rc);
h->on_disk_stats.numbytes = rbuf_ulonglong(&rc);
h->in_memory_stats = h->on_disk_stats;
h->time_of_last_optimize_begin = rbuf_ulonglong(&rc);
h->time_of_last_optimize_end = rbuf_ulonglong(&rc);
h->count_of_optimize_in_progress = rbuf_int(&rc);
h->count_of_optimize_in_progress_read_from_disk = h->count_of_optimize_in_progress;
h->msn_at_start_of_last_completed_optimize = rbuf_msn(&rc);
}
(void)rbuf_int(&rc); //Read in checksum and ignore (already verified).
......@@ -2219,7 +2232,8 @@ deserialize_brtheader_versioned (int fd, struct rbuf *rb, struct brt_header **br
case BRT_LAYOUT_VERSION_14:
h->basementnodesize = 128*1024; // basement nodes added in v15
//fall through on purpose
case BRT_LAYOUT_VERSION_17:
case BRT_LAYOUT_VERSION_18:
case BRT_LAYOUT_VERSION_17: // version 17 never released to customers
case BRT_LAYOUT_VERSION_16: // version 16 never released to customers
case BRT_LAYOUT_VERSION_15: // this will not properly support version 15, we'll fix that on upgrade.
invariant(h->layout_version == BRT_LAYOUT_VERSION);
......
newbrt/brt-test-helpers.c
View file @ f165ee02
......@@ -6,10 +6,11 @@
#include "includes.h"
#include "ule.h"
#include <brt-cachetable-wrappers.h>
#include <brt-flusher.h>
// dummymsn needed to simulate msn because messages are injected at a lower level than toku_brt_root_put_cmd()
#define MIN_DUMMYMSN ((MSN) {(uint64_t)1<<48})
#define MIN_DUMMYMSN ((MSN) {(uint64_t)100000000000})
static MSN dummymsn;
static int testsetup_initialized = 0;
......@@ -31,13 +32,21 @@ next_dummymsn(void) {
BOOL ignore_if_was_already_open;
int toku_testsetup_leaf(BRT brt, BLOCKNUM *blocknum) {
int toku_testsetup_leaf(BRT brt, BLOCKNUM *blocknum, int n_children, char **keys, int *keylens) {
BRTNODE node;
assert(testsetup_initialized);
int r = toku_read_brt_header_and_store_in_cachefile(brt, brt->cf, MAX_LSN, &brt->h, &ignore_if_was_already_open);
if (r!=0) return r;
toku_create_new_brtnode(brt, &node, 0, 1);
BP_STATE(node,0) = PT_AVAIL;
toku_create_new_brtnode(brt, &node, 0, n_children);
int i;
for (i=0; i<n_children; i++) {
BP_STATE(node,i) = PT_AVAIL;
}
for (i=0; i+1<n_children; i++) {
node->childkeys[i] = kv_pair_malloc(keys[i], keylens[i], 0, 0);
node->totalchildkeylens += keylens[i];
}
*blocknum = node->thisnodename;
toku_unpin_brtnode(brt, node);
......@@ -71,7 +80,6 @@ int toku_testsetup_root(BRT brt, BLOCKNUM blocknum) {
int r = toku_read_brt_header_and_store_in_cachefile(brt, brt->cf, MAX_LSN, &brt->h, &ignore_if_was_already_open);
if (r!=0) return r;
brt->h->root = blocknum;
brt->h->root_hash.valid = FALSE;
return 0;
}
......@@ -131,55 +139,22 @@ int toku_testsetup_insert_to_leaf (BRT brt, BLOCKNUM blocknum, char *key, int ke
toku_verify_or_set_counts(node);
assert(node->height==0);
size_t newlesize;
LEAFENTRY leafentry;
OMTVALUE storeddatav;
u_int32_t idx;
DBT keydbt,valdbt;
MSN msn = next_dummymsn();
BRT_MSG_S cmd = {BRT_INSERT, msn, xids_get_root_xids(),
.u.id={toku_fill_dbt(&keydbt, key, keylen),
toku_fill_dbt(&valdbt, val, vallen)}};
//Generate a leafentry (committed insert key,val)
uint childnum = toku_brtnode_which_child(node,
&keydbt,
&brt->h->descriptor, brt->compare_fun);
BASEMENTNODE bn = BLB(node, childnum);
void * maybe_free = 0;
{
int64_t ignoreme;
r = apply_msg_to_leafentry(&cmd, NULL, //No old leafentry
&newlesize, &leafentry,
bn->buffer, &bn->buffer_mempool, &maybe_free,
NULL, NULL, &ignoreme);
assert(r==0);
}
struct cmd_leafval_heaviside_extra be = {brt->compare_fun, &brt->h->descriptor, &keydbt};
r = toku_omt_find_zero(BLB_BUFFER(node, 0), toku_cmd_leafval_heaviside, &be, &storeddatav, &idx);
if (r==0) {
LEAFENTRY storeddata=storeddatav;
// It's already there. So now we have to remove it and put the new one back in.
BLB_NBYTESINBUF(node, 0) -= leafentry_disksize(storeddata);
toku_free(storeddata);
// Now put the new kv in.
toku_omt_set_at(BLB_BUFFER(node, 0), leafentry, idx);
} else {
r = toku_omt_insert(BLB_BUFFER(node, 0), leafentry, toku_cmd_leafval_heaviside, &be, 0);
assert(r==0);
}
// hack to get tests passing. These tests should not be directly inserting into buffers
BLB(node, 0)->max_msn_applied = msn;
BLB_NBYTESINBUF(node, 0) += newlesize;
node->dirty=1;
brtnode_put_cmd (
brt->h->compare_fun,
brt->h->update_fun,
&brt->h->descriptor,
node,
&cmd,
true,
NULL,
NULL
);
toku_verify_or_set_counts(node);
......@@ -194,6 +169,23 @@ testhelper_string_key_cmp(DB *UU(e), const DBT *a, const DBT *b)
return strcmp(s, t);
}
void
toku_pin_node_with_min_bfe(BRTNODE* node, BLOCKNUM b, BRT t)
{
struct brtnode_fetch_extra bfe;
fill_bfe_for_min_read(&bfe, t->h);
toku_pin_brtnode_off_client_thread(
t->h,
b,
toku_cachetable_hash(t->h->cf, b),
&bfe,
0,
NULL,
node
);
}
int toku_testsetup_insert_to_nonleaf (BRT brt, BLOCKNUM blocknum, enum brt_msg_type cmdtype, char *key, int keylen, char *val, int vallen) {
void *node_v;
int r;
......
newbrt/brt-verify.c
View file @ f165ee02
......@@ -12,6 +12,7 @@
*/
#include "includes.h"
#include <brt-flusher.h>
static int
compare_pairs (BRT brt, struct kv_pair *a, struct kv_pair *b) {
......@@ -388,7 +389,7 @@ int
toku_verify_brt_with_progress (BRT brt, int (*progress_callback)(void *extra, float progress), void *progress_extra, int verbose, int keep_on_going) {
assert(brt->h);
u_int32_t root_hash;
CACHEKEY *rootp = toku_calculate_root_offset_pointer(brt, &root_hash);
CACHEKEY *rootp = toku_calculate_root_offset_pointer(brt->h, &root_hash);
int r = toku_verify_brtnode(brt, ZERO_MSN, ZERO_MSN, *rootp, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
if (r == 0) {
toku_brtheader_lock(brt->h);
......
newbrt/brt.c
View file @ f165ee02
This diff is collapsed.
newbrt/brt_layout_version.h
View file @ f165ee02
......@@ -22,6 +22,7 @@ enum brt_layout_version_e {
BRT_LAYOUT_VERSION_16 = 16, // Dr. No: No subtree estimates, partition layout information represented more transparently.
// ALERT ALERT ALERT: version 16 never released to customers, internal and beta use only
BRT_LAYOUT_VERSION_17 = 17, // Dr. No: Add STAT64INFO_S to brt_header
BRT_LAYOUT_VERSION_18 = 18, // Dr. No: Add HOT info to brt_header
BRT_NEXT_VERSION, // the version after the current version
BRT_LAYOUT_VERSION = BRT_NEXT_VERSION-1, // A hack so I don't have to change this line.
BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION_13, // Minimum version supported
......
newbrt/cachetable.c
View file @ f165ee02
......@@ -466,6 +466,19 @@ toku_cachetable_set_lock_unlock_for_io (CACHETABLE ct, void (*ydb_lock_callback)
ct->ydb_unlock_callback = ydb_unlock_callback;
}
void
toku_cachetable_call_ydb_lock(CACHEFILE cf){
if (cf->cachetable->ydb_lock_callback) {
assert(cf->cachetable->ydb_unlock_callback);
cf->cachetable->ydb_lock_callback();
}
}
void
toku_cachetable_call_ydb_unlock(CACHEFILE cf){
if (cf->cachetable->ydb_unlock_callback) cf->cachetable->ydb_unlock_callback();
}
//
// Increment the reference count
// MUST HOLD cachetable lock
......
newbrt/cachetable.h
View file @ f165ee02
......@@ -523,6 +523,8 @@ char * toku_cachetable_get_fname_in_cwd(CACHETABLE ct, const char * fname_in_env
void toku_cachetable_set_lock_unlock_for_io (CACHETABLE ct, void (*ydb_lock_callback)(void), void (*ydb_unlock_callback)(void));
// Effect: When we do I/O we may need to release locks (e.g., the ydb lock). These functions release the lock acquire the lock.
void toku_cachetable_call_ydb_lock(CACHEFILE cf);
void toku_cachetable_call_ydb_unlock(CACHEFILE cf);
void cachefile_kibbutz_enq (CACHEFILE cf, void (*f)(void*), void *extra);
......
newbrt/dbufio.h
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ifndef TOKU_DBUFIO_H
#define TOKU_DBUFIO_H
#ident "$Id: queue.c 20104 2010-05-12 17:22:40Z bkuszmaul $"
#ident "$Id$"
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include <toku_portability.h>
......
newbrt/le-cursor.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id$"
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
......
newbrt/le-cursor.h
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id$"
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
......
newbrt/mempool.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id: mempool.c 19902 2010-05-06 20:41:32Z bkuszmaul $"
#ident "$Id$"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
......
newbrt/mempool.h
View file @ f165ee02
#ifndef _TOKU_MEMPOOL_H
#define _TOKU_MEMPOOL_H
#ident "$Id: mempool.h 19902 2010-05-06 20:41:32Z bkuszmaul $"
#ident "$Id$"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
......
newbrt/nonblocking_mutex.h
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ifndef TOKU_NBMUTEX_H
#define TOKU_NBMUTEX_H
#ident "$Id: rwlock.h 32279 2011-06-29 13:51:57Z bkuszmaul $"
#ident "$Id$"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
......
newbrt/tests/brt-clock-test.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id: brt-serialize-test.c 36450 2011-11-02 20:10:18Z bperlman $"
#ident "$Id$"
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "test.h"
......
newbrt/tests/cachetable-flush-during-cleaner.c
View file @ f165ee02
#ident "$Id: cachetable-simple-verify.c 36579 2011-11-04 20:02:04Z zardosht $"
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id$"
#ident "Copyright (c) 2007-2011 Tokutek Inc. All rights reserved."
#include "includes.h"
#include "test.h"
......
newbrt/tests/make-tree.c
View file @ f165ee02
......@@ -136,7 +136,7 @@ test_make_tree(int height, int fanout, int nperleaf, int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
// set the new root to point to the new tree
*rootp = newroot->thisnodename;
......
newbrt/tests/msnfilter.c
View file @ f165ee02
......@@ -126,7 +126,7 @@ test_msnfilter(int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
BRTNODE newroot = make_node(brt, 0);
......
newbrt/tests/test-brt-overflow.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id$"
#ident "Copyright (c) 2011 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* Test an overflow condition on the leaf. See #632. */
......
newbrt/tests/test-del-inorder.c
View file @ f165ee02
......@@ -34,7 +34,7 @@ doit (void) {
toku_testsetup_initialize(); // must precede any other toku_testsetup calls
r = toku_testsetup_leaf(t, &nodea);
r = toku_testsetup_leaf(t, &nodea, 1, NULL, NULL);
assert(r==0);
r = toku_testsetup_nonleaf(t, 1, &nodeb, 1, &nodea, 0, 0);
......
newbrt/tests/test-dirty-flushes-on-cleaner.c 0 → 100644
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007-2011 Tokutek Inc. All rights reserved."
#ident "$Id$"
/* The goal of this test. Make sure that inserts stay behind deletes. */
#include "test.h"
#include "includes.h"
#include <brt-cachetable-wrappers.h>
#include "brt-flusher.h"
#include "checkpoint.h"
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
enum { NODESIZE = 1024, KSIZE=NODESIZE-100, TOKU_PSIZE=20 };
CACHETABLE ct;
BRT brt;
int fnamelen;
char *fname;
static int update_func(
DB* UU(db),
const DBT* key,
const DBT* old_val,
const DBT* UU(extra),
void (*set_val)(const DBT *new_val, void *set_extra),
void *set_extra)
{
DBT new_val;
assert(old_val->size > 0);
if (verbose) {
printf("applying update to %s\n", (char *)key->data);
}
toku_init_dbt(&new_val);
set_val(&new_val, set_extra);
return 0;
}
static void
doit (void) {
BLOCKNUM node_leaf;
BLOCKNUM node_internal, node_root;
int r;
fnamelen = strlen(__FILE__) + 20;
fname = toku_malloc(fnamelen);
assert(fname!=0);
snprintf(fname, fnamelen, "%s.brt", __FILE__);
r = toku_brt_create_cachetable(&ct, 500*1024*1024, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 1, &brt, NODESIZE, NODESIZE/2, ct, null_txn, toku_builtin_compare_fun, null_db);
assert(r==0);
toku_free(fname);
brt->update_fun = update_func;
brt->h->update_fun = update_func;
toku_testsetup_initialize(); // must precede any other toku_testsetup calls
char* pivots[1];
pivots[0] = toku_strdup("kkkkk");
int pivot_len = 6;
r = toku_testsetup_leaf(brt, &node_leaf, 2, pivots, &pivot_len);
assert(r==0);
r = toku_testsetup_nonleaf(brt, 1, &node_internal, 1, &node_leaf, 0, 0);
assert(r==0);
r = toku_testsetup_nonleaf(brt, 2, &node_root, 1, &node_internal, 0, 0);
assert(r==0);
r = toku_testsetup_root(brt, node_root);
assert(r==0);
//
// at this point we have created a tree with a root, an internal node,
// and two leaf nodes, the pivot being "kkkkk"
//
// now we insert a row into each leaf node
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"a", // key
2, // keylen
"aa",
3
);
assert(r==0);
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"z", // key
2, // keylen
"zz",
3
);
assert(r==0);
char filler[400];
memset(filler, 0, sizeof(filler));
// now we insert filler data so that the rebalance
// keeps it at two nodes
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"b", // key
2, // keylen
filler,
sizeof(filler)
);
assert(r==0);
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"y", // key
2, // keylen
filler,
sizeof(filler)
);
assert(r==0);
//
// now insert a bunch of dummy delete messages
// into the internal node, to get its cachepressure size up
//
for (int i = 0; i < 100000; i++) {
r = toku_testsetup_insert_to_nonleaf (
brt,
node_internal,
BRT_DELETE_ANY,
"jj", // this key does not exist, so its message application should be a no-op
3,
NULL,
0
);
assert(r==0);
}
//
// now insert a broadcast message into the root
//
r = toku_testsetup_insert_to_nonleaf (
brt,
node_root,
BRT_UPDATE_BROADCAST_ALL,
NULL,
0,
NULL,
0
);
assert(r==0);
// now lock and release the leaf node to make sure it is what we expect it to be.
BRTNODE node = NULL;
struct brtnode_fetch_extra bfe;
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_leaf,
toku_cachetable_hash(brt->h->cf, node_leaf),
&bfe,
0,
NULL,
&node
);
assert(node->dirty);
assert(node->n_children == 2);
assert(BP_STATE(node,0) == PT_AVAIL);
assert(BP_STATE(node,1) == PT_AVAIL);
toku_unpin_brtnode_off_client_thread(brt->h, node);
// now do a lookup on one of the keys, this should bring a leaf node up to date
DBT k;
struct check_pair pair = {2, "a", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "a", 2), lookup_checkf, &pair);
assert(r==0);
//
// pin the leaf one more time
// and make sure that one basement
// node is in memory and another is
// on disk
//
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_leaf,
toku_cachetable_hash(brt->h->cf, node_leaf),
&bfe,
0,
NULL,
&node
);
assert(node->dirty);
assert(node->n_children == 2);
assert(BP_STATE(node,0) == PT_AVAIL);
assert(BP_STATE(node,1) == PT_AVAIL);
toku_unpin_brtnode_off_client_thread(brt->h, node);
//
// now let us induce a clean on the internal node
//
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_internal,
toku_cachetable_hash(brt->h->cf, node_internal),
&bfe,
0,
NULL,
&node
);
assert(node->dirty);
// we expect that this flushes its buffer, that
// a merge is not done, and that the lookup
// of values "a" and "z" still works
r = toku_brtnode_cleaner_callback(
node,
node_internal,
toku_cachetable_hash(brt->h->cf, node_internal),
brt->h
);
// verify that node_internal's buffer is empty
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_internal,
toku_cachetable_hash(brt->h->cf, node_internal),
&bfe,
0,
NULL,
&node
);
// check that buffers are empty
assert(toku_bnc_nbytesinbuf(BNC(node, 0)) == 0);
toku_unpin_brtnode_off_client_thread(brt->h, node);
//
// now run a checkpoint to get everything clean,
// and to get the rebalancing to happen
//
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
// check that lookups on the two keys is still good
struct check_pair pair1 = {2, "a", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "a", 2), lookup_checkf, &pair1);
assert(r==0);
struct check_pair pair2 = {2, "z", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "z", 2), lookup_checkf, &pair2);
assert(r==0);
r = toku_close_brt(brt, 0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_free(pivots[0]);
}
int
test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
default_parse_args(argc, argv);
doit();
return 0;
}
newbrt/tests/test-flushes-on-cleaner.c 0 → 100644
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007-2011 Tokutek Inc. All rights reserved."
#ident "$Id$"
/* The goal of this test. Make sure that inserts stay behind deletes. */
#include "test.h"
#include "includes.h"
#include <brt-cachetable-wrappers.h>
#include "brt-flusher.h"
#include "checkpoint.h"
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
enum { NODESIZE = 1024, KSIZE=NODESIZE-100, TOKU_PSIZE=20 };
CACHETABLE ct;
BRT brt;
int fnamelen;
char *fname;
static int update_func(
DB* UU(db),
const DBT* key,
const DBT* old_val,
const DBT* UU(extra),
void (*set_val)(const DBT *new_val, void *set_extra),
void *set_extra)
{
DBT new_val;
assert(old_val->size > 0);
if (verbose) {
printf("applying update to %s\n", (char *)key->data);
}
toku_init_dbt(&new_val);
set_val(&new_val, set_extra);
return 0;
}
static void
doit (void) {
BLOCKNUM node_leaf;
BLOCKNUM node_internal, node_root;
int r;
fnamelen = strlen(__FILE__) + 20;
fname = toku_malloc(fnamelen);
assert(fname!=0);
snprintf(fname, fnamelen, "%s.brt", __FILE__);
r = toku_brt_create_cachetable(&ct, 500*1024*1024, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 1, &brt, NODESIZE, NODESIZE/2, ct, null_txn, toku_builtin_compare_fun, null_db);
assert(r==0);
toku_free(fname);
brt->update_fun = update_func;
brt->h->update_fun = update_func;
toku_testsetup_initialize(); // must precede any other toku_testsetup calls
char* pivots[1];
pivots[0] = toku_strdup("kkkkk");
int pivot_len = 6;
r = toku_testsetup_leaf(brt, &node_leaf, 2, pivots, &pivot_len);
assert(r==0);
r = toku_testsetup_nonleaf(brt, 1, &node_internal, 1, &node_leaf, 0, 0);
assert(r==0);
r = toku_testsetup_nonleaf(brt, 2, &node_root, 1, &node_internal, 0, 0);
assert(r==0);
r = toku_testsetup_root(brt, node_root);
assert(r==0);
//
// at this point we have created a tree with a root, an internal node,
// and two leaf nodes, the pivot being "kkkkk"
//
// now we insert a row into each leaf node
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"a", // key
2, // keylen
"aa",
3
);
assert(r==0);
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"z", // key
2, // keylen
"zz",
3
);
assert(r==0);
char filler[400];
memset(filler, 0, sizeof(filler));
// now we insert filler data so that the rebalance
// keeps it at two nodes
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"b", // key
2, // keylen
filler,
sizeof(filler)
);
assert(r==0);
r = toku_testsetup_insert_to_leaf (
brt,
node_leaf,
"y", // key
2, // keylen
filler,
sizeof(filler)
);
assert(r==0);
//
// now insert a bunch of dummy delete messages
// into the internal node, to get its cachepressure size up
//
for (int i = 0; i < 100000; i++) {
r = toku_testsetup_insert_to_nonleaf (
brt,
node_internal,
BRT_DELETE_ANY,
"jj", // this key does not exist, so its message application should be a no-op
3,
NULL,
0
);
assert(r==0);
}
//
// now insert a broadcast message into the root
//
r = toku_testsetup_insert_to_nonleaf (
brt,
node_root,
BRT_UPDATE_BROADCAST_ALL,
NULL,
0,
NULL,
0
);
assert(r==0);
//
// now run a checkpoint to get everything clean
//
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
// now lock and release the leaf node to make sure it is what we expect it to be.
BRTNODE node = NULL;
struct brtnode_fetch_extra bfe;
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_leaf,
toku_cachetable_hash(brt->h->cf, node_leaf),
&bfe,
0,
NULL,
&node
);
assert(!node->dirty);
assert(node->n_children == 2);
// a hack to get the basement nodes evicted
for (int i = 0; i < 20; i++) {
PAIR_ATTR attr;
toku_brtnode_pe_callback(node, make_pair_attr(0xffffffff), &attr, NULL);
}
// this ensures that when we do the lookups below,
// that the data is read off disk
assert(BP_STATE(node,0) == PT_ON_DISK);
assert(BP_STATE(node,1) == PT_ON_DISK);
toku_unpin_brtnode_off_client_thread(brt->h, node);
// now do a lookup on one of the keys, this should bring a leaf node up to date
DBT k;
struct check_pair pair = {2, "a", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "a", 2), lookup_checkf, &pair);
assert(r==0);
//
// pin the leaf one more time
// and make sure that one basement
// node is in memory and another is
// on disk
//
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_leaf,
toku_cachetable_hash(brt->h->cf, node_leaf),
&bfe,
0,
NULL,
&node
);
assert(!node->dirty);
assert(node->n_children == 2);
assert(BP_STATE(node,0) == PT_AVAIL);
assert(BP_STATE(node,1) == PT_ON_DISK);
toku_unpin_brtnode_off_client_thread(brt->h, node);
//
// now let us induce a clean on the internal node
//
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_internal,
toku_cachetable_hash(brt->h->cf, node_internal),
&bfe,
0,
NULL,
&node
);
assert(!node->dirty);
// we expect that this flushes its buffer, that
// a merge is not done, and that the lookup
// of values "a" and "z" still works
r = toku_brtnode_cleaner_callback(
node,
node_internal,
toku_cachetable_hash(brt->h->cf, node_internal),
brt->h
);
// verify that node_internal's buffer is empty
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
brt->h,
node_internal,
toku_cachetable_hash(brt->h->cf, node_internal),
&bfe,
0,
NULL,
&node
);
// check that buffers are empty
assert(toku_bnc_nbytesinbuf(BNC(node, 0)) == 0);
toku_unpin_brtnode_off_client_thread(brt->h, node);
//
// now run a checkpoint to get everything clean,
// and to get the rebalancing to happen
//
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
// check that lookups on the two keys is still good
struct check_pair pair1 = {2, "a", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "a", 2), lookup_checkf, &pair1);
assert(r==0);
struct check_pair pair2 = {2, "z", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "z", 2), lookup_checkf, &pair2);
assert(r==0);
r = toku_close_brt(brt, 0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_free(pivots[0]);
}
int
test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
default_parse_args(argc, argv);
doit();
return 0;
}
newbrt/tests/test-inc-split.c
View file @ f165ee02
......@@ -65,7 +65,7 @@ doit (int ksize __attribute__((__unused__))) {
toku_testsetup_initialize(); // must precede any other toku_testsetup calls
for (i=0; i<BRT_FANOUT; i++) {
r=toku_testsetup_leaf(t, &cnodes[i]);
r=toku_testsetup_leaf(t, &cnodes[i], 1, NULL, NULL);
assert(r==0);
char key[KSIZE+10];
int keylen = 1+snprintf(key, KSIZE, "%08d%0*d", i*10000+1, KSIZE-9, 0);
......
newbrt/tests/test-merges-on-cleaner.c
View file @ f165ee02
#ident "$Id: test-del-inorder.c 32975 2011-07-11 23:42:51Z leifwalsh $"
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007-2011 Tokutek Inc. All rights reserved."
#ident "$Id: test-merges-on-cleaner.c 38542 2012-01-06 14:06:23Z christianrober $"
/* The goal of this test. Make sure that inserts stay behind deletes. */
#include "test.h"
#include "includes.h"
#include <brt-cachetable-wrappers.h>
#include "brt-flusher.h"
#include "checkpoint.h"
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
......@@ -58,9 +62,9 @@ doit (void) {
toku_testsetup_initialize(); // must precede any other toku_testsetup calls
r = toku_testsetup_leaf(brt, &node_leaf[0]);
r = toku_testsetup_leaf(brt, &node_leaf[0], 1, NULL, NULL);
assert(r==0);
r = toku_testsetup_leaf(brt, &node_leaf[1]);
r = toku_testsetup_leaf(brt, &node_leaf[1], 1, NULL, NULL);
assert(r==0);
char* pivots[1];
......@@ -70,7 +74,7 @@ doit (void) {
r = toku_testsetup_nonleaf(brt, 1, &node_internal, 2, node_leaf, pivots, &pivot_len);
assert(r==0);
r = toku_testsetup_nonleaf(brt, 1, &node_root, 1, &node_internal, 0, 0);
r = toku_testsetup_nonleaf(brt, 2, &node_root, 1, &node_internal, 0, 0);
assert(r==0);
r = toku_testsetup_root(brt, node_root);
......@@ -132,16 +136,21 @@ doit (void) {
);
assert(r==0);
//
// now let us induce a clean on the internal node
//
BRTNODE node;
toku_pin_node_with_min_bfe(&node, node_leaf[1], brt);
// hack to get merge going
BLB_SEQINSERT(node, node->n_children-1) = FALSE;
toku_unpin_brtnode(brt, node);
// now do a lookup on one of the keys, this should bring a leaf node up to date
DBT k;
struct check_pair pair = {2, "a", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "a", 2), lookup_checkf, &pair);
assert(r==0);
//
// now let us induce a clean on the internal node
//
BRTNODE node;
struct brtnode_fetch_extra bfe;
fill_bfe_for_min_read(&bfe, brt->h);
toku_pin_brtnode_off_client_thread(
......@@ -175,12 +184,19 @@ doit (void) {
NULL,
&node
);
// check that no merge happened
assert(node->n_children == 2);
// check that merge happened
assert(node->n_children == 1);
// check that buffers are empty
assert(toku_bnc_nbytesinbuf(BNC(node, 0)) == 0);
assert(toku_bnc_nbytesinbuf(BNC(node, 1)) == 0);
toku_unpin_brtnode_off_client_thread(brt->h, node);
//
// now run a checkpoint to get everything clean,
// and to get the rebalancing to happen
//
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
// check that lookups on the two keys is still good
struct check_pair pair1 = {2, "a", 0, NULL, 0};
r = toku_brt_lookup(brt, toku_fill_dbt(&k, "a", 2), lookup_checkf, &pair1);
......@@ -198,6 +214,7 @@ doit (void) {
int
test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
default_parse_args(argc, argv);
doit();
return 0;
}
newbrt/tests/test-pick-child-to-flush.c 0 → 100644
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id$"
#ident "Copyright (c) 2011 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
/* The goal of this test. Make sure that inserts stay behind deletes. */
#include "test.h"
#include "includes.h"
#include <brt-cachetable-wrappers.h>
#include "brt-flusher.h"
#include "brt-flusher-internal.h"
#include "checkpoint.h"
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
enum { NODESIZE = 1024, KSIZE=NODESIZE-100, TOKU_PSIZE=20 };
CACHETABLE ct;
BRT t;
int fnamelen;
char *fname;
int curr_child_to_flush;
int num_flushes_called;
static int child_to_flush(struct brt_header* UU(h), BRTNODE parent, void* UU(extra)) {
// internal node has 2 children
if (parent->height == 1) {
assert(parent->n_children == 2);
return curr_child_to_flush;
}
// root has 1 child
else if (parent->height == 2) {
assert(parent->n_children == 1);
return 0;
}
else {
assert(FALSE);
}
return curr_child_to_flush;
}
static void update_status(BRTNODE UU(child), int UU(dirtied), void* UU(extra)) {
num_flushes_called++;
}
static bool
dont_destroy_bn(void* UU(extra))
{
return false;
}
static void merge_should_not_happen(struct flusher_advice* UU(fa),
struct brt_header* UU(h),
BRTNODE UU(parent),
int UU(childnum),
BRTNODE UU(child),
void* UU(extra))
{
assert(FALSE);
}
static bool recursively_flush_should_not_happen(BRTNODE UU(child), void* UU(extra)) {
assert(FALSE);
}
static bool always_flush(BRTNODE UU(child), void* UU(extra)) {
return true;
}
static void
doit (void) {
BLOCKNUM node_internal, node_root;
BLOCKNUM node_leaf[2];
int r;
fnamelen = strlen(__FILE__) + 20;
fname = toku_malloc(fnamelen);
assert(fname!=0);
snprintf(fname, fnamelen, "%s.brt", __FILE__);
r = toku_brt_create_cachetable(&ct, 500*1024*1024, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 1, &t, NODESIZE, NODESIZE/2, ct, null_txn, toku_builtin_compare_fun, null_db);
assert(r==0);
toku_free(fname);
toku_testsetup_initialize(); // must precede any other toku_testsetup calls
r = toku_testsetup_leaf(t, &node_leaf[0], 1, NULL, NULL);
assert(r==0);
r = toku_testsetup_leaf(t, &node_leaf[1], 1, NULL, NULL);
assert(r==0);
char* pivots[1];
pivots[0] = toku_strdup("kkkkk");
int pivot_len = 6;
r = toku_testsetup_nonleaf(t, 1, &node_internal, 2, node_leaf, pivots, &pivot_len);
assert(r==0);
r = toku_testsetup_nonleaf(t, 2, &node_root, 1, &node_internal, 0, 0);
assert(r==0);
r = toku_testsetup_root(t, node_root);
assert(r==0);
char filler[900];
memset(filler, 0, sizeof(filler));
// now we insert filler data so that a merge does not happen
r = toku_testsetup_insert_to_leaf (
t,
node_leaf[0],
"b", // key
2, // keylen
filler,
sizeof(filler)
);
assert(r==0);
r = toku_testsetup_insert_to_leaf (
t,
node_leaf[1],
"y", // key
2, // keylen
filler,
sizeof(filler)
);
assert(r==0);
// make buffers in internal node non-empty
r = toku_testsetup_insert_to_nonleaf(
t,
node_internal,
BRT_INSERT,
"a",
2,
NULL,
0
);
assert_zero(r);
r = toku_testsetup_insert_to_nonleaf(
t,
node_internal,
BRT_INSERT,
"z",
2,
NULL,
0
);
assert_zero(r);
//
// now run a checkpoint to get everything clean
//
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
// now with setup done, start the test
// test that if flush_some_child properly honors
// what we say and flushes the child we pick
BRTNODE node = NULL;
toku_pin_node_with_min_bfe(&node, node_internal, t);
toku_assert_entire_node_in_memory(node);
assert(node->n_children == 2);
assert(!node->dirty);
assert(toku_bnc_n_entries(node->bp[0].ptr.u.nonleaf) > 0);
assert(toku_bnc_n_entries(node->bp[1].ptr.u.nonleaf) > 0);
struct flusher_advice fa;
flusher_advice_init(
&fa,
child_to_flush,
dont_destroy_bn,
recursively_flush_should_not_happen,
merge_should_not_happen,
update_status,
default_pick_child_after_split,
NULL
);
curr_child_to_flush = 0;
num_flushes_called = 0;
flush_some_child(t->h, node, &fa);
assert(num_flushes_called == 1);
toku_pin_node_with_min_bfe(&node, node_internal, t);
toku_assert_entire_node_in_memory(node);
assert(node->dirty);
assert(node->n_children == 2);
// child 0 should have empty buffer because it flushed
// child 1 should still have message in buffer
assert(toku_bnc_n_entries(node->bp[0].ptr.u.nonleaf) == 0);
assert(toku_bnc_n_entries(node->bp[1].ptr.u.nonleaf) > 0);
toku_unpin_brtnode(t, node);
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
toku_pin_node_with_min_bfe(&node, node_internal, t);
assert(!node->dirty);
curr_child_to_flush = 1;
num_flushes_called = 0;
flush_some_child(t->h, node, &fa);
assert(num_flushes_called == 1);
toku_pin_node_with_min_bfe(&node, node_internal, t);
assert(node->dirty);
toku_assert_entire_node_in_memory(node);
assert(node->n_children == 2);
// both buffers should be empty now
assert(toku_bnc_n_entries(node->bp[0].ptr.u.nonleaf) == 0);
assert(toku_bnc_n_entries(node->bp[1].ptr.u.nonleaf) == 0);
// now let's do a flush with an empty buffer, make sure it is ok
toku_unpin_brtnode(t, node);
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
toku_pin_node_with_min_bfe(&node, node_internal, t);
assert(!node->dirty);
curr_child_to_flush = 0;
num_flushes_called = 0;
flush_some_child(t->h, node, &fa);
assert(num_flushes_called == 1);
toku_pin_node_with_min_bfe(&node, node_internal, t);
assert(!node->dirty); // nothing was flushed, so node better not be dirty
toku_assert_entire_node_in_memory(node);
assert(node->n_children == 2);
// both buffers should be empty now
assert(toku_bnc_n_entries(node->bp[0].ptr.u.nonleaf) == 0);
assert(toku_bnc_n_entries(node->bp[1].ptr.u.nonleaf) == 0);
toku_unpin_brtnode(t, node);
// now let's start a flush from the root, that always recursively flushes
flusher_advice_init(
&fa,
child_to_flush,
dont_destroy_bn,
always_flush,
merge_should_not_happen,
update_status,
default_pick_child_after_split,
NULL
);
// use a for loop so to get us down both paths
for (int i = 0; i < 2; i++) {
toku_pin_node_with_min_bfe(&node, node_root, t);
toku_assert_entire_node_in_memory(node); // entire root is in memory
curr_child_to_flush = i;
num_flushes_called = 0;
flush_some_child(t->h, node, &fa);
assert(num_flushes_called == 2);
toku_pin_node_with_min_bfe(&node, node_internal, t);
assert(!node->dirty); // nothing was flushed, so node better not be dirty
toku_unpin_brtnode(t, node);
toku_pin_node_with_min_bfe(&node, node_leaf[0], t);
assert(!node->dirty); // nothing was flushed, so node better not be dirty
toku_unpin_brtnode(t, node);
toku_pin_node_with_min_bfe(&node, node_leaf[1], t);
assert(!node->dirty); // nothing was flushed, so node better not be dirty
toku_unpin_brtnode(t, node);
}
// now one more test to show a bug was fixed
// if there is nothing to flush from parent to child,
// and child is not fully in memory, we used to crash
// so, to make sure that is fixed, let's get internal to not
// be fully in memory, and make sure the above test works
// a hack to get internal compressed
r = toku_testsetup_insert_to_nonleaf(
t,
node_internal,
BRT_INSERT,
"c",
2,
NULL,
0
);
assert_zero(r);
r = toku_checkpoint(ct, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
assert_zero(r);
toku_pin_node_with_min_bfe(&node, node_internal, t);
for (int i = 0; i < 20; i++) {
PAIR_ATTR attr;
toku_brtnode_pe_callback(node, make_pair_attr(0xffffffff), &attr, NULL);
}
assert(BP_STATE(node,0) == PT_COMPRESSED);
toku_unpin_brtnode(t, node);
//now let's do the same test as above
toku_pin_node_with_min_bfe(&node, node_root, t);
toku_assert_entire_node_in_memory(node); // entire root is in memory
curr_child_to_flush = 0;
num_flushes_called = 0;
flush_some_child(t->h, node, &fa);
assert(num_flushes_called == 2);
r = toku_close_brt(t, 0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_free(pivots[0]);
}
int
test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
doit();
return 0;
}
newbrt/tests/test3884.c
View file @ f165ee02
......@@ -10,6 +10,7 @@
#include "includes.h"
#include <brt-cachetable-wrappers.h>
#include <brt-flusher.h>
// Some constants to be used in calculations below
static const int nodesize = 1024; // Target max node size
......@@ -27,8 +28,6 @@ static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static const char fname[]= __FILE__ ".brt";
static BRT_STATUS_S my_brt_status;
static int omt_long_cmp(OMTVALUE p, void *q)
{
LEAFENTRY a = p, b = q;
......@@ -172,7 +171,7 @@ test_split_on_boundary(void)
BRTNODE nodea, nodeb;
DBT splitk;
// if we haven't done it right, we should hit the assert in the top of move_leafentries
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL, &my_brt_status);
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL);
verify_basement_node_msns(nodea, dummy_msn_3884);
verify_basement_node_msns(nodeb, dummy_msn_3884);
......@@ -245,7 +244,7 @@ test_split_with_everything_on_the_left(void)
BRTNODE nodea, nodeb;
DBT splitk;
// if we haven't done it right, we should hit the assert in the top of move_leafentries
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL, &my_brt_status);
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL);
toku_unpin_brtnode(brt, nodeb);
r = toku_close_brt(brt, NULL); assert(r == 0);
......@@ -320,7 +319,7 @@ test_split_on_boundary_of_last_node(void)
BRTNODE nodea, nodeb;
DBT splitk;
// if we haven't done it right, we should hit the assert in the top of move_leafentries
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL, &my_brt_status);
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL);
toku_unpin_brtnode(brt, nodeb);
r = toku_close_brt(brt, NULL); assert(r == 0);
......@@ -388,7 +387,7 @@ test_split_at_begin(void)
BRTNODE nodea, nodeb;
DBT splitk;
// if we haven't done it right, we should hit the assert in the top of move_leafentries
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL, &my_brt_status);
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL);
toku_unpin_brtnode(brt, nodeb);
r = toku_close_brt(brt, NULL); assert(r == 0);
......@@ -452,7 +451,7 @@ test_split_at_end(void)
BRTNODE nodea, nodeb;
DBT splitk;
// if we haven't done it right, we should hit the assert in the top of move_leafentries
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL, &my_brt_status);
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL);
toku_unpin_brtnode(brt, nodeb);
r = toku_close_brt(brt, NULL); assert(r == 0);
......@@ -506,7 +505,7 @@ test_split_odd_nodes(void)
BRTNODE nodea, nodeb;
DBT splitk;
// if we haven't done it right, we should hit the assert in the top of move_leafentries
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL, &my_brt_status);
brtleaf_split(brt->h, &sn, &nodea, &nodeb, &splitk, TRUE, 0, NULL);
verify_basement_node_msns(nodea, dummy_msn_3884);
verify_basement_node_msns(nodeb, dummy_msn_3884);
......
newbrt/tests/test4244.c
View file @ f165ee02
......@@ -35,7 +35,7 @@ doit (void) {
toku_testsetup_initialize(); // must precede any other toku_testsetup calls
r = toku_testsetup_leaf(t, &node_leaf);
r = toku_testsetup_leaf(t, &node_leaf, 1, NULL, NULL);
assert(r==0);
r = toku_testsetup_nonleaf(t, 1, &node_internal, 1, &node_leaf, 0, 0);
......
newbrt/tests/verify-bad-msn.c
View file @ f165ee02
......@@ -142,7 +142,7 @@ test_make_tree(int height, int fanout, int nperleaf, int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
// set the new root to point to the new tree
*rootp = newroot->thisnodename;
......
newbrt/tests/verify-bad-pivots.c
View file @ f165ee02
......@@ -112,7 +112,7 @@ test_make_tree(int height, int fanout, int nperleaf, int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
// set the new root to point to the new tree
*rootp = newroot->thisnodename;
......
newbrt/tests/verify-dup-in-leaf.c
View file @ f165ee02
......@@ -66,7 +66,7 @@ test_dup_in_leaf(int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
BRTNODE newroot = make_node(brt, 0);
populate_leaf(newroot, htonl(2), 1);
......
newbrt/tests/verify-dup-pivots.c
View file @ f165ee02
......@@ -112,7 +112,7 @@ test_make_tree(int height, int fanout, int nperleaf, int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
// set the new root to point to the new tree
*rootp = newroot->thisnodename;
......
newbrt/tests/verify-misrouted-msgs.c
View file @ f165ee02
......@@ -127,7 +127,7 @@ test_make_tree(int height, int fanout, int nperleaf, int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
// set the new root to point to the new tree
*rootp = newroot->thisnodename;
......
newbrt/tests/verify-unsorted-leaf.c
View file @ f165ee02
......@@ -66,7 +66,7 @@ test_dup_in_leaf(int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
BRTNODE newroot = make_node(brt, 0);
populate_leaf(newroot, htonl(2), 1);
......
newbrt/tests/verify-unsorted-pivots.c
View file @ f165ee02
......@@ -112,7 +112,7 @@ test_make_tree(int height, int fanout, int nperleaf, int do_verify) {
// discard the old root block
u_int32_t fullhash = 0;
CACHEKEY *rootp;
rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
rootp = toku_calculate_root_offset_pointer(brt->h, &fullhash);
// set the new root to point to the new tree
*rootp = newroot->thisnodename;
......
newbrt/ule-internal.h
View file @ f165ee02
......@@ -7,7 +7,7 @@
#ifndef TOKU_ULE_INTERNAL_H
#define TOKU_ULE_INTERNAL_H
#ident "$Id: ule.h 24600 2010-10-15 15:22:18Z dwells $"
#ident "$Id$"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
......
src/tests/Makefile
View file @ f165ee02
......@@ -136,6 +136,7 @@ BDB_DONTRUN_TESTS = \
hotindexer-simple-abort \
hotindexer-undo-do-test \
hotindexer-with-queries \
hot-optimize-table-tests \
insert-dup-prelock \
isolation \
isolation-read-committed \
......@@ -278,6 +279,7 @@ BDB_DONTRUN_TESTS = \
test_stress4 \
test_stress5 \
test_stress6 \
test_stress7 \
test_stress_with_verify \
test_transactional_descriptor \
test_trans_desc_during_chkpt \
......@@ -355,6 +357,7 @@ DEPENDS_ON_STRESS_HELPERS = \
test_stress4 \
test_stress5 \
test_stress6 \
test_stress7 \
#blank
$(patsubst %,%.tdb,$(DEPENDS_ON_STRESS_HELPERS)): threaded_stress_test_helpers.h
......@@ -682,6 +685,7 @@ test_update_broadcast_stress.tdbrun: VGRIND=
test_update_stress.tdbrun: VGRIND=
stress-test.tdbrun: VGRIND=
stress-test.bdbrun: VGRIND=
hot-optimize-table-tests.tdbrun: VGRIND=
libs:
......
src/tests/hot-optimize-table-tests.c 0 → 100644
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
// hot-optimize-table-tests.c
#include "test.h"
#include "includes.h"
#include <brt-cachetable-wrappers.h>
#include "db.h"
#include "ydb.h"
const int envflags = DB_INIT_MPOOL |
DB_CREATE |
DB_THREAD |
DB_INIT_LOCK |
DB_INIT_LOG |
DB_INIT_TXN |
DB_PRIVATE;
DB_ENV* env;
unsigned int leaf_hits;
// Custom Update Function for our test BRT.
static int
update_func(DB* UU(db),
const DBT* key,
const DBT* old_val,
const DBT* extra,
void (*set_val)(const DBT* new_val, void* set_extra) __attribute__((unused)),
void* UU(set_extra))
{
unsigned int *x_results;
assert(extra->size == sizeof x_results);
x_results = *(unsigned int **) extra->data;
assert(x_results);
assert(old_val->size > 0);
unsigned int* indexptr;
assert(key->size == (sizeof *indexptr));
indexptr = (unsigned int*)key->data;
++leaf_hits;
if (verbose && x_results[*indexptr] != 0) {
printf("x_results = %p, indexptr = %p, *indexptr = %u, x_results[*indexptr] = %u\n", x_results, indexptr, *indexptr, x_results[*indexptr]);
}
assert(x_results[*indexptr] == 0);
x_results[*indexptr]++;
// ++(x_results[*indexptr]);
// memset(&new_val, 0, sizeof(new_val));
// set_val(&new_val, set_extra);
unsigned int i = *indexptr;
if (verbose && ((i + 1) % 50000 == 0)) {
printf("applying update to %u\n", i);
//printf("x_results[] = %u\n", x_results[*indexptr]);
}
return 0;
}
///
static void
hot_test_setup(void)
{
int r = 0;
// Remove any previous environment.
CHK(system("rm -rf " ENVDIR));
// Set up a new TokuDB.
CHK(toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO));
CHK(db_env_create(&env, 0));
env->set_errfile(env, stderr);
r = env->set_default_bt_compare(env, uint_dbt_cmp);CKERR(r);
env->set_update(env, update_func);
CHK(env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO));
}
///
static void
hot_insert_keys(DB* db, unsigned int key_count)
{
int r = 0;
DB_TXN * xact;
unsigned int limit = 1;
if (key_count > 10) {
limit = 100000;
}
// Dummy data.
const unsigned int DUMMY_SIZE = 100;
size_t size = DUMMY_SIZE;
char* dummy = NULL;
dummy = (char*)toku_xmalloc(size);
memset(dummy, 0, size);
// Start the transaction for insertions.
//
r = env->txn_begin(env, 0, &xact, 0); CKERR(r);
unsigned int key;
DBT key_thing;
DBT *keyptr = dbt_init(&key_thing, &key, sizeof(key));
DBT value_thing;
DBT *valueptr = dbt_init(&value_thing, dummy, size);
for (key = 0; key < key_count; ++key)
{
CHK(db->put(db, xact, keyptr, valueptr, 0));
// DEBUG OUTPUT
//
if (verbose && (key + 1) % limit == 0) {
printf("%d Elements inserted.\n", key + 1);
}
}
// Commit the insert transaction.
//
r = xact->commit(xact, 0); CKERR(r);
toku_free(dummy);
}
///
static void
hot_create_db(DB** db, const char* c)
{
int r = 0;
DB_TXN* xact;
verbose ? printf("Creating DB.\n") : 0;
r = env->txn_begin(env, 0, &xact, 0); CKERR(r);
CHK(db_create(db, env, 0));
CHK((*db)->open((*db), xact, c, NULL, DB_BTREE, DB_CREATE, 0666));
r = xact->commit(xact, 0); CKERR(r);
verbose ? printf("DB Created.\n") : 0;
}
///
static void
hot_test(DB* db, unsigned int size)
{
int r = 0;
leaf_hits = 0;
verbose ? printf("Insert some data.\n") : 0;
// Insert our keys to assemble the tree.
hot_insert_keys(db, size);
// Insert Broadcast Message.
verbose ? printf("Insert Broadcast Message.\n") : 0;
unsigned int *XMALLOC_N(size, x_results);
memset(x_results, 0, (sizeof x_results[0]) * size);
DBT extra;
DBT *extrap = dbt_init(&extra, &x_results, sizeof x_results);
DB_TXN * xact;
r = env->txn_begin(env, 0, &xact, 0); CKERR(r);
r = CHK(db->update_broadcast(db, xact, extrap, 0));
r = xact->commit(xact, 0); CKERR(r);
// Flatten the tree.
verbose ? printf("Calling hot optimize...\n") : 0;
r = db->hot_optimize(db, NULL, NULL);
assert(r == 0);
verbose ? printf("HOT Finished!\n") : 0;
for (unsigned int i = 0; i < size; ++i) {
assert(x_results[i] == 1);
}
verbose ? printf("Leaves hit = %u\n", leaf_hits) :0;
toku_free(x_results);
}
///
int
test_main(int argc, char * const argv[])
{
int r = 0;
default_parse_args(argc, argv);
hot_test_setup();
// Create and Open the Database/BRT
DB *db = NULL;
const unsigned int BIG = 4000000;
const unsigned int SMALL = 10;
const unsigned int NONE = 0;
hot_create_db(&db, "none.db");
hot_test(db, NONE);
hot_create_db(&db, "small.db");
hot_test(db, SMALL);
hot_create_db(&db, "big.db");
hot_test(db, BIG);
verbose ? printf("Exiting Test.\n") : 0;
return r;
}
src/tests/recover-ft1.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-ft2.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-ft3.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-ft4.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-ft5.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-ft6.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-ft7.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-ft8.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-test_stress1.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/recover-test_stress2.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35324 2011-10-04 01:48:45Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/simple.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: env_startup.c 20778 2010-05-28 20:38:42Z yfogel $"
#ident "$Id$"
/* Purpose of this test is to verify that a failed assert will
* cause a panic, which should be visible via engine status.
......
src/tests/test_stress5.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress1.c 35109 2011-09-27 18:41:25Z leifwalsh $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/test_stress6.c
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress2.c 35151 2011-09-29 01:32:27Z zardosht $"
#ident "$Id$"
#include "test.h"
#include <stdio.h>
......
src/tests/test_stress7.c 0 → 100644
View file @ f165ee02
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "$Id: test_stress7.c 38515 2012-01-05 20:48:10Z leifwalsh $"
#include "test.h"
#include <stdio.h>
#include <stdlib.h>
#include <toku_pthread.h>
#include <unistd.h>
#include <memory.h>
#include <sys/stat.h>
#include <db.h>
#include "threaded_stress_test_helpers.h"
static void
stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
int n = cli_args->num_elements;
//
// do insertions and queries with a loader lying around doing stuff
//
if (verbose) printf("starting creation of pthreads\n");
const int num_threads = 4 + cli_args->num_update_threads + cli_args->num_ptquery_threads;
struct arg myargs[num_threads];
for (int i = 0; i < num_threads; i++) {
arg_init(&myargs[i], n, dbp, env, cli_args);
}
// make the forward fast scanner
myargs[0].fast = TRUE;
myargs[0].fwd = TRUE;
myargs[0].operation = scan_op;
// make the backward slow scanner
myargs[1].fast = FALSE;
myargs[1].fwd = FALSE;
myargs[1].operation = scan_op;
// make the guy that runs HOT in the background
myargs[2].operation = hot_op;
myargs[3].operation = keyrange_op;
for (int i = 4; i < 4 + cli_args->num_update_threads; ++i) {
myargs[i].operation = update_op;
}
// make the guy that does point queries
for (int i = 4 + cli_args->num_update_threads; i < num_threads; i++) {
myargs[i].operation = ptquery_op;
}
run_workers(myargs, num_threads, cli_args->time_of_test, false);
}
int
test_main(int argc, char *const argv[]) {
struct cli_args args = DEFAULT_ARGS;
// let's make default checkpointing period really slow
args.checkpointing_period = 1;
parse_stress_test_args(argc, argv, &args);
stress_test_main(&args);
return 0;
}
src/tests/threaded_stress_test_helpers.h
View file @ f165ee02
......@@ -555,6 +555,14 @@ static int UU() update_broadcast_op(DB_ENV *UU(env), DB **dbp, DB_TXN *txn, ARG
return r;
}
static int UU() hot_op(DB_ENV *UU(env), DB **dbp, DB_TXN *UU(txn), ARG UU(arg)) {
int r;
DB* db = *dbp;
r = db->hot_optimize(db, NULL, NULL);
CKERR(r);
return r;
}
static int UU() remove_and_recreate_me(DB_ENV *env, DB **dbp, DB_TXN *UU(txn), ARG UU(arg)) {
int r;
r = (*dbp)->close(*dbp, 0); CKERR(r);
......
src/ydb.c
View file @ f165ee02
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7