Skip to content

M
mariadb
Commit 920e14c8 authored by Barry Perlman's avatar Barry Perlman Committed by Yoni Fogel
Browse files
Options

[t:3236] [t:3238] Merge tokudb.3236+3238 to main. 

git-svn-id: file:///svn/toku/tokudb@28070 c7de825b-a66e-492c-adef-691d508d4ae1
parent a8376ef4
Hide whitespace changes
Inline Side-by-side
Showing with 267 additions and 173 deletions
buildheader/db.h_4_1
View file @ 920e14c8
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
buildheader/db.h_4_3
View file @ 920e14c8
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
buildheader/db.h_4_4
View file @ 920e14c8
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
buildheader/db.h_4_5
View file @ 920e14c8
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
buildheader/db.h_4_6
View file @ 920e14c8
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
buildheader/make_db_h.c
View file @ 920e14c8
......@@ -572,10 +572,11 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
printf(" u_int64_t upgrade_header; /* how many brt headers were upgraded? */ \n");
printf(" u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */ \n");
printf(" u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */ \n");
printf(" u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */ \n");
printf(" u_int64_t original_ver; /* original environment version */ \n");
printf(" u_int64_t ver_at_startup; /* environment version at startup */ \n");
printf(" u_int64_t last_lsn_v12; /* last lsn of version 12 environment */ \n");
printf(" char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */ \n");
printf(" u_int64_t last_lsn_v13; /* last lsn of version 13 environment */ \n");
printf(" char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */ \n");
printf(" u_int64_t env_panic; /* non-zero if environment is panicked */ \n");
printf(" u_int64_t logger_panic; /* non-zero if logger is panicked */ \n");
printf(" u_int64_t logger_panic_errno; /* non-zero if environment is panicked */ \n");
......
buildheader/tdb.h
View file @ 920e14c8
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
include/db.h
View file @ 920e14c8
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
newbrt/brt-internal.h
View file @ 920e14c8
......@@ -6,6 +6,14 @@
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// Symbol TOKUDB_REVISION is not defined by fractal-tree makefiles, so
// BUILD_ID of 1000 indicates development build of main, not a release build.
#if defined(TOKUDB_REVISION)
#define BUILD_ID TOKUDB_REVISION
#else
#define BUILD_ID 1000
#endif
#include "brt_layout_version.h"
#include "toku_assert.h"
#include "block_allocator.h"
......@@ -92,6 +100,7 @@ struct brtnode {
int layout_version; // What version of the data structure?
int layout_version_original; // different (<) from layout_version if upgraded from a previous version (useful for debugging)
int layout_version_read_from_disk; // transient, not serialized to disk, (useful for debugging)
uint32_t build_id; // build_id (svn rev number) of software that wrote this node to disk
int height; /* height is always >= 0. 0 for leaf, >0 for nonleaf. */
u_int32_t rand4fingerprint;
u_int32_t local_fingerprint; /* For leaves this is everything in the buffer. For nonleaves, this is everything in the buffers, but does not include child subtree fingerprints. */
......@@ -161,13 +170,17 @@ struct brt_header {
int dirty;
BOOL dictionary_opened; // True once this header has been associated with a dictionary (a brt fully opened)
DICTIONARY_ID dict_id; // unique id for dictionary
int panic; // If nonzero there was a write error. Don't write any more, because it probably only gets worse. This is the error code.
char *panic_string; // A malloced string that can indicate what went wrong.
int panic; // If nonzero there was a write error. Don't write any more, because it probably only gets worse. This is the error code.
char *panic_string; // A malloced string that can indicate what went wrong.
int layout_version;
int layout_version_original; // different (<) from layout_version if upgraded from a previous version (useful for debugging)
int layout_version_read_from_disk; // transient, not serialized to disk
uint32_t build_id; // build_id (svn rev number) of software that wrote this node to disk
uint32_t build_id_original; // build_id of software that created this tree (read from disk, overwritten when written to disk)
uint64_t time_of_creation; // time this tree was created
uint64_t time_of_last_modification; // last time this header was serialized to disk (read from disk, overwritten when written to disk)
BOOL upgrade_brt_performed; // initially FALSE, set TRUE when brt has been fully updated (even though nodes may not have been)
int64_t num_blocks_to_upgrade; // Number of v12 blocks still not newest version. When we release layout 14 we may need to turn this to an array or add more variables.
int64_t num_blocks_to_upgrade; // Number of v13 blocks still not newest version. When we release layout 15 we may need to turn this to an array or add more variables.
unsigned int nodesize;
BLOCKNUM root; // roots of the dictionary
struct remembered_hash root_hash; // hash of the root offset.
......@@ -375,10 +388,10 @@ int toku_brt_remove_now(CACHETABLE ct, DBT* iname_dbt_p);
typedef struct brt_upgrade_status {
u_int64_t header_12; // how many headers upgrade from version 12
u_int64_t nonleaf_12;
u_int64_t leaf_12;
u_int64_t optimized_for_upgrade_12; // how many optimize_for_upgrade messages sent
u_int64_t header_13; // how many headers were upgraded from version 13
u_int64_t nonleaf_13;
u_int64_t leaf_13;
u_int64_t optimized_for_upgrade; // how many optimize_for_upgrade messages were sent
} BRT_UPGRADE_STATUS_S, *BRT_UPGRADE_STATUS;
void toku_brt_get_upgrade_status(BRT_UPGRADE_STATUS);
......
newbrt/brt-serialize.c
View file @ 920e14c8
......@@ -191,7 +191,8 @@ enum {
node_header_overhead = (8+ // magic "tokunode" or "tokuleaf" or "tokuroll"
4+ // layout_version
4), // layout_version_original
4+ // layout_version_original
4), // build_id
extended_node_header_overhead = (4+ // nodesize
4+ // flags
......@@ -294,6 +295,7 @@ serialize_node_header(BRTNODE node, struct wbuf *wbuf) {
invariant(node->layout_version == BRT_LAYOUT_VERSION);
wbuf_nocrc_int(wbuf, node->layout_version);
wbuf_nocrc_int(wbuf, node->layout_version_original);
wbuf_nocrc_uint(wbuf, BUILD_ID);
//printf("%s:%d %lld.calculated_size=%d\n", __FILE__, __LINE__, off, calculated_size);
wbuf_nocrc_uint(wbuf, node->nodesize);
......@@ -755,10 +757,7 @@ deserialize_brtnode_leaf_from_rbuf (BRTNODE result, bytevec magic, struct rbuf *
result->u.l.leaf_stats.dsize = rbuf_ulonglong(rb);
result->u.l.leaf_stats.exact = TRUE;
// TODO: #3228 May need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(result->layout_version == BRT_LAYOUT_VERSION);
if (result->layout_version >= BRT_LAYOUT_VERSION_13) {
if (result->layout_version >= BRT_LAYOUT_VERSION_14) { // this field added in version 14
result->u.l.optimized_for_upgrade = rbuf_int(rb);
}
else {
......@@ -793,9 +792,6 @@ deserialize_brtnode_leaf_from_rbuf (BRTNODE result, bytevec magic, struct rbuf *
u_int32_t start_of_data = rb->ndone;
OMTVALUE *MALLOC_N(n_in_buf, array);
// TODO: #3228 May need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(result->layout_version == BRT_LAYOUT_VERSION);
if (result->layout_version == BRT_LAYOUT_VERSION) {
for (int i=0; i<n_in_buf; i++) {
LEAFENTRY le = (LEAFENTRY)(&rb->buf[rb->ndone]);
......@@ -806,12 +802,12 @@ deserialize_brtnode_leaf_from_rbuf (BRTNODE result, bytevec magic, struct rbuf *
actual_sum += x1764_memory(le, disksize);
}
}
else if (result->layout_version == BRT_LAYOUT_VERSION_12) {
else if (result->layout_version == BRT_LAYOUT_VERSION_13) {
for (int i=0; i<n_in_buf; i++) {
// these two lines and optimized_for_upgrade logic above are only difference in handling
// versions 12 and 13 at this layer (more logic at higher layer)
LEAFENTRY_12 le = (LEAFENTRY_12)(&rb->buf[rb->ndone]);
u_int32_t disksize = leafentry_disksize_12(le);
// versions 13 and 14 at this layer (more logic at higher layer)
LEAFENTRY_13 le = (LEAFENTRY_13)(&rb->buf[rb->ndone]);
u_int32_t disksize = leafentry_disksize_13(le);
rb->ndone += disksize;
invariant(rb->ndone<=rb->size);
......@@ -877,14 +873,11 @@ deserialize_brtnode_from_rbuf (BLOCKNUM blocknum, u_int32_t fullhash, BRTNODE *b
rbuf_literal_bytes(rb, &magic, 8);
result->layout_version = rbuf_int(rb);
// TODO: #3228 May need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(result->layout_version == BRT_LAYOUT_VERSION);
invariant(result->layout_version >= BRT_LAYOUT_MIN_SUPPORTED_VERSION);
invariant(result->layout_version <= BRT_LAYOUT_VERSION);
result->layout_version_original = rbuf_int(rb);
result->layout_version_read_from_disk = result->layout_version;
result->build_id = rbuf_int(rb);
result->nodesize = rbuf_int(rb);
result->thisnodename = blocknum;
......@@ -1020,11 +1013,8 @@ decompress_from_raw_block_into_rbuf_versioned(u_int32_t version, u_int8_t *raw_b
// This function exists solely to accomodate future changes in compression.
int r;
// TODO: #3228 May need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(version == BRT_LAYOUT_VERSION);
switch (version) {
case BRT_LAYOUT_VERSION_12:
case BRT_LAYOUT_VERSION_13:
case BRT_LAYOUT_VERSION:
r = decompress_from_raw_block_into_rbuf(raw_block, raw_block_size, rb, blocknum);
break;
......@@ -1037,9 +1027,6 @@ decompress_from_raw_block_into_rbuf_versioned(u_int32_t version, u_int8_t *raw_b
static int
deserialize_brtnode_from_rbuf_versioned (u_int32_t version, BLOCKNUM blocknum, u_int32_t fullhash, BRTNODE *brtnode, struct brt_header *h, struct rbuf *rb) {
// TODO: #3228 May need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(version == BRT_LAYOUT_VERSION);
int r = 0;
BRTNODE node = NULL;
r = deserialize_brtnode_from_rbuf(blocknum, fullhash, &node, h, rb); // we just filled the node with contents from rbuf
......@@ -1047,8 +1034,8 @@ deserialize_brtnode_from_rbuf_versioned (u_int32_t version, BLOCKNUM blocknum, u
invariant(node);
int upgrade = 0;
switch (version) {
case BRT_LAYOUT_VERSION_12:
invariant(node->layout_version == BRT_LAYOUT_VERSION_12);
case BRT_LAYOUT_VERSION_13:
invariant(node->layout_version == BRT_LAYOUT_VERSION_13);
//Any upgrade necessary.
if (node->height == 0) {
//leaf
......@@ -1064,8 +1051,8 @@ deserialize_brtnode_from_rbuf_versioned (u_int32_t version, BLOCKNUM blocknum, u
r = toku_omt_fetch(omt, i, &v, NULL);
invariant(r==0);
size_t new_memsize, new_disksize;
// Translate packed version 12 leafentry to packed version 13 leafentry
r = toku_le_upgrade_12_13(v, &new_memsize, &new_disksize, &new_les[i]);
// Translate packed version 13 leafentry to packed version 14 leafentry
r = toku_le_upgrade_13_14(v, &new_memsize, &new_disksize, &new_les[i]);
invariant(r==0);
invariant(new_memsize == new_disksize);
incremental_size += OMT_ITEM_OVERHEAD + new_memsize;
......@@ -1099,12 +1086,12 @@ deserialize_brtnode_from_rbuf_versioned (u_int32_t version, BLOCKNUM blocknum, u
toku_free(new_les); // Free array of pointers to new leafentries
//Regenerate nkeys, ndata, dsize
toku_brt_leaf_reset_calc_leaf_stats(node);
toku_sync_fetch_and_increment_uint64(&upgrade_status.leaf_12); // how many leaf nodes upgraded from v12
toku_sync_fetch_and_increment_uint64(&upgrade_status.leaf_13); // how many leaf nodes upgraded from v13
}
else {
toku_sync_fetch_and_increment_uint64(&upgrade_status.nonleaf_12); // how many nonleaf nodes upgraded from v12
toku_sync_fetch_and_increment_uint64(&upgrade_status.nonleaf_13); // how many nonleaf nodes upgraded from v13
}
node->flags &= ~TOKU_DB_VALCMP_BUILTIN_12; // delete obsolete flag
node->flags &= ~TOKU_DB_VALCMP_BUILTIN_13; // delete obsolete flag
node->layout_version = BRT_LAYOUT_VERSION;
upgrade++;
//Fall through on purpose
......@@ -1220,20 +1207,18 @@ toku_maybe_upgrade_brt(BRT t) { // possibly do some work to complete the version
int version = t->h->layout_version_read_from_disk;
// TODO: #3228 May need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(version == BRT_LAYOUT_VERSION);
int upgrade = 0;
if (!t->h->upgrade_brt_performed) { // upgrade may be necessary
switch (version) {
case BRT_LAYOUT_VERSION_12:
case BRT_LAYOUT_VERSION_13:
r = 0;
upgrade++;
//Fall through on purpose.
case BRT_LAYOUT_VERSION:
if (r == 0 && upgrade) {
r = toku_brt_optimize_for_upgrade(t);
toku_sync_fetch_and_increment_uint64(&upgrade_status.optimized_for_upgrade_12);
if (r==0)
toku_sync_fetch_and_increment_uint64(&upgrade_status.optimized_for_upgrade);
}
if (r == 0) {
t->h->upgrade_brt_performed = TRUE; // no further upgrade necessary
......@@ -1320,12 +1305,16 @@ serialize_brt_header_min_size (u_int32_t version) {
u_int32_t size = 0;
// TODO: #3228 WILL need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(version == BRT_LAYOUT_VERSION);
switch(version) {
case BRT_LAYOUT_VERSION_15: // WAS 13
case BRT_LAYOUT_VERSION_14:
size += 8; //TXNID that created
case BRT_LAYOUT_VERSION_13:
size += ( 4 // build_id
+4 // build_id_original
+8 // time_of_creation
+8 // time_of_last_modification
);
// fall through
case BRT_LAYOUT_VERSION_12:
size += (+8 // "tokudata"
+4 // version
......@@ -1362,6 +1351,7 @@ int toku_serialize_brt_header_to_wbuf (struct wbuf *wbuf, struct brt_header *h,
unsigned int size = toku_serialize_brt_header_size (h); // !!! seems silly to recompute the size when the caller knew it. Do we really need the size?
wbuf_literal_bytes(wbuf, "tokudata", 8);
wbuf_network_int (wbuf, h->layout_version); //MUST be in network order regardless of disk order
wbuf_network_int (wbuf, BUILD_ID); //MUST be in network order regardless of disk order
wbuf_network_int (wbuf, size); //MUST be in network order regardless of disk order
wbuf_literal_bytes(wbuf, &toku_byte_order_host, 8); //Must not translate byte order
wbuf_ulonglong(wbuf, h->checkpoint_count);
......@@ -1374,6 +1364,12 @@ int toku_serialize_brt_header_to_wbuf (struct wbuf *wbuf, struct brt_header *h,
wbuf_BLOCKNUM(wbuf, h->root);
wbuf_int(wbuf, h->flags);
wbuf_int(wbuf, h->layout_version_original);
wbuf_int(wbuf, h->build_id_original);
wbuf_ulonglong(wbuf, h->time_of_creation);
{ // time_of_last_modification
uint64_t now = (uint64_t) time(NULL);
wbuf_ulonglong(wbuf, now);
}
wbuf_ulonglong(wbuf, h->num_blocks_to_upgrade);
wbuf_TXNID(wbuf, h->root_xid_that_created);
u_int32_t checksum = x1764_finish(&wbuf->checksum);
......@@ -1572,6 +1568,7 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
toku_list_init(&h->live_brts);
toku_list_init(&h->zombie_brts);
toku_list_init(&h->checkpoint_before_commit_link);
//version MUST be in network order on disk regardless of disk order
h->layout_version = rbuf_network_int(&rc);
//TODO: #1924
......@@ -1579,6 +1576,9 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
invariant(h->layout_version <= BRT_LAYOUT_VERSION);
h->layout_version_read_from_disk = h->layout_version;
//build_id MUST be in network order on disk regardless of disk order
h->build_id = rbuf_network_int(&rc);
//Size MUST be in network order regardless of disk order.
u_int32_t size = rbuf_network_int(&rc);
lazy_assert(size==rc.size);
......@@ -1620,13 +1620,13 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
h->flags = rbuf_int(&rc);
deserialize_descriptor_from(fd, h, &h->descriptor);
h->layout_version_original = rbuf_int(&rc);
h->build_id_original = rbuf_int(&rc);
h->time_of_creation = rbuf_ulonglong(&rc);
h->time_of_last_modification = rbuf_ulonglong(&rc);
h->num_blocks_to_upgrade = rbuf_ulonglong(&rc);
// TODO: #3228 WILL need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(h->layout_version == BRT_LAYOUT_VERSION);
if (h->layout_version >= BRT_LAYOUT_VERSION_13) {
// at this layer, this new field is the only difference between versions 12 and 13
if (h->layout_version >= BRT_LAYOUT_VERSION_14) {
// at this layer, this new field is the only difference between versions 13 and 14
rbuf_TXNID(&rc, &h->root_xid_that_created);
}
(void)rbuf_int(&rc); //Read in checksum and ignore (already verified).
......@@ -1639,22 +1639,19 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
//TODO: When version 14 exists, add case for version 13 that looks like version 12 case,
//TODO: When version 15 exists, add case for version 14 that looks like today's version 13 case,
static int
deserialize_brtheader_versioned (int fd, struct rbuf *rb, struct brt_header **brth, u_int32_t version) {
int rval;
int upgrade = 0;
// TODO: #3228 May need to fix specific version number, certainly delete this assert when upgrade is enabled
lazy_assert(version == BRT_LAYOUT_VERSION);
struct brt_header *h = NULL;
rval = deserialize_brtheader (fd, rb, &h); //deserialize from rbuf and fd into header
if (rval == 0) {
invariant(h);
switch (version) {
case BRT_LAYOUT_VERSION_12:
invariant(h->layout_version == BRT_LAYOUT_VERSION_12);
case BRT_LAYOUT_VERSION_13:
invariant(h->layout_version == BRT_LAYOUT_VERSION_13);
{
//Upgrade root_xid_that_created
//Fake creation during the last checkpoint.
......@@ -1662,10 +1659,10 @@ deserialize_brtheader_versioned (int fd, struct rbuf *rb, struct brt_header **br
}
{
//Deprecate 'TOKU_DB_VALCMP_BUILTIN'. Just remove the flag
h->flags &= ~TOKU_DB_VALCMP_BUILTIN_12;
h->flags &= ~TOKU_DB_VALCMP_BUILTIN_13;
}
h->layout_version++;
toku_sync_fetch_and_increment_uint64(&upgrade_status.header_12); // how many header nodes upgraded from v12
toku_sync_fetch_and_increment_uint64(&upgrade_status.header_13); // how many header nodes upgraded from v13
upgrade++;
//Fall through on purpose
case BRT_LAYOUT_VERSION:
......@@ -1697,6 +1694,7 @@ deserialize_brtheader_from_fd_into_rbuf(int fd, toku_off_t offset_of_header, str
int r = 0;
const int64_t prefix_size = 8 + // magic ("tokudata")
4 + // version
4 + // build_id
4; // size
unsigned char prefix[prefix_size];
rb->buf = NULL;
......@@ -1718,12 +1716,15 @@ deserialize_brtheader_from_fd_into_rbuf(int fd, toku_off_t offset_of_header, str
}
}
u_int32_t version = 0;
u_int32_t build_id = 0;
if (r==0) {
//Version MUST be in network order regardless of disk order.
version = rbuf_network_int(rb);
*version_p = version;
if (version < BRT_LAYOUT_MIN_SUPPORTED_VERSION) r = TOKUDB_DICTIONARY_TOO_OLD; //Cannot use
if (version > BRT_LAYOUT_VERSION) r = TOKUDB_DICTIONARY_TOO_NEW; //Cannot use
//build_id MUST be in network order regardless of disk order.
build_id = rbuf_network_int(rb);
}
u_int32_t size;
if (r==0) {
......@@ -1756,7 +1757,7 @@ deserialize_brtheader_from_fd_into_rbuf(int fd, toku_off_t offset_of_header, str
//We have an rbuf that represents the header.
//Size is within acceptable bounds.
if (r==0) {
//Verify checksum (BRT_LAYOUT_VERSION_12 or later, when checksum function changed)
//Verify checksum (BRT_LAYOUT_VERSION_13 or later, when checksum function changed)
u_int32_t calculated_x1764 = x1764_memory(rb->buf, rb->size-4);
u_int32_t stored_x1764 = toku_dtoh32(*(int*)(rb->buf+rb->size-4));
if (calculated_x1764!=stored_x1764) r = TOKUDB_DICTIONARY_NO_HEADER; //Header useless
......@@ -1881,7 +1882,7 @@ toku_db_badformat(void) {
static size_t
serialize_rollback_log_size(ROLLBACK_LOG_NODE log) {
size_t size = node_header_overhead //8 "tokuroll", 4 version, 4 version_original
size_t size = node_header_overhead //8 "tokuroll", 4 version, 4 version_original, 4 build_id
+8 //TXNID
+8 //sequence
+8 //thislogname
......@@ -1901,6 +1902,7 @@ serialize_rollback_log_node_to_buf(ROLLBACK_LOG_NODE log, char *buf, size_t calc
lazy_assert(log->layout_version == BRT_LAYOUT_VERSION);
wbuf_nocrc_int(&wb, log->layout_version);
wbuf_nocrc_int(&wb, log->layout_version_original);
wbuf_nocrc_uint(&wb, BUILD_ID);
wbuf_nocrc_TXNID(&wb, log->txnid);
wbuf_nocrc_ulonglong(&wb, log->sequence);
wbuf_nocrc_BLOCKNUM(&wb, log->thislogname);
......@@ -2001,6 +2003,7 @@ deserialize_rollback_log_from_rbuf (BLOCKNUM blocknum, u_int32_t fullhash, ROLLB
lazy_assert(result->layout_version == BRT_LAYOUT_VERSION);
result->layout_version_original = rbuf_int(rb);
result->layout_version_read_from_disk = result->layout_version;
result->build_id = rbuf_int(rb);
result->dirty = FALSE;
//TODO: Maybe add descriptor (or just descriptor version) here eventually?
//TODO: This is hard.. everything is shared in a single dictionary.
......
newbrt/brt.c
View file @ 920e14c8
......@@ -3166,8 +3166,10 @@ brt_init_header (BRT t, TOKUTXN txn) {
// allocate and initialize a brt header.
// t->cf is not set to anything.
int toku_brt_alloc_init_header(BRT t, TOKUTXN txn) {
static int
brt_alloc_init_header(BRT t, TOKUTXN txn) {
int r;
uint64_t now = (uint64_t) time(NULL);
r = brtheader_alloc(&t->h);
if (r != 0) {
......@@ -3180,6 +3182,12 @@ int toku_brt_alloc_init_header(BRT t, TOKUTXN txn) {
t->h->layout_version_original = BRT_LAYOUT_VERSION;
t->h->layout_version_read_from_disk = BRT_LAYOUT_VERSION; // fake, prevent unnecessary upgrade logic
t->h->build_id = BUILD_ID;
t->h->build_id_original = BUILD_ID;
t->h->time_of_creation = now;
t->h->time_of_last_modification = 0;
memset(&t->h->descriptor, 0, sizeof(t->h->descriptor));
r = brt_init_header(t, txn);
......@@ -3375,7 +3383,7 @@ brt_open(BRT t, const char *fname_in_env, int is_create, int only_create, CACHET
if (is_create) {
r = toku_read_brt_header_and_store_in_cachefile(t->cf, max_acceptable_lsn, &t->h, &was_already_open);
if (r==TOKUDB_DICTIONARY_NO_HEADER) {
r = toku_brt_alloc_init_header(t, txn);
r = brt_alloc_init_header(t, txn);
if (r != 0) goto died_after_read_and_pin;
}
else if (r!=0) {
......
newbrt/brt.h
View file @ 920e14c8
......@@ -179,7 +179,7 @@ enum brt_header_flags {
//TOKU_DB_DUP = (1<<0), //Obsolete #2862
//TOKU_DB_DUPSORT = (1<<1), //Obsolete #2862
TOKU_DB_KEYCMP_BUILTIN = (1<<2),
TOKU_DB_VALCMP_BUILTIN_12 = (1<<3),
TOKU_DB_VALCMP_BUILTIN_13 = (1<<3),
};
int toku_brt_keyrange (BRT brt, DBT *key, u_int64_t *less, u_int64_t *equal, u_int64_t *greater) __attribute__ ((warn_unused_result));
......
newbrt/brt_layout_version.h
View file @ 920e14c8
......@@ -16,12 +16,11 @@ enum brt_layout_version_e {
BRT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from brt layer
BRT_LAYOUT_VERSION_11 = 11, // Diff from 10 to 11: Nested transaction leafentries (completely redesigned). BRT_CMDs on disk now support XIDS (multiple txnids) instead of exactly one.
BRT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added BRT_CMD 'BRT_INSERT_NO_OVERWRITE', compressed block format, num old blocks
BRT_LAYOUT_VERSION_13 = 13, // RESERVED
BRT_LAYOUT_VERSION_14 = 14, // RESERVED
BRT_LAYOUT_VERSION_15 = 15, // Diff from 12 to 15: Added MVCC, deprecated TOKU_DB_VALCMP_BUILTIN(_12)
BRT_LAYOUT_VERSION_13 = 13, // Diff from 12 to 13: Fixed loader pivot bug, added build_id to every node, timestamps to brtheader
BRT_LAYOUT_VERSION_14 = 14, // Diff from 13 to 14: Added MVCC, deprecated TOKU_DB_VALCMP_BUILTIN(_13)
BRT_NEXT_VERSION, // the version after the current version
BRT_LAYOUT_VERSION = BRT_NEXT_VERSION-1, // A hack so I don't have to change this line.
BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION_15 // Minimum version supported
BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION_13 // Minimum version supported
};
#endif
newbrt/brtdump.c
View file @ 920e14c8
......@@ -6,6 +6,27 @@
#include "includes.h"
static void
format_time(const uint64_t time_int, char *buf) {
time_t timer = (time_t) time_int;
ctime_r(&timer, buf);
size_t len = strlen(buf);
assert(len < 26);
char end;
assert(len>=1);
end = buf[len-1];
while (end == '\n' || end == '\r') {
buf[len-1] = '\0';
len--;
assert(len>=1);
end = buf[len-1];
}
}
static int dump_data = 1;
static CACHETABLE ct;
......@@ -76,6 +97,7 @@ static void
dump_header (int f, struct brt_header **header, CACHEFILE cf) {
struct brt_header *h;
int r;
char timestr[26];
r = toku_deserialize_brtheader_from (f, MAX_LSN, &h);
assert(r==0);
h->cf = cf;
......@@ -83,6 +105,13 @@ dump_header (int f, struct brt_header **header, CACHEFILE cf) {
printf(" layout_version=%d\n", h->layout_version);
printf(" layout_version_original=%d\n", h->layout_version_original);
printf(" layout_version_read_from_disk=%d\n", h->layout_version_read_from_disk);
printf(" build_id=%d\n", h->build_id);
printf(" build_id_original=%d\n", h->build_id_original);
format_time(h->time_of_creation, timestr);
printf(" time_of_creation= %ld %s\n", h->time_of_creation, timestr);
format_time(h->time_of_last_modification, timestr);
printf(" time_of_last_modification=%ld %s\n", h->time_of_last_modification, timestr);
printf(" dirty=%d\n", h->dirty);
printf(" nodesize=%u\n", h->nodesize);
printf(" unnamed_root=%" PRId64 "\n", h->root.b);
......@@ -119,6 +148,7 @@ dump_node (int f, BLOCKNUM blocknum, struct brt_header *h) {
printf(" layout_version=%d\n", n->layout_version);
printf(" layout_version_original=%d\n", n->layout_version_original);
printf(" layout_version_read_from_disk=%d\n", n->layout_version_read_from_disk);
printf(" build_id=%d\n", n->build_id);
printf(" rand4fp =%08x\n", n->rand4fingerprint);
printf(" localfp =%08x\n", n->local_fingerprint);
if (n->height>0) {
......
newbrt/brtloader.c
View file @ 920e14c8
......@@ -2226,6 +2226,7 @@ static struct leaf_buf *start_leaf (struct dbout *out, const DESCRIPTOR UU(desc)
putbuf_bytes(&lbuf->dbuf, "tokuleaf", 8);
putbuf_int32(&lbuf->dbuf, layout_version);
putbuf_int32(&lbuf->dbuf, layout_version); // layout_version original
putbuf_int32(&lbuf->dbuf, BUILD_ID); // build_id (svn rev number) of software that wrote this node to disk
putbuf_int32(&lbuf->dbuf, target_nodesize);
putbuf_int32(&lbuf->dbuf, flags);
......@@ -2874,6 +2875,7 @@ static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progr
int n_uncompressed_bytes_at_beginning = (8 // tokuleaf
+4 // layout version
+4 // layout version original
+4 // build_id
);
int uncompressed_len = lbuf->dbuf.off - n_uncompressed_bytes_at_beginning;
......@@ -2905,12 +2907,14 @@ static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progr
// cppy the uncompressed header to the compressed buffer
memcpy(compressed_buf, lbuf->dbuf.buf, n_uncompressed_bytes_at_beginning);
int uncompressed_header_size = n_uncompressed_bytes_at_beginning + sizeof(n_sub_blocks);
// serialize the sub block header
memcpy(compressed_buf+16, &n_sub_blocks, 4);
memcpy(compressed_buf+n_uncompressed_bytes_at_beginning, &n_sub_blocks, 4);
for (int i = 0; i < n_sub_blocks; i++) {
memcpy(compressed_buf+20+12*i+0, &sub_block[i].compressed_size, 4);
memcpy(compressed_buf+20+12*i+4, &sub_block[i].uncompressed_size, 4);
memcpy(compressed_buf+20+12*i+8, &sub_block[i].xsum, 4);
memcpy(compressed_buf+uncompressed_header_size+12*i+0, &sub_block[i].compressed_size, 4);
memcpy(compressed_buf+uncompressed_header_size+12*i+4, &sub_block[i].uncompressed_size, 4);
memcpy(compressed_buf+uncompressed_header_size+12*i+8, &sub_block[i].xsum, 4);
}
// compute the header checksum and serialize it
......@@ -2994,12 +2998,15 @@ write_header (struct dbout *out, long long translation_location_on_disk, long lo
struct brt_header h; memset(&h, 0, sizeof h);
h.layout_version = BRT_LAYOUT_VERSION;
h.layout_version_original = BRT_LAYOUT_VERSION;
h.build_id = BUILD_ID;
h.build_id_original = BUILD_ID;
h.time_of_creation = (uint64_t) time(NULL);
h.checkpoint_count = 1;
h.checkpoint_lsn = load_lsn;
h.nodesize = target_nodesize;
h.root = root_blocknum_on_disk;
h.flags = 0;
h.layout_version_original = BRT_LAYOUT_VERSION;
h.root_xid_that_created = root_xid_that_created;
unsigned int size = toku_serialize_brt_header_size (&h);
......@@ -3136,6 +3143,7 @@ static void write_nonleaf_node (BRTLOADER bl, struct dbout *out, int64_t blocknu
node->thisnodename = make_blocknum(blocknum_of_new_node);
node->layout_version = BRT_LAYOUT_VERSION;
node->layout_version_original = BRT_LAYOUT_VERSION;
node->build_id = BUILD_ID;
node->height=height;
node->u.n.n_children = n_children;
node->flags = 0;
......
newbrt/leafentry.h
View file @ 920e14c8
......@@ -114,7 +114,7 @@ struct __attribute__ ((__packed__)) leafentry {
typedef struct leafentry *LEAFENTRY;
typedef struct leafentry_12 *LEAFENTRY_12;
typedef struct leafentry_13 *LEAFENTRY_13;
u_int32_t toku_le_crc(LEAFENTRY v);
......@@ -190,9 +190,9 @@ int le_iterate_val(LEAFENTRY le, LE_ITERATE_CALLBACK f, void** valpp, u_int32_t
size_t
leafentry_disksize_12(LEAFENTRY_12 le);
leafentry_disksize_13(LEAFENTRY_13 le);
int
toku_le_upgrade_12_13(LEAFENTRY_12 old_leafentry, // NULL if there was no stored data.
toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry, // NULL if there was no stored data.
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY *new_leafentry_p);
......
newbrt/logger.h
View file @ 920e14c8
......@@ -17,7 +17,7 @@ enum {
//After 2 we linked the log version to the BRT_LAYOUT VERSION.
//So it went from 2 to 13 (3-12 do not exist)
TOKU_LOG_VERSION = BRT_LAYOUT_VERSION,
TOKU_LOG_MIN_SUPPORTED_VERSION = TOKU_LOG_VERSION_2,
TOKU_LOG_MIN_SUPPORTED_VERSION = BRT_LAYOUT_MIN_SUPPORTED_VERSION,
};
#define ROLLBACK_CACHEFILE_NAME "tokudb.rollback"
......
newbrt/rollback.h
View file @ 920e14c8
......@@ -52,6 +52,7 @@ struct rollback_log_node {
int layout_version;
int layout_version_original;
int layout_version_read_from_disk;
uint32_t build_id; // build_id (svn rev number) of software that wrote this node to disk
int dirty;
TXNID txnid; // Which transaction made this?
uint64_t sequence; // Which rollback log in the sequence is this?
......
newbrt/ule.c
View file @ 920e14c8
......@@ -2111,15 +2111,15 @@ bool transaction_open(TXNID xid) {
#if TOKU_WINDOWS
#pragma pack(push, 1)
#endif
struct __attribute__ ((__packed__)) leafentry_12 {
struct __attribute__ ((__packed__)) leafentry_13 {
u_int8_t num_xrs;
u_int32_t keylen;
u_int32_t innermost_inserted_vallen;
union {
struct __attribute__ ((__packed__)) leafentry_committed_12 {
struct __attribute__ ((__packed__)) leafentry_committed_13 {
u_int8_t key_val[0]; //Actual key, then actual val
} comm;
struct __attribute__ ((__packed__)) leafentry_provisional_12 {
struct __attribute__ ((__packed__)) leafentry_provisional_13 {
u_int8_t innermost_type;
TXNID xid_outermost_uncommitted;
u_int8_t key_val_xrs[]; //Actual key,
......@@ -2135,7 +2135,7 @@ struct __attribute__ ((__packed__)) leafentry_12 {
//Requires:
// Leafentry that ule represents should not be destroyed (is not just all deletes)
static size_t
le_memsize_from_ule_12 (ULE ule) {
le_memsize_from_ule_13 (ULE ule) {
uint32_t num_uxrs = ule->num_cuxrs + ule->num_puxrs;
assert(num_uxrs);
size_t rval;
......@@ -2165,16 +2165,16 @@ le_memsize_from_ule_12 (ULE ule) {
return rval;
}
//This function is mostly copied from 4.1.1
// Note, number of transaction records in version 12 has been replaced by separate counters in version 13 (MVCC),
//This function is mostly copied from 4.1.1 (which is version 12, same as 13 except that only 13 is upgradable).
// Note, number of transaction records in version 13 has been replaced by separate counters in version 14 (MVCC),
// one counter for committed transaction records and one counter for provisional transaction records. When
// upgrading a version 12 le to version 13, the number of committed transaction records is always set to one (1)
// upgrading a version 13 le to version 14, the number of committed transaction records is always set to one (1)
// and the number of provisional transaction records is set to the original number of transaction records
// minus one. The bottom transaction record is assumed to be a committed value. (If there is no committed
// value then the bottom transaction record of version 12 is a committed delete.)
// value then the bottom transaction record of version 13 is a committed delete.)
// This is the only change from the 4.1.1 code. The rest of the leafentry is read as is.
static void
le_unpack_12(ULE ule, LEAFENTRY_12 le) {
le_unpack_13(ULE ule, LEAFENTRY_13 le) {
//Read num_uxrs
uint8_t num_xrs = le->num_xrs;
assert(num_xrs > 0);
......@@ -2272,29 +2272,29 @@ le_unpack_12(ULE ule, LEAFENTRY_12 le) {
assert(found_innermost_insert);
}
#if ULE_DEBUG
size_t memsize = le_memsize_from_ule_12(ule);
size_t memsize = le_memsize_from_ule_13(ule);
assert(p == ((u_int8_t*)le) + memsize);
#endif
}
size_t
leafentry_disksize_12(LEAFENTRY_12 le) {
leafentry_disksize_13(LEAFENTRY_13 le) {
ULE_S ule;
le_unpack_12(&ule, le);
size_t memsize = le_memsize_from_ule_12(&ule);
le_unpack_13(&ule, le);
size_t memsize = le_memsize_from_ule_13(&ule);
ule_cleanup(&ule);
return memsize;
}
int
toku_le_upgrade_12_13(LEAFENTRY_12 old_leafentry,
toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry,
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY *new_leafentry_p) {
ULE_S ule;
int rval;
invariant(old_leafentry);
le_unpack_12(&ule, old_leafentry);
le_unpack_13(&ule, old_leafentry);
rval = le_pack(&ule, // create packed leafentry
new_leafentry_memorysize,
new_leafentry_disksize,
......
src/tests/Makefile
View file @ 920e14c8
......@@ -38,9 +38,9 @@ NONSTANDARD_SRCS= \
$(RECOVER_SRCS) \
$(LOADER_SRCS) \
# TODO: #3228
# SRCS = $(sort $(wildcard *.c))
SRCS = $(sort $(filter-out $(TRANSPARENT_UPGRADE_SRCS),$(wildcard *.c)))
SRCS = $(sort $(wildcard *.c))
# To patch out upgrade tests, replace line above with line below
#SRCS = $(sort $(filter-out $(TRANSPARENT_UPGRADE_SRCS),$(wildcard *.c)))
#end
......@@ -719,9 +719,8 @@ preload-db-nested.tdbrun: VGRIND=
upgrade-test-4.tdbrun: VGRIND=
# TODO: #3228 Restore loader-stress-test3 when ready
#loader-stress-test.loader: $(patsubst %,loader-stress-test%.tdbrun, 0 1 2 3)
loader-stress-test.loader: $(patsubst %,loader-stress-test%.tdbrun, 0 1 2)
loader-stress-test.loader: $(patsubst %,loader-stress-test%.tdbrun, 0 1 2 3)
true
loader-stress-test0.tdbrun: loader-stress-test.tdb$(BINSUF)
......
src/tests/loader-stress-test.c
View file @ 920e14c8
......@@ -7,7 +7,7 @@
*
* NOTE: This test is used for upgrade testing as well as for exercising the loader.
* Changes should not be made gratuitously.
* The 4.1.1 version of this test was used to create many of the preloaded
* The 4.2.0 version of this test was used to create many of the preloaded
* environments in the <svn-top>/tokudb/tokudb.data directory.
*/
......@@ -444,7 +444,7 @@ char *env_dir = ENVDIR; // the default env_dir.
char *tmp_subdir = "tmp.subdir";
#define OLDDATADIR "../../../../tokudb.data/"
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.emptydictionaries.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.emptydictionaries.cleanshutdown";
static void setup(void) {
int r;
......
src/tests/preload-db.c
View file @ 920e14c8
......@@ -151,10 +151,13 @@ static void run_test(void)
r = env->close(env, 0); CKERR(r);
toku_free(dbs);
/*********** DO NOT TRIM LOGFILES: Trimming logfiles defeats purpose of upgrade tests which must handle untrimmed logfiles.
// reopen, then close environment to trim logfiles
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, env_dir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
***********/
}
// ------------ infrastructure ----------
......
src/tests/upgrade-test-1.c
View file @ 920e14c8
......@@ -28,9 +28,9 @@ int flat = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir_flat = OLDDATADIR "env_preload.4.1.1.flat.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
char *db_v4_dir_flat = OLDDATADIR "env_preload.4.2.0.flat.cleanshutdown";
......
src/tests/upgrade-test-2.c
View file @ 920e14c8
......@@ -26,8 +26,8 @@ int littlenode = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
static void upgrade_test_2(DB **dbs) {
......
src/tests/upgrade-test-3.c
View file @ 920e14c8
......@@ -3,7 +3,7 @@
#ident "$Id$"
// Purpose of this test is to verify that dictionaries created with 4.1.1
// Purpose of this test is to verify that dictionaries created with 4.2.0
// can be properly truncated with TokuDB version 5.x or later.
......@@ -31,8 +31,8 @@ int littlenode = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
static void upgrade_test_3(DB **dbs) {
......
src/tests/upgrade-test-4.c
View file @ 920e14c8
......@@ -47,8 +47,8 @@ int littlenode = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
enum {ROWS_PER_TRANSACTION=10000};
......
src/tests/upgrade-test-5.c
View file @ 920e14c8
......@@ -19,7 +19,7 @@
DB_ENV *env;
enum {MAX_NAME=128};
int NUM_DBS=1;
int NUM_ROWS=50000;
int NUM_ROWS=100000;
int SRC_VERSION = 4;
#define MAXDEPTH 64
......@@ -27,7 +27,7 @@ int SRC_VERSION = 4;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db-nested.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.nested.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.nested.cleanshutdown";
static void
......
src/tests/upgrade-test-6.c
View file @ 920e14c8
......@@ -3,9 +3,9 @@
#ident "$Id$"
// Purpose of this test is to verify that a dictionary created by the 4.1.1
// Purpose of this test is to verify that a dictionary created by the 4.2.0
// loader can be properly read with 5.0.
// This file was derived from the 4.1.1 version of loader-stress-test.c,
// This file was derived from the 4.2.0 version of loader-stress-test.c,
// which was used to create the dictionary.
// This test only reads (and upgrades) the dictionary, it does not load it.
......@@ -212,7 +212,7 @@ char *env_dir = ENVDIR; // the default env_dir.
char *tmp_subdir = "tmp.subdir";
#define OLDDATADIR "../../../../tokudb.data/"
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.loader250kd1.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.loader250kd1.cleanshutdown";
static void setup(void) {
......
src/tests/upgrade_simple.c
View file @ 920e14c8
......@@ -31,13 +31,15 @@ static void test_shutdown(void);
static char *env_dir = ENVDIR; // the default env_dir.
static char * dir_v4_clean = "env_simple.4.1.1.cleanshutdown";
static char * dir_v4_dirty = "env_simple.4.1.1.dirtyshutdown";
static char * dir_v4_dirty_multilogfile = OLDDATADIR "env_preload.4.1.1.multilog.dirtyshutdown";
static char * dir_v41_clean = "env_simple.4.1.1.cleanshutdown";
static char * dir_v42_clean = "env_simple.4.2.0.cleanshutdown";
static char * dir_v42_dirty = "env_simple.4.2.0.dirtyshutdown";
static char * dir_v41_dirty_multilogfile = OLDDATADIR "env_preload.4.1.1.multilog.dirtyshutdown";
static char * dir_v42_dirty_multilogfile = OLDDATADIR "env_preload.4.2.0.multilog.dirtyshutdown";
static void
setup (u_int32_t flags, BOOL clean, char * src_db_dir) {
setup (u_int32_t flags, BOOL clean, BOOL too_old, char * src_db_dir) {
int r;
int len = 256;
char syscmd[len];
......@@ -61,8 +63,12 @@ setup (u_int32_t flags, BOOL clean, char * src_db_dir) {
r=env->open(env, ENVDIR, flags, mode);
if (clean)
CKERR(r);
else
CKERR2(r, TOKUDB_UPGRADE_FAILURE);
else {
if (too_old)
CKERR2(r, TOKUDB_DICTIONARY_TOO_OLD);
else
CKERR2(r, TOKUDB_UPGRADE_FAILURE);
}
}
......@@ -81,18 +87,29 @@ test_env_startup(void) {
flags = FLAGS_LOG;
setup(flags, TRUE, dir_v4_clean);
setup(flags, TRUE, FALSE, dir_v42_clean);
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, TRUE, dir_v41_clean);
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, FALSE, dir_v42_dirty);
if (verbose) {
printf("\n\nEngine status after aborted env->open() will have some garbage values:\n");
}
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, dir_v4_dirty);
setup(flags, FALSE, TRUE, dir_v41_dirty_multilogfile);
if (verbose) {
printf("\n\nEngine status after aborted env->open() will have some garbage values:\n");
}
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, dir_v4_dirty_multilogfile);
setup(flags, FALSE, FALSE, dir_v42_dirty_multilogfile);
if (verbose) {
printf("\n\nEngine status after aborted env->open() will have some garbage values:\n");
}
......
src/ydb.c
View file @ 920e14c8
......@@ -35,6 +35,7 @@ const char *toku_copyright_string = "Copyright (c) 2007-2009 Tokutek Inc. All r
#include "brtloader.h"
#include "log_header.h"
#ifdef TOKUTRACE
#define DB_ENV_CREATE_FUN db_env_create_toku10
#define DB_CREATE_FUN db_create_toku10
......@@ -491,20 +492,20 @@ db_use_builtin_key_cmp(DB *db) {
// Following keys added in version 12
static const char * orig_env_ver_key = "original_version";
static const char * curr_env_ver_key = "current_version";
// Following keys added in version 13, add more keys for future versions
// Following keys added in version 14, add more keys for future versions
static const char * creation_time_key = "creation_time";
static const char * last_lsn_of_v12_key = "last_lsn_of_v12";
static const char * upgrade_v13_time_key = "upgrade_v13_time";
static const char * upgrade_v13_footprint_key = "upgrade_v13_footprint";
static const char * last_lsn_of_v13_key = "last_lsn_of_v13";
static const char * upgrade_v14_time_key = "upgrade_v14_time";
static const char * upgrade_v14_footprint_key = "upgrade_v14_footprint";
// Values read from (or written into) persistent environment,
// kept here for read-only access from engine status.
static uint32_t persistent_original_env_version;
static uint32_t persistent_stored_env_version_at_startup; // read from curr_env_ver_key, prev version as of this startup
static time_t persistent_creation_time;
static uint64_t persistent_last_lsn_of_v12;
static time_t persistent_upgrade_v13_time;
static uint64_t persistent_upgrade_v13_footprint;
static uint64_t persistent_last_lsn_of_v13;
static time_t persistent_upgrade_v14_time;
static uint64_t persistent_upgrade_v14_footprint;
// Requires: persistent environment dictionary is already open.
// Input arg is lsn of clean shutdown of previous version,
......@@ -532,21 +533,21 @@ maybe_upgrade_persistent_environment_dictionary(DB_ENV * env, DB_TXN * txn, LSN
r = toku_db_put(persistent_environment, txn, &key, &val, DB_YESOVERWRITE);
assert(r==0);
uint64_t last_lsn_of_v12_d = toku_htod64(last_lsn_of_clean_shutdown_read_from_log.lsn);
toku_fill_dbt(&key, last_lsn_of_v12_key, strlen(last_lsn_of_v12_key));
toku_fill_dbt(&val, &last_lsn_of_v12_d, sizeof(last_lsn_of_v12_d));
uint64_t last_lsn_of_v13_d = toku_htod64(last_lsn_of_clean_shutdown_read_from_log.lsn);
toku_fill_dbt(&key, last_lsn_of_v13_key, strlen(last_lsn_of_v13_key));
toku_fill_dbt(&val, &last_lsn_of_v13_d, sizeof(last_lsn_of_v13_d));
r = toku_db_put(persistent_environment, txn, &key, &val, DB_YESOVERWRITE);
assert(r==0);
time_t upgrade_v13_time_d = toku_htod64(time(NULL));
toku_fill_dbt(&key, upgrade_v13_time_key, strlen(upgrade_v13_time_key));
toku_fill_dbt(&val, &upgrade_v13_time_d, sizeof(upgrade_v13_time_d));
time_t upgrade_v14_time_d = toku_htod64(time(NULL));
toku_fill_dbt(&key, upgrade_v14_time_key, strlen(upgrade_v14_time_key));
toku_fill_dbt(&val, &upgrade_v14_time_d, sizeof(upgrade_v14_time_d));
r = toku_db_put(persistent_environment, txn, &key, &val, DB_NOOVERWRITE);
assert(r==0);
uint64_t upgrade_v13_footprint_d = toku_htod64(toku_log_upgrade_get_footprint());
toku_fill_dbt(&key, upgrade_v13_footprint_key, strlen(upgrade_v13_footprint_key));
toku_fill_dbt(&val, &upgrade_v13_footprint_d, sizeof(upgrade_v13_footprint_d));
uint64_t upgrade_v14_footprint_d = toku_htod64(toku_log_upgrade_get_footprint());
toku_fill_dbt(&key, upgrade_v14_footprint_key, strlen(upgrade_v14_footprint_key));
toku_fill_dbt(&val, &upgrade_v14_footprint_d, sizeof(upgrade_v14_footprint_d));
r = toku_db_put(persistent_environment, txn, &key, &val, DB_NOOVERWRITE);
assert(r==0);
}
......@@ -576,8 +577,7 @@ capture_persistent_env_contents (DB_ENV * env, DB_TXN * txn) {
assert(persistent_original_env_version <= curr_env_version);
// make no assertions about timestamps, clock may have been reset
// TODO: #3228 May need to fix specific version number to whenever this new info was added.
if (persistent_original_env_version >= BRT_LAYOUT_VERSION_15) {
if (persistent_original_env_version >= BRT_LAYOUT_VERSION_14) {
toku_fill_dbt(&key, creation_time_key, strlen(creation_time_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
......@@ -588,23 +588,23 @@ capture_persistent_env_contents (DB_ENV * env, DB_TXN * txn) {
if (persistent_original_env_version != curr_env_version) {
// an upgrade was performed at some time, capture info about the upgrade
toku_fill_dbt(&key, last_lsn_of_v12_key, strlen(last_lsn_of_v12_key));
toku_fill_dbt(&key, last_lsn_of_v13_key, strlen(last_lsn_of_v13_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
assert(r == 0);
persistent_last_lsn_of_v12 = toku_dtoh64(*(uint32_t*)val.data);
persistent_last_lsn_of_v13 = toku_dtoh64(*(uint32_t*)val.data);
toku_fill_dbt(&key, upgrade_v13_time_key, strlen(upgrade_v13_time_key));
toku_fill_dbt(&key, upgrade_v14_time_key, strlen(upgrade_v14_time_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
assert(r == 0);
persistent_upgrade_v13_time = toku_dtoh64((*(time_t*)val.data));
persistent_upgrade_v14_time = toku_dtoh64((*(time_t*)val.data));
toku_fill_dbt(&key, upgrade_v13_footprint_key, strlen(upgrade_v13_footprint_key));
toku_fill_dbt(&key, upgrade_v14_footprint_key, strlen(upgrade_v14_footprint_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
assert(r == 0);
persistent_upgrade_v13_footprint = toku_dtoh64((*(uint64_t*)val.data));
persistent_upgrade_v14_footprint = toku_dtoh64((*(uint64_t*)val.data));
}
}
......@@ -1966,16 +1966,17 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat, char * env_panic_st
// is provided in six least significant decimal digits, footprint of
// upgrade performed when environment was actually upgraded is provided
// in most significant decimal digits.
// If ver_at_startup == 12, then the footprint will have the same value in
// If ver_at_startup == 13, then the footprint will have the same value in
// upper and lower digits.
engstat->upgrade_env_status = (persistent_upgrade_v13_footprint * 1000000) + upgrade_footprint;
engstat->upgrade_header = brt_upgrade_stat.header_12;
engstat->upgrade_nonleaf = brt_upgrade_stat.nonleaf_12;
engstat->upgrade_leaf = brt_upgrade_stat.leaf_12;
engstat->upgrade_env_status = (persistent_upgrade_v14_footprint * 1000000) + upgrade_footprint;
engstat->upgrade_header = brt_upgrade_stat.header_13;
engstat->upgrade_nonleaf = brt_upgrade_stat.nonleaf_13;
engstat->upgrade_leaf = brt_upgrade_stat.leaf_13;
engstat->optimized_for_upgrade = brt_upgrade_stat.optimized_for_upgrade;
engstat->original_ver = persistent_original_env_version;
engstat->ver_at_startup = persistent_stored_env_version_at_startup;
engstat->last_lsn_v12 = persistent_last_lsn_of_v12;
format_time(&persistent_upgrade_v13_time, engstat->upgrade_v13_time);
engstat->last_lsn_v13 = persistent_last_lsn_of_v13;
format_time(&persistent_upgrade_v14_time, engstat->upgrade_v14_time);
}
}
return r;
......@@ -1992,6 +1993,9 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
char panicstring[stringsize];
int n = 0; // number of characters printed so far
n = snprintf(buff, bufsiz - n, "BUILD_ID = %d\n", BUILD_ID);
int r = env_get_engine_status(env, &engstat, panicstring, stringsize);
if (strlen(panicstring)) {
......@@ -2126,10 +2130,11 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
n += snprintf(buff + n, bufsiz - n, "upgrade_header %"PRIu64"\n", engstat.upgrade_header);
n += snprintf(buff + n, bufsiz - n, "upgrade_nonleaf %"PRIu64"\n", engstat.upgrade_nonleaf);
n += snprintf(buff + n, bufsiz - n, "upgrade_leaf %"PRIu64"\n", engstat.upgrade_leaf);
n += snprintf(buff + n, bufsiz - n, "optimized_for_upgrade_14 %"PRIu64"\n", engstat.optimized_for_upgrade);
n += snprintf(buff + n, bufsiz - n, "original_ver %"PRIu64"\n", engstat.original_ver);
n += snprintf(buff + n, bufsiz - n, "ver_at_startup %"PRIu64"\n", engstat.ver_at_startup);
n += snprintf(buff + n, bufsiz - n, "last_lsn_v12 %"PRIu64"\n", engstat.last_lsn_v12);
n += snprintf(buff + n, bufsiz - n, "upgrade_v13_time %s \n", engstat.upgrade_v13_time);
n += snprintf(buff + n, bufsiz - n, "last_lsn_v13 %"PRIu64"\n", engstat.last_lsn_v13);
n += snprintf(buff + n, bufsiz - n, "upgrade_v14_time %s \n", engstat.upgrade_v14_time);
}
if (n > bufsiz) {
char * errmsg = "BUFFER TOO SMALL\n";
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7