Commit 920e14c8 authored by Barry Perlman's avatar Barry Perlman Committed by Yoni Fogel

[t:3236] [t:3238] Merge tokudb.3236+3238 to main.

git-svn-id: file:///svn/toku/tokudb@28070 c7de825b-a66e-492c-adef-691d508d4ae1
parent a8376ef4
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
......@@ -572,10 +572,11 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
printf(" u_int64_t upgrade_header; /* how many brt headers were upgraded? */ \n");
printf(" u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */ \n");
printf(" u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */ \n");
printf(" u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */ \n");
printf(" u_int64_t original_ver; /* original environment version */ \n");
printf(" u_int64_t ver_at_startup; /* environment version at startup */ \n");
printf(" u_int64_t last_lsn_v12; /* last lsn of version 12 environment */ \n");
printf(" char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */ \n");
printf(" u_int64_t last_lsn_v13; /* last lsn of version 13 environment */ \n");
printf(" char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */ \n");
printf(" u_int64_t env_panic; /* non-zero if environment is panicked */ \n");
printf(" u_int64_t logger_panic; /* non-zero if logger is panicked */ \n");
printf(" u_int64_t logger_panic_errno; /* non-zero if environment is panicked */ \n");
......
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
......@@ -178,10 +178,11 @@ typedef struct __toku_engine_status {
u_int64_t upgrade_header; /* how many brt headers were upgraded? */
u_int64_t upgrade_nonleaf; /* how many brt nonleaf nodes were upgraded? */
u_int64_t upgrade_leaf; /* how many brt leaf nodes were upgraded? */
u_int64_t optimized_for_upgrade; /* how many optimized_for_upgrade messages were broadcast */
u_int64_t original_ver; /* original environment version */
u_int64_t ver_at_startup; /* environment version at startup */
u_int64_t last_lsn_v12; /* last lsn of version 12 environment */
char upgrade_v13_time[26]; /* timestamp of when upgrade to version 13 environment was done */
u_int64_t last_lsn_v13; /* last lsn of version 13 environment */
char upgrade_v14_time[26]; /* timestamp of when upgrade to version 14 environment was done */
u_int64_t env_panic; /* non-zero if environment is panicked */
u_int64_t logger_panic; /* non-zero if logger is panicked */
u_int64_t logger_panic_errno; /* non-zero if environment is panicked */
......
......@@ -6,6 +6,14 @@
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
// Symbol TOKUDB_REVISION is not defined by fractal-tree makefiles, so
// BUILD_ID of 1000 indicates development build of main, not a release build.
#if defined(TOKUDB_REVISION)
#define BUILD_ID TOKUDB_REVISION
#else
#define BUILD_ID 1000
#endif
#include "brt_layout_version.h"
#include "toku_assert.h"
#include "block_allocator.h"
......@@ -92,6 +100,7 @@ struct brtnode {
int layout_version; // What version of the data structure?
int layout_version_original; // different (<) from layout_version if upgraded from a previous version (useful for debugging)
int layout_version_read_from_disk; // transient, not serialized to disk, (useful for debugging)
uint32_t build_id; // build_id (svn rev number) of software that wrote this node to disk
int height; /* height is always >= 0. 0 for leaf, >0 for nonleaf. */
u_int32_t rand4fingerprint;
u_int32_t local_fingerprint; /* For leaves this is everything in the buffer. For nonleaves, this is everything in the buffers, but does not include child subtree fingerprints. */
......@@ -166,8 +175,12 @@ struct brt_header {
int layout_version;
int layout_version_original; // different (<) from layout_version if upgraded from a previous version (useful for debugging)
int layout_version_read_from_disk; // transient, not serialized to disk
uint32_t build_id; // build_id (svn rev number) of software that wrote this node to disk
uint32_t build_id_original; // build_id of software that created this tree (read from disk, overwritten when written to disk)
uint64_t time_of_creation; // time this tree was created
uint64_t time_of_last_modification; // last time this header was serialized to disk (read from disk, overwritten when written to disk)
BOOL upgrade_brt_performed; // initially FALSE, set TRUE when brt has been fully updated (even though nodes may not have been)
int64_t num_blocks_to_upgrade; // Number of v12 blocks still not newest version. When we release layout 14 we may need to turn this to an array or add more variables.
int64_t num_blocks_to_upgrade; // Number of v13 blocks still not newest version. When we release layout 15 we may need to turn this to an array or add more variables.
unsigned int nodesize;
BLOCKNUM root; // roots of the dictionary
struct remembered_hash root_hash; // hash of the root offset.
......@@ -375,10 +388,10 @@ int toku_brt_remove_now(CACHETABLE ct, DBT* iname_dbt_p);
typedef struct brt_upgrade_status {
u_int64_t header_12; // how many headers upgrade from version 12
u_int64_t nonleaf_12;
u_int64_t leaf_12;
u_int64_t optimized_for_upgrade_12; // how many optimize_for_upgrade messages sent
u_int64_t header_13; // how many headers were upgraded from version 13
u_int64_t nonleaf_13;
u_int64_t leaf_13;
u_int64_t optimized_for_upgrade; // how many optimize_for_upgrade messages were sent
} BRT_UPGRADE_STATUS_S, *BRT_UPGRADE_STATUS;
void toku_brt_get_upgrade_status(BRT_UPGRADE_STATUS);
......
This diff is collapsed.
......@@ -3166,8 +3166,10 @@ brt_init_header (BRT t, TOKUTXN txn) {
// allocate and initialize a brt header.
// t->cf is not set to anything.
int toku_brt_alloc_init_header(BRT t, TOKUTXN txn) {
static int
brt_alloc_init_header(BRT t, TOKUTXN txn) {
int r;
uint64_t now = (uint64_t) time(NULL);
r = brtheader_alloc(&t->h);
if (r != 0) {
......@@ -3180,6 +3182,12 @@ int toku_brt_alloc_init_header(BRT t, TOKUTXN txn) {
t->h->layout_version_original = BRT_LAYOUT_VERSION;
t->h->layout_version_read_from_disk = BRT_LAYOUT_VERSION; // fake, prevent unnecessary upgrade logic
t->h->build_id = BUILD_ID;
t->h->build_id_original = BUILD_ID;
t->h->time_of_creation = now;
t->h->time_of_last_modification = 0;
memset(&t->h->descriptor, 0, sizeof(t->h->descriptor));
r = brt_init_header(t, txn);
......@@ -3375,7 +3383,7 @@ brt_open(BRT t, const char *fname_in_env, int is_create, int only_create, CACHET
if (is_create) {
r = toku_read_brt_header_and_store_in_cachefile(t->cf, max_acceptable_lsn, &t->h, &was_already_open);
if (r==TOKUDB_DICTIONARY_NO_HEADER) {
r = toku_brt_alloc_init_header(t, txn);
r = brt_alloc_init_header(t, txn);
if (r != 0) goto died_after_read_and_pin;
}
else if (r!=0) {
......
......@@ -179,7 +179,7 @@ enum brt_header_flags {
//TOKU_DB_DUP = (1<<0), //Obsolete #2862
//TOKU_DB_DUPSORT = (1<<1), //Obsolete #2862
TOKU_DB_KEYCMP_BUILTIN = (1<<2),
TOKU_DB_VALCMP_BUILTIN_12 = (1<<3),
TOKU_DB_VALCMP_BUILTIN_13 = (1<<3),
};
int toku_brt_keyrange (BRT brt, DBT *key, u_int64_t *less, u_int64_t *equal, u_int64_t *greater) __attribute__ ((warn_unused_result));
......
......@@ -16,12 +16,11 @@ enum brt_layout_version_e {
BRT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from brt layer
BRT_LAYOUT_VERSION_11 = 11, // Diff from 10 to 11: Nested transaction leafentries (completely redesigned). BRT_CMDs on disk now support XIDS (multiple txnids) instead of exactly one.
BRT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added BRT_CMD 'BRT_INSERT_NO_OVERWRITE', compressed block format, num old blocks
BRT_LAYOUT_VERSION_13 = 13, // RESERVED
BRT_LAYOUT_VERSION_14 = 14, // RESERVED
BRT_LAYOUT_VERSION_15 = 15, // Diff from 12 to 15: Added MVCC, deprecated TOKU_DB_VALCMP_BUILTIN(_12)
BRT_LAYOUT_VERSION_13 = 13, // Diff from 12 to 13: Fixed loader pivot bug, added build_id to every node, timestamps to brtheader
BRT_LAYOUT_VERSION_14 = 14, // Diff from 13 to 14: Added MVCC, deprecated TOKU_DB_VALCMP_BUILTIN(_13)
BRT_NEXT_VERSION, // the version after the current version
BRT_LAYOUT_VERSION = BRT_NEXT_VERSION-1, // A hack so I don't have to change this line.
BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION_15 // Minimum version supported
BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION_13 // Minimum version supported
};
#endif
......@@ -6,6 +6,27 @@
#include "includes.h"
static void
format_time(const uint64_t time_int, char *buf) {
time_t timer = (time_t) time_int;
ctime_r(&timer, buf);
size_t len = strlen(buf);
assert(len < 26);
char end;
assert(len>=1);
end = buf[len-1];
while (end == '\n' || end == '\r') {
buf[len-1] = '\0';
len--;
assert(len>=1);
end = buf[len-1];
}
}
static int dump_data = 1;
static CACHETABLE ct;
......@@ -76,6 +97,7 @@ static void
dump_header (int f, struct brt_header **header, CACHEFILE cf) {
struct brt_header *h;
int r;
char timestr[26];
r = toku_deserialize_brtheader_from (f, MAX_LSN, &h);
assert(r==0);
h->cf = cf;
......@@ -83,6 +105,13 @@ dump_header (int f, struct brt_header **header, CACHEFILE cf) {
printf(" layout_version=%d\n", h->layout_version);
printf(" layout_version_original=%d\n", h->layout_version_original);
printf(" layout_version_read_from_disk=%d\n", h->layout_version_read_from_disk);
printf(" build_id=%d\n", h->build_id);
printf(" build_id_original=%d\n", h->build_id_original);
format_time(h->time_of_creation, timestr);
printf(" time_of_creation= %ld %s\n", h->time_of_creation, timestr);
format_time(h->time_of_last_modification, timestr);
printf(" time_of_last_modification=%ld %s\n", h->time_of_last_modification, timestr);
printf(" dirty=%d\n", h->dirty);
printf(" nodesize=%u\n", h->nodesize);
printf(" unnamed_root=%" PRId64 "\n", h->root.b);
......@@ -119,6 +148,7 @@ dump_node (int f, BLOCKNUM blocknum, struct brt_header *h) {
printf(" layout_version=%d\n", n->layout_version);
printf(" layout_version_original=%d\n", n->layout_version_original);
printf(" layout_version_read_from_disk=%d\n", n->layout_version_read_from_disk);
printf(" build_id=%d\n", n->build_id);
printf(" rand4fp =%08x\n", n->rand4fingerprint);
printf(" localfp =%08x\n", n->local_fingerprint);
if (n->height>0) {
......
......@@ -2226,6 +2226,7 @@ static struct leaf_buf *start_leaf (struct dbout *out, const DESCRIPTOR UU(desc)
putbuf_bytes(&lbuf->dbuf, "tokuleaf", 8);
putbuf_int32(&lbuf->dbuf, layout_version);
putbuf_int32(&lbuf->dbuf, layout_version); // layout_version original
putbuf_int32(&lbuf->dbuf, BUILD_ID); // build_id (svn rev number) of software that wrote this node to disk
putbuf_int32(&lbuf->dbuf, target_nodesize);
putbuf_int32(&lbuf->dbuf, flags);
......@@ -2874,6 +2875,7 @@ static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progr
int n_uncompressed_bytes_at_beginning = (8 // tokuleaf
+4 // layout version
+4 // layout version original
+4 // build_id
);
int uncompressed_len = lbuf->dbuf.off - n_uncompressed_bytes_at_beginning;
......@@ -2905,12 +2907,14 @@ static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progr
// cppy the uncompressed header to the compressed buffer
memcpy(compressed_buf, lbuf->dbuf.buf, n_uncompressed_bytes_at_beginning);
int uncompressed_header_size = n_uncompressed_bytes_at_beginning + sizeof(n_sub_blocks);
// serialize the sub block header
memcpy(compressed_buf+16, &n_sub_blocks, 4);
memcpy(compressed_buf+n_uncompressed_bytes_at_beginning, &n_sub_blocks, 4);
for (int i = 0; i < n_sub_blocks; i++) {
memcpy(compressed_buf+20+12*i+0, &sub_block[i].compressed_size, 4);
memcpy(compressed_buf+20+12*i+4, &sub_block[i].uncompressed_size, 4);
memcpy(compressed_buf+20+12*i+8, &sub_block[i].xsum, 4);
memcpy(compressed_buf+uncompressed_header_size+12*i+0, &sub_block[i].compressed_size, 4);
memcpy(compressed_buf+uncompressed_header_size+12*i+4, &sub_block[i].uncompressed_size, 4);
memcpy(compressed_buf+uncompressed_header_size+12*i+8, &sub_block[i].xsum, 4);
}
// compute the header checksum and serialize it
......@@ -2994,12 +2998,15 @@ write_header (struct dbout *out, long long translation_location_on_disk, long lo
struct brt_header h; memset(&h, 0, sizeof h);
h.layout_version = BRT_LAYOUT_VERSION;
h.layout_version_original = BRT_LAYOUT_VERSION;
h.build_id = BUILD_ID;
h.build_id_original = BUILD_ID;
h.time_of_creation = (uint64_t) time(NULL);
h.checkpoint_count = 1;
h.checkpoint_lsn = load_lsn;
h.nodesize = target_nodesize;
h.root = root_blocknum_on_disk;
h.flags = 0;
h.layout_version_original = BRT_LAYOUT_VERSION;
h.root_xid_that_created = root_xid_that_created;
unsigned int size = toku_serialize_brt_header_size (&h);
......@@ -3136,6 +3143,7 @@ static void write_nonleaf_node (BRTLOADER bl, struct dbout *out, int64_t blocknu
node->thisnodename = make_blocknum(blocknum_of_new_node);
node->layout_version = BRT_LAYOUT_VERSION;
node->layout_version_original = BRT_LAYOUT_VERSION;
node->build_id = BUILD_ID;
node->height=height;
node->u.n.n_children = n_children;
node->flags = 0;
......
......@@ -114,7 +114,7 @@ struct __attribute__ ((__packed__)) leafentry {
typedef struct leafentry *LEAFENTRY;
typedef struct leafentry_12 *LEAFENTRY_12;
typedef struct leafentry_13 *LEAFENTRY_13;
u_int32_t toku_le_crc(LEAFENTRY v);
......@@ -190,9 +190,9 @@ int le_iterate_val(LEAFENTRY le, LE_ITERATE_CALLBACK f, void** valpp, u_int32_t
size_t
leafentry_disksize_12(LEAFENTRY_12 le);
leafentry_disksize_13(LEAFENTRY_13 le);
int
toku_le_upgrade_12_13(LEAFENTRY_12 old_leafentry, // NULL if there was no stored data.
toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry, // NULL if there was no stored data.
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY *new_leafentry_p);
......
......@@ -17,7 +17,7 @@ enum {
//After 2 we linked the log version to the BRT_LAYOUT VERSION.
//So it went from 2 to 13 (3-12 do not exist)
TOKU_LOG_VERSION = BRT_LAYOUT_VERSION,
TOKU_LOG_MIN_SUPPORTED_VERSION = TOKU_LOG_VERSION_2,
TOKU_LOG_MIN_SUPPORTED_VERSION = BRT_LAYOUT_MIN_SUPPORTED_VERSION,
};
#define ROLLBACK_CACHEFILE_NAME "tokudb.rollback"
......
......@@ -52,6 +52,7 @@ struct rollback_log_node {
int layout_version;
int layout_version_original;
int layout_version_read_from_disk;
uint32_t build_id; // build_id (svn rev number) of software that wrote this node to disk
int dirty;
TXNID txnid; // Which transaction made this?
uint64_t sequence; // Which rollback log in the sequence is this?
......
......@@ -2111,15 +2111,15 @@ bool transaction_open(TXNID xid) {
#if TOKU_WINDOWS
#pragma pack(push, 1)
#endif
struct __attribute__ ((__packed__)) leafentry_12 {
struct __attribute__ ((__packed__)) leafentry_13 {
u_int8_t num_xrs;
u_int32_t keylen;
u_int32_t innermost_inserted_vallen;
union {
struct __attribute__ ((__packed__)) leafentry_committed_12 {
struct __attribute__ ((__packed__)) leafentry_committed_13 {
u_int8_t key_val[0]; //Actual key, then actual val
} comm;
struct __attribute__ ((__packed__)) leafentry_provisional_12 {
struct __attribute__ ((__packed__)) leafentry_provisional_13 {
u_int8_t innermost_type;
TXNID xid_outermost_uncommitted;
u_int8_t key_val_xrs[]; //Actual key,
......@@ -2135,7 +2135,7 @@ struct __attribute__ ((__packed__)) leafentry_12 {
//Requires:
// Leafentry that ule represents should not be destroyed (is not just all deletes)
static size_t
le_memsize_from_ule_12 (ULE ule) {
le_memsize_from_ule_13 (ULE ule) {
uint32_t num_uxrs = ule->num_cuxrs + ule->num_puxrs;
assert(num_uxrs);
size_t rval;
......@@ -2165,16 +2165,16 @@ le_memsize_from_ule_12 (ULE ule) {
return rval;
}
//This function is mostly copied from 4.1.1
// Note, number of transaction records in version 12 has been replaced by separate counters in version 13 (MVCC),
//This function is mostly copied from 4.1.1 (which is version 12, same as 13 except that only 13 is upgradable).
// Note, number of transaction records in version 13 has been replaced by separate counters in version 14 (MVCC),
// one counter for committed transaction records and one counter for provisional transaction records. When
// upgrading a version 12 le to version 13, the number of committed transaction records is always set to one (1)
// upgrading a version 13 le to version 14, the number of committed transaction records is always set to one (1)
// and the number of provisional transaction records is set to the original number of transaction records
// minus one. The bottom transaction record is assumed to be a committed value. (If there is no committed
// value then the bottom transaction record of version 12 is a committed delete.)
// value then the bottom transaction record of version 13 is a committed delete.)
// This is the only change from the 4.1.1 code. The rest of the leafentry is read as is.
static void
le_unpack_12(ULE ule, LEAFENTRY_12 le) {
le_unpack_13(ULE ule, LEAFENTRY_13 le) {
//Read num_uxrs
uint8_t num_xrs = le->num_xrs;
assert(num_xrs > 0);
......@@ -2272,29 +2272,29 @@ le_unpack_12(ULE ule, LEAFENTRY_12 le) {
assert(found_innermost_insert);
}
#if ULE_DEBUG
size_t memsize = le_memsize_from_ule_12(ule);
size_t memsize = le_memsize_from_ule_13(ule);
assert(p == ((u_int8_t*)le) + memsize);
#endif
}
size_t
leafentry_disksize_12(LEAFENTRY_12 le) {
leafentry_disksize_13(LEAFENTRY_13 le) {
ULE_S ule;
le_unpack_12(&ule, le);
size_t memsize = le_memsize_from_ule_12(&ule);
le_unpack_13(&ule, le);
size_t memsize = le_memsize_from_ule_13(&ule);
ule_cleanup(&ule);
return memsize;
}
int
toku_le_upgrade_12_13(LEAFENTRY_12 old_leafentry,
toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry,
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY *new_leafentry_p) {
ULE_S ule;
int rval;
invariant(old_leafentry);
le_unpack_12(&ule, old_leafentry);
le_unpack_13(&ule, old_leafentry);
rval = le_pack(&ule, // create packed leafentry
new_leafentry_memorysize,
new_leafentry_disksize,
......
......@@ -38,9 +38,9 @@ NONSTANDARD_SRCS= \
$(RECOVER_SRCS) \
$(LOADER_SRCS) \
# TODO: #3228
# SRCS = $(sort $(wildcard *.c))
SRCS = $(sort $(filter-out $(TRANSPARENT_UPGRADE_SRCS),$(wildcard *.c)))
SRCS = $(sort $(wildcard *.c))
# To patch out upgrade tests, replace line above with line below
#SRCS = $(sort $(filter-out $(TRANSPARENT_UPGRADE_SRCS),$(wildcard *.c)))
#end
......@@ -719,9 +719,8 @@ preload-db-nested.tdbrun: VGRIND=
upgrade-test-4.tdbrun: VGRIND=
# TODO: #3228 Restore loader-stress-test3 when ready
#loader-stress-test.loader: $(patsubst %,loader-stress-test%.tdbrun, 0 1 2 3)
loader-stress-test.loader: $(patsubst %,loader-stress-test%.tdbrun, 0 1 2)
loader-stress-test.loader: $(patsubst %,loader-stress-test%.tdbrun, 0 1 2 3)
true
loader-stress-test0.tdbrun: loader-stress-test.tdb$(BINSUF)
......
......@@ -7,7 +7,7 @@
*
* NOTE: This test is used for upgrade testing as well as for exercising the loader.
* Changes should not be made gratuitously.
* The 4.1.1 version of this test was used to create many of the preloaded
* The 4.2.0 version of this test was used to create many of the preloaded
* environments in the <svn-top>/tokudb/tokudb.data directory.
*/
......@@ -444,7 +444,7 @@ char *env_dir = ENVDIR; // the default env_dir.
char *tmp_subdir = "tmp.subdir";
#define OLDDATADIR "../../../../tokudb.data/"
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.emptydictionaries.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.emptydictionaries.cleanshutdown";
static void setup(void) {
int r;
......
......@@ -151,10 +151,13 @@ static void run_test(void)
r = env->close(env, 0); CKERR(r);
toku_free(dbs);
/*********** DO NOT TRIM LOGFILES: Trimming logfiles defeats purpose of upgrade tests which must handle untrimmed logfiles.
// reopen, then close environment to trim logfiles
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, env_dir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
***********/
}
// ------------ infrastructure ----------
......
......@@ -28,9 +28,9 @@ int flat = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir_flat = OLDDATADIR "env_preload.4.1.1.flat.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
char *db_v4_dir_flat = OLDDATADIR "env_preload.4.2.0.flat.cleanshutdown";
......
......@@ -26,8 +26,8 @@ int littlenode = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
static void upgrade_test_2(DB **dbs) {
......
......@@ -3,7 +3,7 @@
#ident "$Id$"
// Purpose of this test is to verify that dictionaries created with 4.1.1
// Purpose of this test is to verify that dictionaries created with 4.2.0
// can be properly truncated with TokuDB version 5.x or later.
......@@ -31,8 +31,8 @@ int littlenode = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
static void upgrade_test_3(DB **dbs) {
......
......@@ -47,8 +47,8 @@ int littlenode = 0;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.1.1.node4k.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.cleanshutdown";
char *db_v4_dir_node4k = OLDDATADIR "env_preload.4.2.0.node4k.cleanshutdown";
enum {ROWS_PER_TRANSACTION=10000};
......
......@@ -19,7 +19,7 @@
DB_ENV *env;
enum {MAX_NAME=128};
int NUM_DBS=1;
int NUM_ROWS=50000;
int NUM_ROWS=100000;
int SRC_VERSION = 4;
#define MAXDEPTH 64
......@@ -27,7 +27,7 @@ int SRC_VERSION = 4;
char *env_dir = ENVDIR; // the default env_dir.
char *db_v5_dir = "dir.preload-db-nested.c.tdb";
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.nested.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.nested.cleanshutdown";
static void
......
......@@ -3,9 +3,9 @@
#ident "$Id$"
// Purpose of this test is to verify that a dictionary created by the 4.1.1
// Purpose of this test is to verify that a dictionary created by the 4.2.0
// loader can be properly read with 5.0.
// This file was derived from the 4.1.1 version of loader-stress-test.c,
// This file was derived from the 4.2.0 version of loader-stress-test.c,
// which was used to create the dictionary.
// This test only reads (and upgrades) the dictionary, it does not load it.
......@@ -212,7 +212,7 @@ char *env_dir = ENVDIR; // the default env_dir.
char *tmp_subdir = "tmp.subdir";
#define OLDDATADIR "../../../../tokudb.data/"
char *db_v4_dir = OLDDATADIR "env_preload.4.1.1.loader250kd1.cleanshutdown";
char *db_v4_dir = OLDDATADIR "env_preload.4.2.0.loader250kd1.cleanshutdown";
static void setup(void) {
......
......@@ -31,13 +31,15 @@ static void test_shutdown(void);
static char *env_dir = ENVDIR; // the default env_dir.
static char * dir_v4_clean = "env_simple.4.1.1.cleanshutdown";
static char * dir_v4_dirty = "env_simple.4.1.1.dirtyshutdown";
static char * dir_v4_dirty_multilogfile = OLDDATADIR "env_preload.4.1.1.multilog.dirtyshutdown";
static char * dir_v41_clean = "env_simple.4.1.1.cleanshutdown";
static char * dir_v42_clean = "env_simple.4.2.0.cleanshutdown";
static char * dir_v42_dirty = "env_simple.4.2.0.dirtyshutdown";
static char * dir_v41_dirty_multilogfile = OLDDATADIR "env_preload.4.1.1.multilog.dirtyshutdown";
static char * dir_v42_dirty_multilogfile = OLDDATADIR "env_preload.4.2.0.multilog.dirtyshutdown";
static void
setup (u_int32_t flags, BOOL clean, char * src_db_dir) {
setup (u_int32_t flags, BOOL clean, BOOL too_old, char * src_db_dir) {
int r;
int len = 256;
char syscmd[len];
......@@ -61,8 +63,12 @@ setup (u_int32_t flags, BOOL clean, char * src_db_dir) {
r=env->open(env, ENVDIR, flags, mode);
if (clean)
CKERR(r);
else {
if (too_old)
CKERR2(r, TOKUDB_DICTIONARY_TOO_OLD);
else
CKERR2(r, TOKUDB_UPGRADE_FAILURE);
}
}
......@@ -81,18 +87,29 @@ test_env_startup(void) {
flags = FLAGS_LOG;
setup(flags, TRUE, dir_v4_clean);
setup(flags, TRUE, FALSE, dir_v42_clean);
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, TRUE, dir_v41_clean);
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, FALSE, dir_v42_dirty);
if (verbose) {
printf("\n\nEngine status after aborted env->open() will have some garbage values:\n");
}
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, dir_v4_dirty);
setup(flags, FALSE, TRUE, dir_v41_dirty_multilogfile);
if (verbose) {
printf("\n\nEngine status after aborted env->open() will have some garbage values:\n");
}
print_engine_status(env);
test_shutdown();
setup(flags, FALSE, dir_v4_dirty_multilogfile);
setup(flags, FALSE, FALSE, dir_v42_dirty_multilogfile);
if (verbose) {
printf("\n\nEngine status after aborted env->open() will have some garbage values:\n");
}
......
......@@ -35,6 +35,7 @@ const char *toku_copyright_string = "Copyright (c) 2007-2009 Tokutek Inc. All r
#include "brtloader.h"
#include "log_header.h"
#ifdef TOKUTRACE
#define DB_ENV_CREATE_FUN db_env_create_toku10
#define DB_CREATE_FUN db_create_toku10
......@@ -491,20 +492,20 @@ db_use_builtin_key_cmp(DB *db) {
// Following keys added in version 12
static const char * orig_env_ver_key = "original_version";
static const char * curr_env_ver_key = "current_version";
// Following keys added in version 13, add more keys for future versions
// Following keys added in version 14, add more keys for future versions
static const char * creation_time_key = "creation_time";
static const char * last_lsn_of_v12_key = "last_lsn_of_v12";
static const char * upgrade_v13_time_key = "upgrade_v13_time";
static const char * upgrade_v13_footprint_key = "upgrade_v13_footprint";
static const char * last_lsn_of_v13_key = "last_lsn_of_v13";
static const char * upgrade_v14_time_key = "upgrade_v14_time";
static const char * upgrade_v14_footprint_key = "upgrade_v14_footprint";
// Values read from (or written into) persistent environment,
// kept here for read-only access from engine status.
static uint32_t persistent_original_env_version;
static uint32_t persistent_stored_env_version_at_startup; // read from curr_env_ver_key, prev version as of this startup
static time_t persistent_creation_time;
static uint64_t persistent_last_lsn_of_v12;
static time_t persistent_upgrade_v13_time;
static uint64_t persistent_upgrade_v13_footprint;
static uint64_t persistent_last_lsn_of_v13;
static time_t persistent_upgrade_v14_time;
static uint64_t persistent_upgrade_v14_footprint;
// Requires: persistent environment dictionary is already open.
// Input arg is lsn of clean shutdown of previous version,
......@@ -532,21 +533,21 @@ maybe_upgrade_persistent_environment_dictionary(DB_ENV * env, DB_TXN * txn, LSN
r = toku_db_put(persistent_environment, txn, &key, &val, DB_YESOVERWRITE);
assert(r==0);
uint64_t last_lsn_of_v12_d = toku_htod64(last_lsn_of_clean_shutdown_read_from_log.lsn);
toku_fill_dbt(&key, last_lsn_of_v12_key, strlen(last_lsn_of_v12_key));
toku_fill_dbt(&val, &last_lsn_of_v12_d, sizeof(last_lsn_of_v12_d));
uint64_t last_lsn_of_v13_d = toku_htod64(last_lsn_of_clean_shutdown_read_from_log.lsn);
toku_fill_dbt(&key, last_lsn_of_v13_key, strlen(last_lsn_of_v13_key));
toku_fill_dbt(&val, &last_lsn_of_v13_d, sizeof(last_lsn_of_v13_d));
r = toku_db_put(persistent_environment, txn, &key, &val, DB_YESOVERWRITE);
assert(r==0);
time_t upgrade_v13_time_d = toku_htod64(time(NULL));
toku_fill_dbt(&key, upgrade_v13_time_key, strlen(upgrade_v13_time_key));
toku_fill_dbt(&val, &upgrade_v13_time_d, sizeof(upgrade_v13_time_d));
time_t upgrade_v14_time_d = toku_htod64(time(NULL));
toku_fill_dbt(&key, upgrade_v14_time_key, strlen(upgrade_v14_time_key));
toku_fill_dbt(&val, &upgrade_v14_time_d, sizeof(upgrade_v14_time_d));
r = toku_db_put(persistent_environment, txn, &key, &val, DB_NOOVERWRITE);
assert(r==0);
uint64_t upgrade_v13_footprint_d = toku_htod64(toku_log_upgrade_get_footprint());
toku_fill_dbt(&key, upgrade_v13_footprint_key, strlen(upgrade_v13_footprint_key));
toku_fill_dbt(&val, &upgrade_v13_footprint_d, sizeof(upgrade_v13_footprint_d));
uint64_t upgrade_v14_footprint_d = toku_htod64(toku_log_upgrade_get_footprint());
toku_fill_dbt(&key, upgrade_v14_footprint_key, strlen(upgrade_v14_footprint_key));
toku_fill_dbt(&val, &upgrade_v14_footprint_d, sizeof(upgrade_v14_footprint_d));
r = toku_db_put(persistent_environment, txn, &key, &val, DB_NOOVERWRITE);
assert(r==0);
}
......@@ -576,8 +577,7 @@ capture_persistent_env_contents (DB_ENV * env, DB_TXN * txn) {
assert(persistent_original_env_version <= curr_env_version);
// make no assertions about timestamps, clock may have been reset
// TODO: #3228 May need to fix specific version number to whenever this new info was added.
if (persistent_original_env_version >= BRT_LAYOUT_VERSION_15) {
if (persistent_original_env_version >= BRT_LAYOUT_VERSION_14) {
toku_fill_dbt(&key, creation_time_key, strlen(creation_time_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
......@@ -588,23 +588,23 @@ capture_persistent_env_contents (DB_ENV * env, DB_TXN * txn) {
if (persistent_original_env_version != curr_env_version) {
// an upgrade was performed at some time, capture info about the upgrade
toku_fill_dbt(&key, last_lsn_of_v12_key, strlen(last_lsn_of_v12_key));
toku_fill_dbt(&key, last_lsn_of_v13_key, strlen(last_lsn_of_v13_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
assert(r == 0);
persistent_last_lsn_of_v12 = toku_dtoh64(*(uint32_t*)val.data);
persistent_last_lsn_of_v13 = toku_dtoh64(*(uint32_t*)val.data);
toku_fill_dbt(&key, upgrade_v13_time_key, strlen(upgrade_v13_time_key));
toku_fill_dbt(&key, upgrade_v14_time_key, strlen(upgrade_v14_time_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
assert(r == 0);
persistent_upgrade_v13_time = toku_dtoh64((*(time_t*)val.data));
persistent_upgrade_v14_time = toku_dtoh64((*(time_t*)val.data));
toku_fill_dbt(&key, upgrade_v13_footprint_key, strlen(upgrade_v13_footprint_key));
toku_fill_dbt(&key, upgrade_v14_footprint_key, strlen(upgrade_v14_footprint_key));
toku_init_dbt(&val);
r = toku_db_get(persistent_environment, txn, &key, &val, 0);
assert(r == 0);
persistent_upgrade_v13_footprint = toku_dtoh64((*(uint64_t*)val.data));
persistent_upgrade_v14_footprint = toku_dtoh64((*(uint64_t*)val.data));
}
}
......@@ -1966,16 +1966,17 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat, char * env_panic_st
// is provided in six least significant decimal digits, footprint of
// upgrade performed when environment was actually upgraded is provided
// in most significant decimal digits.
// If ver_at_startup == 12, then the footprint will have the same value in
// If ver_at_startup == 13, then the footprint will have the same value in
// upper and lower digits.
engstat->upgrade_env_status = (persistent_upgrade_v13_footprint * 1000000) + upgrade_footprint;
engstat->upgrade_header = brt_upgrade_stat.header_12;
engstat->upgrade_nonleaf = brt_upgrade_stat.nonleaf_12;
engstat->upgrade_leaf = brt_upgrade_stat.leaf_12;
engstat->upgrade_env_status = (persistent_upgrade_v14_footprint * 1000000) + upgrade_footprint;
engstat->upgrade_header = brt_upgrade_stat.header_13;
engstat->upgrade_nonleaf = brt_upgrade_stat.nonleaf_13;
engstat->upgrade_leaf = brt_upgrade_stat.leaf_13;
engstat->optimized_for_upgrade = brt_upgrade_stat.optimized_for_upgrade;
engstat->original_ver = persistent_original_env_version;
engstat->ver_at_startup = persistent_stored_env_version_at_startup;
engstat->last_lsn_v12 = persistent_last_lsn_of_v12;
format_time(&persistent_upgrade_v13_time, engstat->upgrade_v13_time);
engstat->last_lsn_v13 = persistent_last_lsn_of_v13;
format_time(&persistent_upgrade_v14_time, engstat->upgrade_v14_time);
}
}
return r;
......@@ -1992,6 +1993,9 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
char panicstring[stringsize];
int n = 0; // number of characters printed so far
n = snprintf(buff, bufsiz - n, "BUILD_ID = %d\n", BUILD_ID);
int r = env_get_engine_status(env, &engstat, panicstring, stringsize);
if (strlen(panicstring)) {
......@@ -2126,10 +2130,11 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
n += snprintf(buff + n, bufsiz - n, "upgrade_header %"PRIu64"\n", engstat.upgrade_header);
n += snprintf(buff + n, bufsiz - n, "upgrade_nonleaf %"PRIu64"\n", engstat.upgrade_nonleaf);
n += snprintf(buff + n, bufsiz - n, "upgrade_leaf %"PRIu64"\n", engstat.upgrade_leaf);
n += snprintf(buff + n, bufsiz - n, "optimized_for_upgrade_14 %"PRIu64"\n", engstat.optimized_for_upgrade);
n += snprintf(buff + n, bufsiz - n, "original_ver %"PRIu64"\n", engstat.original_ver);
n += snprintf(buff + n, bufsiz - n, "ver_at_startup %"PRIu64"\n", engstat.ver_at_startup);
n += snprintf(buff + n, bufsiz - n, "last_lsn_v12 %"PRIu64"\n", engstat.last_lsn_v12);
n += snprintf(buff + n, bufsiz - n, "upgrade_v13_time %s \n", engstat.upgrade_v13_time);
n += snprintf(buff + n, bufsiz - n, "last_lsn_v13 %"PRIu64"\n", engstat.last_lsn_v13);
n += snprintf(buff + n, bufsiz - n, "upgrade_v14_time %s \n", engstat.upgrade_v14_time);
}
if (n > bufsiz) {
char * errmsg = "BUFFER TOO SMALL\n";
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment