Skip to content

M
mariadb
Commit b6e764a0 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

Convert from crc to x1764. Fixes #1021. 

git-svn-id: file:///svn/tokudb@5363 c7de825b-a66e-492c-adef-691d508d4ae1
parent fb55d78f
Expand all Show whitespace changes
Inline Side-by-side
Showing with 196 additions and 321 deletions
newbrt/Makefile
View file @ b6e764a0
......@@ -36,7 +36,7 @@ FORMAT=-Wmissing-format-attribute
endif
CFLAGS = -Wall -Wextra -Wcast-align -Wbad-function-cast -Wmissing-noreturn $(FORMAT) $(OPTFLAGS) -g3 -ggdb3 $(GCOV_FLAGS) $(PROF_FLAGS) -Werror $(FPICFLAGS) $(SHADOW) $(VISIBILITY)
LDFLAGS = $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS) -lz
LDFLAGS = $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS)
CPPFLAGS += -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE=500
# Add -Wconversion
......@@ -55,9 +55,7 @@ BINS= brtdump \
build default: bins libs tdb-recover tdb_logprint $(TEST_OFILES)
cd tests;$(MAKE) build
# Put crc first to make -combine work right
BRT_SOURCES = \
crc \
bread \
brt-serialize \
brt-verify \
......@@ -77,6 +75,7 @@ BRT_SOURCES = \
roll \
toku_assert \
ybt \
x1764 \
trace_mem \
# keep this line so I can ha vea \ on the previous line
......@@ -119,7 +118,7 @@ check-fanout:
let BRT_FANOUT=BRT_FANOUT+1; \
done
BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h omt.h brt.h brt-search.h brttypes.h ybt.h log.h ../include/db.h kv-pair.h memory.h crc.h mempool.h leafentry.h
BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h omt.h brt.h brt-search.h brttypes.h ybt.h log.h ../include/db.h kv-pair.h memory.h mempool.h leafentry.h
brt-test-helpers.o: $(BRT_INTERNAL_H_INCLUDES) toku_assert.h
logformat: logformat.c toku_assert.c
brt-serialize-test.o: $(BRT_INTERNAL_H_INCLUDES)
......
newbrt/brt-internal.h
View file @ b6e764a0
......@@ -7,7 +7,6 @@
#include "cachetable.h"
#include "fifo.h"
#include "brt.h"
#include "crc.h"
#include "list.h"
#include "mempool.h"
#include "kv-pair.h"
......@@ -195,8 +194,8 @@ static const BRTNODE null_brtnode=0;
//extern u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, int vallen);
//extern u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp);
extern u_int32_t toku_calccrc32_cmd (u_int32_t type, TXNID xid, const void *key, u_int32_t keylen, const void *val, u_int32_t vallen);
extern u_int32_t toku_calccrc32_cmdstruct (BRT_CMD cmd);
extern u_int32_t toku_calc_fingerprint_cmd (u_int32_t type, TXNID xid, const void *key, u_int32_t keylen, const void *val, u_int32_t vallen);
extern u_int32_t toku_calc_fingerprint_cmdstruct (BRT_CMD cmd);
// How long is the pivot key?
unsigned int toku_brt_pivot_key_len (BRT, struct kv_pair *); // Given the tree
......@@ -250,7 +249,8 @@ int toku_verify_brtnode (BRT brt, DISKOFF off, bytevec lorange, ITEMLEN lolen, b
enum brt_layout_version_e {
BRT_LAYOUT_VERSION_5 = 5,
BRT_LAYOUT_VERSION_6 = 6, // Diff from 5 to 6: Add leafentry_estimate
BRT_LAYOUT_VERSION_7 = 7, // Diff from 6 to 7: Add exact-bit to leafentry_estimate #818, add magic to header #22, add per-subdataase flags #333
BRT_LAYOUT_VERSION_7 = 7, // Diff from 6 to 7: Add exact-bit to leafentry_estimate #818, add magic to header #22, add per-subdatase flags #333
BRT_LAYOUT_VERSION_8 = 8, // Diff from 7 to 8: Use murmur instead of crc32. We are going to make a simplification and stop supporting version 7 and before.
BRT_ANTEULTIMATE_VERSION, // the version after the most recent version
BRT_LAYOUT_VERSION = BRT_ANTEULTIMATE_VERSION-1 // A hack so I don't have to change this line.
};
......
newbrt/brt-serialize.c
View file @ b6e764a0
......@@ -120,7 +120,7 @@ void toku_serialize_brtnode_to (int fd, DISKOFF off, BRTNODE node) {
wbuf_literal_bytes(&w, "toku", 4);
if (node->height==0) wbuf_literal_bytes(&w, "leaf", 4);
else wbuf_literal_bytes(&w, "node", 4);
wbuf_int(&w, BRT_LAYOUT_VERSION_7);
wbuf_int(&w, BRT_LAYOUT_VERSION);
wbuf_ulonglong(&w, node->log_lsn.lsn);
//printf("%s:%d %lld.calculated_size=%d\n", __FILE__, __LINE__, off, calculated_size);
wbuf_uint(&w, calculated_size);
......@@ -176,7 +176,7 @@ void toku_serialize_brtnode_to (int fd, DISKOFF off, BRTNODE node) {
wbuf_TXNID(&w, xid);
wbuf_bytes(&w, key, keylen);
wbuf_bytes(&w, data, datalen);
check_local_fingerprint+=node->rand4fingerprint*toku_calccrc32_cmd(type, xid, key, keylen, data, datalen);
check_local_fingerprint+=node->rand4fingerprint*toku_calc_fingerprint_cmd(type, xid, key, keylen, data, datalen);
}));
}
//printf("%s:%d check_local_fingerprint=%8x\n", __FILE__, __LINE__, check_local_fingerprint);
......@@ -193,7 +193,10 @@ void toku_serialize_brtnode_to (int fd, DISKOFF off, BRTNODE node) {
wbuf_int(&w, crc32(toku_null_crc, w.buf, w.ndone));
#endif
#ifdef CRC_INCR
wbuf_uint(&w, w.crc32);
{
u_int32_t checksum = x1764_finish(&w.checksum);
wbuf_uint(&w, checksum);
}
#endif
if (!node->ever_been_written)
......@@ -274,9 +277,8 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, u_int32_t fullhash, BRTN
result->layout_version = rbuf_int(&rc);
{
switch (result->layout_version) {
case BRT_LAYOUT_VERSION_5:
case BRT_LAYOUT_VERSION_6:
case BRT_LAYOUT_VERSION_7: goto ok_layout_version;
case BRT_LAYOUT_VERSION_8: goto ok_layout_version;
// Don't support older versions.
}
r=DB_BADFORMAT;
goto died1;
......@@ -311,11 +313,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, u_int32_t fullhash, BRTN
u_int32_t childfp = rbuf_int(&rc);
BNC_SUBTREE_FINGERPRINT(result, i)= childfp;
check_subtree_fingerprint += childfp;
if (result->layout_version>BRT_LAYOUT_VERSION_5) {
BNC_SUBTREE_LEAFENTRY_ESTIMATE(result, i)=rbuf_ulonglong(&rc);
} else {
BNC_SUBTREE_LEAFENTRY_ESTIMATE(result, i)=0;
}
}
for (i=0; i<result->u.n.n_children-1; i++) {
if (result->flags & TOKU_DB_DUPSORT) {
......@@ -364,7 +362,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, u_int32_t fullhash, BRTN
TXNID xid = rbuf_ulonglong(&rc);
rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
rbuf_bytes(&rc, &val, &vallen);
check_local_fingerprint += result->rand4fingerprint * toku_calccrc32_cmd(type, xid, key, keylen, val, vallen);
check_local_fingerprint += result->rand4fingerprint * toku_calc_fingerprint_cmd(type, xid, key, keylen, val, vallen);
//printf("Found %s,%s\n", (char*)key, (char*)val);
{
r=toku_fifo_enq(BNC_BUFFER(result, cnum), key, keylen, val, vallen, type, xid); /* Copies the data into the hash table. */
......@@ -403,7 +401,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, u_int32_t fullhash, BRTN
assert(rc.ndone<=rc.size);
array[i]=(OMTVALUE)le;
actual_sum += toku_crc32(toku_null_crc, le, disksize);
actual_sum += x1764_memory(le, disksize);
}
u_int32_t end_of_data = rc.ndone;
result->u.l.n_bytes_in_buffer += end_of_data-start_of_data + n_in_buf*OMT_ITEM_OVERHEAD;
......@@ -434,10 +432,11 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, u_int32_t fullhash, BRTN
if (n_read_so_far+4!=rc.size) {
r = DB_BADFORMAT; goto died_21;
}
uint32_t crc = toku_crc32(toku_null_crc, rc.buf, n_read_so_far);
uint32_t crc = x1764_memory(rc.buf, n_read_so_far);
uint32_t storedcrc = rbuf_int(&rc);
if (crc!=storedcrc) {
printf("Bad CRC\n");
printf("%s:%d crc=%08x stored=%08x\n", __FILE__, __LINE__, crc, storedcrc);
assert(0);//this is wrong!!!
r = DB_BADFORMAT;
goto died_21;
......@@ -560,91 +559,6 @@ int toku_serialize_brt_header_to (int fd, struct brt_header *h) {
return r;
}
static int deserialize_brtheader_6_or_earlier (int fd, DISKOFF off, struct brt_header **brth, u_int32_t fullhash) {
// Deserialize a brt header from version 6 or earlier.
struct brt_header *MALLOC(h);
if (h==0) return errno;
h->fullhash = fullhash;
int ret=-1;
if (0) { died0: toku_free(h); return ret; }
int size;
int sizeagain;
h->layout_version = BRT_LAYOUT_VERSION_6;
{
uint32_t size_n;
ssize_t r = pread(fd, &size_n, sizeof(size_n), off);
assert(r==sizeof(size_n)); // we already read it earlier.
size = ntohl(size_n);
}
struct rbuf rc;
rc.buf = toku_malloc(size);
if (rc.buf == NULL) { ret = ENOMEM; goto died0; }
if (0) { died1: toku_free(rc.buf); goto died0; }
rc.size=size;
if (rc.size<=0) {ret = EINVAL; goto died1;}
rc.ndone=0;
{
ssize_t r = pread(fd, rc.buf, size, off);
if (r!=size) {ret = EINVAL; goto died1;}
}
h->dirty=0;
sizeagain = rbuf_int(&rc);
if (sizeagain!=size) {ret = EINVAL; goto died1;}
u_int32_t flags_for_all = rbuf_int(&rc);
h->nodesize = rbuf_int(&rc);
h->freelist = rbuf_diskoff(&rc);
h->unused_memory = rbuf_diskoff(&rc);
h->n_named_roots = rbuf_int(&rc);
if (h->n_named_roots>=0) {
int i;
MALLOC_N(h->n_named_roots, h->flags_array);
for (i=0; i<h->n_named_roots; i++) h->flags_array[i]=flags_for_all;
MALLOC_N(h->n_named_roots, h->roots);
MALLOC_N(h->n_named_roots, h->root_hashes);
if (h->n_named_roots > 0 && (h->roots == NULL || h->root_hashes==NULL)) {ret = ENOMEM; goto died1;}
if (0) {
died2:
toku_free(h->roots);
toku_free(h->root_hashes);
goto died1;
}
MALLOC_N(h->n_named_roots, h->names);
if (h->n_named_roots > 0 && h->names == NULL) {ret = ENOMEM; goto died2;}
if (0) {
died3:
toku_free(h->names);
for (i = 0; i < h->n_named_roots; i++) {
if (h->names[i]) toku_free(h->names[i]);
}
goto died2;
}
for (i=0; i<h->n_named_roots; i++) {
bytevec nameptr;
unsigned int len;
h->root_hashes[i].valid = FALSE;
h->roots[i] = rbuf_diskoff(&rc);
rbuf_bytes(&rc, &nameptr, &len);
if (strlen(nameptr)+1!=len) {ret = EINVAL; goto died3;}
h->names[i] = toku_memdup(nameptr,len);
if (len > 0 && h->names[i] == NULL) {ret = ENOMEM; goto died3;}
}
} else {
MALLOC_N(1, h->flags_array);
MALLOC_N(1, h->roots);
MALLOC_N(1, h->root_hashes);
h->flags_array[0]=flags_for_all;
h->roots[0] = rbuf_diskoff(&rc);
h->root_hashes[0].valid = FALSE;
h->names = 0;
}
if (rc.ndone!=rc.size) {ret = EINVAL; goto died3;}
toku_free(rc.buf);
*brth = h;
return 0;
}
int deserialize_brtheader_7_or_later(u_int32_t size, int fd, DISKOFF off, struct brt_header **brth, u_int32_t fullhash) {
// We already know the first 8 bytes are "tokudata", and we read in the size.
struct brt_header *MALLOC(h);
......@@ -665,7 +579,7 @@ int deserialize_brtheader_7_or_later(u_int32_t size, int fd, DISKOFF off, struct
h->dirty=0;
h->layout_version = rbuf_int(&rc);
h->nodesize = rbuf_int(&rc);
assert(h->layout_version==BRT_LAYOUT_VERSION_7);
assert(h->layout_version==BRT_LAYOUT_VERSION_8);
h->freelist = rbuf_diskoff(&rc);
h->unused_memory = rbuf_diskoff(&rc);
h->n_named_roots = rbuf_int(&rc);
......@@ -713,12 +627,9 @@ int toku_deserialize_brtheader_from (int fd, DISKOFF off, u_int32_t fullhash, st
if (r==0) return -1;
if (r<0) return errno;
if (r!=12) return EINVAL;
if (memcmp(magic,"tokudata",8)==0) {
// It's version 7 or later
assert(memcmp(magic,"tokudata",8)==0);
// It's version 7 or later, and the magi clooks OK
return deserialize_brtheader_7_or_later(ntohl(*(int*)(&magic[8])), fd, off, brth, fullhash);
} else {
return deserialize_brtheader_6_or_earlier(fd, off, brth, fullhash);
}
}
unsigned int toku_brt_pivot_key_len (BRT brt, struct kv_pair *pk) {
......
newbrt/brt-test-helpers.c
View file @ b6e764a0
......@@ -137,7 +137,7 @@ int toku_testsetup_insert_to_nonleaf (BRT brt, DISKOFF diskoff, enum brt_cmd_typ
r = toku_fifo_enq(BNC_BUFFER(node, childnum), key, keylen, val, vallen, cmdtype, (TXNID)0);
assert(r==0);
u_int32_t fdelta = node->rand4fingerprint * toku_calccrc32_cmd(cmdtype, (TXNID)0, key, keylen, val, vallen);
u_int32_t fdelta = node->rand4fingerprint * toku_calc_fingerprint_cmd(cmdtype, (TXNID)0, key, keylen, val, vallen);
node->local_fingerprint += fdelta;
*subtree_fingerprint += fdelta;
int sizediff = keylen + vallen + KEY_VALUE_OVERHEAD + BRT_CMD_OVERHEAD;
......
newbrt/brt-verify.c
View file @ b6e764a0
......@@ -26,7 +26,7 @@ static void verify_local_fingerprint (BRTNODE node) {
for (i=0; i<node->u.n.n_children; i++)
FIFO_ITERATE(BNC_BUFFER(node,i), key, keylen, data, datalen, type, xid,
({
fp += node->rand4fingerprint * toku_calccrc32_cmd(type, xid, key, keylen, data, datalen);
fp += node->rand4fingerprint * toku_calc_fingerprint_cmd(type, xid, key, keylen, data, datalen);
}));
assert(fp==node->local_fingerprint);
} else {
......
newbrt/brt.c
View file @ b6e764a0
......@@ -391,7 +391,7 @@ static int insert_to_buffer_in_nonleaf (BRTNODE node, int childnum, DBT *k, DBT
int r = toku_fifo_enq(BNC_BUFFER(node,childnum), k->data, k->size, v->data, v->size, type, xid);
if (r!=0) return r;
// printf("%s:%d fingerprint %08x -> ", __FILE__, __LINE__, node->local_fingerprint);
node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_cmd(type, xid, k->data, k->size, v->data, v->size);
node->local_fingerprint += node->rand4fingerprint*toku_calc_fingerprint_cmd(type, xid, k->data, k->size, v->data, v->size);
// printf(" %08x\n", node->local_fingerprint);
BNC_NBYTESINBUF(node,childnum) += n_bytes_added;
node->u.n.n_bytes_in_buffers += n_bytes_added;
......@@ -566,7 +566,7 @@ static int log_and_save_brtenq(TOKULOGGER logger, BRT t, BRTNODE node, int child
BYTESTRING keybs = {.len=keylen, .data=(char*)key};
BYTESTRING databs = {.len=datalen, .data=(char*)data};
u_int32_t old_fingerprint = *fingerprint;
u_int32_t fdiff=node->rand4fingerprint*toku_calccrc32_cmd(type, xid, key, keylen, data, datalen);
u_int32_t fdiff=node->rand4fingerprint*toku_calc_fingerprint_cmd(type, xid, key, keylen, data, datalen);
u_int32_t new_fingerprint = old_fingerprint + fdiff;
//printf("%s:%d node=%lld fingerprint old=%08x new=%08x diff=%08x xid=%lld\n", __FILE__, __LINE__, (long long)node->thisnodename, old_fingerprint, new_fingerprint, fdiff, (long long)xid);
*fingerprint = new_fingerprint;
......@@ -631,7 +631,7 @@ static int brt_nonleaf_split (BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *node
if (fr!=0) break;
int n_bytes_moved = keylen+datalen + KEY_VALUE_OVERHEAD + BRT_CMD_OVERHEAD;
u_int32_t old_from_fingerprint = node->local_fingerprint;
u_int32_t delta = toku_calccrc32_cmd(type, xid, key, keylen, data, datalen);
u_int32_t delta = toku_calc_fingerprint_cmd(type, xid, key, keylen, data, datalen);
u_int32_t new_from_fingerprint = old_from_fingerprint - node->rand4fingerprint*delta;
if (r!=0) return r;
if (t->txn_that_created != xid) {
......@@ -816,7 +816,7 @@ static int push_a_brt_cmd_down (BRT t, BRTNODE node, BRTNODE child, int childnum
DBT *v = cmd->u.id.val;
//if (debug) printf("%s:%d %*sinserted down child_did_split=%d\n", __FILE__, __LINE__, debug, "", child_did_split);
u_int32_t old_fingerprint = node->local_fingerprint;
u_int32_t new_fingerprint = old_fingerprint - node->rand4fingerprint*toku_calccrc32_cmdstruct(cmd);
u_int32_t new_fingerprint = old_fingerprint - node->rand4fingerprint*toku_calc_fingerprint_cmdstruct(cmd);
node->local_fingerprint = new_fingerprint;
if (t->txn_that_created != cmd->xid) {
int r = toku_log_brtdeq(logger, &node->log_lsn, 0, toku_cachefile_filenum(t->cf), node->thisnodename, childnum);
......@@ -910,7 +910,7 @@ static int handle_split_of_child (BRT t, BRTNODE node, int childnum,
FIFO_ITERATE(old_h, skey, skeylen, sval, svallen, type, xid,
({
u_int32_t old_fingerprint = node->local_fingerprint;
u_int32_t new_fingerprint = old_fingerprint - node->rand4fingerprint*toku_calccrc32_cmd(type, xid, skey, skeylen, sval, svallen);
u_int32_t new_fingerprint = old_fingerprint - node->rand4fingerprint*toku_calc_fingerprint_cmd(type, xid, skey, skeylen, sval, svallen);
if (t->txn_that_created != xid) {
r = toku_log_brtdeq(logger, &node->log_lsn, 0, toku_cachefile_filenum(t->cf), node->thisnodename, childnum);
assert(r==0);
......@@ -1950,7 +1950,7 @@ static void verify_local_fingerprint_nonleaf (BRTNODE node) {
for (i=0; i<node->u.n.n_children; i++)
FIFO_ITERATE(BNC_BUFFER(node,i), key, keylen, data, datalen, type, xid,
({
fp += node->rand4fingerprint * toku_calccrc32_cmd(type, xid, key, keylen, data, datalen);
fp += node->rand4fingerprint * toku_calc_fingerprint_cmd(type, xid, key, keylen, data, datalen);
}));
assert(fp==node->local_fingerprint);
}
......
newbrt/crc.c deleted 100644 → 0
View file @ fb55d78f
#include <sys/types.h>
#include <zlib.h>
// hack: include crc.h below so we can deprecate the call to crc32
inline u_int32_t toku_crc32 (u_int32_t oldcrc32, const void *data, u_int32_t len) {
if (len==0) return oldcrc32;
else return crc32((unsigned long)oldcrc32, data, (uInt)len);
}
// Hack
#include "crc.h"
newbrt/crc.h deleted 100644 → 0
View file @ fb55d78f
#ifndef TOKU_CRC_H
#define TOKU_CRC_H
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include <sys/types.h>
#include <zlib.h>
// zlib crc32 has a bug: If len==0 then it should return oldcrc32, but crc32 returns 0.
inline u_int32_t toku_crc32 (u_int32_t oldcrc32, const void *data, u_int32_t len);
static const u_int32_t toku_null_crc = 0;
// Don't use crc32, use toku_crc32 to avoid that bug.
ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)) __attribute__((deprecated));
#endif
newbrt/fingerprint.c
View file @ b6e764a0
......@@ -4,17 +4,17 @@
#include <arpa/inet.h>
#include "brt-internal.h"
#include "toku_assert.h"
#include "x1764.h"
// Calculate the fingerprint for a kvpair
static inline u_int32_t toku_calc_more_crc32_kvpair (u_int32_t crc, const void *key, int keylen, const void *val, int vallen) {
static void toku_calc_more_murmur_kvpair (struct x1764 *mm, const void *key, int keylen, const void *val, int vallen) {
int i;
i = htonl(keylen);
crc = toku_crc32(crc, (void*)&i, 4);
crc = toku_crc32(crc, key, keylen);
x1764_add(mm, (void*)&i, 4);
x1764_add(mm, key, keylen);
i = htonl(vallen);
crc = toku_crc32(crc, (void*)&i, 4);
crc = toku_crc32(crc, val, vallen);
return crc;
x1764_add(mm, (void*)&i, 4);
x1764_add(mm, val, vallen);
}
#if 0
......@@ -28,18 +28,20 @@ u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp) {
}
#endif
u_int32_t toku_calccrc32_cmd (u_int32_t type, TXNID xid, const void *key, u_int32_t keylen, const void *val, u_int32_t vallen) {
u_int32_t toku_calc_fingerprint_cmd (u_int32_t type, TXNID xid, const void *key, u_int32_t keylen, const void *val, u_int32_t vallen) {
unsigned char type_c = type;
unsigned int a = htonl(xid>>32);
unsigned int b = htonl(xid&0xffffffff);
return toku_calc_more_crc32_kvpair(toku_crc32(toku_crc32(toku_crc32(toku_null_crc,
&type_c, 1),
&a, 4),
&b, 4),
key, keylen, val, vallen);
struct x1764 mm;
x1764_init(&mm);
x1764_add(&mm, &type_c, 1);
x1764_add(&mm, &a, 4);
x1764_add(&mm, &b, 4);
toku_calc_more_murmur_kvpair(&mm, key, keylen, val, vallen);
return x1764_finish(&mm);
}
u_int32_t toku_calccrc32_cmdstruct (BRT_CMD cmd) {
u_int32_t toku_calc_fingerprint_cmdstruct (BRT_CMD cmd) {
switch (cmd->type) {
case BRT_INSERT:
case BRT_DELETE_ANY:
......@@ -48,7 +50,7 @@ u_int32_t toku_calccrc32_cmdstruct (BRT_CMD cmd) {
case BRT_COMMIT_BOTH:
case BRT_ABORT_ANY:
case BRT_ABORT_BOTH:
return toku_calccrc32_cmd (cmd->type, cmd->xid, cmd->u.id.key->data, cmd->u.id.key->size, cmd->u.id.val->data, cmd->u.id.val->size);
return toku_calc_fingerprint_cmd (cmd->type, cmd->xid, cmd->u.id.key->data, cmd->u.id.key->size, cmd->u.id.val->data, cmd->u.id.val->size);
case BRT_NONE:
return 0;
}
......
newbrt/leafentry.c
View file @ b6e764a0
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brttypes.h"
#include "crc.h"
#include "leafentry.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -14,7 +13,7 @@
#include <string.h>
u_int32_t toku_le_crc(LEAFENTRY v) {
return toku_crc32(toku_null_crc, v, leafentry_memsize(v));
return x1764_memory(v, leafentry_memsize(v));
}
int le_committed (u_int32_t klen, void* kval, u_int32_t dlen, void* dval, u_int32_t *resultsize, u_int32_t *disksize, LEAFENTRY *result) {
......@@ -148,27 +147,27 @@ u_int32_t toku_logsizeof_LEAFENTRY (LEAFENTRY le) {
return leafentry_disksize(le);
}
int toku_fread_LEAFENTRY(FILE *f, LEAFENTRY *le, u_int32_t *crc, u_int32_t *len) {
int toku_fread_LEAFENTRY(FILE *f, LEAFENTRY *le, struct x1764 *checksum, u_int32_t *len) {
assert(0);
u_int8_t state;
int r = toku_fread_u_int8_t (f, &state, crc, len); if (r!=0) return r;
int r = toku_fread_u_int8_t (f, &state, checksum, len); if (r!=0) return r;
TXNID xid;
BYTESTRING a,b,c;
u_int32_t memsize, disksize;
switch ((enum le_state)state) {
case LE_COMMITTED:
r = toku_fread_BYTESTRING(f, &a, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &a, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, checksum, len); if (r!=0) return r;
r = le_committed(a.len, a.data, b.len, b.data,
&memsize, &disksize, le);
toku_free_BYTESTRING(a);
toku_free_BYTESTRING(b);
return r;
case LE_BOTH:
r = toku_fread_TXNID(f, &xid, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &a, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &c, crc, len); if (r!=0) return r;
r = toku_fread_TXNID(f, &xid, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &a, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &c, checksum, len); if (r!=0) return r;
r = le_both(xid, a.len, a.data, b.len, b.data, c.len, c.data,
&memsize, &disksize, le);
toku_free_BYTESTRING(a);
......@@ -176,18 +175,18 @@ int toku_fread_LEAFENTRY(FILE *f, LEAFENTRY *le, u_int32_t *crc, u_int32_t *len)
toku_free_BYTESTRING(c);
return r;
case LE_PROVDEL:
r = toku_fread_TXNID(f, &xid, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &a, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, crc, len); if (r!=0) return r;
r = toku_fread_TXNID(f, &xid, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &a, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, checksum, len); if (r!=0) return r;
r = le_provdel(xid, a.len, a.data, b.len, b.data,
&memsize, &disksize, le);
toku_free_BYTESTRING(a);
toku_free_BYTESTRING(b);
return r;
case LE_PROVPAIR:
r = toku_fread_TXNID(f, &xid, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &a, crc, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, crc, len); if (r!=0) return r;
r = toku_fread_TXNID(f, &xid, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &a, checksum, len); if (r!=0) return r;
r = toku_fread_BYTESTRING(f, &b, checksum, len); if (r!=0) return r;
r = le_provpair(xid, a.len, a.data, b.len, b.data,
&memsize, &disksize, le);
toku_free_BYTESTRING(a);
......@@ -244,9 +243,9 @@ int print_leafentry (FILE *outf, LEAFENTRY v) {
LESWITCHCALL(v, print, outf);
}
int toku_logprint_LEAFENTRY (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format __attribute__((__unused__))) {
int toku_logprint_LEAFENTRY (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format __attribute__((__unused__))) {
LEAFENTRY v;
int r = toku_fread_LEAFENTRY(inf, &v, crc, len);
int r = toku_fread_LEAFENTRY(inf, &v, checksum, len);
if (r!=0) return r;
fprintf(outf, " %s=", fieldname);
print_leafentry(outf, v);
......
newbrt/leafentry.h
View file @ b6e764a0
......@@ -29,6 +29,7 @@
#include "brttypes.h"
#include "rbuf.h"
#include "x1764.h"
#include <arpa/inet.h>
u_int32_t toku_le_crc(LEAFENTRY v);
......@@ -116,8 +117,8 @@ u_int32_t leafentry_disksize (LEAFENTRY le); // this is the same as logsizeof_LE
u_int32_t toku_logsizeof_LEAFENTRY(LEAFENTRY le);
void wbuf_LEAFENTRY(struct wbuf *w, LEAFENTRY le);
void rbuf_LEAFENTRY(struct rbuf *r, u_int32_t *resultsize, u_int32_t *disksize, LEAFENTRY *le);
int toku_fread_LEAFENTRY(FILE *f, LEAFENTRY *le, u_int32_t *crc, u_int32_t *len); // read a leafentry from a log
int toku_logprint_LEAFENTRY(FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format); // read a leafentry from a log and then print it in human-readable form.
int toku_fread_LEAFENTRY(FILE *f, LEAFENTRY *le, struct x1764 *, u_int32_t *len); // read a leafentry from a log
int toku_logprint_LEAFENTRY(FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *format); // read a leafentry from a log and then print it in human-readable form.
void toku_free_LEAFENTRY(LEAFENTRY le);
int print_leafentry (FILE *outf, LEAFENTRY v); // Print a leafentry out in human-readable form.
......
newbrt/log.c
View file @ b6e764a0
This diff is collapsed.
newbrt/log.h
View file @ b6e764a0
......@@ -9,6 +9,7 @@
#include "brttypes.h"
#include "memory.h"
#include "bread.h"
#include "x1764.h"
struct logbytes;
struct logbytes {
......@@ -49,28 +50,28 @@ int toku_logger_log_newbrtnode (TOKUTXN txn, FILENUM filenum, DISKOFF offset, u_
int toku_logger_fsync (TOKULOGGER logger);
int toku_fread_u_int8_t (FILE *f, u_int8_t *v, u_int32_t *crc, u_int32_t *len);
int toku_fread_u_int8_t (FILE *f, u_int8_t *v, struct x1764 *, u_int32_t *len);
int toku_fread_u_int32_t_nocrclen (FILE *f, u_int32_t *v);
int toku_fread_u_int32_t (FILE *f, u_int32_t *v, u_int32_t *crc, u_int32_t *len);
int toku_fread_LSN (FILE *f, LSN *lsn, u_int32_t *crc, u_int32_t *len);
int toku_fread_FILENUM (FILE *f, FILENUM *filenum, u_int32_t *crc, u_int32_t *len);
int toku_fread_DISKOFF (FILE *f, DISKOFF *diskoff, u_int32_t *crc, u_int32_t *len);
int toku_fread_TXNID (FILE *f, TXNID *txnid, u_int32_t *crc, u_int32_t *len);
int toku_fread_u_int32_t (FILE *f, u_int32_t *v, struct x1764 *, u_int32_t *len);
int toku_fread_LSN (FILE *f, LSN *lsn, struct x1764 *, u_int32_t *len);
int toku_fread_FILENUM (FILE *f, FILENUM *filenum, struct x1764 *, u_int32_t *len);
int toku_fread_DISKOFF (FILE *f, DISKOFF *diskoff, struct x1764 *, u_int32_t *len);
int toku_fread_TXNID (FILE *f, TXNID *txnid, struct x1764 *, u_int32_t *len);
// fills in the bs with malloced data.
int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, u_int32_t *crc, u_int32_t *len);
int toku_fread_LOGGEDBRTHEADER(FILE *f, LOGGEDBRTHEADER *v, u_int32_t *crc, u_int32_t *len);
int toku_fread_INTPAIRARRAY (FILE *f, INTPAIRARRAY *v, u_int32_t *crc, u_int32_t *len);
int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_DISKOFF (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_INTPAIRARRAY (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, struct x1764 *, u_int32_t *len);
int toku_fread_LOGGEDBRTHEADER(FILE *f, LOGGEDBRTHEADER *v, struct x1764 *, u_int32_t *len);
int toku_fread_INTPAIRARRAY (FILE *f, INTPAIRARRAY *v, struct x1764 *, u_int32_t *len);
int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_DISKOFF (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
int toku_logprint_INTPAIRARRAY (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *, u_int32_t *len, const char *);
// Useful thing for printing a bytestring.
void toku_print_BYTESTRING (FILE *outf, u_int32_t len, char *data);
......
newbrt/logformat.c
View file @ b6e764a0
......@@ -375,17 +375,17 @@ void generate_log_writer (void) {
void generate_log_reader (void) {
DO_LOGTYPES(lt, ({
fprintf(cf, "static int toku_log_fread_%s (FILE *infile, struct logtype_%s *data, u_int32_t crc)", lt->name, lt->name);
fprintf(cf, "static int toku_log_fread_%s (FILE *infile, struct logtype_%s *data, struct x1764 *checksum)", lt->name, lt->name);
fprintf(cf, " {\n");
fprintf(cf, " int r=0;\n");
fprintf(cf, " u_int32_t actual_len=5; // 1 for the command, 4 for the first len.\n");
fprintf(cf, " r=toku_fread_%-16s(infile, &data->%-16s, &crc, &actual_len); if (r!=0) return r;\n", "LSN", "lsn");
fprintf(cf, " r=toku_fread_%-16s(infile, &data->%-16s, checksum, &actual_len); if (r!=0) return r;\n", "LSN", "lsn");
DO_FIELDS(ft, lt,
fprintf(cf, " r=toku_fread_%-16s(infile, &data->%-16s, &crc, &actual_len); if (r!=0) return r;\n", ft->type, ft->name));
fprintf(cf, " u_int32_t crc_in_file, len_in_file;\n");
fprintf(cf, " r=toku_fread_u_int32_t_nocrclen(infile, &crc_in_file); actual_len+=4; if (r!=0) return r;\n");
fprintf(cf, " r=toku_fread_%-16s(infile, &data->%-16s, checksum, &actual_len); if (r!=0) return r;\n", ft->type, ft->name));
fprintf(cf, " u_int32_t checksum_in_file, len_in_file;\n");
fprintf(cf, " r=toku_fread_u_int32_t_nocrclen(infile, &checksum_in_file); actual_len+=4; if (r!=0) return r;\n");
fprintf(cf, " r=toku_fread_u_int32_t_nocrclen(infile, &len_in_file); actual_len+=4; if (r!=0) return r;\n");
fprintf(cf, " if (crc_in_file!=crc || len_in_file!=actual_len) return DB_BADFORMAT;\n");
fprintf(cf, " if (checksum_in_file!=x1764_finish(checksum) || len_in_file!=actual_len) return DB_BADFORMAT;\n");
fprintf(cf, " return 0;\n");
fprintf(cf, "}\n\n");
}));
......@@ -393,17 +393,19 @@ void generate_log_reader (void) {
fprintf(hf, ";\n");
fprintf(cf, " {\n");
fprintf(cf, " u_int32_t len1; int r;\n");
fprintf(cf, " u_int32_t crc=0,ignorelen=0;\n");
fprintf(cf, " r = toku_fread_u_int32_t(infile, &len1,&crc,&ignorelen); if (r!=0) return r;\n");
fprintf(cf, " u_int32_t ignorelen=0;\n");
fprintf(cf, " struct x1764 checksum;\n");
fprintf(cf, " x1764_init(&checksum);\n");
fprintf(cf, " r = toku_fread_u_int32_t(infile, &len1, &checksum, &ignorelen); if (r!=0) return r;\n");
fprintf(cf, " int cmd=fgetc(infile);\n");
fprintf(cf, " if (cmd==EOF) return EOF;\n");
fprintf(cf, " char cmdchar = cmd;\n");
fprintf(cf, " crc = toku_crc32(crc, &cmdchar, 1);\n");
fprintf(cf, " x1764_add(&checksum, &cmdchar, 1);\n");
fprintf(cf, " le->cmd=cmd;\n");
fprintf(cf, " switch ((enum lt_cmd)cmd) {\n");
DO_LOGTYPES(lt, ({
fprintf(cf, " case LT_%s:\n", lt->name);
fprintf(cf, " return toku_log_fread_%s (infile, &le->u.%s, crc);\n", lt->name, lt->name);
fprintf(cf, " return toku_log_fread_%s (infile, &le->u.%s, &checksum);\n", lt->name, lt->name);
}));
fprintf(cf, " };\n");
fprintf(cf, " return DB_BADFORMAT;\n"); // Should read past the record using the len field.
......@@ -417,14 +419,16 @@ void generate_logprint (void) {
fprintf(cf, " {\n");
fprintf(cf, " int cmd, r;\n");
fprintf(cf, " u_int32_t len1, crc_in_file;\n");
fprintf(cf, " u_int32_t crc = 0, ignorelen=0;\n");
fprintf(cf, " r=toku_fread_u_int32_t(f, &len1, &crc, &ignorelen);\n");
fprintf(cf, " u_int32_t ignorelen=0;\n");
fprintf(cf, " struct x1764 checksum;\n");
fprintf(cf, " x1764_init(&checksum);\n");
fprintf(cf, " r=toku_fread_u_int32_t(f, &len1, &checksum, &ignorelen);\n");
fprintf(cf, " if (r==EOF) return EOF;\n");
fprintf(cf, " cmd=fgetc(f);\n");
fprintf(cf, " if (cmd==EOF) return DB_BADFORMAT;\n");
fprintf(cf, " u_int32_t len_in_file, len=1+4; // cmd + len1\n");
fprintf(cf, " char charcmd = cmd;\n");
fprintf(cf, " crc = toku_crc32(crc, &charcmd, 1);\n");
fprintf(cf, " x1764_add(&checksum, &charcmd, 1);\n");
fprintf(cf, " switch ((enum lt_cmd)cmd) {\n");
DO_LOGTYPES(lt, ({ if (strlen(lt->name)>maxnamelen) maxnamelen=strlen(lt->name); }));
DO_LOGTYPES(lt, ({
......@@ -434,20 +438,23 @@ void generate_logprint (void) {
fprintf(cf, " fprintf(outf, \"%%-%ds \", \"%s\");\n", maxnamelen, lt->name);
if (isprint(cmd)) fprintf(cf," fprintf(outf, \" '%c':\");\n", cmd);
else fprintf(cf," fprintf(outf, \"0%03o:\");\n", cmd);
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"lsn\", &crc, &len, 0); if (r!=0) return r;\n", "LSN");
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"lsn\", &checksum, &len, 0); if (r!=0) return r;\n", "LSN");
DO_FIELDS(ft, lt, ({
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"%s\", &crc, &len,", ft->type, ft->name);
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"%s\", &checksum, &len,", ft->type, ft->name);
if (ft->format) fprintf(cf, "\"%s\"", ft->format);
else fprintf(cf, "0");
fprintf(cf, "); if (r!=0) return r;\n");
}));
fprintf(cf, " {\n");
fprintf(cf, " u_int32_t actual_murmur = x1764_finish(&checksum);\n");
fprintf(cf, " r = toku_fread_u_int32_t_nocrclen (f, &crc_in_file); len+=4; if (r!=0) return r;\n");
fprintf(cf, " fprintf(outf, \" crc=%%08x\", crc_in_file);\n");
fprintf(cf, " if (crc_in_file!=crc) fprintf(outf, \" actual_crc=%%08x\", crc);\n");
fprintf(cf, " if (crc_in_file!=actual_murmur) fprintf(outf, \" actual_fingerprint=%%08x\", actual_murmur);\n");
fprintf(cf, " r = toku_fread_u_int32_t_nocrclen (f, &len_in_file); len+=4; if (r!=0) return r;\n");
fprintf(cf, " fprintf(outf, \" len=%%d\", len_in_file);\n");
fprintf(cf, " if (len_in_file!=len) fprintf(outf, \" actual_len=%%d\", len);\n");
fprintf(cf, " if (len_in_file!=len || crc_in_file!=crc) return DB_BADFORMAT;\n");
fprintf(cf, " if (len_in_file!=len || crc_in_file!=actual_murmur) return DB_BADFORMAT;\n");
fprintf(cf, " };\n");
fprintf(cf, " fprintf(outf, \"\\n\");\n");
fprintf(cf, " return 0;;\n\n");
}));
......
newbrt/recover.c
View file @ b6e764a0
......@@ -288,7 +288,7 @@ void toku_recover_brtdeq (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t c
r = toku_fifo_peek(BNC_BUFFER(node, childnum), &actual_key, &actual_keylen, &actual_data, &actual_datalen, &actual_type, &actual_xid);
assert(r==0);
u_int32_t sizediff = actual_keylen + actual_datalen + KEY_VALUE_OVERHEAD + BRT_CMD_OVERHEAD;
node->local_fingerprint -= node->rand4fingerprint * toku_calccrc32_cmd(actual_type, actual_xid, actual_key, actual_keylen, actual_data, actual_datalen);
node->local_fingerprint -= node->rand4fingerprint * toku_calc_fingerprint_cmd(actual_type, actual_xid, actual_key, actual_keylen, actual_data, actual_datalen);
node->log_lsn = lsn;
node->u.n.n_bytes_in_buffers -= sizediff;
BNC_NBYTESINBUF(node, childnum) -= sizediff;
......@@ -306,7 +306,7 @@ void toku_recover_brtenq (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t c
//printf("enq: %lld expected_old_fingerprint=%08x actual=%08x new=%08x\n", diskoff, oldfingerprint, node->local_fingerprint, newfingerprint);
r = toku_fifo_enq(BNC_BUFFER(node, childnum), key.data, key.len, data.data, data.len, typ, xid);
assert(r==0);
node->local_fingerprint += node->rand4fingerprint * toku_calccrc32_cmd(typ, xid, key.data, key.len, data.data, data.len);
node->local_fingerprint += node->rand4fingerprint * toku_calc_fingerprint_cmd(typ, xid, key.data, key.len, data.data, data.len);
node->log_lsn = lsn;
u_int32_t sizediff = key.len + data.len + KEY_VALUE_OVERHEAD + BRT_CMD_OVERHEAD;
r = toku_cachetable_unpin(cf, diskoff, node->fullhash, 1, toku_serialize_brtnode_size(node));
......
newbrt/tests/Makefile
View file @ b6e764a0
......@@ -33,7 +33,7 @@ FORMAT=-Wmissing-format-attribute
endif
CFLAGS = -Wall -Wextra -Wcast-align -Wbad-function-cast -Wmissing-noreturn $(FORMAT) $(OPTFLAGS) -g3 -ggdb3 $(GCOV_FLAGS) $(PROF_FLAGS) -Werror $(FPICFLAGS) $(SHADOW) $(VISIBILITY)
LDFLAGS = $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS) -lz
LDFLAGS = $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS)
CPPFLAGS += -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE=500 -I..
# Add -Wconversion
......@@ -87,6 +87,7 @@ REGRESSION_TESTS = \
test-leafentry \
test_oexcl \
test_toku_malloc_plain_free \
x1764-test \
ybt-test \
# This line intentially kept commented so I can have a \ on the end of the previous line
......@@ -97,7 +98,6 @@ BINS = $(REGRESSION_TESTS) \
# This line intentially kept commented so I can have a \ on the end of the previous line
CHECKS = \
load_version_6 \
benchmarktest_256 \
test-assertA test-assertB \
$(REGRESSION_TESTS) \
......@@ -113,10 +113,6 @@ check_fail:
check_ok:
test 0 = 0 $(SUMMARIZE_CMD)
check_load_version_6:
(cp bench.db.ver6 bench.db.ver6.tmp && \
$(VGRIND) ../brtdump bench.db.ver6.tmp > /dev/null ) $(SUMMARIZE_CMD)
check_benchmarktest_256: benchmark-test
$(VGRIND) ./benchmark-test $(VERBVERBOSE) --valsize 256 --verify 1 $(SUMMARIZE_CMD)
......
newbrt/tests/brt-serialize-test.c
View file @ b6e764a0
......@@ -28,7 +28,7 @@ static void test_serialize(void) {
sn.thisnodename = sn.nodesize*20;
sn.disk_lsn.lsn = 789;
sn.log_lsn.lsn = 123456;
sn.layout_version = BRT_LAYOUT_VERSION_7;
sn.layout_version = BRT_LAYOUT_VERSION;
sn.height = 1;
sn.rand4fingerprint = randval;
sn.local_fingerprint = 0;
......@@ -46,9 +46,9 @@ static void test_serialize(void) {
BNC_SUBTREE_LEAFENTRY_ESTIMATE(&sn, 1) = random() + (((long long)random())<<32);
r = toku_fifo_create(&BNC_BUFFER(&sn,0)); assert(r==0);
r = toku_fifo_create(&BNC_BUFFER(&sn,1)); assert(r==0);
r = toku_fifo_enq(BNC_BUFFER(&sn,0), "a", 2, "aval", 5, BRT_NONE, (TXNID)0); assert(r==0); sn.local_fingerprint += randval*toku_calccrc32_cmd(BRT_NONE, (TXNID)0, "a", 2, "aval", 5);
r = toku_fifo_enq(BNC_BUFFER(&sn,0), "b", 2, "bval", 5, BRT_NONE, (TXNID)123); assert(r==0); sn.local_fingerprint += randval*toku_calccrc32_cmd(BRT_NONE, (TXNID)123, "b", 2, "bval", 5);
r = toku_fifo_enq(BNC_BUFFER(&sn,1), "x", 2, "xval", 5, BRT_NONE, (TXNID)234); assert(r==0); sn.local_fingerprint += randval*toku_calccrc32_cmd(BRT_NONE, (TXNID)234, "x", 2, "xval", 5);
r = toku_fifo_enq(BNC_BUFFER(&sn,0), "a", 2, "aval", 5, BRT_NONE, (TXNID)0); assert(r==0); sn.local_fingerprint += randval*toku_calc_fingerprint_cmd(BRT_NONE, (TXNID)0, "a", 2, "aval", 5);
r = toku_fifo_enq(BNC_BUFFER(&sn,0), "b", 2, "bval", 5, BRT_NONE, (TXNID)123); assert(r==0); sn.local_fingerprint += randval*toku_calc_fingerprint_cmd(BRT_NONE, (TXNID)123, "b", 2, "bval", 5);
r = toku_fifo_enq(BNC_BUFFER(&sn,1), "x", 2, "xval", 5, BRT_NONE, (TXNID)234); assert(r==0); sn.local_fingerprint += randval*toku_calc_fingerprint_cmd(BRT_NONE, (TXNID)234, "x", 2, "xval", 5);
BNC_NBYTESINBUF(&sn, 0) = 2*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
sn.u.n.n_bytes_in_buffers = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
......@@ -60,7 +60,7 @@ static void test_serialize(void) {
assert(dn->thisnodename==nodesize*20);
assert(dn->disk_lsn.lsn==123456);
assert(dn->layout_version ==BRT_LAYOUT_VERSION_7);
assert(dn->layout_version ==BRT_LAYOUT_VERSION);
assert(dn->height == 1);
assert(dn->rand4fingerprint==randval);
assert(dn->u.n.n_children==2);
......
newbrt/wbuf.h
View file @ b6e764a0
......@@ -3,18 +3,13 @@
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#include "x1764.h"
#include "memory.h"
#include "toku_assert.h"
#include <errno.h>
#include <string.h>
//#define CRC_NO
#define CRC_INCR
//#define CRC_ATEND
#ifndef CRC_NO
#include "crc.h"
#endif
/* When serializing a value, write it into a buffer. */
/* This code requires that the buffer be big enough to hold whatever you put into it. */
......@@ -24,27 +19,21 @@ struct wbuf {
unsigned char *buf;
unsigned int size;
unsigned int ndone;
#ifdef CRC_INCR
u_int32_t crc32; // A 32-bit CRC of everything written so foar.
#endif
struct x1764 checksum; // The checksumx state
};
static inline void wbuf_init (struct wbuf *w, void *buf, DISKOFF size) {
w->buf=buf;
w->size=size;
w->ndone=0;
#ifdef CRC_INCR
w->crc32 = toku_crc32(toku_null_crc, Z_NULL, 0);
#endif
x1764_init(&w->checksum);
}
/* Write a character. */
static inline void wbuf_char (struct wbuf *w, unsigned int ch) {
assert(w->ndone<w->size);
w->buf[w->ndone++]=ch;
#ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone-1], 1);
#endif
x1764_add(&w->checksum, &w->buf[w->ndone-1], 1);
}
static void wbuf_int (struct wbuf *w, int32_t i) {
......@@ -63,9 +52,7 @@ static void wbuf_int (struct wbuf *w, int32_t i) {
#else
*(u_int32_t*)(&w->buf[w->ndone]) = htonl(i);
#endif
#ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], 4);
#endif
x1764_add(&w->checksum, &w->buf[w->ndone], 4);
w->ndone += 4;
#endif
}
......@@ -80,9 +67,7 @@ static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, u_int32_
#else
assert(w->ndone + nbytes <= w->size);
memcpy(w->buf + w->ndone, bytes, (size_t)nbytes);
#ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], nbytes);
#endif
x1764_add(&w->checksum, &w->buf[w->ndone], nbytes);
w->ndone += nbytes;
#endif
......
src/Makefile
View file @ b6e764a0
......@@ -82,7 +82,7 @@ RANGETREE_BINS = range_tree/rangetree.o
LOCKTREE_BINS = lock_tree/locktree.o lock_tree/rth.o lock_tree/lth.o lock_tree/idlth.o lock_tree/db_id.o $(RANGETREE_BINS)
$(LIBRARY): $(DBBINS) | buildlocktrees
$(CC) $(CPPFLAGS) $^ $(LOCKTREE_BINS) $(SHARED) -o $@ $(CFLAGS) -lz $(RPATHNAME)
$(CC) $(CPPFLAGS) $^ $(LOCKTREE_BINS) $(SHARED) -o $@ $(CFLAGS) $(RPATHNAME)
$(LIBNAME).a: $(DBBINS) | buildlocktrees
$(AR) cr $@ $^ $(LOCKTREE_BINS)
......@@ -90,7 +90,7 @@ $(LIBNAME).a: $(DBBINS) | buildlocktrees
$(LIBNAME).a(ydb.o): ydb.o
$(TLIBRARY): $(TDBBINS) | buildlocktrees
$(CC) $(CPPFLAGS) $^ $(LOCKTREE_BINS) $(SHARED) -o $@ $(CFLAGS) -lz $(RPATHNAME)
$(CC) $(CPPFLAGS) $^ $(LOCKTREE_BINS) $(SHARED) -o $@ $(CFLAGS) $(RPATHNAME)
$(TLIBNAME).a: $(TDBBINS) | buildlocktrees
$(AR) cr $@ $^ $(LOCKTREE_BINS)
......
src/Makefile.include
View file @ b6e764a0
......@@ -21,4 +21,4 @@ $(ROOT)lib/libtokudb.a: $(SRC_OFILES)
$(AR) rv $@ $^
$(ROOT)lib/libtokudb.$(LIBEXT): $(SRC_OFILES)
cc $^ $(SHARED) -o $@ $(CFLAGS) -lz $(RPATHNAME)
cc $^ $(SHARED) -o $@ $(CFLAGS) $(RPATHNAME)
src/lock_tree/tests/Makefile
View file @ b6e764a0
......@@ -118,13 +118,13 @@ foo:
echo ../locktree.h test.h $(LT_LINEAR)
%.lin: %.c ../locktree.h test.h $(LT_LINEAR)
$(CC) -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR) $(NEWBRT_BINS) -lz
$(CC) -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR) $(NEWBRT_BINS)
%.tlin: %.c ../locktree.h test.h $(LT_TLINEAR)
$(CC) -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS -lz
$(CC) -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.tlog: %.c ../locktree.h test.h $(LT_TLOG)
$(CC) -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS -lz
$(CC) -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.log: %.c ../locktree.h test.h $(LT_LOG)
$(CC) -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG) $(NEWBRT_BINS) -lz
$(CC) -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG) $(NEWBRT_BINS)
clean:
rm -f $(ALL_TESTS) *.o *.gcno *.gcda *.gcov
......
src/range_tree/tests/Makefile
View file @ b6e764a0
......@@ -15,7 +15,7 @@ OPTFLAGS=-O0
# GCOV_FLAGS = -fprofile-arcs -ftest-coverage
CFLAGS = -W -Wall -Wextra -Werror $(OPTFLAGS) -g3 -ggdb3 $(GCOV_FLAGS)
CFLAGS += -Wbad-function-cast -Wcast-align -Wconversion -Waggregate-return
CFLAGS += -Wmissing-noreturn -Wmissing-format-attribute -lz
CFLAGS += -Wmissing-noreturn -Wmissing-format-attribute
CPPFLAGS += -I../ -I../../../newbrt -I../../../include
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7