Commit e5a97fb2 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul Committed by Yoni Fogel

Merge all mainline patches into the 1195 branch. Fixes #1243.

Details:  I merged with
{{{
svn merge -r6585:7747 https://svn.tokutek.com/tokudb/toku/tokudb
}}}
then resolved conflicts.  The most interesting conflict was the double inclusion of the {{{BLOCK_ALLOCATOR_HEADER_RESERVE}}} for {{{brt-internal.h}}}.   The subversion merger got confused because the patch was applied with an extra space at the end of each line.


git-svn-id: file:///svn/toku/tokudb.1243@7750 c7de825b-a66e-492c-adef-691d508d4ae1
parent 222e3a36
......@@ -7,6 +7,10 @@ LDFLAGS = -lz
SRCS = $(wildcard *.cpp)
OBJS = $(patsubst %.cpp, %.o, $(SRCS))
ifeq ($(CC),icc)
CXXFLAGS += -diag-disable 981
endif
LIBNAME = libtokudb_cxx
default: install build
......@@ -18,6 +22,7 @@ check:
install: $(LIBNAME).a
cp $< ../lib/
$(OBJS): ../include/db_cxx.h
test1: test1.o dbt.o db.o dbenv.o ../lib/libdb.a
$(LIBNAME).a: $(OBJS)
......
......@@ -20,6 +20,10 @@ CPPFLAGS = -I../ -I../../include
CXXFLAGS = -Wall $(OPTFLAGS) -g $(GCOV_FLAGS)
LDLIBS = ../../lib/libtokudb_cxx.a ../../lib/libtokudb.a -lz -lpthread
ifeq ($(CC),icc)
CXXFLAGS += -diag-disable 981
endif
ifneq ($(OSX),)
VGRIND=
else
......
......@@ -20,6 +20,10 @@ OPTFLAGS = -O2
CFLAGS = -Wall -Werror -g $(OPTFLAGS) $(GCOV_FLAGS) $(PROF_FLAGS)
# CFLAGS += -pg
ifeq ($(CC),icc)
CFLAGS += -diag-disable 981
endif
LDFLAGS += -lpthread
ifdef BDBDIR
BDB_CPPFLAGS = -I$(BDBDIR)/include
......@@ -33,7 +37,7 @@ TDB_LDFLAGS = -L../lib -ltokudb -Wl,-rpath,$(PWD)/../lib -lpthread -lz
TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) $(TARGET_TDB) scanscan-tokudb
TARGETS = $(TARGET_BDB) scanscan-bdb $(TARGET_TDB) scanscan-tokudb
default: build
build: $(TARGETS)
......@@ -74,6 +78,8 @@ scanscan-tokudb: scanscan.c
endif
db-benchmark-test-bdb: db-benchmark-test.c
$(CC) $(CFLAGS) $(BDB_CPPFLAGS) $(BDB_LDFLAGS) $< -o $@ -DDIRSUF=bdb
scanscan-bdb: scanscan.c
$(CC) $(CFLAGS) $(BDB_CPPFLAGS) $(BDB_LDFLAGS) $< -o $@ -DDIRSUF=bdb
PARGS =
ptest0 ptest1 ptest2 ptest3 ptest4 ptest5 ptest6 ptest7: db-benchmark-test-tokudb
......
......@@ -53,13 +53,15 @@ static void do_prelock(DB* db, DB_TXN* txn) {
#if !defined(NO_DB_PRELOCKED)
int r = db->pre_acquire_table_lock(db, txn);
assert(r==0);
#else
db = db; txn = txn;
#endif
}
}
#define STRINGIFY2(s) #s
#define STRINGIFY(s) STRINGIFY2(s)
const char *dbdir = "./bench." STRINGIFY(DIRSUF) "/"; /* DIRSUF is passed in as a -D argument to the compiler. */;
const char *dbdir = "./bench." STRINGIFY(DIRSUF) "/"; /* DIRSUF is passed in as a -D argument to the compiler. */
char *dbfilename = "bench.db";
char *dbname;
......@@ -68,7 +70,7 @@ DB *db;
DB_TXN *tid=0;
void setup (void) {
static void setup (void) {
int r;
{
......@@ -133,7 +135,7 @@ void setup (void) {
}
void shutdown (void) {
static void shutdown (void) {
int r;
if (do_transactions && singlex) {
......@@ -146,13 +148,13 @@ void shutdown (void) {
assert(r == 0);
}
void long_long_to_array (unsigned char *a, int array_size, unsigned long long l) {
static void long_long_to_array (unsigned char *a, int array_size, unsigned long long l) {
int i;
for (i=0; i<8 && i<array_size; i++)
a[i] = (l>>(56-8*i))&0xff;
}
DBT *fill_dbt(DBT *dbt, const void *data, int size) {
static DBT *fill_dbt(DBT *dbt, const void *data, int size) {
memset(dbt, 0, sizeof *dbt);
dbt->size = size;
dbt->data = (void *) data;
......@@ -160,7 +162,7 @@ DBT *fill_dbt(DBT *dbt, const void *data, int size) {
}
// Fill array with 0's if compressibilty==-1, otherwise fill array with data that is likely to compress by a factor of compressibility.
void fill_array (unsigned char *data, int size) {
static void fill_array (unsigned char *data, int size) {
memset(data, 0, size);
if (compressibility>0) {
int i;
......@@ -170,7 +172,7 @@ void fill_array (unsigned char *data, int size) {
}
}
void insert (long long v) {
static void insert (long long v) {
unsigned char kc[keysize], vc[valsize];
DBT kt, vt;
fill_array(kc, sizeof kc);
......@@ -191,7 +193,7 @@ void insert (long long v) {
}
}
void serial_insert_from (long long from) {
static void serial_insert_from (long long from) {
long long i;
if (do_transactions && !singlex) {
int r = dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
......@@ -212,11 +214,11 @@ void serial_insert_from (long long from) {
}
}
long long llrandom (void) {
static long long llrandom (void) {
return (((long long)(random()))<<32) + random();
}
void random_insert_below (long long below) {
static void random_insert_below (long long below) {
long long i;
if (do_transactions && !singlex) {
int r = dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
......@@ -231,11 +233,11 @@ void random_insert_below (long long below) {
}
}
double tdiff (struct timeval *a, struct timeval *b) {
static double tdiff (struct timeval *a, struct timeval *b) {
return (a->tv_sec-b->tv_sec)+1e-6*(a->tv_usec-b->tv_usec);
}
void biginsert (long long n_elements, struct timeval *starttime) {
static void biginsert (long long n_elements, struct timeval *starttime) {
long long i;
struct timeval t1,t2;
int iteration;
......@@ -261,7 +263,7 @@ void biginsert (long long n_elements, struct timeval *starttime) {
const long long default_n_items = 1LL<<22;
int print_usage (const char *argv0) {
static int print_usage (const char *argv0) {
fprintf(stderr, "Usage:\n");
fprintf(stderr, " %s [-x] [--keysize KEYSIZE] [--valsize VALSIZE] [--noserial] [--norandom] [ n_iterations ]\n", argv0);
fprintf(stderr, " where\n");
......@@ -389,7 +391,7 @@ int main (int argc, const char *argv[]) {
printf("Total time %9.6fs for %lld insertions = %8.0f/s\n", tdiff(&t3, &t1),
(!noserial+!norandom)*total_n_items, (!noserial+!norandom)*total_n_items/tdiff(&t3, &t1));
}
#ifdef TOKUDB
#if 0 && defined TOKUDB
if (verbose) {
extern unsigned long toku_get_maxrss(void);
printf("maxrss=%.2fMB\n", toku_get_maxrss()/256.0);
......
......@@ -16,7 +16,7 @@ u_int32_t lock_flag = 0;
long limitcount=-1;
u_int32_t cachesize = 127*1024*1024;
void parse_args (int argc, const char *argv[]) {
static void parse_args (int argc, const char *argv[]) {
pname=argv[0];
argc--;
argv++;
......@@ -32,8 +32,12 @@ void parse_args (int argc, const char *argv[]) {
if (specified_run_mode && run_mode!=RUN_VERIFY) goto two_modes;
run_mode = RUN_HWC;
} else if (strcmp(*argv, "--prelock")==0) prelock=1;
else if (strcmp(*argv, "--prelockflag")==0) { prelockflag=1; lock_flag = DB_PRELOCKED; }
#if defined(DB_PRELOCKED)
else if (strcmp(*argv, "--prelockflag")==0) { prelockflag=1; lock_flag = DB_PRELOCKED; }
#endif
#if defined(DB_PRELOCKED_WRITE)
else if (strcmp(*argv, "--prelockwriteflag")==0) { prelockflag=1; lock_flag = DB_PRELOCKED_WRITE; }
#endif
else if (strcmp(*argv, "--nox")==0) { do_txns=0; }
else if (strcmp(*argv, "--count")==0) {
char *end;
......@@ -69,12 +73,12 @@ DB_TXN *tid=0;
#define STRINGIFY2(s) #s
#define STRINGIFY(s) STRINGIFY2(s)
const char *dbdir = "./bench." STRINGIFY(DIRSUF) "/"; /* DIRSUF is passed in as a -D argument to the compiler. */;
const char *dbdir = "./bench." STRINGIFY(DIRSUF) "/"; /* DIRSUF is passed in as a -D argument to the compiler. */
int env_open_flags_yesx = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOG|DB_INIT_LOCK;
int env_open_flags_nox = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL;
char *dbfilename = "bench.db";
void setup (void) {
static void setup (void) {
int r;
r = db_env_create(&env, 0); assert(r==0);
r = env->set_cachesize(env, 0, cachesize, 1); assert(r==0);
......@@ -84,6 +88,7 @@ void setup (void) {
r = env->txn_begin(env, 0, &tid, 0); assert(r==0);
}
r = db->open(db, tid, dbfilename, NULL, DB_BTREE, 0, 0644); assert(r==0);
#if defined(DB_PRELOCKED)
if (prelock) {
r = db->pre_acquire_read_lock(db,
tid,
......@@ -91,29 +96,32 @@ void setup (void) {
db->dbt_pos_infty(), db->dbt_pos_infty());
assert(r==0);
}
#endif
}
void shutdown (void) {
static void shutdown (void) {
int r;
r = db->close(db, 0); assert(r==0);
if (do_txns) {
r = tid->commit(tid, 0); assert(r==0);
}
r = db->close(db, 0); assert(r==0);
r = env->close(env, 0); assert(r==0);
#if 0
{
extern unsigned long toku_get_maxrss(void);
printf("maxrss=%.2fMB\n", toku_get_maxrss()/256.0);
}
#endif
}
double gettime (void) {
static double gettime (void) {
struct timeval tv;
int r = gettimeofday(&tv, 0);
assert(r==0);
return tv.tv_sec + 1e-6*tv.tv_usec;
}
void scanscan_hwc (void) {
static void scanscan_hwc (void) {
int r;
int counter=0;
for (counter=0; counter<2; counter++) {
......@@ -141,17 +149,20 @@ void scanscan_hwc (void) {
}
}
#if defined(TOKUDB)
struct extra_count {
long long totalbytes;
int rowcounter;
};
void counttotalbytes (DBT const *key, DBT const *data, void *extrav) {
static void counttotalbytes (DBT const *key, DBT const *data, void *extrav) {
struct extra_count *e=extrav;
e->totalbytes += key->size + data->size;
e->rowcounter++;
}
void scanscan_lwc (void) {
static void scanscan_lwc (void) {
int r;
int counter=0;
for (counter=0; counter<2; counter++) {
......@@ -175,12 +186,17 @@ void scanscan_lwc (void) {
}
}
#endif
struct extra_verify {
long long totalbytes;
int rowcounter;
DBT k,v; // the k and v are gotten using the old cursor
};
void checkbytes (DBT const *key, DBT const *data, void *extrav) {
#if defined(TOKUDB)
static void checkbytes (DBT const *key, DBT const *data, void *extrav) {
struct extra_verify *e=extrav;
e->totalbytes += key->size + data->size;
e->rowcounter++;
......@@ -192,8 +208,9 @@ void checkbytes (DBT const *key, DBT const *data, void *extrav) {
assert(e->v.data != data->data);
}
#endif
void scanscan_verify (void) {
static void scanscan_verify (void) {
int r;
int counter=0;
for (counter=0; counter<2; counter++) {
......@@ -213,11 +230,14 @@ void scanscan_verify (void) {
c_get_flags |= lock_flag;
}
while (1) {
int r1,r2;
r2 = dbc1->c_get(dbc1, &v.k, &v.v, c_get_flags);
r1 = dbc2->c_getf_next(dbc2, f_flags, checkbytes, &v);
int r2 = dbc1->c_get(dbc1, &v.k, &v.v, c_get_flags);
#if defined(TOKUDB)
int r1 = dbc2->c_getf_next(dbc2, f_flags, checkbytes, &v);
assert(r1==r2);
if (r1) break;
#else
if (r2) break;
#endif
}
r = dbc1->c_close(dbc1); assert(r==0);
r = dbc2->c_close(dbc2); assert(r==0);
......@@ -234,15 +254,17 @@ int main (int argc, const char *argv[]) {
setup();
switch (run_mode) {
case RUN_HWC: scanscan_hwc(); goto ok;
case RUN_LWC: scanscan_lwc(); goto ok;
case RUN_VERIFY: scanscan_verify(); goto ok;
case RUN_HWC: scanscan_hwc(); break;
#if defined(TOKUDB)
case RUN_LWC: scanscan_lwc(); break;
#endif
case RUN_VERIFY: scanscan_verify(); break;
default: assert(0);
}
assert(0);
ok:
shutdown();
#ifdef TOKUDB
#if 0 && defined TOKUDB
if (0) {
extern void print_hash_histogram (void) __attribute__((__visibility__("default")));
print_hash_histogram();
......
......@@ -283,10 +283,9 @@ void toku_brtheader_free (struct brt_header *h);
int toku_brtheader_close (CACHEFILE cachefile, void *header_v);
#define BLOCK_ALLOCATOR_ALIGNMENT 4096
// How much must be reserved at the beginning for the block?
// The actual header is 8+4+4+8+8_4+8+ the length of the db names + 1 pointer for each root.
// So 4096 should be enough.
#define BLOCK_ALLOCATOR_HEADER_RESERVE 4096
// How much must be reserved at the beginning for the block?
// The actual header is 8+4+4+8+8_4+8+ the length of the db names + 1 pointer for each root.
// So 4096 should be enough.
#define BLOCK_ALLOCATOR_HEADER_RESERVE 4096
#endif
......@@ -167,7 +167,7 @@ static void
dump_block_translation(struct brt_header *h, u_int64_t offset) {
if (offset < h->translated_blocknum_limit) {
struct block_translation_pair *bx = &h->block_translation[offset];
printf("%"PRIu64": %"PRId64" %"PRId64"\n", offset, bx->diskoff, bx->size);
printf("%" PRIu64 ": %" PRId64 " %" PRId64 "\n", offset, bx->diskoff, bx->size);
}
}
......@@ -208,11 +208,11 @@ dump_fragmentation(int f, struct brt_header *h) {
fragsizes += bx[i+1].diskoff - (bx[i].diskoff + bx[i].size);
}
free(bx);
printf("translated_blocknum_limit: %"PRIu64"\n", h->translated_blocknum_limit);
printf("leafblocks: %"PRIu64"\n", leafblocks);
printf("blocksizes: %"PRIu64"\n", blocksizes);
printf("leafsizes: %"PRIu64"\n", leafsizes);
printf("fragsizes: %"PRIu64"\n", fragsizes);
printf("translated_blocknum_limit: %" PRIu64 "\n", h->translated_blocknum_limit);
printf("leafblocks: %" PRIu64 "\n", leafblocks);
printf("blocksizes: %" PRIu64 "\n", blocksizes);
printf("leafsizes: %" PRIu64 "\n", leafsizes);
printf("fragsizes: %" PRIu64 "\n", fragsizes);
printf("fragmentation: %.1f%%\n", 100. * ((double)fragsizes / (double)(fragsizes + blocksizes)));
}
......
......@@ -980,13 +980,13 @@ int toku_maybe_spill_rollbacks (TOKUTXN txn) {
assert((ssize_t)w.ndone==bufsize);
txn->oldest_logentry = txn->newest_logentry = 0;
if (txn->rollentry_fd<0) {
const char filenamepart[] = "/__rolltmp.XXXXXX";
int fnamelen = strlen(txn->logger->directory)+sizeof(filenamepart);
const char filenamepart[] = "/__rolltmp.";
int fnamelen = strlen(txn->logger->directory)+sizeof(filenamepart)+16;
assert(txn->rollentry_filename==0);
txn->rollentry_filename = toku_malloc(fnamelen);
if (txn->rollentry_filename==0) return errno;
snprintf(txn->rollentry_filename, fnamelen, "%s%s", txn->logger->directory, filenamepart);
txn->rollentry_fd = mkstemp(txn->rollentry_filename);
snprintf(txn->rollentry_filename, fnamelen, "%s%s%.16"PRIx64, txn->logger->directory, filenamepart, txn->txnid64);
txn->rollentry_fd = open(txn->rollentry_filename, O_CREAT+O_RDWR+O_EXCL, 0600);
if (txn->rollentry_fd==-1) return errno;
}
ssize_t r = write_it(txn->rollentry_fd, buf, w.ndone);
......
......@@ -221,7 +221,7 @@ int toku_omt_iterate(OMT omt, int (*f)(OMTVALUE, u_int32_t, void*), void*v);
// Performance: time=O(i+\log N) where i is the number of times f is called, and N is the number of elements in omt.
// Rational: Although the functional iterator requires defining another function (as opposed to C++ style iterator), it is much easier to read.
int toku_omt_insert_at(OMT omt, OMTVALUE value, u_int32_t index);
int toku_omt_insert_at(OMT omt, OMTVALUE value, u_int32_t idx);
// Effect: Increases indexes of all items at slot >= index by 1.
// Insert value into the position at index.
//
......@@ -233,7 +233,7 @@ int toku_omt_insert_at(OMT omt, OMTVALUE value, u_int32_t index);
// Performance: time=O(\log N) amortized time.
// Rationale: Some future implementation may be O(\log N) worst-case time, but O(\log N) amortized is good enough for now.
int toku_omt_set_at (OMT omt, OMTVALUE value, u_int32_t index);
int toku_omt_set_at (OMT omt, OMTVALUE value, u_int32_t idx);
// Effect: Replaces the item at index with value.
// Returns:
// 0 success
......@@ -242,7 +242,7 @@ int toku_omt_set_at (OMT omt, OMTVALUE value, u_int32_t index);
// Performance: time=O(\log N)
// Rationale: The BRT needs to be able to replace a value with another copy of the same value (allocated in a different location)
int toku_omt_insert(OMT omt, OMTVALUE value, int(*h)(OMTVALUE, void*v), void *v, u_int32_t *index);
int toku_omt_insert(OMT omt, OMTVALUE value, int(*h)(OMTVALUE, void*v), void *v, u_int32_t *idx);
// Effect: Insert value into the OMT.
// If there is some i such that $h(V_i, v)=0$ then returns DB_KEYEXIST.
// Otherwise, let i be the minimum value such that $h(V_i, v)>0$.
......@@ -260,7 +260,7 @@ int toku_omt_insert(OMT omt, OMTVALUE value, int(*h)(OMTVALUE, void*v), void *v,
// Performance: time=O(\log N) amortized.
// Rationale: Some future implementation may be O(\log N) worst-case time, but O(\log N) amortized is good enough for now.
int toku_omt_delete_at(OMT omt, u_int32_t index);
int toku_omt_delete_at(OMT omt, u_int32_t idx);
// Effect: Delete the item in slot index.
// Decreases indexes of all items at slot >= index by 1.
// Returns
......@@ -270,7 +270,7 @@ int toku_omt_delete_at(OMT omt, u_int32_t index);
// Rationale: To delete an item, first find its index using toku_omt_find, then delete it.
// Performance: time=O(\log N) amortized.
void toku_omt_cursor_set_index(OMTCURSOR c, u_int32_t index);
void toku_omt_cursor_set_index(OMTCURSOR c, u_int32_t idx);
// Effect:
// Set the abstract index.
// Requires:
......@@ -291,14 +291,14 @@ int toku_omt_fetch (OMT V, u_int32_t i, OMTVALUE *v, OMTCURSOR c);
// function, the function must remove c's association with the old
// OMT, and associate it with the new OMT.
int toku_omt_find_zero(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, OMTVALUE *value, u_int32_t *index, OMTCURSOR c);
int toku_omt_find_zero(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, OMTVALUE *value, u_int32_t *idx, OMTCURSOR c);
// Effect: Find the smallest i such that h(V_i, extra)>=0
// If there is such an i and h(V_i,extra)==0 then set *index=i and return 0.
// If there is such an i and h(V_i,extra)>0 then set *index=i and return DB_NOTFOUND.
// If there is no such i then set *index=toku_omt_size(V) and return DB_NOTFOUND.
// Requires: index!=NULL
int toku_omt_find(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, int direction, OMTVALUE *value, u_int32_t *index, OMTCURSOR c);
int toku_omt_find(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, int direction, OMTVALUE *value, u_int32_t *idx, OMTCURSOR c);
// Effect:
// If direction >0 then find the smallest i such that h(V_i,extra)>0.
// If direction <0 then find the largest i such that h(V_i,extra)<0.
......@@ -359,7 +359,7 @@ int toku_omt_find(OMT V, int (*h)(OMTVALUE, void*extra), void*extra, int directi
// -...-0...0+...+
// AC B
int toku_omt_split_at(OMT omt, OMT *newomt, u_int32_t index);
int toku_omt_split_at(OMT omt, OMT *newomt, u_int32_t idx);
// Effect: Create a new OMT, storing it in *newomt.
// The values to the right of index (starting at index) are moved to *newomt.
// Requires: omt != NULL
......@@ -422,7 +422,7 @@ int toku_omt_cursor_next (OMTCURSOR c, OMTVALUE *v);
// Performance: time=O(log N) worst case, expected time=O(1) for a randomly
// chosen initial position.
int toku_omt_cursor_current_index(OMTCURSOR c, u_int32_t *index);
int toku_omt_cursor_current_index(OMTCURSOR c, u_int32_t *idx);
// Effect: Stores c's offset in *index.
// Requires: index != NULL
// Returns
......
......@@ -230,11 +230,13 @@ static void test_chaining (void) {
test_mutex_destroy();
}
#if 0
static void __attribute__((__noreturn__))
usage (const char *progname) {
fprintf(stderr, "Usage:\n %s [-v] [-q]\n", progname);
exit(1);
}
#endif
int main (int argc, const char *argv[]) {
default_parse_args(argc, argv);
......
......@@ -29,6 +29,7 @@ void x1764_init(struct x1764 *l) {
l->input=0;
l->n_input_bytes=0;
}
void x1764_add (struct x1764 *l, const void *vbuf, int len) {
if (PRINT) printf("%d: n_input_bytes=%d len=%d\n", __LINE__, l->n_input_bytes, len);
int n_input_bytes = l->n_input_bytes;
......
......@@ -22,13 +22,13 @@ size_from (u_int32_t gbytes, u_int32_t bytes) {
return ((u_int64_t)gbytes << 30) + bytes;
}
static void
static inline void
size_to (u_int64_t s, u_int32_t *gbytes, u_int32_t *bytes) {
*gbytes = s >> 30;
*bytes = s & ((1<<30) - 1);
}
static void
static inline void
expect_le (u_int64_t a, u_int32_t gbytes, u_int32_t bytes) {
u_int64_t b = size_from(gbytes, bytes);
if (a != b && verbose)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment