Added DB_UNKNOWN to db.h

You can use DB_UNKNOWN as a dbtype (as long as you don't use DB_EXCL) in dbopen,
and then it will load the flags from the file.

tokudb_dump
    Now supports duplicates/sorted duplicates.
tokudb_load will give a warning about any duplicate duplicate loads
    but will continue with the rest of the data.

Closes #148
Closes #166

git-svn-id: file:///svn/tokudb@1086 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_4

@@ -28,7 +28,8 @@ typedef struct __toku_db_txn_stat DB_TXN_STAT;
 typedef struct __toku_dbc DBC;
 typedef struct __toku_dbt DBT;
 typedef enum {
- DB_BTREE=1
+ DB_BTREE=1,
+ DB_UNKNOWN=5
 } DBTYPE;
 #ifndef _TOKUDB_WRAP_H
 #define DB_VERB_DEADLOCK 1

buildheader/make_db_h.c

@@ -14,7 +14,8 @@
 void print_dbtype(void) {
     /* DBTYPE is mentioned by db_open.html */
     printf("typedef enum {\n");
-    printf(" DB_BTREE=%d\n", DB_BTREE);
+    printf(" DB_BTREE=%d,\n", DB_BTREE);
+    printf(" DB_UNKNOWN=%d\n", DB_UNKNOWN);
     printf("} DBTYPE;\n");
 }
 #if 0

include/db.h

@@ -28,7 +28,8 @@ typedef struct __toku_db_txn_stat DB_TXN_STAT;
 typedef struct __toku_dbc DBC;
 typedef struct __toku_dbt DBT;
 typedef enum {
- DB_BTREE=1
+ DB_BTREE=1,
+ DB_UNKNOWN=5
 } DBTYPE;
 #ifndef _TOKUDB_WRAP_H
 #define DB_VERB_DEADLOCK 1

newbrt/Makefile

@@ -3,7 +3,7 @@
 
 # GCOV_FLAGS = -fprofile-arcs -ftest-coverage
 # PROF_FLAGS = -pg
-OPTFLAGS = -O2
+OPTFLAGS = -O0
 
 ifeq ($(CYGWIN),cygwin)
 else

newbrt/brt.c

@@ -1515,7 +1515,7 @@ int toku_brt_set_dup_compare(BRT brt, int (*dup_compare)(DB *, const DBT*, const
     return 0;
 }
 
-int toku_brt_open(BRT t, const char *fname, const char *fname_in_env, const char *dbname, int is_create, int only_create, CACHETABLE cachetable, TOKUTXN txn) {
+int toku_brt_open(BRT t, const char *fname, const char *fname_in_env, const char *dbname, int is_create, int only_create, int load_flags, CACHETABLE cachetable, TOKUTXN txn) {
 
     /* If dbname is NULL then we setup to hold a single tree.  Otherwise we setup an array. */
     int r;
@@ -1526,6 +1526,7 @@ int toku_brt_open(BRT t, const char *fname, const char *fname_in_env, const char
     if (0) { died0:  assert(r); return r; }
 
     assert(is_create || !only_create);
+    assert(!load_flags || !only_create);
     if (dbname) {
 	malloced_name = toku_strdup(dbname);
 	if (malloced_name==0) {
@@ -1635,7 +1636,8 @@ int toku_brt_open(BRT t, const char *fname, const char *fname_in_env, const char
     found_it:
         t->nodesize = t->h->nodesize;                 /* inherit the pagesize from the file */
         if (t->flags != t->h->flags) {                /* flags must match */
-            r = EINVAL; goto died1;
+            if (load_flags) t->flags = t->h->flags;
+            else {r = EINVAL; goto died1;}
         }
     }
     assert(t->h);
@@ -1696,7 +1698,8 @@ int toku_open_brt (const char *fname, const char *dbname, int is_create, BRT *ne
 	      int (*compare_fun)(DB*,const DBT*,const DBT*), DB *db) {
     BRT brt;
     int r;
-    int only_create = 0;
+    const int only_create = 0;
+    const int load_flags  = 0;
 
     r = toku_brt_create(&brt);
     if (r != 0)
@@ -1705,7 +1708,7 @@ int toku_open_brt (const char *fname, const char *dbname, int is_create, BRT *ne
     toku_brt_set_bt_compare(brt, compare_fun);
     brt->db = db;
 
-    r = toku_brt_open(brt, fname, fname, dbname, is_create, only_create, cachetable, txn);
+    r = toku_brt_open(brt, fname, fname, dbname, is_create, only_create, load_flags, cachetable, txn);
     if (r != 0) {
         return r;
     }

newbrt/brt.h

@@ -21,7 +21,7 @@ int toku_brt_get_nodesize(BRT, unsigned int *nodesize);
 int toku_brt_set_bt_compare(BRT, int (*bt_compare)(DB *, const DBT*, const DBT*));
 int toku_brt_set_dup_compare(BRT, int (*dup_compare)(DB *, const DBT*, const DBT*));
 int brt_set_cachetable(BRT, CACHETABLE);
-int toku_brt_open(BRT, const char *fname, const char *fname_in_env, const char *dbname, int is_create, int only_create, CACHETABLE ct, TOKUTXN txn);
+int toku_brt_open(BRT, const char *fname, const char *fname_in_env, const char *dbname, int is_create, int only_create, int load_flags, CACHETABLE ct, TOKUTXN txn);
 int toku_brt_remove_subdb(BRT brt, const char *dbname, u_int32_t flags);
 
 int toku_brt_insert (BRT, DBT *, DBT *, TOKUTXN);

src/ydb.c

@@ -1113,9 +1113,9 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db
 
     int openflags = 0;
     int r;
-    if (dbtype!=DB_BTREE) return EINVAL;
-
+    if (dbtype!=DB_BTREE && dbtype!=DB_UNKNOWN) return EINVAL;
     if ((flags & DB_EXCL) && !(flags & DB_CREATE)) return EINVAL;
+    if (dbtype==DB_UNKNOWN && (flags & DB_EXCL)) return EINVAL;
 
     if (db->i->full_fname)
         return -1;              /* It was already open. */
@@ -1159,7 +1159,7 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db
     db->i->open_mode = mode;
 
     r = toku_brt_open(db->i->brt, db->i->full_fname, fname, dbname,
-		 flags & DB_CREATE, flags & DB_EXCL,
+		 flags & DB_CREATE, flags & DB_EXCL, dbtype==DB_UNKNOWN,
 		 db->dbenv->i->cachetable,
 		 txn ? txn->i->tokutxn : NULL_TXN);
     if (r != 0)

utils/tokudb_dump.c

@@ -50,8 +50,8 @@ int main(int argc, char *argv[]) {
    memset(&g, 0, sizeof(g));
    g.leadingspace   = true;
    //TODO: Uncomment when DB_UNKNOWN + db->get_type are implemented.
-   //g.dbtype         = DB_UNKNOWN;
-   g.dbtype         = DB_BTREE;
+   g.dbtype         = DB_UNKNOWN;
+   //g.dbtype         = DB_BTREE;
    g.progname       = argv[0];
    g.header         = true;
    g.footer         = true;
@@ -312,15 +312,14 @@ int dump_header()
       printf("\n");      
    }
    //TODO: Uncomment when db->get_flags is implemented
-   /*
    if ((retval = db->get_flags(db, &flags)) != 0) {
       ERROR(retval, "DB->get_flags");
       goto error;
    }
-   DUMP_IGNORED_FLAG(DB_CHKSUM,    "chksum=1");
-   DUMP_FLAG(        DB_DUP,       "duplicates=1");
-   DUMP_IGNORED_FLAG(DB_DUPSORT,   "dupsort=1");
-   DUMP_IGNORED_FLAG(DB_RECNUM,    "recnum=1");*/
+   DUMP_IGNORED_FLAG(DB_CHKSUM,    "chksum=1\n");
+   DUMP_FLAG(        DB_DUP,       "duplicates=1\n");
+   DUMP_FLAG(        DB_DUPSORT,   "dupsort=1\n");
+   DUMP_IGNORED_FLAG(DB_RECNUM,    "recnum=1\n");
    printf("HEADER=END\n");
    
    if (ferror(stdout)) goto error;

utils/tokudb_load.c

@@ -796,7 +796,7 @@ int insert_pair(DBT* key, DBT* data)
    if (retval != 0) {
       //TODO: Check for transaction failures/etc.. retry if necessary.
       ERROR(retval, "DB->put");
-      goto error;
+      if (!(retval == DB_KEYEXIST && g.overwritekeys)) goto error;
    }
    return EXIT_SUCCESS;
 error: