Addresses #1510 Merge tokud.main+1510 back into main

svn merge --accept=postpone -r 10895:10898 ../tokudb.main+1510
Previously on tokudb.main+1510:
svn merge --accept=postpone -r 10317:10892 ../tokudb.1510

git-svn-id: file:///svn/toku/tokudb@10899 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -176,6 +176,14 @@ struct __toku_db_key_range {
 struct __toku_db_lsn {
   char __toku_dummy0[8];  /* Padding at the end */ 
 };
+struct __toku_dbt {
+  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
+  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
+  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
+  char __toku_dummy0[8];
+  u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
+  /* 4 more bytes of alignment in the 64-bit case. */
+};
 struct __toku_db {
   struct __toku_db_internal *i;
   int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact);
@@ -188,7 +196,9 @@ struct __toku_db {
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
   int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
   int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
-  void* __toku_dummy0[24];
+  DBT descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
+  int (*set_descriptor) (DB*, const DBT*) /* set row/dictionary descriptor for a db.  Available only while db is open */;
+  void* __toku_dummy0[22];
   char __toku_dummy1[96];
   void *api_internal; /* 32-bit offset=236 size=4, 64=bit offset=376 size=8 */
   void* __toku_dummy2[5];
@@ -280,14 +290,6 @@ struct __toku_dbc {
   int (*c_get) (DBC *, DBT *, DBT *, u_int32_t); /* 32-bit offset=204 size=4, 64=bit offset=304 size=8 */
   void* __toku_dummy3[11]; /* Padding at the end */ 
 };
-struct __toku_dbt {
-  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
-  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
-  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
-  char __toku_dummy0[8];
-  u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
-  /* 4 more bytes of alignment in the 64-bit case. */
-};
 #ifdef _TOKUDB_WRAP_H
 #define txn_begin txn_begin_tokudb
 #endif

buildheader/db.h_4_3

@@ -186,6 +186,14 @@ struct __toku_db_key_range {
 struct __toku_db_lsn {
   char __toku_dummy0[8];  /* Padding at the end */ 
 };
+struct __toku_dbt {
+  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
+  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
+  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
+  char __toku_dummy0[8];
+  u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
+  /* 4 more bytes of alignment in the 64-bit case. */
+};
 struct __toku_db {
   struct __toku_db_internal *i;
   int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact);
@@ -198,7 +206,9 @@ struct __toku_db {
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
   int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
   int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
-  void* __toku_dummy0[27];
+  DBT descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
+  int (*set_descriptor) (DB*, const DBT*) /* set row/dictionary descriptor for a db.  Available only while db is open */;
+  void* __toku_dummy0[25];
   char __toku_dummy1[96];
   void *api_internal; /* 32-bit offset=248 size=4, 64=bit offset=400 size=8 */
   void* __toku_dummy2[5];
@@ -296,14 +306,6 @@ struct __toku_dbc {
   int (*c_get) (DBC *, DBT *, DBT *, u_int32_t); /* 32-bit offset=204 size=4, 64=bit offset=296 size=8 */
   void* __toku_dummy3[10]; /* Padding at the end */ 
 };
-struct __toku_dbt {
-  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
-  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
-  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
-  char __toku_dummy0[8];
-  u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
-  /* 4 more bytes of alignment in the 64-bit case. */
-};
 #ifdef _TOKUDB_WRAP_H
 #define txn_begin txn_begin_tokudb
 #endif

buildheader/db.h_4_4

@@ -189,6 +189,14 @@ struct __toku_db_key_range {
 struct __toku_db_lsn {
   char __toku_dummy0[8];  /* Padding at the end */ 
 };
+struct __toku_dbt {
+  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
+  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
+  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
+  char __toku_dummy0[8];
+  u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
+  /* 4 more bytes of alignment in the 64-bit case. */
+};
 struct __toku_db {
   struct __toku_db_internal *i;
   int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact);
@@ -201,7 +209,9 @@ struct __toku_db {
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
   int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
   int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
-  void* __toku_dummy0[29];
+  DBT descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
+  int (*set_descriptor) (DB*, const DBT*) /* set row/dictionary descriptor for a db.  Available only while db is open */;
+  void* __toku_dummy0[27];
   char __toku_dummy1[96];
   void *api_internal; /* 32-bit offset=256 size=4, 64=bit offset=416 size=8 */
   void* __toku_dummy2[5];
@@ -303,14 +313,6 @@ struct __toku_dbc {
   int (*c_get) (DBC *, DBT *, DBT *, u_int32_t); /* 32-bit offset=204 size=4, 64=bit offset=304 size=8 */
   void* __toku_dummy3[10]; /* Padding at the end */ 
 };
-struct __toku_dbt {
-  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
-  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
-  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
-  char __toku_dummy0[8];
-  u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
-  /* 4 more bytes of alignment in the 64-bit case. */
-};
 #ifdef _TOKUDB_WRAP_H
 #define txn_begin txn_begin_tokudb
 #endif

buildheader/db.h_4_5

@@ -188,6 +188,15 @@ struct __toku_db_key_range {
 struct __toku_db_lsn {
   char __toku_dummy0[8];  /* Padding at the end */ 
 };
+struct __toku_dbt {
+  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
+  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
+  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
+  void* __toku_dummy0[1];
+  char __toku_dummy1[8];
+  u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
+  /* 4 more bytes of alignment in the 64-bit case. */
+};
 struct __toku_db {
   struct __toku_db_internal *i;
   int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact);
@@ -200,7 +209,9 @@ struct __toku_db {
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
   int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
   int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
-  void* __toku_dummy0[32];
+  DBT descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
+  int (*set_descriptor) (DB*, const DBT*) /* set row/dictionary descriptor for a db.  Available only while db is open */;
+  void* __toku_dummy0[30];
   char __toku_dummy1[96];
   void *api_internal; /* 32-bit offset=268 size=4, 64=bit offset=440 size=8 */
   void* __toku_dummy2[5];
@@ -302,15 +313,6 @@ struct __toku_dbc {
   int (*c_get) (DBC *, DBT *, DBT *, u_int32_t); /* 32-bit offset=220 size=4, 64=bit offset=336 size=8 */
   void* __toku_dummy3[10]; /* Padding at the end */ 
 };
-struct __toku_dbt {
-  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
-  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
-  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
-  void* __toku_dummy0[1];
-  char __toku_dummy1[8];
-  u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
-  /* 4 more bytes of alignment in the 64-bit case. */
-};
 #ifdef _TOKUDB_WRAP_H
 #define txn_begin txn_begin_tokudb
 #endif

buildheader/db.h_4_6

@@ -191,6 +191,15 @@ struct __toku_db_key_range {
 struct __toku_db_lsn {
   char __toku_dummy0[8];  /* Padding at the end */ 
 };
+struct __toku_dbt {
+  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
+  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
+  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
+  void* __toku_dummy0[1];
+  char __toku_dummy1[8];
+  u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
+  /* 4 more bytes of alignment in the 64-bit case. */
+};
 struct __toku_db {
   struct __toku_db_internal *i;
   int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact);
@@ -204,7 +213,9 @@ struct __toku_db {
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
   int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
   int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
-  void* __toku_dummy1[36];
+  DBT descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
+  int (*set_descriptor) (DB*, const DBT*) /* set row/dictionary descriptor for a db.  Available only while db is open */;
+  void* __toku_dummy1[34];
   char __toku_dummy2[80];
   void *api_internal; /* 32-bit offset=276 size=4, 64=bit offset=464 size=8 */
   void* __toku_dummy3[5];
@@ -307,15 +318,6 @@ struct __toku_dbc {
   int (*c_get) (DBC *, DBT *, DBT *, u_int32_t); /* 32-bit offset=260 size=4, 64=bit offset=416 size=8 */
   void* __toku_dummy3[10]; /* Padding at the end */ 
 };
-struct __toku_dbt {
-  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
-  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
-  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
-  void* __toku_dummy0[1];
-  char __toku_dummy1[8];
-  u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
-  /* 4 more bytes of alignment in the 64-bit case. */
-};
 #ifdef _TOKUDB_WRAP_H
 #define txn_begin txn_begin_tokudb
 #endif

buildheader/make_db_h.c

@@ -329,6 +329,9 @@ int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__un
     assert(sizeof(db_lsn_fields32)==sizeof(db_lsn_fields64));
     print_struct("db_lsn", 0, db_lsn_fields32, db_lsn_fields64, sizeof(db_lsn_fields32)/sizeof(db_lsn_fields32[0]), 0);
 
+    assert(sizeof(dbt_fields32)==sizeof(dbt_fields64));
+    print_struct("dbt", 0, dbt_fields32, dbt_fields64, sizeof(dbt_fields32)/sizeof(dbt_fields32[0]), 0);
+
     assert(sizeof(db_fields32)==sizeof(db_fields64));
     {
 	const char *extra[]={"int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact)",
@@ -339,6 +342,8 @@ int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__un
 			     "const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/",
                              "int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */",
                              "int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */",
+                             "DBT descriptor /* saved row/dictionary descriptor for aiding in comparisons */",
+                             "int (*set_descriptor) (DB*, const DBT*) /* set row/dictionary descriptor for a db.  Available only while db is open */",
 			     NULL};
 	print_struct("db", 1, db_fields32, db_fields64, sizeof(db_fields32)/sizeof(db_fields32[0]), extra);
     }
@@ -379,9 +384,6 @@ int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__un
 	print_struct("dbc", 1, dbc_fields32, dbc_fields64, sizeof(dbc_fields32)/sizeof(dbc_fields32[0]), extra);
     }
 
-    assert(sizeof(dbt_fields32)==sizeof(dbt_fields64));
-    print_struct("dbt", 0, dbt_fields32, dbt_fields64, sizeof(dbt_fields32)/sizeof(dbt_fields32[0]), 0);
-
     printf("#ifdef _TOKUDB_WRAP_H\n#define txn_begin txn_begin_tokudb\n#endif\n");
     printf("int db_env_create(DB_ENV **, u_int32_t) %s;\n", VISIBLE);
     printf("int db_create(DB **, DB_ENV *, u_int32_t) %s;\n", VISIBLE);

include/db.h

@@ -191,6 +191,15 @@ struct __toku_db_key_range {
 struct __toku_db_lsn {
   char __toku_dummy0[8];  /* Padding at the end */ 
 };
+struct __toku_dbt {
+  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
+  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
+  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
+  void* __toku_dummy0[1];
+  char __toku_dummy1[8];
+  u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
+  /* 4 more bytes of alignment in the 64-bit case. */
+};
 struct __toku_db {
   struct __toku_db_internal *i;
   int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact);
@@ -204,7 +213,9 @@ struct __toku_db {
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
   int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
   int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
-  void* __toku_dummy1[36];
+  DBT descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
+  int (*set_descriptor) (DB*, const DBT*) /* set row/dictionary descriptor for a db.  Available only while db is open */;
+  void* __toku_dummy1[34];
   char __toku_dummy2[80];
   void *api_internal; /* 32-bit offset=276 size=4, 64=bit offset=464 size=8 */
   void* __toku_dummy3[5];
@@ -307,15 +318,6 @@ struct __toku_dbc {
   int (*c_get) (DBC *, DBT *, DBT *, u_int32_t); /* 32-bit offset=260 size=4, 64=bit offset=416 size=8 */
   void* __toku_dummy3[10]; /* Padding at the end */ 
 };
-struct __toku_dbt {
-  void*data; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
-  u_int32_t size; /* 32-bit offset=4 size=4, 64=bit offset=8 size=4 */
-  u_int32_t ulen; /* 32-bit offset=8 size=4, 64=bit offset=12 size=4 */
-  void* __toku_dummy0[1];
-  char __toku_dummy1[8];
-  u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
-  /* 4 more bytes of alignment in the 64-bit case. */
-};
 #ifdef _TOKUDB_WRAP_H
 #define txn_begin txn_begin_tokudb
 #endif

newbrt/block_allocator.h

@@ -11,6 +11,11 @@
 //  The actual header is 8+4+4+8+8_4+8+ the length of the db names + 1 pointer for each root.
 //  So 4096 should be enough.
 #define BLOCK_ALLOCATOR_HEADER_RESERVE 4096
+#if (BLOCK_ALLOCATOR_HEADER_RESERVE % BLOCK_ALLOCATOR_ALIGNMENT) != 0
+#error
+#endif
+
+#define BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE (2*BLOCK_ALLOCATOR_HEADER_RESERVE)
 
 // A block allocator manages the allocation of variable-sized blocks.
 // The translation of block numbers to addresses is handled elsewhere.

newbrt/block_table.h

@@ -7,56 +7,57 @@ typedef struct block_table *BLOCK_TABLE;
 
 //Needed by tests, brtdump
 struct block_translation_pair {
-    DISKOFF diskoff; // When in free list, set to the next free block.  In this case it's really a BLOCKNUM.
+    union { // If in the freelist, use next_free_blocknum, otherwise diskoff.
+        DISKOFF  diskoff; 
+        BLOCKNUM next_free_blocknum;
+    } u;
     DISKOFF size;    // set to 0xFFFFFFFFFFFFFFFF for free
 };
 
-void toku_block_realloc(BLOCK_TABLE bt, BLOCKNUM b, u_int64_t size, u_int64_t *offset, int *dirty);
-void toku_block_alloc(BLOCK_TABLE bt, u_int64_t size, u_int64_t *offset);
-void toku_block_free(BLOCK_TABLE bt, u_int64_t offset);
-DISKOFF toku_block_get_offset(BLOCK_TABLE bt, BLOCKNUM b);
-DISKOFF toku_block_get_size(BLOCK_TABLE bt, BLOCKNUM b);
-void toku_block_get_offset_size(BLOCK_TABLE bt, BLOCKNUM b, DISKOFF *offset, DISKOFF *size);
-int toku_allocate_diskblocknumber(BLOCK_TABLE bt, BLOCKNUM *res, int *dirty);
-int toku_free_diskblocknumber(BLOCK_TABLE bt, BLOCKNUM *b, int *dirty);
-void toku_verify_diskblocknumber_allocated(BLOCK_TABLE bt, BLOCKNUM b);
-void toku_block_verify_no_free_blocks(BLOCK_TABLE bt);
-u_int64_t toku_block_allocator_allocated_limit(BLOCK_TABLE bt);
-void toku_block_dump_translation_table(FILE *f, BLOCK_TABLE bt);
-void toku_block_dump_translation(BLOCK_TABLE bt, u_int64_t offset);
-
+void toku_blocktable_create_new(BLOCK_TABLE *btp);
+void toku_blocktable_create_from_buffer(BLOCK_TABLE *btp, DISKOFF location_on_disk, DISKOFF size_on_disk, unsigned char *translation_buffer);
 void toku_blocktable_destroy(BLOCK_TABLE *btp);
-void toku_blocktable_debug_set_translation(BLOCK_TABLE bt,
-        u_int64_t limit,
-        struct block_translation_pair *table);
-void toku_blocktable_create(BLOCK_TABLE *btp,
-        BLOCKNUM free_blocks,
-        BLOCKNUM unused_blocks,
-        u_int64_t translated_blocknum_limit,
-        u_int64_t block_translation_address_on_disk,
-        unsigned char *buffer);
-void toku_blocktable_create_from_loggedheader(BLOCK_TABLE *btp, LOGGEDBRTHEADER);
-void toku_blocktable_create_new(BLOCK_TABLE *bt);
-
-BLOCKNUM toku_block_get_unused_blocks(BLOCK_TABLE bt);
-BLOCKNUM toku_block_get_free_blocks(BLOCK_TABLE bt);
-u_int64_t toku_block_get_translated_blocknum_limit(BLOCK_TABLE bt);
-
-void toku_block_memcpy_translation_table(BLOCK_TABLE bt, size_t n, void *p);
-
-//Unlocked/multi ops
+
 void toku_block_lock_for_multiple_operations(BLOCK_TABLE bt);
 void toku_block_unlock_for_multiple_operations(BLOCK_TABLE bt);
 
-void toku_block_realloc_translation_unlocked(BLOCK_TABLE bt);
-void toku_block_wbuf_free_blocks_unlocked(BLOCK_TABLE bt, struct wbuf *wbuf);
-void toku_block_wbuf_unused_blocks_unlocked(BLOCK_TABLE bt, struct wbuf *wbuf);
-void toku_block_wbuf_translated_blocknum_limit_unlocked(BLOCK_TABLE bt, struct wbuf *wbuf);
-void toku_block_wbuf_block_translation_address_on_disk_unlocked(BLOCK_TABLE bt, struct wbuf *wbuf);
-void toku_block_wbuf_init_and_fill_unlocked(BLOCK_TABLE bt, struct wbuf *w,
-                                            u_int64_t *size, u_int64_t *address);
+void toku_block_translation_note_start_checkpoint_unlocked(BLOCK_TABLE bt);
+void toku_block_translation_note_end_checkpoint(BLOCK_TABLE bt);
+void toku_block_translation_note_failed_checkpoint(BLOCK_TABLE bt);
+void toku_block_translation_note_skipped_checkpoint(BLOCK_TABLE bt);
+
+//Blocknums
+void toku_allocate_blocknum(BLOCK_TABLE bt, BLOCKNUM *res, struct brt_header * h);
+void toku_free_blocknum(BLOCK_TABLE bt, BLOCKNUM *b, struct brt_header * h);
+void toku_verify_blocknum_allocated(BLOCK_TABLE bt, BLOCKNUM b);
+void toku_block_verify_no_free_blocknums(BLOCK_TABLE bt);
+void toku_realloc_fifo_on_disk_unlocked (BLOCK_TABLE, DISKOFF size, DISKOFF *offset);
+void toku_get_fifo_offset_on_disk(BLOCK_TABLE bt, DISKOFF *offset);
+
+//Blocks and Blocknums
+void toku_blocknum_realloc_on_disk(BLOCK_TABLE bt, BLOCKNUM b, DISKOFF size, DISKOFF *offset, struct brt_header * h, BOOL for_checkpoint);
+void toku_translate_blocknum_to_offset_size(BLOCK_TABLE bt, BLOCKNUM b, DISKOFF *offset, DISKOFF *size);
+
+//Serialization
+void toku_serialize_translation_to_wbuf_unlocked(BLOCK_TABLE bt, struct wbuf *w, int64_t *address, int64_t *size);
+
+//DEBUG ONLY (brtdump included), tests included
+void toku_blocknum_dump_translation(BLOCK_TABLE bt, BLOCKNUM b);
+void toku_dump_translation_table(FILE *f, BLOCK_TABLE bt);
+void toku_block_alloc(BLOCK_TABLE bt, u_int64_t size, u_int64_t *offset);
+void toku_block_free(BLOCK_TABLE bt, u_int64_t offset);
+typedef int(*BLOCKTABLE_CALLBACK)(BLOCKNUM b, int64_t size, int64_t address, void *extra);
+enum translation_type {TRANSLATION_NONE=0,
+                       TRANSLATION_CURRENT,
+                       TRANSLATION_INPROGRESS,
+                       TRANSLATION_CHECKPOINTED,
+                       TRANSLATION_DEBUG};
 
+int toku_blocktable_iterate(BLOCK_TABLE bt, enum translation_type type, BLOCKTABLE_CALLBACK f, void *extra, BOOL data_only, BOOL used_only); 
+void toku_blocktable_internal_fragmentation(BLOCK_TABLE bt, int64_t *total_sizep, int64_t *used_sizep);
 
+//ROOT FIFO (To delete)
+u_int64_t toku_block_allocator_allocated_limit(BLOCK_TABLE bt);
 
 #endif
 

newbrt/brt-internal.h

+/* -*- mode: C; c-basic-offset: 4 -*- */
 #ifndef BRT_INTERNAL_H
 #define BRT_INTERNAL_H
 
@@ -148,7 +149,20 @@ struct remembered_hash {
 
 // The brt_header is not managed by the cachetable.  Instead, it hangs off the cachefile as userdata.
 
+enum brtheader_type {BRTHEADER_CURRENT=1, BRTHEADER_CHECKPOINT_INPROGRESS};
+
+struct descriptor {
+    struct simple_dbt sdbt;
+    BLOCKNUM          b;
+};
+
 struct brt_header {
+    enum brtheader_type type;
+    struct brt_header * checkpoint_header;
+    u_int64_t checkpoint_count; // Free-running counter incremented once per checkpoint (toggling LSB).
+                                // LSB indicates which header location is used on disk so this
+                                // counter is effectively a boolean which alternates with each checkpoint.
+    LSN checkpoint_lsn;         // LSN of creation of "checkpoint-begin" record in log.  
     int refcount;
     int dirty;
     int panic; // If nonzero there was a write error.  Don't write any more, because it probably only gets worse.  This is the error code.
@@ -160,6 +174,7 @@ struct brt_header {
     BLOCKNUM *roots;            // An array of the roots of the various dictionaries.  Element 0 holds the element if no subdatabases allowed.
     struct remembered_hash *root_hashes;     // an array of hashes of the root offsets.
     unsigned int *flags_array;  // an array of flags.  Element 0 holds the element if no subdatabases allowed.
+    struct descriptor *descriptors;  // an array of descriptors.  Element 0 holds the element if no subdatabases allowed.
 
     FIFO fifo; // all the abort and commit commands.  If the header gets flushed to disk, we write the fifo contents beyond the unused_memory.
 
@@ -180,6 +195,8 @@ struct brt {
     unsigned int nodesize;
     unsigned int flags;
     unsigned int did_set_flags;
+    unsigned int did_set_descriptor;
+    DBT          temp_descriptor;
     int (*compare_fun)(DB*,const DBT*,const DBT*);
     int (*dup_compare)(DB*,const DBT*,const DBT*);
     DB *db;           // To pass to the compare fun
@@ -193,7 +210,7 @@ struct brt {
 };
 
 /* serialization code */
-int toku_serialize_brtnode_to(int fd, BLOCKNUM, BRTNODE node, struct brt_header *h, int n_workitems, int n_threads);
+int toku_serialize_brtnode_to(int fd, BLOCKNUM, BRTNODE node, struct brt_header *h, int n_workitems, int n_threads, BOOL for_checkpoint);
 int toku_deserialize_brtnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/, BRTNODE *brtnode, struct brt_header *h);
 unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
 int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
@@ -202,10 +219,12 @@ void toku_verify_counts(BRTNODE);
 
 int toku_serialize_brt_header_size (struct brt_header *h);
 int toku_serialize_brt_header_to (int fd, struct brt_header *h);
-int toku_serialize_brt_header_to_wbuf (struct wbuf *, struct brt_header *h);
-int toku_deserialize_brtheader_from (int fd, BLOCKNUM off, struct brt_header **brth);
+int toku_serialize_brt_header_to_wbuf (struct wbuf *, struct brt_header *h, int64_t address_translation, int64_t size_translation);
+int toku_deserialize_brtheader_from (int fd, struct brt_header **brth);
+int toku_serialize_descriptor_contents_to_fd(int fd, DBT *desc, DISKOFF offset);
 
-int toku_serialize_fifo_at (int fd, toku_off_t freeoff, FIFO fifo); // Write a fifo into a disk, without worrying about fitting it into a block.  This write is done at the end of the file.
+int toku_serialize_fifo_at (int fd, toku_off_t freeoff, FIFO fifo, u_int64_t fifo_size); // Write a fifo into a disk, without worrying about fitting it into a block.  This write is done at the end of the file.
+u_int64_t toku_fifo_get_serialized_size (FIFO fifo);
 
 void toku_brtnode_free (BRTNODE *node);
 
@@ -222,7 +241,7 @@ struct brtenv {
 //    SPINLOCK  checkpointing;
 };
 
-extern void toku_brtnode_flush_callback (CACHEFILE cachefile, BLOCKNUM nodename, void *brtnode_v, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p);
+extern void toku_brtnode_flush_callback (CACHEFILE cachefile, BLOCKNUM nodename, void *brtnode_v, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p, BOOL for_checkpoint);
 extern int toku_brtnode_fetch_callback (CACHEFILE cachefile, BLOCKNUM nodename, u_int32_t fullhash, void **brtnode_pv, long *sizep, void*extraargs, LSN *written_lsn);
 extern int toku_brt_alloc_init_header(BRT t, const char *dbname);
 extern int toku_read_brt_header_and_store_in_cachefile (CACHEFILE cf, struct brt_header **header);
@@ -308,14 +327,16 @@ enum brt_layout_version_e {
     BRT_LAYOUT_VERSION_7 = 7,   // Diff from 6 to 7:  Add exact-bit to leafentry_estimate #818, add magic to header #22, add per-subdatase flags #333
     BRT_LAYOUT_VERSION_8 = 8,   // Diff from 7 to 8:  Use murmur instead of crc32.  We are going to make a simplification and stop supporting version 7 and before.  Current As of Beta 1.0.6
     BRT_LAYOUT_VERSION_9 = 9,   // Diff from 8 to 9:  Variable-sized blocks and compression.
-    BRT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates.
+    BRT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header
     BRT_ANTEULTIMATE_VERSION,   // the version after the most recent version
     BRT_LAYOUT_VERSION   = BRT_ANTEULTIMATE_VERSION-1 // A hack so I don't have to change this line.
 };
 
 void toku_brtheader_free (struct brt_header *h);
 int toku_brtheader_close (CACHEFILE cachefile, void *header_v, char **error_string);
+int toku_brtheader_begin_checkpoint (CACHEFILE cachefile, LSN checkpoint_lsn, void *header_v);
 int toku_brtheader_checkpoint (CACHEFILE cachefile, void *header_v);
+int toku_brtheader_end_checkpoint (CACHEFILE cachefile, void *header_v);
 
 int toku_db_badformat(void);
 

newbrt/brt.h

@@ -30,6 +30,7 @@ int toku_open_brt (const char *fname, const char *dbname, int is_create, BRT *,
 
 int toku_brt_create(BRT *);
 int toku_brt_set_flags(BRT, unsigned int flags);
+int toku_brt_set_descriptor (BRT t, const DBT *descriptor);
 int toku_brt_get_flags(BRT, unsigned int *flags);
 int toku_brt_set_nodesize(BRT, unsigned int nodesize);
 int toku_brt_get_nodesize(BRT, unsigned int *nodesize);

newbrt/brtdump.c

@@ -24,16 +24,12 @@ static void
 dump_header (int f, struct brt_header **header) {
     struct brt_header *h;
     int r;
-    r = toku_deserialize_brtheader_from (f, header_blocknum, &h); assert(r==0);
+    r = toku_deserialize_brtheader_from (f, &h); assert(r==0);
     printf("brtheader:\n");
     if (h->layout_version==BRT_LAYOUT_VERSION_6) printf(" layout_version<=6\n");
     else printf(" layout_version=%d\n", h->layout_version);
     printf(" dirty=%d\n", h->dirty);
     printf(" nodesize=%u\n", h->nodesize);
-    BLOCKNUM free_blocks = toku_block_get_free_blocks(h->blocktable);
-    BLOCKNUM unused_blocks = toku_block_get_unused_blocks(h->blocktable);
-    printf(" free_blocks=%" PRId64 "\n", free_blocks.b);
-    printf(" unused_memory=%" PRId64 "\n", unused_blocks.b);
     if (h->n_named_roots==-1) {
 	printf(" unnamed_root=%" PRId64 "\n", h->roots[0].b);
 	printf(" flags=%u\n", h->flags_array[0]);
@@ -91,7 +87,7 @@ dump_node (int f, BLOCKNUM blocknum, struct brt_header *h) {
     assert(n!=0);
     printf("brtnode\n");
     DISKOFF disksize, diskoffset;
-    toku_block_get_offset_size(h->blocktable, blocknum, &diskoffset, &disksize);
+    toku_translate_blocknum_to_offset_size(h->blocktable, blocknum, &diskoffset, &disksize);
     printf(" diskoffset  =%" PRId64 "\n", diskoffset);
     printf(" disksize    =%" PRId64 "\n", disksize);
     printf(" nodesize    =%u\n", n->nodesize);
@@ -172,55 +168,53 @@ dump_node (int f, BLOCKNUM blocknum, struct brt_header *h) {
 
 static void 
 dump_block_translation(struct brt_header *h, u_int64_t offset) {
-    toku_block_dump_translation(h->blocktable, offset);
+    toku_blocknum_dump_translation(h->blocktable, make_blocknum(offset));
 }
 
+typedef struct {
+    int f;
+    struct brt_header *h;
+    u_int64_t blocksizes;
+    u_int64_t leafsizes;
+    u_int64_t leafblocks;
+} frag_help_extra;
+
 static int
-bxpcmp(const void *a, const void *b) {
-    const struct block_translation_pair *bxpa = a;
-    const struct block_translation_pair *bxpb = b;
-    if (bxpa->diskoff < bxpb->diskoff) return -1;
-    if (bxpa->diskoff > bxpb->diskoff) return +1;
+fragmentation_helper(BLOCKNUM b, int64_t size, int64_t UU(address), void *extra) {
+    frag_help_extra *info = extra;
+    BRTNODE n;
+    int r = toku_deserialize_brtnode_from(info->f, b, 0 /*pass zero for hash, it doesn't matter*/, &n, info->h);
+    if (r==0) {
+        info->blocksizes += size;
+        if (n->height == 0) {
+            info->leafsizes += size;
+            info->leafblocks++;
+        }
+	toku_brtnode_free(&n);
+    }
     return 0;
 }
 
 static void 
 dump_fragmentation(int f, struct brt_header *h) {
-    u_int64_t blocksizes = 0;
-    u_int64_t leafsizes = 0; 
-    u_int64_t leafblocks = 0;
-    u_int64_t fragsizes = 0;
-    u_int64_t i;
-    u_int64_t limit = toku_block_get_translated_blocknum_limit(h->blocktable);
-    for (i = 0; i < limit; i++) {
-        BRTNODE n;
-	BLOCKNUM blocknum = make_blocknum(i);
-        int r = toku_deserialize_brtnode_from (f, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, h);
-	if (r != 0) continue;
+    frag_help_extra info;
+    memset(&info, 0, sizeof(info));
+    info.f = f;
+    info.h = h;
+    toku_blocktable_iterate(h->blocktable, TRANSLATION_CHECKPOINTED,
+                            fragmentation_helper, &info, TRUE, TRUE);
+    int64_t used_space;
+    int64_t total_space;
+    toku_blocktable_internal_fragmentation(h->blocktable, &total_space, &used_space);
+    int64_t fragsizes = total_space - used_space;
 
-        DISKOFF size = toku_block_get_size(h->blocktable, blocknum);
-        blocksizes += size;
-	if (n->height == 0) {
-	    leafsizes += size;
-	    leafblocks += 1;
-	}
-	toku_brtnode_free(&n);
-    }
-    size_t n = limit * sizeof (struct block_translation_pair);
-    struct block_translation_pair *bx = toku_malloc(n);
-    toku_block_memcpy_translation_table(h->blocktable, n, bx);
-    qsort(bx, limit, sizeof (struct block_translation_pair), bxpcmp);
-    for (i = 0; i < limit - 1; i++) {
-        // printf("%lu %lu %lu\n", i, bx[i].diskoff, bx[i].size);
-        fragsizes += bx[i+1].diskoff - (bx[i].diskoff + bx[i].size);
-    }
-    toku_free(bx);
-    printf("translated_blocknum_limit: %" PRIu64 "\n", limit);
-    printf("leafblocks: %" PRIu64 "\n", leafblocks);
-    printf("blocksizes: %" PRIu64 "\n", blocksizes);
-    printf("leafsizes: %" PRIu64 "\n", leafsizes);
-    printf("fragsizes: %" PRIu64 "\n", fragsizes);
-    printf("fragmentation: %.1f%%\n", 100. * ((double)fragsizes / (double)(fragsizes + blocksizes)));
+    printf("leafblocks: %" PRIu64 "\n", info.leafblocks);
+    printf("blocksizes: %" PRIu64 "\n", info.blocksizes);
+    printf("used size: %" PRId64 "\n",  used_space);
+    printf("total size: %" PRId64 "\n", total_space);
+    printf("leafsizes: %" PRIu64 "\n", info.leafsizes);
+    printf("fragsizes: %" PRId64 "\n", fragsizes);
+    printf("fragmentation: %.1f%%\n", 100. * ((double)fragsizes / (double)(total_space)));
 }
 
 static void
@@ -273,6 +267,18 @@ usage(const char *arg0) {
     return 1;
 }
 
+typedef struct __dump_node_extra {
+    int f;
+    struct brt_header *h;
+} dump_node_extra;
+
+static int
+dump_node_wrapper(BLOCKNUM b, int64_t UU(size), int64_t UU(address), void *extra) {
+    dump_node_extra *info = extra;
+    dump_node(info->f, b, info->h);
+    return 0;
+}
+
 int 
 main (int argc, const char *argv[]) {
     const char *arg0 = argv[0];
@@ -339,28 +345,19 @@ main (int argc, const char *argv[]) {
             }
         }
     } else {
-	BLOCKNUM blocknum;
 	printf("Block translation:");
 
-        u_int64_t limit = toku_block_get_translated_blocknum_limit(h->blocktable);
-        BLOCKNUM unused_blocks = toku_block_get_unused_blocks(h->blocktable);
-        size_t bx_size = limit * sizeof (struct block_translation_pair);
-        struct block_translation_pair *bx = toku_malloc(bx_size);
-        toku_block_memcpy_translation_table(h->blocktable, bx_size, bx);
-
+        toku_dump_translation_table(stdout, h->blocktable);
 
-	for (blocknum.b=0; blocknum.b< unused_blocks.b; blocknum.b++) {
-	    printf(" %" PRId64 ":", blocknum.b);
-	    if (bx[blocknum.b].size == -1) printf("free");
-	    else printf("%" PRId64 ":%" PRId64, bx[blocknum.b].diskoff, bx[blocknum.b].size);
-	}
-	for (blocknum.b=1; blocknum.b<unused_blocks.b; blocknum.b++) {
-	    if (bx[blocknum.b].size != -1)
-		dump_node(f, blocknum, h);
-        }
-        toku_free(bx);
+        struct __dump_node_extra info;
+        info.f = f;
+        info.h = h;
+        toku_blocktable_iterate(h->blocktable, TRANSLATION_CHECKPOINTED,
+                                dump_node_wrapper, &info, TRUE, TRUE);
     }
     toku_brtheader_free(h);
     toku_malloc_cleanup();
     return 0;
 }
+
+

newbrt/brttypes.h

@@ -25,7 +25,6 @@ typedef u_int64_t TXNID;
 typedef struct s_blocknum { int64_t b; } BLOCKNUM; // make a struct so that we will notice type problems.
 
 static inline BLOCKNUM make_blocknum(int64_t b) { BLOCKNUM result={b}; return result; }
-static const BLOCKNUM header_blocknum = {0};
 
 typedef struct {
     u_int32_t len;

newbrt/cachetable.h

@@ -37,7 +37,11 @@ int toku_cachefile_of_filenum (CACHETABLE t, FILENUM filenum, CACHEFILE *cf);
 
 // Checkpoint the cachetable.
 // Effects: ?
-int toku_cachetable_checkpoint (CACHETABLE ct);
+int toku_cachetable_checkpoint (CACHETABLE ct, TOKULOGGER);
+
+// Does an fsync of a cachefile.
+// Handles the case where cf points to /dev/null
+int toku_cachefile_fsync(CACHEFILE cf);
 
 // Close the cachetable.
 // Effects: All of the memory objects are flushed to disk, and the cachetable is
@@ -59,8 +63,9 @@ void toku_cachefile_get_workqueue_load (CACHEFILE, int *n_in_queue, int *n_threa
 // The flush callback is called when a key value pair is being written to storage and possibly removed from the cachetable.
 // When write_me is true, the value should be written to storage.
 // When keep_me is false, the value should be freed.
+// When for_checkpoint is true, this was a 'pending' write
 // Returns: 0 if success, otherwise an error number.
-typedef void (*CACHETABLE_FLUSH_CALLBACK)(CACHEFILE, CACHEKEY key, void *value, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p);
+typedef void (*CACHETABLE_FLUSH_CALLBACK)(CACHEFILE, CACHEKEY key, void *value, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p, BOOL for_checkpoint);
 
 // The fetch callback is called when a thread is attempting to get and pin a memory
 // object and it is not in the cachetable.
@@ -68,8 +73,9 @@ typedef void (*CACHETABLE_FLUSH_CALLBACK)(CACHEFILE, CACHEKEY key, void *value,
 // associated with the key are returned.
 typedef int (*CACHETABLE_FETCH_CALLBACK)(CACHEFILE, CACHEKEY key, u_int32_t fullhash, void **value, long *sizep, void *extraargs, LSN *written_lsn);
 
-void toku_cachefile_set_userdata(CACHEFILE cf, void *userdata, int (*close_userdata)(CACHEFILE, void*, char **/*error_string*/), int (*checkpoint_userdata)(CACHEFILE, void*));
+void toku_cachefile_set_userdata(CACHEFILE cf, void *userdata, int (*close_userdata)(CACHEFILE, void*, char **/*error_string*/), int (*checkpoint_userdata)(CACHEFILE, void*), int (*begin_checkpoint_userdata)(CACHEFILE, LSN, void*), int (*end_checkpoint_userdata)(CACHEFILE, void*));
 // Effect: Store some cachefile-specific user data.  When the last reference to a cachefile is closed, we call close_userdata().
+// Before starting a checkpoint, we call checkpoint_prepare_userdata().
 // When the cachefile needs to be checkpointed, we call checkpoint_userdata().
 // If userdata is already non-NULL, then we simply overwrite it.
 
@@ -209,15 +215,6 @@ void toku_cachetable_verify (CACHETABLE t);
 // Not for use in production, but useful for testing.
 void toku_cachetable_print_hash_histogram (void) __attribute__((__visibility__("default")));
 
-int toku_graceful_open(const char *db_fname, BOOL *is_dirtyp);
-int toku_graceful_close(CACHEFILE cf);
-int toku_graceful_dirty(CACHEFILE cf);
-int toku_graceful_delete(const char *db_fname);
-void toku_graceful_lock_init(void);
-void toku_graceful_lock_destroy(void);
-
-void toku_graceful_fill_names(const char *db_fname, char *cleanbuf, size_t cleansize, char *dirtybuf, size_t dirtysize);
-
 #define TOKU_CACHETABLE_DO_EVICT_FROM_WRITER 0
 
 void toku_cachetable_maybe_flush_some(CACHETABLE ct);

newbrt/log-internal.h

@@ -69,7 +69,7 @@ struct tokulogger {
     // To access these, you must have the output lock
     LSN written_lsn; // the last lsn written
     LSN fsynced_lsn; // What is the LSN of the highest fsynced log entry
-    LSN checkpoint_lsns[2]; // What are the LSNs of the most recent checkpoints.  checkpoint_lsn[0] is the most recent one.
+    LSN checkpoint_lsn;     // What is the LSN of the most recent completed checkpoint.
     long long next_log_file_number;
     char buf[LOGGER_BUF_SIZE]; // used to marshall logbytes so we can use only a single write
     int n_in_file;

newbrt/log.c

@@ -101,7 +101,7 @@ int toku_logger_create (TOKULOGGER *resultp) {
     result->n_in_buf=0;
     result->n_in_file=0;
     result->directory=0;
-    result->checkpoint_lsns[0]=result->checkpoint_lsns[1]=(LSN){0};
+    result->checkpoint_lsn=(LSN){0};
     result->write_block_size = BRT_DEFAULT_NODE_SIZE; // default logging size is the same as the default brt block size
     *resultp=result;
     r = ml_init(&result->input_lock); if (r!=0) goto died0;
@@ -510,6 +510,7 @@ int toku_logger_commit (TOKUTXN txn, int nosync, void(*yield)(void*yieldv), void
     return r;
 }
 
+#if 0
 int toku_logger_log_checkpoint (TOKULOGGER logger) {
     if (logger->is_panicked) return EINVAL;
     int r = toku_cachetable_checkpoint(logger->ct);
@@ -518,6 +519,7 @@ int toku_logger_log_checkpoint (TOKULOGGER logger) {
     logger->checkpoint_lsns[0]=logger->lsn;
     return toku_log_checkpoint(logger, (LSN*)0, 1);
 }
+#endif
 
 int toku_logger_txn_begin (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER logger) {
     if (logger->is_panicked) return EINVAL;
@@ -975,8 +977,7 @@ int toku_logger_log_archive (TOKULOGGER logger, char ***logs_p, int flags) {
     // Count the total number of bytes, because we have to return a single big array.  (That's the BDB interface.  Bleah...)
     LSN earliest_lsn_seen={(unsigned long long)(-1LL)};
     r = peek_at_log(logger, all_logs[all_n_logs-1], &earliest_lsn_seen); // try to find the lsn that's in the most recent log
-    if ((earliest_lsn_seen.lsn <= logger->checkpoint_lsns[0].lsn)&&
-	(earliest_lsn_seen.lsn <= logger->checkpoint_lsns[1].lsn)&&
+    if ((earliest_lsn_seen.lsn <= logger->checkpoint_lsn.lsn)&&
 	(earliest_lsn_seen.lsn <= oldest_live_txn_lsn.lsn)) {
 	i=all_n_logs-1;
     } else {
@@ -985,8 +986,7 @@ int toku_logger_log_archive (TOKULOGGER logger, char ***logs_p, int flags) {
 	    if (r!=0) continue; // In case of error, just keep going
 	
 	    //printf("%s:%d file=%s firstlsn=%lld checkpoint_lsns={%lld %lld}\n", __FILE__, __LINE__, all_logs[i], (long long)earliest_lsn_seen.lsn, (long long)logger->checkpoint_lsns[0].lsn, (long long)logger->checkpoint_lsns[1].lsn);
-	    if ((earliest_lsn_seen.lsn <= logger->checkpoint_lsns[0].lsn)&&
-		(earliest_lsn_seen.lsn <= logger->checkpoint_lsns[1].lsn)&&
+	    if ((earliest_lsn_seen.lsn <= logger->checkpoint_lsn.lsn)&&
 		(earliest_lsn_seen.lsn <= oldest_live_txn_lsn.lsn)) {
 		break;
 	    }
@@ -1190,3 +1190,27 @@ toku_logger_txn_rolltmp_raw_count(TOKUTXN txn, u_int64_t *raw_count)
     *raw_count = txn->rollentry_raw_count;
     return 0;
 }
+
+int
+toku_logger_iterate_over_live_txns (TOKULOGGER logger, int (*f)(TOKULOGGER, TOKUTXN, void*), void *v)
+{
+    struct list *l;
+    for (l = list_head(&logger->live_txns); l != &logger->live_txns; l = l->next) {
+	TOKUTXN txn = list_struct(l, struct tokutxn, live_txns_link);
+	int r = f(logger, txn, v);
+	if (r!=0) return r;
+    }
+    return 0;
+}
+
+TOKUTXN
+toku_logger_txn_parent (TOKUTXN txn)
+{
+    return txn->parent;
+}
+
+void
+toku_logger_note_checkpoint(TOKULOGGER logger, LSN lsn)
+{
+    logger->checkpoint_lsn = lsn;
+}

newbrt/log.h

@@ -207,4 +207,13 @@ char *toku_strdup_in_rollback(TOKUTXN txn, const char *s);
 // A hook for testing #1572.  Sets the amount that txn's are incremented by.
 void toku_set_lsn_increment (uint64_t incr) __attribute__((__visibility__("default")));
 
+int toku_logger_iterate_over_live_txns (TOKULOGGER logger, int (*f)(TOKULOGGER, TOKUTXN, void*), void *v);
+// Effect: Call f on each open transaction (root transactions and non-root transactions) in no particular order.
+// If f returns nonzero, then this function returns immediately (without calling f any more), and returns that nonzero value.
+// Otherwise returns 0.
+
+TOKUTXN toku_logger_txn_parent (TOKUTXN);
+
+void toku_logger_note_checkpoint(TOKULOGGER, LSN);
+
 #endif

newbrt/logformat.c

@@ -88,7 +88,14 @@ const struct logtype rollbacks[] = {
 };
 
 const struct logtype logtypes[] = {
-    {"checkpoint", 'x', FA{NULLFIELD}},
+    // Records produced by checkpoints
+    {"begin_checkpoint", 'x', FA{NULLFIELD}},
+    {"end_checkpoint",   'X', FA{{"TXNID", "txnid", 0}, NULLFIELD}},
+    {"fassociate",  'f', FA{{"FILENUM", "filenum", 0},
+			    {"BYTESTRING", "fname", 0},
+			    NULLFIELD}},
+    {"xstillopen", 's', FA{{"TXNID", "txnid", 0}, NULLFIELD}}, // only record root transactions
+    // Reords produced by transactions
     {"commit", 'C', FA{{"TXNID", "txnid", 0},NULLFIELD}},
     {"xabort", 'q', FA{{"TXNID", "txnid", 0},NULLFIELD}},
     {"xbegin", 'b', FA{{"TXNID", "parenttxnid", 0},NULLFIELD}},

newbrt/rbuf.h

@@ -75,6 +75,11 @@ static inline DISKOFF rbuf_diskoff (struct rbuf *r) {
     return rbuf_ulonglong(r);
 }
 
+static inline LSN rbuf_lsn (struct rbuf *r) {
+    LSN lsn = {rbuf_ulonglong(r)};
+    return lsn;
+}
+
 static inline BLOCKNUM rbuf_blocknum (struct rbuf *r) {
     BLOCKNUM result = make_blocknum(rbuf_longlong(r));
     return result;

newbrt/recover.c

@@ -103,23 +103,8 @@ toku_recover_note_cachefile (FILENUM fnum, CACHEFILE cf, BRT brt) {
     return 0;
 }
 
-#define CLEANSUFFIX ".clean"
-#define DIRTYSUFFIX ".dirty"
-
 static void
 internal_toku_recover_fopen_or_fcreate (int flags, int mode, char *fixedfname, FILENUM filenum) {
-    // If .dirty file exists rename it to .clean
-    int slen = strlen(fixedfname);
-    char cleanname[slen + sizeof(CLEANSUFFIX)];
-    char dirtyname[slen + sizeof(DIRTYSUFFIX)];
-    toku_graceful_fill_names(fixedfname, cleanname, sizeof(cleanname), dirtyname, sizeof(dirtyname));
-    toku_struct_stat tmpbuf;
-    BOOL clean_exists = toku_stat(cleanname, &tmpbuf)==0;
-    BOOL dirty_exists = toku_stat(dirtyname, &tmpbuf)==0;
-    if (dirty_exists) {
-	if (clean_exists) { int r = unlink(dirtyname);            assert(r==0); }
-	else              { int r = rename(dirtyname, cleanname); assert(r==0); }
-    }
     CACHEFILE cf;
     int fd = open(fixedfname, O_RDWR|O_BINARY|flags, mode);
     assert(fd>=0);
@@ -181,8 +166,8 @@ static void toku_recover_fheader (LSN UU(lsn), TXNID UU(txnid),FILENUM filenum,
     XMALLOC(h->flags_array);
     h->flags_array[0] = header.flags;
     h->nodesize = header.nodesize;
-    toku_blocktable_create_from_loggedheader(&h->blocktable,
-					     header);
+    //toku_blocktable_create_from_loggedheader(&h->blocktable, header);
+    assert(0); //create from loggedheader disabled for now. //TODO: #1605
     assert(h->blocktable);
     h->n_named_roots = header.n_named_roots;
     r=toku_fifo_create(&h->fifo);
@@ -211,7 +196,7 @@ static void toku_recover_fheader (LSN UU(lsn), TXNID UU(txnid),FILENUM filenum,
     pair->brt->h = h;
     pair->brt->nodesize = h->nodesize;
     pair->brt->flags    = h->nodesize;
-    toku_cachefile_set_userdata(pair->cf, pair->brt->h, toku_brtheader_close, toku_brtheader_checkpoint);
+    toku_cachefile_set_userdata(pair->cf, pair->brt->h, toku_brtheader_close, toku_brtheader_checkpoint, toku_brtheader_begin_checkpoint, toku_brtheader_end_checkpoint);
 }
 
 static void toku_recover_deqrootentry (LSN lsn __attribute__((__unused__)), FILENUM filenum) {
@@ -291,7 +276,20 @@ toku_recover_changeunnamedroot (LSN UU(lsn), FILENUM filenum, BLOCKNUM UU(oldroo
 static void
 toku_recover_changenamedroot (LSN UU(lsn), FILENUM UU(filenum), BYTESTRING UU(name), BLOCKNUM UU(oldroot), BLOCKNUM UU(newroot)) { assert(0); }
 
-static int toku_recover_checkpoint (LSN UU(lsn)) {
+static int toku_recover_begin_checkpoint (LSN UU(lsn)) {
+    return 0;
+}
+
+
+static int toku_recover_end_checkpoint (LSN UU(lsn), TXNID UU(txnid)) {
+    return 0;
+}
+
+static int toku_recover_fassociate (LSN UU(lsn), FILENUM UU(filenum), BYTESTRING UU(fname)) {
+    return 0;
+}
+
+static int toku_recover_xstillopen (LSN UU(lsn), TXNID UU(txnid)) {
     return 0;
 }
 

newbrt/roll.c

@@ -47,8 +47,6 @@ toku_rollback_fcreate (TXNID      UU(xid),
     }
     r = unlink(full_fname);
     assert(r==0);
-    r = toku_graceful_delete(full_fname);
-    assert(r==0);
     toku_free(fname);
     return 0;
 }

newbrt/tests/Makefile

@@ -29,10 +29,11 @@ endif
 # For very verbose output do 
 #   make VERBOSE=2
 
-
+#test1305 is first, since it is the longest test.  Thus reducing the makespan on parallel checks
 # Put these one-per-line so that if we insert a new one the svn diff can understand it better.
 # Also keep them sorted.
 REGRESSION_TESTS_RAW = \
+	test1305 \
 	block_allocator_test \
 	bread-test \
 	brt-serialize-test \
@@ -59,6 +60,7 @@ REGRESSION_TESTS_RAW = \
 	cachetable-count-pinned-test \
 	cachetable-debug-test \
 	cachetable-debug-test \
+	cachetable-checkpoint-pending \
 	cachetable-checkpoint-test \
 	cachetable-prefetch-maybegetandpin-test \
 	cachetable-prefetch2-test \
@@ -86,7 +88,6 @@ REGRESSION_TESTS_RAW = \
 	omt-cursor-test \
 	omt-test \
 	shortcut \
-	test1305 \
 	test1308a \
 	test1626 \
 	test-assert \
@@ -131,8 +132,8 @@ check_ok:
 # Don't run 1305 under valgrind.  It takes way too long.
 check_test1305: VGRIND=
 
-ifeq ($(SKIP_VERY_SLOW_TESTS),1)
-check_test1305 check_test1308a:
+ifeq ($(SKIP_1305),1)
+check_test1305:
 	@echo SKIPPED SLOW TEST $@
 endif
 

newbrt/tests/benchmark-test.c

@@ -28,7 +28,7 @@ static BRT t;
 
 static void setup (void) {
     int r;
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);         assert(r==0);
     r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, NULL_TXN, toku_default_compare_fun, (DB*)0); assert(r==0);
 }

newbrt/tests/bread-test.c

@@ -29,7 +29,7 @@ char wrotedata[RECORDS*RECORDLEN];
 static void
 test (int seed) {
     srandom(seed);
-    unlink_file_and_bit(FNAME);
+    unlink(FNAME);
     int i;
     {
         int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
@@ -69,7 +69,7 @@ test (int seed) {
 
     { int r=close_bread_without_closing_fd(br); assert(r==0); }
     { int r=close(fd); assert(r==0); }
-    unlink_file_and_bit(FNAME);
+    unlink(FNAME);
 }
 
 int

newbrt/tests/brt-serialize-sub-block-test.c

@@ -23,7 +23,7 @@ static void test_sub_block(int n) {
     BRT brt;
     int i;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     error = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(error == 0);

newbrt/tests/brt-serialize-test.c

@@ -55,21 +55,29 @@ static void test_serialize(void) {
 
     struct brt *XMALLOC(brt);
     struct brt_header *XMALLOC(brt_h);
-    struct block_translation_pair *XMALLOC_N(21, btps);
-    memset(btps, 0, sizeof(btps));
     brt->h = brt_h;
+    brt_h->type = BRTHEADER_CURRENT;
     brt_h->panic = 0; brt_h->panic_string = 0;
     toku_blocktable_create_new(&brt_h->blocktable);
-    toku_blocktable_debug_set_translation(brt_h->blocktable, 1, btps);
-    btps[20].diskoff = 4096;
-    btps[20].size    = 100;
+    //Want to use block #20
+    BLOCKNUM b = make_blocknum(0);
+    while (b.b < 20) {
+        toku_allocate_blocknum(brt_h->blocktable, &b, brt_h);
+    }
+    assert(b.b == 20);
+
     {
-	u_int64_t b;
-        toku_block_alloc(brt_h->blocktable, 100, &b);
-	assert(b==4096);
+        DISKOFF offset;
+        DISKOFF size;
+        toku_blocknum_realloc_on_disk(brt_h->blocktable, b, 100, &offset, brt_h, FALSE);
+        assert(offset==BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+
+        toku_translate_blocknum_to_offset_size(brt_h->blocktable, b, &offset, &size);
+        assert(offset == BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
+        assert(size   == 100);
     }
     
-    r = toku_serialize_brtnode_to(fd, make_blocknum(20), &sn, brt->h, 1, 1);  
+    r = toku_serialize_brtnode_to(fd, make_blocknum(20), &sn, brt->h, 1, 1, FALSE);  
     assert(r==0);
     
     r = toku_deserialize_brtnode_from(fd, make_blocknum(20), 0/*pass zero for hash*/, &dn, brt_h);
@@ -128,7 +136,7 @@ static void test_serialize(void) {
     toku_free(sn.u.n.childinfos);
     toku_free(sn.u.n.childkeys);
 
-    toku_block_free(brt_h->blocktable, 4096);
+    toku_block_free(brt_h->blocktable, BLOCK_ALLOCATOR_TOTAL_HEADER_RESERVE);
     toku_blocktable_destroy(&brt_h->blocktable);
     toku_free(brt_h);
     toku_free(brt);

newbrt/tests/brt-test-cursor-2.c

@@ -29,7 +29,7 @@ static void test_multiple_brt_cursor_dbts(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursors[n];
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);

newbrt/tests/brt-test-cursor.c

@@ -84,7 +84,7 @@ static void test_brt_cursor_first(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_first:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -125,7 +125,7 @@ static void test_brt_cursor_last(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_last:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -166,7 +166,7 @@ static void test_brt_cursor_first_last(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_first_last:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -211,7 +211,7 @@ static void test_brt_cursor_rfirst(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_rfirst:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -279,7 +279,7 @@ static void test_brt_cursor_walk(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_walk:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -345,7 +345,7 @@ static void test_brt_cursor_rwalk(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_rwalk:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -429,7 +429,7 @@ static void test_brt_cursor_rand(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_rand:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -483,7 +483,7 @@ static void test_brt_cursor_split(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_split:%d %p\n", n, db);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -559,7 +559,7 @@ static void test_multiple_brt_cursors(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursors[n];
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -605,7 +605,7 @@ static void test_multiple_brt_cursor_walk(int n, DB *db) {
     const int ncursors = n/cursor_gap;
     BRT_CURSOR cursors[ncursors];
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     int nodesize = 1<<12;
     int h = log16(n);
@@ -685,7 +685,7 @@ static void test_brt_cursor_set(int n, int cursor_op, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -757,7 +757,7 @@ static void test_brt_cursor_set_range(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -821,7 +821,7 @@ static void test_brt_cursor_delete(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     error = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(error == 0);
@@ -882,7 +882,7 @@ static void test_brt_cursor_get_both(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     error = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(error == 0);

newbrt/tests/brt-test-named-db.c

@@ -15,7 +15,7 @@ static void test_named_db (void) {
     DBT k,v;
 
     if (verbose) printf("test_named_db\n");
-    unlink_file_and_bit(n0);
+    unlink(n0);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);           assert(r==0);
     r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);

newbrt/tests/brt-test.c

@@ -16,7 +16,7 @@ static void test_dump_empty_db (void) {
     toku_memory_check=1;
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     if (verbose) toku_dump_brt(stdout, t);
@@ -33,8 +33,8 @@ static void test_multiple_files_of_size (int size) {
     BRT t0,t1;
     int r,i;
     if (verbose) printf("test_multiple_files_of_size(%d)\n", size);
-    unlink_file_and_bit(n0);
-    unlink_file_and_bit(n1);
+    unlink(n0);
+    unlink(n1);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);   assert(r==0);
     r = toku_open_brt(n0, 0, 1, &t0, size, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
@@ -94,8 +94,8 @@ static void test_multiple_dbs (void) {
     int r;
     DBT k,v;
     if (verbose) printf("test_multiple_dbs: ");
-    unlink_file_and_bit(n0);
-    unlink_file_and_bit(n1);
+    unlink(n0);
+    unlink(n1);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);        assert(r==0);
     r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
@@ -136,7 +136,7 @@ static void test_multiple_dbs_many (void) {
     BRT trees[MANYN];
     if (verbose) printf("test_multiple_dbs_many:\n");
     toku_memory_check_all_free();
-    unlink_file_and_bit(name);
+    unlink(name);
     r = toku_brt_create_cachetable(&ct, (MANYN+4), ZERO_LSN, NULL_LOGGER);     assert(r==0);
     for (i=0; i<MANYN; i++) {
 	char dbname[20];
@@ -166,7 +166,7 @@ static void test_multiple_brts_one_db_one_file (void) {
     BRT trees[MANYN];
     if (verbose) printf("test_multiple_brts_one_db_one_file:");
     toku_memory_check_all_free();
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create_cachetable(&ct, 32, ZERO_LSN, NULL_LOGGER); assert(r==0);
     for (i=0; i<MANYN; i++) {
 	r = toku_open_brt(fname, 0, (i==0), &trees[i], 1<<12, ct, null_txn, toku_default_compare_fun, null_db);
@@ -203,7 +203,7 @@ static void  test_read_what_was_written (void) {
 
     if (verbose) printf("test_read_what_was_written(): "); fflush(stdout);
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
     toku_memory_check_all_free();
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);       assert(r==0);
@@ -320,7 +320,7 @@ static void test_cursor_last_empty(void) {
     BRT_CURSOR cursor=0;
     int r;
     if (verbose) printf("%s", __FUNCTION__);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     toku_memory_check_all_free();
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);       assert(r==0);
@@ -354,7 +354,7 @@ static void test_cursor_next (void) {
     int r;
     DBT kbt, vbt;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);       assert(r==0);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
@@ -422,7 +422,7 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
     int r;
     unsigned int i;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
     toku_memory_check_all_free();
 
     {
@@ -530,7 +530,7 @@ static void test_large_kv(int bsize, int ksize, int vsize) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, bsize, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -576,7 +576,7 @@ static void test_brt_delete_empty() {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -604,7 +604,7 @@ static void test_brt_delete_present(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -666,7 +666,7 @@ static void test_brt_delete_not_present(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -712,7 +712,7 @@ static void test_brt_delete_cursor_first(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -805,7 +805,7 @@ static void test_insert_delete_lookup(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -853,7 +853,7 @@ static void test_brt_delete_both(int n) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, TOKU_DB_DUP + TOKU_DB_DUPSORT); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -957,7 +957,7 @@ static void test_new_brt_cursor_first(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1010,7 +1010,7 @@ static void test_new_brt_cursor_last(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1064,7 +1064,7 @@ static void test_new_brt_cursor_next(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1108,7 +1108,7 @@ static void test_new_brt_cursor_prev(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1152,7 +1152,7 @@ static void test_new_brt_cursor_current(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1235,7 +1235,7 @@ static void test_new_brt_cursor_set_range(int n, int dup_mode) {
     BRT_CURSOR cursor=0;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&brt); assert(r == 0);
     r = toku_brt_set_flags(brt, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(brt, 4096); assert(r == 0);
@@ -1294,7 +1294,7 @@ static void test_new_brt_cursor_set(int n, int cursor_op, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
 

newbrt/tests/brt-test0.c

@@ -18,7 +18,7 @@ static void test0 (void) {
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
     if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     //printf("%s:%d test0\n", __FILE__, __LINE__);

newbrt/tests/brt-test1.c

@@ -17,7 +17,7 @@ static void test1 (void) {
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);

newbrt/tests/brt-test2.c

@@ -17,7 +17,7 @@ static void test2 (int memcheck, int limit) {
     if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
     assert(r==0);

newbrt/tests/brt-test3.c

@@ -19,7 +19,7 @@ static void test3 (int nodesize, int count, int memcheck) {
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
     gettimeofday(&t0, 0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     for (i=0; i<count; i++) {

newbrt/tests/brt-test4.c

@@ -16,7 +16,7 @@ static void test4 (int nodesize, int count, int memcheck) {
     int i;
     CACHETABLE ct;
     gettimeofday(&t0, 0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     toku_memory_check=memcheck;
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);         assert(r==0);

newbrt/tests/brt-test5.c

@@ -18,7 +18,7 @@ static void test5 (void) {
     toku_memory_check_all_free();
     MALLOC_N(limit,values);
     for (i=0; i<limit; i++) values[i]=-1;
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);        assert(r==0);
     r = toku_open_brt(fname, 0, 1, &t, 1<<12, ct, null_txn, toku_default_compare_fun, null_db);   assert(r==0);
     for (i=0; i<limit/2; i++) {

newbrt/tests/cachetable-checkpoint-pending.c

+// Make sure that the pending stuff gets checkpointed, but subsequent changes don't, even with concurrent updates.
+
+#include <stdio.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include "test.h"
+#include "cachetable.h"
+
+static int N; // how many items in the table
+static CACHEFILE cf;
+static CACHETABLE ct;
+int    *values;
+
+static const int item_size = sizeof(int);
+
+static volatile int n_flush, n_write_me, n_keep_me, n_fetch;
+
+static void
+sleep_random (void)
+{
+    struct timespec req = {.tv_sec  = 0,
+			   .tv_nsec = random()%1000000};
+    nanosleep(&req, NULL);
+}
+
+int expect_value = 42; // initially 42, later 43
+
+static void
+flush (CACHEFILE UU(thiscf), CACHEKEY UU(key), void *value, void *UU(extraargs), long size, BOOL write_me, BOOL keep_me, LSN UU(modified_lsn), BOOL UU(rename_p), BOOL UU(for_checkpoint))
+{
+    // printf("f");
+    assert(size == item_size);
+    int *v = value;
+    if (*v!=expect_value) printf("got %d expect %d\n", *v, expect_value);
+    assert(*v==expect_value);
+    (void)__sync_fetch_and_add(&n_flush, 1);
+    if (write_me) (void)__sync_fetch_and_add(&n_write_me, 1);
+    if (keep_me)  (void)__sync_fetch_and_add(&n_keep_me,  1);
+    sleep_random();
+}
+
+static int
+fetch (CACHEFILE UU(thiscf), CACHEKEY UU(key), u_int32_t UU(fullhash), void **UU(value), long *UU(sizep), void *UU(extraargs), LSN *UU(written_lsn))
+{
+    assert(0); // should not be called
+    return 0;
+}
+
+static void*
+do_update (void *UU(ignore))
+{
+    while (n_flush==0); // wait until the first checkpoint ran
+    int i;
+    for (i=0; i<N; i++) {
+	CACHEKEY key = make_blocknum(i);
+        u_int32_t hi = toku_cachetable_hash(cf, key);
+        void *vv;
+	long size;
+        int r = toku_cachetable_get_and_pin(cf, key, hi, &vv, &size, flush, fetch, 0);
+	//printf("g");
+	assert(r==0);
+	assert(size==sizeof(int));
+	int *v = vv;
+	assert(*v==42);
+	*v = 43;
+	//printf("[%d]43\n", i);
+	r = toku_cachetable_unpin(cf, key, hi, CACHETABLE_DIRTY, item_size);
+	sleep_random();
+    }
+    return 0;
+}
+
+static void*
+do_checkpoint (void *UU(v))
+{
+    int r = toku_cachetable_checkpoint(ct, NULL);
+    assert(r == 0);
+    return 0;
+}
+
+// put n items into the cachetable, mark them dirty, and then concurently
+//   do a checkpoint (in which the callback functions are slow)
+//   replace the n items with new values
+// make sure that the stuff that was checkpointed includes only the old versions
+// then do a flush and make sure the new items are written
+
+static void cachetable_checkpoint_pending(void) {
+    if (verbose) printf("%s:%d n=%d\n", __FUNCTION__, __LINE__, N);
+    const int test_limit = N;
+    int r;
+    r = toku_create_cachetable(&ct, test_limit*sizeof(int), ZERO_LSN, NULL_LOGGER); assert(r == 0);
+    char fname1[] = __FILE__ "test1.dat";
+    unlink(fname1);
+    r = toku_cachetable_openf(&cf, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
+
+    // Insert items into the cachetable. All dirty.
+    int i;
+    for (i=0; i<N; i++) {
+        CACHEKEY key = make_blocknum(i);
+        u_int32_t hi = toku_cachetable_hash(cf, key);
+	values[i] = 42;
+        r = toku_cachetable_put(cf, key, hi, &values[i], sizeof(int), flush, fetch, 0);
+        assert(r == 0);
+
+        r = toku_cachetable_unpin(cf, key, hi, CACHETABLE_DIRTY, item_size);
+        assert(r == 0);
+    }
+
+    // the checkpoint should cause n writes, but since n <= the cachetable size,
+    // all items should be kept in the cachetable
+    n_flush = n_write_me = n_keep_me = n_fetch = 0; expect_value = 42;
+    //printf("E42\n");
+    pthread_t checkpoint_thread, update_thread;
+    r = pthread_create(&checkpoint_thread, NULL, do_checkpoint, NULL);  assert(r==0);
+    r = pthread_create(&update_thread,     NULL, do_update,     NULL);  assert(r==0);
+    r = pthread_join(checkpoint_thread, 0);                             assert(r==0);
+    r = pthread_join(update_thread, 0);                                 assert(r==0);
+    
+    assert(n_flush == N && n_write_me == N && n_keep_me == N);
+
+    // after the checkpoint, all of the items should be 43
+    //printf("E43\n");
+    n_flush = n_write_me = n_keep_me = n_fetch = 0; expect_value = 43;
+    r = toku_cachetable_checkpoint(ct, NULL);
+    assert(r == 0);
+    assert(n_flush == N && n_write_me == N && n_keep_me == N);
+
+    // a subsequent checkpoint should cause no flushes, or writes since all of the items are clean
+    n_flush = n_write_me = n_keep_me = n_fetch = 0;
+    r = toku_cachetable_checkpoint(ct, NULL);
+    assert(r == 0);
+    assert(n_flush == 0 && n_write_me == 0 && n_keep_me == 0);
+
+    r = toku_cachefile_close(&cf, NULL_LOGGER, 0); assert(r == 0 && cf == 0);
+    r = toku_cachetable_close(&ct); assert(r == 0 && ct == 0);
+}
+
+int
+test_main(int argc, const char *argv[]) {
+    {
+	struct timeval tv;
+	gettimeofday(&tv, 0);
+	srandom(tv.tv_sec * 1000000 + tv.tv_usec);
+    }	
+    {
+	int i;
+	for (i=1; i<argc; i++) {
+	    if (strcmp(argv[i], "-v") == 0) {
+		verbose++;
+		continue;
+	    }
+	}
+    }
+    for (N=1; N<=128; N*=2) {
+	int myvalues[N];
+	values = myvalues;
+        cachetable_checkpoint_pending();
+	//printf("\n");
+    }
+    return 0;
+}

newbrt/tests/cachetable-checkpoint-test.c

@@ -9,7 +9,7 @@ static const int item_size = 1;
 
 static int n_flush, n_write_me, n_keep_me, n_fetch;
 
-static void flush(CACHEFILE cf, CACHEKEY key, void *value, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p) {
+static void flush(CACHEFILE cf, CACHEKEY key, void *value, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p, BOOL UU(for_checkpoint)) {
     cf = cf; key = key; value = value; extraargs = extraargs; modified_lsn = modified_lsn; rename_p = rename_p;
     // assert(key == make_blocknum((long)value));
     assert(size == item_size);
@@ -37,7 +37,7 @@ static void cachetable_checkpoint_test(int n, enum cachetable_dirty dirty) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 
@@ -67,7 +67,7 @@ static void cachetable_checkpoint_test(int n, enum cachetable_dirty dirty) {
     // the checkpoint should cause n writes, but since n <= the cachetable size,
     // all items should be kept in the cachetable
     n_flush = n_write_me = n_keep_me = n_fetch = 0;
-    r = toku_cachetable_checkpoint(ct);
+    r = toku_cachetable_checkpoint(ct, NULL);
     assert(r == 0);
     assert(n_flush == n && n_write_me == n && n_keep_me == n);
 
@@ -93,12 +93,11 @@ static void cachetable_checkpoint_test(int n, enum cachetable_dirty dirty) {
         assert(its_size == item_size);
     }
 
-    // a subsequent checkpoint should cause n flushes, but no writes since all
-    // of the items are clean
+    // a subsequent checkpoint should cause no flushes, or writes since all of the items are clean
     n_flush = n_write_me = n_keep_me = n_fetch = 0;
-    r = toku_cachetable_checkpoint(ct);
+    r = toku_cachetable_checkpoint(ct, NULL);
     assert(r == 0);
-    assert(n_flush == n && n_write_me == 0 && n_keep_me == n);
+    assert(n_flush == 0 && n_write_me == 0 && n_keep_me == 0);
 
     r = toku_cachefile_close(&f1, NULL_LOGGER, 0); assert(r == 0 && f1 == 0);
     r = toku_cachetable_close(&ct); assert(r == 0 && ct == 0);

newbrt/tests/cachetable-count-pinned-test.c

@@ -10,7 +10,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     /* Do nothing */
 }
@@ -34,7 +35,7 @@ cachetable_count_pinned_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-debug-test.c

@@ -10,7 +10,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     /* Do nothing */
 }
@@ -34,7 +35,7 @@ cachetable_debug_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-fd-test.c

@@ -9,7 +9,7 @@ cachetable_fd_test (void) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE cf;
     r = toku_cachetable_openf(&cf, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 
@@ -17,7 +17,7 @@ cachetable_fd_test (void) {
 
     // test set to good fd succeeds
     char fname2[] = __FILE__ "test2.data";
-    unlink_file_and_bit(fname2);
+    unlink(fname2);
     int fd2 = open(fname2, O_RDWR | O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd2 >= 0 && fd1 != fd2);
     r = toku_cachefile_set_fd(cf, fd2, fname2); assert(r == 0);
     assert(toku_cachefile_fd(cf) == fd2);

newbrt/tests/cachetable-flush-test.c

@@ -10,7 +10,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     /* Do nothing */
 }
@@ -34,12 +35,12 @@ test_cachetable_flush (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 
     char fname2[] = __FILE__ "test2.dat";
-    unlink_file_and_bit(fname2);
+    unlink(fname2);
     CACHEFILE f2;
     r = toku_cachetable_openf(&f2, ct, fname2, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-getandpin-test.c

@@ -10,7 +10,8 @@ flush (CACHEFILE cf     __attribute__((__unused__)),
        BOOL write_me    __attribute__((__unused__)),
        BOOL keep_me     __attribute__((__unused__)),
        LSN lsn          __attribute__((__unused__)),
-       BOOL rename_p    __attribute__((__unused__))
+       BOOL rename_p    __attribute__((__unused__)),
+       BOOL for_checkpoint    __attribute__((__unused__))
        ) {
     assert((long) key.b == size);
     if (!keep_me) toku_free(v);
@@ -43,7 +44,7 @@ cachetable_getandpin_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test_getandpin.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-prefetch-checkpoint-test.c

@@ -13,7 +13,7 @@ const int item_size = 1;
 
 int n_flush, n_write_me, n_keep_me, n_fetch;
 
-static void flush(CACHEFILE cf, CACHEKEY key, void *value, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p) {
+static void flush(CACHEFILE cf, CACHEKEY key, void *value, void *extraargs, long size, BOOL write_me, BOOL keep_me, LSN modified_lsn, BOOL rename_p, BOOL UU(for_checkpoint)) {
     cf = cf; key = key; value = value; extraargs = extraargs; modified_lsn = modified_lsn; rename_p = rename_p;
     // assert(key == make_blocknum((long)value));
     assert(size == item_size);
@@ -78,7 +78,7 @@ static void cachetable_prefetch_checkpoint_test(int n, enum cachetable_dirty dir
     // the checkpoint should cause n writes, but since n <= the cachetable size,
     // all items should be kept in the cachetable
     n_flush = n_write_me = n_keep_me = n_fetch = 0;
-    r = toku_cachetable_checkpoint(ct);
+    r = toku_cachetable_checkpoint(ct, NULL);
     assert(r == 0);
     assert(n_flush == n && n_write_me == n && n_keep_me == n);
 
@@ -104,12 +104,11 @@ static void cachetable_prefetch_checkpoint_test(int n, enum cachetable_dirty dir
         assert(its_size == item_size);
     }
 
-    // a subsequent checkpoint should cause n flushes, but no writes since all
-    // of the items are clean
+    // a subsequent checkpoint should cause no flushes, or writes since all of the items are clean
     n_flush = n_write_me = n_keep_me = n_fetch = 0;
-    r = toku_cachetable_checkpoint(ct);
+    r = toku_cachetable_checkpoint(ct, NULL);
     assert(r == 0);
-    assert(n_flush == n && n_write_me == 0 && n_keep_me == n);
+    assert(n_flush == 0 && n_write_me == 0 && n_keep_me == 0);
 
     r = toku_cachefile_close(&f1, NULL_LOGGER, 0); assert(r == 0 && f1 == 0);
     r = toku_cachetable_close(&ct); assert(r == 0 && ct == 0);

newbrt/tests/cachetable-prefetch-close-fail-test.c

@@ -13,7 +13,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     assert(w == FALSE);
 }

newbrt/tests/cachetable-prefetch-close-leak-test.c

@@ -13,7 +13,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     assert(w == FALSE && v != NULL);
     toku_free(v);

newbrt/tests/cachetable-prefetch-close-test.c

@@ -13,7 +13,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     assert(w == FALSE);
 }

newbrt/tests/cachetable-prefetch-getandpin-fail-test.c

@@ -13,7 +13,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     assert(w == FALSE);
 }

newbrt/tests/cachetable-prefetch-getandpin-test.c

@@ -13,7 +13,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     assert(w == FALSE);
 }

newbrt/tests/cachetable-prefetch-maybegetandpin-test.c

@@ -13,7 +13,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     assert(w == FALSE);
 }

newbrt/tests/cachetable-prefetch2-test.c

@@ -14,7 +14,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     assert(w == FALSE);
 }

newbrt/tests/cachetable-put-test.c

@@ -10,7 +10,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     /* Do nothing */
 }
@@ -34,7 +35,7 @@ cachetable_put_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-rename-test.c

@@ -45,7 +45,8 @@ static void r_flush (CACHEFILE f      __attribute__((__unused__)),
 		     BOOL write_me    __attribute__((__unused__)),
 		     BOOL keep_me,
 		     LSN modified_lsn __attribute__((__unused__)),
-		     BOOL rename_p    __attribute__((__unused__))) {
+		     BOOL rename_p    __attribute__((__unused__)),
+		     BOOL for_checkpoint    __attribute__((__unused__))) {
     int i;
     //printf("Flush\n");
     if (keep_me) return;
@@ -88,7 +89,7 @@ static void test_rename (void) {
     test_mutex_init();
     const char fname[] = __FILE__ "rename.dat";
     r=toku_create_cachetable(&t, KEYLIMIT, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r==0);
     for (i=0; i<TRIALLIMIT; i++) {

newbrt/tests/cachetable-scan.c

@@ -19,7 +19,8 @@ static void f_flush (CACHEFILE f,
 		     BOOL write_me,
 		     BOOL keep_me,
 		     LSN  modified_lsn __attribute__((__unused__)),
-		     BOOL rename_p     __attribute__((__unused__))) {
+		     BOOL rename_p     __attribute__((__unused__)),
+		     BOOL for_checkpoint     __attribute__((__unused__))) {
     assert(size==BLOCKSIZE);
     if (write_me) {
 	int r = pwrite(toku_cachefile_fd(f), value, BLOCKSIZE, key.b);

newbrt/tests/cachetable-test.c

@@ -108,7 +108,8 @@ static void flush (CACHEFILE f,
 		   BOOL write_me __attribute__((__unused__)),
 		   BOOL keep_me __attribute__((__unused__)),
 		   LSN modified_lsn __attribute__((__unused__)),
-		   BOOL rename_p __attribute__((__unused__))) {
+		   BOOL rename_p __attribute__((__unused__)),
+		   BOOL for_checkpoint __attribute__((__unused__))) {
     struct item *it = value;
     int i;
 
@@ -173,7 +174,7 @@ static void test0 (void) {
 
     r=toku_create_cachetable(&t, 5, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r==0);
 
@@ -303,7 +304,8 @@ static void flush_n (CACHEFILE f __attribute__((__unused__)), CACHEKEY key __att
 		     void *extra  __attribute__((__unused__)),
                      long size __attribute__((__unused__)),
 		     BOOL write_me __attribute__((__unused__)),    BOOL keep_me __attribute__((__unused__)),
-		     LSN modified_lsn __attribute__((__unused__)), BOOL rename_p __attribute__ ((__unused__))) {
+		     LSN modified_lsn __attribute__((__unused__)), BOOL rename_p __attribute__ ((__unused__)),
+		     BOOL for_checkpoint __attribute__ ((__unused__))) {
     int *v = value;
     assert(*v==0);
 }
@@ -327,7 +329,7 @@ static void test_nested_pin (void) {
     char fname[] = __FILE__ "test_ct.dat";
     r = toku_create_cachetable(&t, 1, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r==0);
     expect_f = f;
@@ -368,7 +370,8 @@ static void null_flush (CACHEFILE cf     __attribute__((__unused__)),
                         BOOL write_me    __attribute__((__unused__)),
                         BOOL keep_me     __attribute__((__unused__)),
                         LSN modified_lsn __attribute__((__unused__)),
-                        BOOL rename_p    __attribute__((__unused__))) {
+                        BOOL rename_p    __attribute__((__unused__)),
+                        BOOL for_checkpoint __attribute__((__unused__))) {
 }
 
 static int add123_fetch (CACHEFILE cf, CACHEKEY key, u_int32_t fullhash, void **value, long *sizep __attribute__((__unused__)), void*extraargs, LSN *written_lsn) {
@@ -397,8 +400,8 @@ static void test_multi_filehandles (void) {
     char fname3[]= __FILE__ "test3_ct.dat";
     int r;
     void *v;
-    unlink_file_and_bit(fname1);
-    unlink_file_and_bit(fname2);
+    unlink(fname1);
+    unlink(fname2);
 
     r = toku_create_cachetable(&t, 4, ZERO_LSN, NULL_LOGGER);          assert(r==0);
     r = toku_cachetable_openf(&f1, t, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);   assert(r==0);
@@ -443,7 +446,8 @@ static void test_dirty_flush(CACHEFILE f,
 			     BOOL do_write,
 			     BOOL keep,
 			     LSN modified_lsn __attribute__((__unused__)),
-			     BOOL rename_p __attribute__((__unused__))) {
+			     BOOL rename_p __attribute__((__unused__)),
+			     BOOL for_checkpoint __attribute__((__unused__))) {
     if (verbose) printf("test_dirty_flush %p %" PRId64 " %p %ld %u %u\n", f, key.b, value, size, (unsigned)do_write, (unsigned)keep);
 }
 
@@ -468,7 +472,7 @@ static void test_dirty() {
     assert(r == 0);
 
     char *fname = __FILE__ "test.dat";
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r == 0);
 
@@ -568,7 +572,8 @@ static void test_size_flush_callback(CACHEFILE f,
 				     BOOL do_write,
 				     BOOL keep,
 				     LSN modified_lsn __attribute__((__unused__)),
-				     BOOL rename_p __attribute__((__unused__))) {
+				     BOOL rename_p __attribute__((__unused__)),
+				     BOOL for_checkpoint __attribute__((__unused__))) {
     if (test_size_debug && verbose) printf("test_size_flush %p %" PRId64 " %p %ld %u %u\n", f, key.b, value, size, (unsigned)do_write, (unsigned)keep);
     if (keep) {
         if (do_write) {
@@ -595,7 +600,7 @@ static void test_size_resize() {
     assert(r == 0);
 
     char *fname = __FILE__ "test.dat";
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r == 0);
 
@@ -650,7 +655,7 @@ static void test_size_flush() {
     assert(r == 0);
 
     char *fname = __FILE__ "test.dat";
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r == 0);
 

newbrt/tests/cachetable-test2.c

@@ -102,7 +102,8 @@ static void flush_forchain (CACHEFILE f            __attribute__((__unused__)),
 			    BOOL      write_me     __attribute__((__unused__)),
 			    BOOL      keep_me      __attribute__((__unused__)),
 			    LSN       modified_lsn __attribute__((__unused__)),
-			    BOOL      rename_p     __attribute__((__unused__))) {
+			    BOOL      rename_p     __attribute__((__unused__)),
+			    BOOL      for_checkpoint     __attribute__((__unused__))) {
     if (keep_me) return;
     int *v = value;
     //toku_cachetable_print_state(ct);
@@ -156,7 +157,7 @@ static void test_chaining (void) {
     for (i=0; i<N_FILES; i++) {
 	r = snprintf(fname[i], FILENAME_LEN, __FILE__ ".%ld.dat", i);
 	assert(r>0 && r<FILENAME_LEN);
-	unlink_file_and_bit(fname[i]);
+	unlink(fname[i]);
 	r = toku_cachetable_openf(&f[i], ct, fname[i], O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);   assert(r==0);
 	}
     for (i=0; i<N_PRESENT_LIMIT; i++) {

newbrt/tests/cachetable-unpin-and-remove-test.c

@@ -38,7 +38,7 @@ cachetable_unpin_and_remove_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, table_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, 0777); assert(r == 0);
 
@@ -101,7 +101,7 @@ cachetable_put_evict_remove_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, table_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, 0777); assert(r == 0);
 

newbrt/tests/cachetable-unpin-test.c

@@ -10,7 +10,8 @@ flush (CACHEFILE f __attribute__((__unused__)),
        BOOL w      __attribute__((__unused__)),
        BOOL keep   __attribute__((__unused__)),
        LSN m       __attribute__((__unused__)),
-       BOOL r      __attribute__((__unused__))
+       BOOL r      __attribute__((__unused__)),
+       BOOL c      __attribute__((__unused__))
        ) {
     /* Do nothing */
 }
@@ -34,7 +35,7 @@ cachetable_unpin_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink_file_and_bit(fname1);
+    unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/dup-delete-all.c

@@ -18,7 +18,7 @@ static DB * const null_db = 0;
 static void test_delete_all (void) {
     char fname[]= __FILE__ ".brt";
     u_int32_t limit =200;
-    unlink_file_and_bit(fname);
+    unlink(fname);
     CACHETABLE ct;
     int r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                                assert(r==0);
     BRT t;

newbrt/tests/keyrange-dupsort-unflat.c

@@ -13,7 +13,7 @@ static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     u_int64_t limit=100;
     u_int64_t ilimit=100;
-    unlink_file_and_bit(fname);
+    unlink(fname);
     CACHETABLE ct;
     // set the cachetable to size 1 so that things won't fit.
     int r = toku_brt_create_cachetable(&ct, 1, ZERO_LSN, NULL_LOGGER);                                assert(r==0);

newbrt/tests/keyrange-dupsort.c

@@ -13,7 +13,7 @@ static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     u_int64_t limit=100;
     u_int64_t ilimit=100;
-    unlink_file_and_bit(fname);
+    unlink(fname);
     CACHETABLE ct;
     int r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                                assert(r==0);
     BRT t;

newbrt/tests/keyrange-unflat.c

@@ -13,7 +13,7 @@ static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     const u_int64_t limit=10000;
     u_int64_t permute[limit];
-    unlink_file_and_bit(fname);
+    unlink(fname);
     CACHETABLE ct;
     // set the cachetable to size 1 so that things won't fit.
     int r = toku_brt_create_cachetable(&ct, 1, ZERO_LSN, NULL_LOGGER);                                assert(r==0);

newbrt/tests/keyrange.c

@@ -12,7 +12,7 @@ static DB * const null_db = 0;
 static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     u_int64_t limit=30000;
-    unlink_file_and_bit(fname);
+    unlink(fname);
     CACHETABLE ct;
     int r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                                assert(r==0);
     BRT t;

newbrt/tests/shortcut.c

@@ -18,7 +18,7 @@ test_main (int argc __attribute__((__unused__)), const char *argv[]  __attribute
     DB a_db;
     DB *db = &a_db;
 
-    unlink_file_and_bit(fname);
+    unlink(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                               assert(r==0);
     r = toku_open_brt(fname, 0, 1, &brt, 1<<12, ct, null_txn, test_brt_cursor_keycompare, db);   assert(r==0);

newbrt/tests/test-brt-delete-both.c

@@ -19,7 +19,7 @@ doit (void) {
     DBT k,v;
     if (verbose) printf("%s\n", __FUNCTION__);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create(&t); assert(r==0);
     r = toku_brt_open(t, fname, fname, 0, 1, 1, ct, null_txn, (DB*)0); assert(r==0);
 

newbrt/tests/test-brt-overflow.c

@@ -17,7 +17,7 @@ test_overflow (void) {
     CACHETABLE ct;
     u_int32_t nodesize = 1<<20; 
     int r;
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);         assert(r==0);
     r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
 

newbrt/tests/test-del-inorder.c

@@ -27,7 +27,7 @@ doit (void) {
 
     snprintf(fname, fnamelen, "%s.brt", __FILE__);
     r = toku_brt_create_cachetable(&ct, 16*1024, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, NODESIZE, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     toku_free(fname);

newbrt/tests/test-dump-brt.c

@@ -13,7 +13,7 @@ test_main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__u
     BRT t;
     CACHETABLE ct;
     FILE *f = fopen("test-dump-brt.out", "w");
-    unlink_file_and_bit(n);
+    unlink(n);
     assert(f);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);   assert(r==0);
     r = toku_open_brt(n, 0, 1, &t, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);

newbrt/tests/test-inc-split.c

@@ -56,7 +56,7 @@ doit (int ksize __attribute__((__unused__))) {
 
     snprintf(fname, fnamelen, "%s.brt", __FILE__);
     r = toku_brt_create_cachetable(&ct, 16*1024, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(fname);
+    unlink(fname);
     r = toku_open_brt(fname, 0, 1, &t, NODESIZE, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     toku_free(fname);

newbrt/tests/test.h

@@ -67,17 +67,6 @@ brt_lookup_and_fail_nodup (BRT t, char *keystring)
 
 int verbose=0;
 
-void
-unlink_file_and_bit(const char *name) {
-    char dirty[strlen(name) + sizeof(".dirty")];
-    char clean[strlen(name) + sizeof(".clean")];
-    sprintf(dirty, "%s.dirty", name);
-    sprintf(clean, "%s.clean", name);
-    unlink(name);
-    unlink(dirty);
-    unlink(clean);
-}
-
 static inline void
 default_parse_args (int argc, const char *argv[]) {
     const char *progname=argv[0];

newbrt/tests/test1305.c

@@ -24,7 +24,7 @@
 
 static void
 test (u_int64_t fsize) {
-    unlink_file_and_bit(FNAME);
+    unlink(FNAME);
     // Create a file of size fsize.  Fill it with 8-byte values which are integers, in order)
     assert(fsize%(N_BIGINTS*sizeof(u_int64_t)) == 0); // Make sure the fsize is a multiple of the buffer size.
     u_int64_t i = 0;
@@ -94,7 +94,7 @@ assert(sizeof(buf) == N_BIGINTS * BIGINT_SIZE);
     }
     //printf("Did %" PRIu64 "\n", fsize);
     //system("ls -l " FNAME);
-    unlink_file_and_bit(FNAME);
+    unlink(FNAME);
 }
 
 int

newbrt/tests/test1308a.c

@@ -15,7 +15,7 @@
 int
 test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__)))
 {
-    unlink_file_and_bit(FNAME);
+    unlink(FNAME);
     
     int fd;
     {
@@ -45,6 +45,6 @@ test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute_
     assert(file_size==file_size2);
     close(fd);
 
-    unlink_file_and_bit(FNAME);
+    unlink(FNAME);
     return 0;
 }

newbrt/tests/test1626.c

@@ -29,7 +29,7 @@ test_main (int argc , const char *argv[]) {
     int r;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink_file_and_bit(FNAME);
+    unlink(FNAME);
     BRT one;
     BRT two;
     BRT three;

newbrt/tests/test_oexcl.c

@@ -9,7 +9,7 @@
 
 int
 test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
-    unlink_file_and_bit(fname);
+    unlink(fname);
     int fd0 = open (fname, O_RDWR|O_CREAT|O_EXCL, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(fd0>=0);
     int fd1 = open (fname, O_RDWR|O_CREAT|O_EXCL, S_IRWXU|S_IRWXG|S_IRWXO);

src/tests/Makefile

@@ -80,6 +80,7 @@ BDB_DONTRUN_TESTS = \
 	test_txn_abort9 \
 	test_txn_close_open_commit \
 	test_txn_commit8 \
+	test_db_descriptor \
 #\ ends prev line
 
 ifeq ($(OS_CHOICE),windows)

src/tests/checkpoint1.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
+#include "test.h"
+#include <sys/stat.h>
+
+/* basic checkpoint testing.  Do things end up in the log? */
+
+DB_ENV *env;
+DB *db;
+DB_TXN *txn;
+
+static void
+insert(int i)
+{
+    char hello[30], there[30];
+    snprintf(hello, sizeof(hello), "hello%d", i);
+    snprintf(there, sizeof(there), "there%d", i);
+    DBT key, val;
+    int r=db->put(db, txn,
+		  dbt_init(&key, hello, strlen(hello)+1),
+		  dbt_init(&val, there, strlen(there)+1),
+		  DB_YESOVERWRITE);
+    CKERR(r);
+}
+
+static void
+checkpoint1 (void)
+{
+    system("rm -rf " ENVDIR);
+    toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);
+
+    int r;
+    r = db_env_create(&env, 0);                                                     CKERR(r);
+
+    r = env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
+    r = db_create(&db, env, 0);                                                     CKERR(r);
+
+    r=env->txn_begin(env, 0, &txn, 0);                                              CKERR(r);
+    r=db->open(db, txn, "foo.db", 0, DB_BTREE, DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
+    r=txn->commit(txn, 0);                                                          CKERR(r);
+
+    r=env->txn_begin(env, 0, &txn, 0);                                              CKERR(r);
+    insert(0);
+    r=env->txn_checkpoint(env, 0, 0, 0);                                            CKERR(r);
+    r=txn->commit(txn, 0);                                                          CKERR(r);
+    r=db->close(db, 0);                                                             CKERR(r);
+    r=env->close(env, 0);                                                           CKERR(r);
+}
+
+int
+test_main (int argc, const char *argv[])
+{
+    parse_args(argc, argv);
+    checkpoint1();
+    return 0;
+}
+

src/tests/test_abort1.c

@@ -62,21 +62,9 @@ test_db_open_aborts (void) {
 	r=toku_stat(ENVDIR "/foo.db", &buf);
 	assert(r!=0);
 	assert(errno==ENOENT);
-	r=toku_stat(ENVDIR "/foo.db.clean", &buf);
-	assert(r!=0);
-	assert(errno==ENOENT);
-	r=toku_stat(ENVDIR "/foo.db.dirty", &buf);
-	assert(r!=0);
-	assert(errno==ENOENT);
     }
 
     r=db->close(db, 0);       assert(r==0);
-    r=toku_stat(ENVDIR "/foo.db.clean", &buf);
-    assert(r!=0);
-    assert(errno==ENOENT);
-    r=toku_stat(ENVDIR "/foo.db.dirty", &buf);
-    assert(r!=0);
-    assert(errno==ENOENT);
     r=env->close(env, 0);     assert(r==0);
 }
 

src/tests/test_db_remove.c

@@ -15,6 +15,10 @@ void test_db_remove (void) {
     r=toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); assert(r==0);
 
     // create the DB
+    r = db_create(&db1, null_env, 0);                                  assert(r == 0);
+    r = db1->open(db1, null_txn, fname, 0, DB_BTREE, DB_CREATE, 0666); assert(r == 0);
+    r = db1->close(db1, 0); assert(r == 0); //Header has been written to disk
+
     r = db_create(&db1, null_env, 0);                                  assert(r == 0);
     r = db1->open(db1, null_txn, fname, 0, DB_BTREE, DB_CREATE, 0666); assert(r == 0);