#4171 fsync the correct directory in the brtloader refs[t:4171]

git-svn-id: file:///svn/toku/tokudb@37212 c7de825b-a66e-492c-adef-691d508d4ae1

linux/memory.c

-../windows/memory.c
\ No newline at end of file

linux/memory.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007-2011 Tokutek Inc.  All rights reserved."
+
+#include <toku_portability.h>
+#include "memory.h"
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <malloc.h>
+#include "toku_assert.h"
+#include "toku_pthread.h"
+
+static malloc_fun_t  t_malloc  = 0;
+static malloc_fun_t  t_xmalloc = 0;
+static free_fun_t    t_free    = 0;
+static realloc_fun_t t_realloc = 0;
+static realloc_fun_t t_xrealloc = 0;
+
+static MEMORY_STATUS_S status;
+
+static size_t
+my_malloc_usable_size(void *p) {
+    return p == NULL ? 0 : malloc_usable_size(p);
+}
+
+void 
+toku_memory_get_status(MEMORY_STATUS s) {
+    *s = status;
+}
+
+// max_in_use may be slightly off because use of max_in_use is not thread-safe.
+// It is not worth the overhead to make it completely accurate.
+static inline void 
+set_max(uint64_t sum_used, uint64_t sum_freed) {
+    uint64_t in_use = (sum_used - sum_freed);
+    if ((!(in_use & 0x8000000000000000)) // if wrap due to another thread, ignore bogus "negative" value
+	&& (in_use > status.max_in_use)) {
+	status.max_in_use = in_use;
+    }
+}
+
+void *toku_malloc(size_t size) {
+    void *p = t_malloc ? t_malloc(size) : os_malloc(size);
+    if (p) {
+        size_t used = my_malloc_usable_size(p);
+        __sync_add_and_fetch(&status.malloc_count, 1);
+        __sync_add_and_fetch(&status.requested,size);
+        __sync_add_and_fetch(&status.used, used);
+        set_max(status.used, status.freed);
+    } else {
+        __sync_add_and_fetch(&status.malloc_fail, 1);
+    }
+    return p;
+}
+
+void *
+toku_calloc(size_t nmemb, size_t size) {
+    size_t newsize = nmemb * size;
+    void *p = toku_malloc(newsize);
+    if (p) memset(p, 0, newsize);
+    return p;
+}
+
+void *
+toku_realloc(void *p, size_t size) {
+    size_t used_orig = p ? my_malloc_usable_size(p) : 0;
+    void *q = t_realloc ? t_realloc(p, size) : os_realloc(p, size);
+    if (q) {
+	size_t used = my_malloc_usable_size(q);
+	__sync_add_and_fetch(&status.realloc_count, 1);
+	__sync_add_and_fetch(&status.requested, size);
+	__sync_add_and_fetch(&status.used, used);
+	__sync_add_and_fetch(&status.freed, used_orig);
+	set_max(status.used, status.freed);
+    } else {
+	__sync_add_and_fetch(&status.realloc_fail, 1);
+    }
+    return q;
+}
+
+void *
+toku_memdup(const void *v, size_t len) {
+    void *p = toku_malloc(len);
+    if (p) memcpy(p, v,len);
+    return p;
+}
+
+char *
+toku_strdup(const char *s) {
+    return toku_memdup(s, strlen(s)+1);
+}
+
+void
+toku_free(void *p) {
+    if (p) {
+	size_t used = my_malloc_usable_size(p);
+	__sync_add_and_fetch(&status.free_count, 1);
+	__sync_add_and_fetch(&status.freed, used);
+	if (t_free)
+	    t_free(p);
+	else
+	    os_free(p);
+    }
+}
+
+void
+toku_free_n(void* p, size_t size __attribute__((unused))) {
+    toku_free(p);
+}
+
+void *
+toku_xmalloc(size_t size) {
+    void *p = t_xmalloc ? t_xmalloc(size) : os_malloc(size);
+    if (p == NULL)  // avoid function call in common case
+        resource_assert(p);
+    size_t used = my_malloc_usable_size(p);
+    __sync_add_and_fetch(&status.malloc_count, 1);
+    __sync_add_and_fetch(&status.requested, size);
+    __sync_add_and_fetch(&status.used, used);
+    set_max(status.used, status.freed);
+    return p;
+}
+
+void *
+toku_xcalloc(size_t nmemb, size_t size) {
+    size_t newsize = nmemb * size;
+    void *vp = toku_xmalloc(newsize);
+    if (vp) memset(vp, 0, newsize);
+    return vp;
+}
+
+void *
+toku_xrealloc(void *v, size_t size) {
+    size_t used_orig = v ? my_malloc_usable_size(v) : 0;
+    void *p = t_xrealloc ? t_xrealloc(v, size) : os_realloc(v, size);
+    if (p == 0)  // avoid function call in common case
+        resource_assert(p);
+    size_t used = my_malloc_usable_size(p);
+    __sync_add_and_fetch(&status.realloc_count, 1);
+    __sync_add_and_fetch(&status.requested, size);
+    __sync_add_and_fetch(&status.used, used);
+    __sync_add_and_fetch(&status.freed, used_orig);
+    set_max(status.used, status.freed);
+    return p;
+}
+
+size_t 
+toku_malloc_usable_size(void *p) {
+    return p == NULL ? 0 : malloc_usable_size(p);
+}
+
+void *
+toku_xmemdup (const void *v, size_t len) {
+    void *p = toku_xmalloc(len);
+    memcpy(p, v, len);
+    return p;
+}
+
+char *
+toku_xstrdup (const char *s) {
+    return toku_xmemdup(s, strlen(s)+1);
+}
+
+void
+toku_set_func_malloc(malloc_fun_t f) {
+    t_malloc = f;
+    t_xmalloc = f;
+}
+
+void
+toku_set_func_xmalloc_only(malloc_fun_t f) {
+    t_xmalloc = f;
+}
+
+void
+toku_set_func_malloc_only(malloc_fun_t f) {
+    t_malloc = f;
+}
+
+void
+toku_set_func_realloc(realloc_fun_t f) {
+    t_realloc = f;
+    t_xrealloc = f;
+}
+
+void
+toku_set_func_xrealloc_only(realloc_fun_t f) {
+    t_xrealloc = f;
+}
+
+void
+toku_set_func_realloc_only(realloc_fun_t f) {
+    t_realloc = f;
+
+}
+
+void
+toku_set_func_free(free_fun_t f) {
+    t_free = f;
+}
+
+#include <valgrind/drd.h>
+void __attribute__((constructor)) toku_memory_drd_ignore(void);
+void
+toku_memory_drd_ignore(void) {
+    DRD_IGNORE_VAR(status);
+}

linux/toku_assert.c

-../windows/toku_assert.c
\ No newline at end of file

linux/toku_assert.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007-2010 Tokutek Inc.  All rights reserved."
+
+#include <toku_portability.h>
+#include "toku_assert.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <malloc.h>
+#if !TOKU_WINDOWS
+#include <execinfo.h>
+#endif
+
+
+#if !TOKU_WINDOWS
+#define N_POINTERS 1000
+// These are statically allocated so that the backtrace can run without any calls to malloc()
+static void *backtrace_pointers[N_POINTERS];
+#endif
+
+// Function pointers are zero by default so asserts can be used by brt-layer tests without an environment.
+static int (*toku_maybe_get_engine_status_text_p)(char* buff, int buffsize) = 0;
+static void (*toku_maybe_set_env_panic_p)(int code, char* msg) = 0;
+
+void toku_assert_set_fpointers(int (*toku_maybe_get_engine_status_text_pointer)(char*, int), 
+			       void (*toku_maybe_set_env_panic_pointer)(int, char*)) {
+    toku_maybe_get_engine_status_text_p = toku_maybe_get_engine_status_text_pointer;
+    toku_maybe_set_env_panic_p = toku_maybe_set_env_panic_pointer;
+}
+
+void (*do_assert_hook)(void) = NULL;
+
+static void toku_do_backtrace_abort(void) __attribute__((noreturn));
+
+static void 
+toku_do_backtrace_abort(void) {
+
+    // backtrace
+#if !TOKU_WINDOWS
+    int n = backtrace(backtrace_pointers, N_POINTERS);
+    fprintf(stderr, "Backtrace: (Note: toku_do_assert=0x%p)\n", toku_do_assert); fflush(stderr);
+    backtrace_symbols_fd(backtrace_pointers, n, fileno(stderr));
+#endif
+
+    fflush(stderr);
+    
+    if (toku_maybe_get_engine_status_text_p) {
+	int r;
+	int buffsize = 1024 * 32;
+	char buff[buffsize];
+	
+	r = toku_maybe_get_engine_status_text_p(buff, buffsize);  
+	fprintf(stderr, "Engine status:\n%s\n", buff);
+    }
+    else
+	fprintf(stderr, "Engine status function not available\n");
+    fprintf(stderr, "Memory usage:\n");
+    fflush(stderr);	    // just in case malloc_stats() crashes, we still want engine status (and to know that malloc_stats() failed)
+    malloc_stats();
+    fflush(stderr);	    
+    
+
+#if TOKU_WINDOWS
+    //Following commented methods will not always end the process (could hang).
+    //They could be unacceptable for other reasons as well (popups,
+    //flush buffers before quitting, etc)
+    //  abort()
+    //  assert(FALSE) (assert.h assert)
+    //  raise(SIGABRT)
+    //  divide by 0
+    //  null dereference
+    //  _exit
+    //  exit
+    //  ExitProcess
+    TerminateProcess(GetCurrentProcess(), 134); //Only way found so far to unconditionally
+    //Terminate the process
+#endif
+
+    if (do_assert_hook) do_assert_hook();
+
+    abort();
+}
+
+
+static void
+set_panic_if_not_panicked(int caller_errno, char * msg) {
+    int code = caller_errno ? caller_errno : -1;
+    if (toku_maybe_set_env_panic_p) {
+	toku_maybe_set_env_panic_p(code, msg);
+    }
+}
+
+
+#define MSGLEN 1024
+
+void 
+toku_do_assert_fail (const char *expr_as_string, const char *function, const char *file, int line, int caller_errno) {
+    char msg[MSGLEN];
+    snprintf(msg, MSGLEN, "%s:%d %s: Assertion `%s' failed (errno=%d)\n", file, line, function, expr_as_string, caller_errno);
+    perror(msg);
+    set_panic_if_not_panicked(caller_errno, msg);
+    toku_do_backtrace_abort();
+}
+
+void 
+toku_do_assert_zero_fail (uintptr_t expr, const char *expr_as_string, const char *function, const char *file, int line, int caller_errno) {
+    char msg[MSGLEN];
+    snprintf(msg, MSGLEN, "%s:%d %s: Assertion `%s == 0' failed (errno=%d) (%s=%"PRIuPTR")\n", file, line, function, expr_as_string, caller_errno, expr_as_string, expr);
+    perror(msg);
+    set_panic_if_not_panicked(caller_errno, msg);
+    toku_do_backtrace_abort();
+}
+
+void 
+toku_do_assert(int expr, const char *expr_as_string, const char *function, const char* file, int line, int caller_errno) {
+    if (expr == 0)
+        toku_do_assert_fail(expr_as_string, function, file, line, caller_errno);
+}
+

newbrt/brtloader.c

@@ -350,12 +350,12 @@ void toku_brtloader_internal_destroy (BRTLOADER bl, BOOL is_error) {
     toku_free(bl->dbs);
     toku_free(bl->descriptors);
     toku_free(bl->root_xids_that_created);
-    for (int i = 0; i < bl->N; i++) {
-	if (bl->new_fnames_in_env) 
+    if (bl->new_fnames_in_env) {
+        for (int i = 0; i < bl->N; i++)
             toku_free((char*)bl->new_fnames_in_env[i]);
+        toku_free(bl->new_fnames_in_env);
     }
     toku_free(bl->extracted_datasizes);
-    toku_free(bl->new_fnames_in_env);
     toku_free(bl->bt_compare_funs);
     toku_free((char*)bl->temp_file_template);
     brtloader_fi_destroy(&bl->file_infos, is_error);
@@ -2644,22 +2644,18 @@ static int toku_brt_loader_close_internal (BRTLOADER bl)
         result = update_progress(PROGRESS_MAX, bl, "done");
     else {
         int remaining_progress = PROGRESS_MAX;
-        for (int i=0; i<bl->N; i++) {
-            char * fname_in_cwd = toku_cachetable_get_fname_in_cwd(bl->cachetable, bl->new_fnames_in_env[i]);
+        for (int i = 0; i < bl->N; i++) {
             // Take the unallocated progress and divide it among the unfinished jobs.
             // This calculation allocates all of the PROGRESS_MAX bits of progress to some job.
             int allocate_here = remaining_progress/(bl->N - i);
             remaining_progress -= allocate_here;
-            //printf("%s:%d do_i(%d)\n", __FILE__, __LINE__, i);
             BL_TRACE(blt_close);
-            result = loader_do_i(bl, i, bl->dbs[i], bl->bt_compare_funs[i], bl->descriptors[i], fname_in_cwd,
-                                 allocate_here
-                                 );
+            char *fname_in_cwd = toku_cachetable_get_fname_in_cwd(bl->cachetable, bl->new_fnames_in_env[i]);
+            result = loader_do_i(bl, i, bl->dbs[i], bl->bt_compare_funs[i], bl->descriptors[i], fname_in_cwd, allocate_here);
             toku_free(fname_in_cwd);
-            if (result!=0) goto error;
-            toku_free((void*)bl->new_fnames_in_env[i]);
-            bl->new_fnames_in_env[i] = NULL;
-            invariant(0<=bl->progress && bl->progress <= PROGRESS_MAX);
+            if (result != 0) 
+                goto error;
+            invariant(0 <= bl->progress && bl->progress <= PROGRESS_MAX);
         }
 	if (result==0) invariant(remaining_progress==0);