Skip to content

M
mariadb
Commit 800921a7 authored by Zardosht Kasheff's avatar Zardosht Kasheff
Browse files
Options

refs #61, 

 - have closed cachefiles not immedietely free pairs, but set them to the side
 - leave freeing of pairs to the evictor and/or shutdown
 - should a cachefile be reopened before all pairs are freed,
   the pairs belonging to that cachefile are reintegrated into the cachetable
parent dfb30afc
Hide whitespace changes
Inline Side-by-side
Showing with 755 additions and 136 deletions
ft/cachetable-internal.h
View file @ 800921a7
......@@ -380,7 +380,10 @@ public:
toku_pthread_rwlock_t m_pending_lock_cheap;
void init();
void destroy();
void evict(PAIR pair);
void evict_completely(PAIR pair);
void evict_from_cachetable(PAIR pair);
void evict_from_cachefile(PAIR pair);
void add_to_cachetable_only(PAIR p);
void put(PAIR pair);
PAIR find_pair(CACHEFILE file, CACHEKEY key, uint32_t hash);
void pending_pairs_remove (PAIR p);
......@@ -404,7 +407,6 @@ public:
private:
void pair_remove (PAIR p);
void cf_pairs_remove (PAIR p);
void remove_from_hash_chain(PAIR p);
void add_to_cf_list (PAIR p);
void add_to_clock (PAIR p);
......@@ -426,16 +428,25 @@ public:
int cachefile_of_iname_in_env(const char *iname_in_env, CACHEFILE *cf);
int cachefile_of_filenum(FILENUM filenum, CACHEFILE *cf);
void add_cf_unlocked(CACHEFILE newcf);
void add_stale_cf(CACHEFILE newcf);
void remove_cf(CACHEFILE cf);
void remove_stale_cf_unlocked(CACHEFILE cf);
FILENUM reserve_filenum();
uint32_t get_new_hash_id_unlocked();
CACHEFILE find_cachefile_unlocked(struct fileid* fileid);
CACHEFILE find_stale_cachefile_unlocked(struct fileid* fileid);
void verify_unused_filenum(FILENUM filenum);
bool evict_some_stale_pair(evictor* ev);
void free_stale_data(evictor* ev);
// access to these fields are protected by the lock
CACHEFILE m_active_head; // head of CACHEFILEs that are active
CACHEFILE m_stale_head; // head of CACHEFILEs that are stale
CACHEFILE m_stale_tail; // tail of CACHEFILEs that are stale
FILENUM m_next_filenum_to_use;
uint32_t m_next_hash_id_to_use;
toku_pthread_rwlock_t m_lock; // this field is publoc so we are still POD
private:
CACHEFILE find_cachefile_in_list_unlocked(CACHEFILE start, struct fileid* fileid);
};
......@@ -500,7 +511,7 @@ const int EVICTION_PERIOD = 1;
//
class evictor {
public:
void init(long _size_limit, pair_list* _pl, KIBBUTZ _kibbutz, uint32_t eviction_period);
void init(long _size_limit, pair_list* _pl, cachefile_list* _cf_list, KIBBUTZ _kibbutz, uint32_t eviction_period);
void destroy();
void add_pair_attr(PAIR_ATTR attr);
void remove_pair_attr(PAIR_ATTR attr);
......@@ -533,6 +544,7 @@ private:
int64_t unsafe_read_size_evicting(void) const;
pair_list* m_pl;
cachefile_list* m_cf_list;
int64_t m_size_current; // the sum of the sizes of the pairs in the cachetable
// changes to these two values are protected
// by ev_thread_lock
......
ft/cachetable.cc
View file @ 800921a7
......@@ -305,7 +305,7 @@ void toku_cachetable_create(CACHETABLE *result, long size_limit, LSN UU(initial_
int checkpointing_nworkers = (num_processors/4) ? num_processors/4 : 1;
ct->checkpointing_kibbutz = toku_kibbutz_create(checkpointing_nworkers);
// must be done after creating ct_kibbutz
ct->ev.init(size_limit, &ct->list, ct->ct_kibbutz, EVICTION_PERIOD);
ct->ev.init(size_limit, &ct->list, &ct->cf_list, ct->ct_kibbutz, EVICTION_PERIOD);
ct->cp.init(&ct->list, logger, &ct->ev, &ct->cf_list);
ct->cl.init(1, &ct->list, ct); // by default, start with one iteration
ct->env_dir = toku_xstrdup(".");
......@@ -356,7 +356,8 @@ int toku_cachefile_of_filenum (CACHETABLE ct, FILENUM filenum, CACHEFILE *cf) {
// If something goes wrong, close the fd. After this, the caller shouldn't close the fd, but instead should close the cachefile.
int toku_cachetable_openfd (CACHEFILE *cfptr, CACHETABLE ct, int fd, const char *fname_in_env) {
FILENUM filenum = toku_cachetable_reserve_filenum(ct);
return toku_cachetable_openfd_with_filenum(cfptr, ct, fd, fname_in_env, filenum);
bool was_open;
return toku_cachetable_openfd_with_filenum(cfptr, ct, fd, fname_in_env, filenum, &was_open);
}
// Get a unique filenum from the cachetable
......@@ -390,7 +391,7 @@ static void create_new_cachefile(
int toku_cachetable_openfd_with_filenum (CACHEFILE *cfptr, CACHETABLE ct, int fd,
const char *fname_in_env,
FILENUM filenum) {
FILENUM filenum, bool* was_open) {
int r;
CACHEFILE newcf;
struct fileid fileid;
......@@ -405,6 +406,7 @@ int toku_cachetable_openfd_with_filenum (CACHEFILE *cfptr, CACHETABLE ct, int fd
ct->cf_list.write_lock();
CACHEFILE existing_cf = ct->cf_list.find_cachefile_unlocked(&fileid);
if (existing_cf) {
*was_open = true;
// Reuse an existing cachefile and close the caller's fd, whose
// responsibility has been passed to us.
r = close(fd);
......@@ -413,7 +415,32 @@ int toku_cachetable_openfd_with_filenum (CACHEFILE *cfptr, CACHETABLE ct, int fd
r = 0;
goto exit;
}
*was_open = false;
ct->cf_list.verify_unused_filenum(filenum);
// now let's try to find it in the stale cachefiles
existing_cf = ct->cf_list.find_stale_cachefile_unlocked(&fileid);
// found the stale file,
if (existing_cf) {
// fix up the fields in the cachefile
existing_cf->filenum = filenum;
existing_cf->fd = fd;
existing_cf->fname_in_env = toku_xstrdup(fname_in_env);
bjm_init(&existing_cf->bjm);
// now we need to move all the PAIRs in it back into the cachetable
for (PAIR curr_pair = existing_cf->cf_head; curr_pair; curr_pair = curr_pair->cf_next) {
pair_lock(curr_pair);
ct->list.add_to_cachetable_only(curr_pair);
pair_unlock(curr_pair);
}
// move the cachefile back to the list of active cachefiles
ct->cf_list.remove_stale_cf_unlocked(existing_cf);
ct->cf_list.add_cf_unlocked(existing_cf);
*cfptr = existing_cf;
r = 0;
goto exit;
}
create_new_cachefile(
ct,
filenum,
......@@ -433,7 +460,7 @@ int toku_cachetable_openfd_with_filenum (CACHEFILE *cfptr, CACHETABLE ct, int fd
return r;
}
static void cachetable_flush_cachefile (CACHETABLE, CACHEFILE cf);
static void cachetable_flush_cachefile (CACHETABLE, CACHEFILE cf, bool evict_completely);
//TEST_ONLY_FUNCTION
int toku_cachetable_openf (CACHEFILE *cfptr, CACHETABLE ct, const char *fname_in_env, int flags, mode_t mode) {
......@@ -459,6 +486,13 @@ toku_cachefile_get_fd (CACHEFILE cf) {
return cf->fd;
}
static void cachefile_destroy(CACHEFILE cf) {
if (cf->free_userdata) {
cf->free_userdata(cf, cf->userdata);
}
toku_free(cf);
}
void toku_cachefile_close(CACHEFILE *cfp, bool oplsn_valid, LSN oplsn) {
CACHEFILE cf = *cfp;
CACHETABLE ct = cf->cachetable;
......@@ -470,26 +504,34 @@ void toku_cachefile_close(CACHEFILE *cfp, bool oplsn_valid, LSN oplsn) {
// note_pin_by_checkpoint callback.
assert(!cf->for_checkpoint);
// Flush the cachefile and remove all of its pairs from the cachetable
cachetable_flush_cachefile(ct, cf);
// Flush the cachefile and remove all of its pairs from the cachetable,
// but keep the PAIRs linked in the cachefile. We will store the cachefile
// away in case it gets opened immedietely
//
// if we are unlinking on close, then we want to evict completely,
// otherwise, we will keep the PAIRs and cachefile around in case
// a subsequent open comes soon
cachetable_flush_cachefile(ct, cf, cf->unlink_on_close);
// Call the close userdata callback to notify the client this cachefile
// and its underlying file are going to be closed
if (cf->close_userdata) {
invariant(cf->free_userdata);
cf->close_userdata(cf, cf->fd, cf->userdata, oplsn_valid, oplsn);
cf->free_userdata(cf, cf->userdata);
}
ct->cf_list.remove_cf(cf);
bjm_destroy(cf->bjm);
cf->bjm = NULL;
// fsync and close the fd.
toku_file_fsync_without_accounting(cf->fd);
int r = close(cf->fd);
assert(r == 0);
cf->fd = -1;
// destroy the parts of the cachefile
// that do not persist across opens/closes
bjm_destroy(cf->bjm);
cf->bjm = NULL;
cf->filenum = FILENUM_NONE;
// remove the cf from the list of active cachefiles
ct->cf_list.remove_cf(cf);
// Unlink the file if the bit was set
if (cf->unlink_on_close) {
......@@ -499,7 +541,17 @@ void toku_cachefile_close(CACHEFILE *cfp, bool oplsn_valid, LSN oplsn) {
toku_free(fname_in_cwd);
}
toku_free(cf->fname_in_env);
toku_free(cf);
cf->fname_in_env = NULL;
// we destroy the cf if the unlink bit was set or if no PAIRs exist
// if no PAIRs exist, there is no sense in keeping the cachefile around
bool destroy_cf = cf->unlink_on_close || (cf->cf_head == NULL);
if (destroy_cf) {
cachefile_destroy(cf);
}
else {
ct->cf_list.add_stale_cf(cf);
}
}
// This hash function comes from Jenkins: http://burtleburtle.net/bob/c/lookup3.c
......@@ -534,12 +586,12 @@ static void pair_touch (PAIR p) {
p->count = (p->count < CLOCK_SATURATION) ? p->count+1 : CLOCK_SATURATION;
}
// Remove a pair from the cachetable
// Remove a pair from the cachetable, requires write list lock to be held and p->mutex to be held
// Effects: the pair is removed from the LRU list and from the cachetable's hash table.
// The size of the objects in the cachetable is adjusted by the size of the pair being
// removed.
static void cachetable_remove_pair (pair_list* list, evictor* ev, PAIR p) {
list->evict(p);
list->evict_completely(p);
ev->remove_pair_attr(p->attr);
}
......@@ -2366,17 +2418,29 @@ static void flush_pair_for_close_on_background_thread(
pair_unlock(p);
}
static void remove_pair_for_close(PAIR p, CACHETABLE ct) {
static void remove_pair_for_close(PAIR p, CACHETABLE ct, bool completely) {
pair_lock(p);
assert(p->value_rwlock.users() == 0);
assert(nb_mutex_users(&p->disk_nb_mutex) == 0);
assert(!p->cloned_value_data);
assert(p->dirty == CACHETABLE_CLEAN);
assert(p->refcount == 0);
// TODO: maybe break up this function
// so that write lock does not need to be held for entire
// free
cachetable_maybe_remove_and_free_pair(&ct->list, &ct->ev, p);
if (completely) {
// TODO: maybe break up this function
// so that write lock does not need to be held for entire
// free
cachetable_maybe_remove_and_free_pair(&ct->list, &ct->ev, p);
}
else {
// if we are not evicting completely,
// we only want to remove the PAIR from the cachetable,
// that is, remove from the hashtable and various linked
// list, but we will keep the PAIRS and the linked list
// in the cachefile intact, as they will be cached away
// in case an open comes soon.
ct->list.evict_from_cachetable(p);
pair_unlock(p);
}
}
// helper function for cachetable_flush_cachefile, which happens on a close
......@@ -2410,18 +2474,37 @@ static void write_dirty_pairs_for_close(CACHETABLE ct, CACHEFILE cf) {
bjm_destroy(bjm);
}
static void remove_all_pairs_for_close(CACHETABLE ct, CACHEFILE cf) {
static void remove_all_pairs_for_close(CACHETABLE ct, CACHEFILE cf, bool evict_completely) {
ct->list.write_list_lock();
if (cf) {
while (cf->num_pairs > 0) {
PAIR p = cf->cf_head;
remove_pair_for_close(p, ct);
}
if (evict_completely) {
// if we are evicting completely, then the PAIRs will
// be removed from the linked list managed by the
// cachefile, so this while loop works
while (cf->num_pairs > 0) {
PAIR p = cf->cf_head;
remove_pair_for_close(p, ct, evict_completely);
}
}
else {
// on the other hand, if we are not evicting completely,
// then the cachefile's linked list stays intact, and we must
// iterate like this.
for (PAIR p = cf->cf_head; p; p = p->cf_next) {
remove_pair_for_close(p, ct, evict_completely);
}
}
}
else {
while (ct->list.m_n_in_table > 0) {
PAIR p = ct->list.m_checkpoint_head;
remove_pair_for_close(p, ct);
// if there is no cachefile, then we better
// be evicting completely because we have no
// cachefile to save the PAIRs to. At least,
// we have no guarantees that the cachefile
// will remain good
invariant(evict_completely);
remove_pair_for_close(p, ct, true);
}
}
ct->list.write_list_unlock();
......@@ -2463,7 +2546,7 @@ static void verify_cachefile_flushed(CACHETABLE ct UU(), CACHEFILE cf UU()) {
// of being used by a checkpoint. If a checkpoint is currently happening,
// it does NOT include this cachefile.
//
static void cachetable_flush_cachefile(CACHETABLE ct, CACHEFILE cf) {
static void cachetable_flush_cachefile(CACHETABLE ct, CACHEFILE cf, bool evict_completely) {
//
// Because work on a kibbutz is always done by the client thread,
// and this function assumes that no client thread is doing any work
......@@ -2478,7 +2561,7 @@ static void cachetable_flush_cachefile(CACHETABLE ct, CACHEFILE cf) {
write_dirty_pairs_for_close(ct, cf);
// now that everything is clean, get rid of everything
remove_all_pairs_for_close(ct, cf);
remove_all_pairs_for_close(ct, cf, evict_completely);
verify_cachefile_flushed(ct, cf);
}
......@@ -2496,7 +2579,8 @@ void toku_cachetable_close (CACHETABLE *ctp) {
CACHETABLE ct = *ctp;
ct->cp.destroy();
ct->cl.destroy();
cachetable_flush_cachefile(ct, NULL);
ct->cf_list.free_stale_data(&ct->ev);
cachetable_flush_cachefile(ct, NULL, true);
ct->ev.destroy();
ct->list.destroy();
ct->cf_list.destroy();
......@@ -3185,6 +3269,21 @@ void toku_pair_list_set_lock_size(uint32_t num_locks) {
PAIR_LOCK_SIZE = num_locks;
}
static void evict_pair_from_cachefile(PAIR p) {
CACHEFILE cf = p->cachefile;
if (p->cf_next) {
p->cf_next->cf_prev = p->cf_prev;
}
if (p->cf_prev) {
p->cf_prev->cf_next = p->cf_next;
}
else if (p->cachefile->cf_head == p) {
cf->cf_head = p->cf_next;
}
p->cf_prev = p->cf_next = NULL;
cf->num_pairs--;
}
// Allocates the hash table of pairs inside this pair list.
//
void pair_list::init() {
......@@ -3234,36 +3333,54 @@ void pair_list::destroy() {
toku_free(m_mutexes);
}
// This places the given pair inside of the pair list.
//
// requires caller to have grabbed write lock on list.
// requires caller to have p->mutex held as well
//
void pair_list::put(PAIR p) {
// adds a PAIR to the cachetable's structures,
// but does NOT add it to the list maintained by
// the cachefile
void pair_list::add_to_cachetable_only(PAIR p) {
// sanity check to make sure that the PAIR does not already exist
PAIR pp = this->find_pair(p->cachefile, p->key, p->fullhash);
assert(pp == NULL);
this->add_to_clock(p);
this->add_to_cf_list(p);
this->add_to_hash_chain(p);
m_n_in_table++;
}
// This removes the given pair from the pair list.
// This places the given pair inside of the pair list.
//
// requires caller to have grabbed write lock on list.
// requires caller to have p->mutex held as well
//
void pair_list::put(PAIR p) {
this->add_to_cachetable_only(p);
this->add_to_cf_list(p);
}
// This removes the given pair from completely from the pair list.
//
void pair_list::evict(PAIR p) {
// requires caller to have grabbed write lock on list, and p->mutex held
//
void pair_list::evict_completely(PAIR p) {
this->evict_from_cachetable(p);
this->evict_from_cachefile(p);
}
// Removes the PAIR from the cachetable's lists,
// but does NOT impact the list maintained by the cachefile
void pair_list::evict_from_cachetable(PAIR p) {
this->pair_remove(p);
this->pending_pairs_remove(p);
this->cf_pairs_remove(p);
this->remove_from_hash_chain(p);
assert(m_n_in_table > 0);
m_n_in_table--;
}
// Removes the PAIR from the cachefile's list of PAIRs
void pair_list::evict_from_cachefile(PAIR p) {
evict_pair_from_cachefile(p);
}
//
// Remove pair from linked list for cleaner/clock
//
......@@ -3292,8 +3409,8 @@ void pair_list::pair_remove (PAIR p) {
}
p->clock_prev->clock_next = p->clock_next;
p->clock_next->clock_prev = p->clock_prev;
}
p->clock_prev = p->clock_next = NULL;
}
//Remove a pair from the list of pairs that were marked with the
......@@ -3315,23 +3432,6 @@ void pair_list::pending_pairs_remove (PAIR p) {
p->pending_prev = p->pending_next = NULL;
}
// add the pair to the linked list that of PAIRs belonging
// to the same cachefile.
void pair_list::cf_pairs_remove(PAIR p) {
CACHEFILE cf = p->cachefile;
if (p->cf_next) {
p->cf_next->cf_prev = p->cf_prev;
}
if (p->cf_prev) {
p->cf_prev->cf_next = p->cf_next;
}
else if (p->cachefile->cf_head == p) {
cf->cf_head = p->cf_next;
}
p->cf_prev = p->cf_next = NULL;
cf->num_pairs--;
}
void pair_list::remove_from_hash_chain(PAIR p) {
// Remove it from the hash chain.
unsigned int h = p->fullhash&(m_table_size - 1);
......@@ -3347,6 +3447,7 @@ void pair_list::remove_from_hash_chain(PAIR p) {
// remove p from the singular linked list
curr->hash_chain = p->hash_chain;
}
p->hash_chain = NULL;
}
// Returns a pair from the pair list, using the given
......@@ -3559,7 +3660,7 @@ static void *eviction_thread(void *evictor_v) {
// Starts the eviction thread, assigns external object references,
// and initializes all counters and condition variables.
//
void evictor::init(long _size_limit, pair_list* _pl, KIBBUTZ _kibbutz, uint32_t eviction_period) {
void evictor::init(long _size_limit, pair_list* _pl, cachefile_list* _cf_list, KIBBUTZ _kibbutz, uint32_t eviction_period) {
TOKU_VALGRIND_HG_DISABLE_CHECKING(&m_ev_thread_is_running, sizeof m_ev_thread_is_running);
TOKU_VALGRIND_HG_DISABLE_CHECKING(&m_size_evicting, sizeof m_size_evicting);
......@@ -3597,6 +3698,7 @@ void evictor::init(long _size_limit, pair_list* _pl, KIBBUTZ _kibbutz, uint32_t
m_long_wait_pressure_time = create_partitioned_counter();
m_pl = _pl;
m_cf_list = _cf_list;
m_kibbutz = _kibbutz;
toku_mutex_init(&m_ev_thread_lock, NULL);
toku_cond_init(&m_flow_control_cond, NULL);
......@@ -3806,41 +3908,44 @@ void evictor::run_eviction(){
// release ev_thread_lock so that eviction may run without holding mutex
toku_mutex_unlock(&m_ev_thread_lock);
m_pl->read_list_lock();
PAIR curr_in_clock = m_pl->m_clock_head;
// if nothing to evict, we need to exit
if (!curr_in_clock) {
m_pl->read_list_unlock();
toku_mutex_lock(&m_ev_thread_lock);
exited_early = true;
goto exit;
}
if (num_pairs_examined_without_evicting > m_pl->m_n_in_table) {
// we have a cycle where everything in the clock is in use
// do not return an error
// just let memory be overfull
// first try to do an eviction from stale cachefiles
bool some_eviction_ran = m_cf_list->evict_some_stale_pair(this);
if (!some_eviction_ran) {
m_pl->read_list_lock();
PAIR curr_in_clock = m_pl->m_clock_head;
// if nothing to evict, we need to exit
if (!curr_in_clock) {
m_pl->read_list_unlock();
toku_mutex_lock(&m_ev_thread_lock);
exited_early = true;
goto exit;
}
if (num_pairs_examined_without_evicting > m_pl->m_n_in_table) {
// we have a cycle where everything in the clock is in use
// do not return an error
// just let memory be overfull
m_pl->read_list_unlock();
toku_mutex_lock(&m_ev_thread_lock);
exited_early = true;
goto exit;
}
bool eviction_run = run_eviction_on_pair(curr_in_clock);
if (eviction_run) {
// reset the count
num_pairs_examined_without_evicting = 0;
}
else {
num_pairs_examined_without_evicting++;
}
// at this point, either curr_in_clock is still in the list because it has not been fully evicted,
// and we need to move ct->m_clock_head over. Otherwise, curr_in_clock has been fully evicted
// and we do NOT need to move ct->m_clock_head, as the removal of curr_in_clock
// modified ct->m_clock_head
if (m_pl->m_clock_head && (m_pl->m_clock_head == curr_in_clock)) {
m_pl->m_clock_head = m_pl->m_clock_head->clock_next;
}
m_pl->read_list_unlock();
toku_mutex_lock(&m_ev_thread_lock);
exited_early = true;
goto exit;
}
bool eviction_run = run_eviction_on_pair(curr_in_clock);
if (eviction_run) {
// reset the count
num_pairs_examined_without_evicting = 0;
}
else {
num_pairs_examined_without_evicting++;
}
// at this point, either curr_in_clock is still in the list because it has not been fully evicted,
// and we need to move ct->m_clock_head over. Otherwise, curr_in_clock has been fully evicted
// and we do NOT need to move ct->m_clock_head, as the removal of curr_in_clock
// modified ct->m_clock_head
if (m_pl->m_clock_head && (m_pl->m_clock_head == curr_in_clock)) {
m_pl->m_clock_head = m_pl->m_clock_head->clock_next;
}
m_pl->read_list_unlock();
toku_mutex_lock(&m_ev_thread_lock);
}
......@@ -4553,6 +4658,8 @@ static_assert(std::is_pod<cachefile_list>::value, "cachefile_list isn't POD");
void cachefile_list::init() {
m_active_head = NULL;
m_stale_head = NULL;
m_stale_tail = NULL;
m_next_filenum_to_use.fileid = 0;
m_next_hash_id_to_use = 0;
toku_pthread_rwlock_init(&m_lock, NULL);
......@@ -4611,6 +4718,8 @@ int cachefile_list::cachefile_of_filenum(FILENUM filenum, CACHEFILE *cf) {
}
void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
invariant(cf->next == NULL);
invariant(cf->prev == NULL);
cf->next = m_active_head;
cf->prev = NULL;
if (m_active_head) {
......@@ -4619,6 +4728,22 @@ void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
m_active_head = cf;
}
void cachefile_list::add_stale_cf(CACHEFILE cf) {
write_lock();
invariant(cf->next == NULL);
invariant(cf->prev == NULL);
cf->next = m_stale_head;
cf->prev = NULL;
if (m_stale_head) {
m_stale_head->prev = cf;
}
m_stale_head = cf;
if (m_stale_tail == NULL) {
m_stale_tail = cf;
}
write_unlock();
}
void cachefile_list::remove_cf(CACHEFILE cf) {
write_lock();
......@@ -4633,9 +4758,32 @@ void cachefile_list::remove_cf(CACHEFILE cf) {
invariant(cf->prev == NULL);
m_active_head = cf->next;
}
cf->prev = NULL;
cf->next = NULL;
write_unlock();
}
void cachefile_list::remove_stale_cf_unlocked(CACHEFILE cf) {
invariant(m_stale_head != NULL);
invariant(m_stale_tail != NULL);
if (cf->next) {
cf->next->prev = cf->prev;
}
if (cf->prev) {
cf->prev->next = cf->next;
}
if (cf == m_stale_head) {
invariant(cf->prev == NULL);
m_stale_head = cf->next;
}
if (cf == m_stale_tail) {
invariant(cf->next == NULL);
m_stale_tail = cf->prev;
}
cf->prev = NULL;
cf->next = NULL;
}
FILENUM cachefile_list::reserve_filenum() {
CACHEFILE extant;
FILENUM filenum;
......@@ -4660,9 +4808,13 @@ uint32_t cachefile_list::get_new_hash_id_unlocked() {
return retval;
}
CACHEFILE cachefile_list::find_cachefile_unlocked(struct fileid* fileid) {
CACHEFILE cachefile_list::find_cachefile_in_list_unlocked(
CACHEFILE start,
struct fileid* fileid
)
{
CACHEFILE retval = NULL;
for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
for (CACHEFILE extant = start; extant; extant = extant->next) {
if (toku_fileids_are_equal(&extant->fileid, fileid)) {
// Clients must serialize cachefile open, close, and unlink
// So, during open, we should never see a closing cachefile
......@@ -4676,12 +4828,75 @@ exit:
return retval;
}
CACHEFILE cachefile_list::find_cachefile_unlocked(struct fileid* fileid) {
return find_cachefile_in_list_unlocked(m_active_head, fileid);
}
CACHEFILE cachefile_list::find_stale_cachefile_unlocked(struct fileid* fileid) {
return find_cachefile_in_list_unlocked(m_stale_head, fileid);
}
void cachefile_list::verify_unused_filenum(FILENUM filenum) {
for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
invariant(extant->filenum.fileid != filenum.fileid);
}
}
// returns true if some eviction ran, false otherwise
bool cachefile_list::evict_some_stale_pair(evictor* ev) {
PAIR p = NULL;
CACHEFILE cf_to_destroy = NULL;
write_lock();
if (m_stale_tail == NULL) {
write_unlock();
return false;
}
p = m_stale_tail->cf_head;
// we should not have a cf in the stale list
// that does not have any pairs
paranoid_invariant(p != NULL);
evict_pair_from_cachefile(p);
// now that we have evicted something,
// let's check if the cachefile is needed anymore
if (m_stale_tail->cf_head == NULL) {
cf_to_destroy = m_stale_tail;
remove_stale_cf_unlocked(m_stale_tail);
}
write_unlock();
ev->remove_pair_attr(p->attr);
cachetable_free_pair(p);
if (cf_to_destroy) {
cachefile_destroy(cf_to_destroy);
}
return true;
}
void cachefile_list::free_stale_data(evictor* ev) {
write_lock();
while (m_stale_tail != NULL) {
PAIR p = m_stale_tail->cf_head;
// we should not have a cf in the stale list
// that does not have any pairs
paranoid_invariant(p != NULL);
evict_pair_from_cachefile(p);
ev->remove_pair_attr(p->attr);
cachetable_free_pair(p);
// now that we have evicted something,
// let's check if the cachefile is needed anymore
if (m_stale_tail->cf_head == NULL) {
CACHEFILE cf_to_destroy = m_stale_tail;
remove_stale_cf_unlocked(m_stale_tail);
cachefile_destroy(cf_to_destroy);
}
}
write_unlock();
}
void __attribute__((__constructor__)) toku_cachetable_helgrind_ignore(void);
void
......
ft/cachetable.h
View file @ 800921a7
......@@ -164,10 +164,10 @@ int toku_cachetable_openf(CACHEFILE *,CACHETABLE, const char *fname_in_env, int
// Bind a file to a new cachefile object.
int toku_cachetable_openfd(CACHEFILE *,CACHETABLE, int fd,
const char *fname_relative_to_env);
const char *fname_relative_to_env);
int toku_cachetable_openfd_with_filenum (CACHEFILE *,CACHETABLE, int fd,
const char *fname_in_env,
FILENUM filenum);
const char *fname_in_env,
FILENUM filenum, bool* was_open);
// reserve a unique filenum
FILENUM toku_cachetable_reserve_filenum(CACHETABLE ct);
......
ft/ft-ops.cc
View file @ 800921a7
......@@ -3612,6 +3612,7 @@ ft_handle_open(FT_HANDLE ft_h, const char *fname_in_env, int is_create, int only
FILENUM reserved_filenum;
reserved_filenum = use_filenum;
fname_in_cwd = toku_cachetable_get_fname_in_cwd(cachetable, fname_in_env);
bool was_already_open;
{
int fd = -1;
r = ft_open_file(fname_in_cwd, &fd);
......@@ -3631,13 +3632,12 @@ ft_handle_open(FT_HANDLE ft_h, const char *fname_in_env, int is_create, int only
if (r) { goto exit; }
}
if (r) { goto exit; }
r=toku_cachetable_openfd_with_filenum(&cf, cachetable, fd, fname_in_env, reserved_filenum);
r=toku_cachetable_openfd_with_filenum(&cf, cachetable, fd, fname_in_env, reserved_filenum, &was_already_open);
if (r) { goto exit; }
}
assert(ft_h->options.nodesize>0);
bool was_already_open;
if (is_create) {
r = toku_read_ft_and_store_in_cachefile(ft_h, cf, max_acceptable_lsn, &ft, &was_already_open);
r = toku_read_ft_and_store_in_cachefile(ft_h, cf, max_acceptable_lsn, &ft);
if (r==TOKUDB_DICTIONARY_NO_HEADER) {
toku_ft_create(&ft, &ft_h->options, cf, txn);
}
......@@ -3653,7 +3653,7 @@ ft_handle_open(FT_HANDLE ft_h, const char *fname_in_env, int is_create, int only
// so it is ok for toku_read_ft_and_store_in_cachefile to have read
// the header via toku_read_ft_and_store_in_cachefile
} else {
r = toku_read_ft_and_store_in_cachefile(ft_h, cf, max_acceptable_lsn, &ft, &was_already_open);
r = toku_read_ft_and_store_in_cachefile(ft_h, cf, max_acceptable_lsn, &ft);
if (r) { goto exit; }
}
if (!ft_h->did_set_flags) {
......
ft/ft.cc
View file @ 800921a7
......@@ -464,7 +464,7 @@ void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn) {
}
// TODO: (Zardosht) get rid of brt parameter
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header, bool* was_open)
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header)
// If the cachefile already has the header, then just get it.
// If the cachefile has not been initialized, then don't modify anything.
// max_acceptable_lsn is the latest acceptable checkpointed version of the file.
......@@ -473,13 +473,11 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_ac
FT h;
if ((h = (FT) toku_cachefile_get_userdata(cf))!=0) {
*header = h;
*was_open = true;
assert(brt->options.update_fun == h->update_fun);
assert(brt->options.compare_fun == h->compare_fun);
return 0;
}
}
*was_open = false;
FT h = nullptr;
int r;
{
......
ft/ft.h
View file @ 800921a7
......@@ -112,7 +112,7 @@ void toku_ft_release_reflock(FT ft);
void toku_ft_create(FT *ftp, FT_OPTIONS options, CACHEFILE cf, TOKUTXN txn);
void toku_ft_free (FT h);
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header, bool* was_open);
int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_acceptable_lsn, FT *header);
void toku_ft_note_ft_handle_open(FT ft, FT_HANDLE live);
bool toku_ft_needed_unlocked(FT ft);
......
ft/tests/cachetable-checkpointer-class.cc
View file @ 800921a7
......@@ -230,7 +230,7 @@ void checkpointer_test::test_pending_bits() {
m_cp.turn_on_pending_bits();
assert(p.checkpoint_pending);
m_cp.m_list->evict(&p);
m_cp.m_list->evict_completely(&p);
//
// 3. Many hash chain entries.
......@@ -251,7 +251,7 @@ void checkpointer_test::test_pending_bits() {
uint32_t full_hash = toku_cachetable_hash(&cf, key);
PAIR pp = m_cp.m_list->find_pair(&cf, key, full_hash);
assert(pp);
m_cp.m_list->evict(pp);
m_cp.m_list->evict_completely(pp);
}
ctbl.list.destroy();
......@@ -389,7 +389,7 @@ void checkpointer_test::test_end_checkpoint() {
uint32_t full_hash = toku_cachetable_hash(&cf, key);
PAIR pp = m_cp.m_list->find_pair(&cf, key, full_hash);
assert(pp);
m_cp.m_list->evict(pp);
m_cp.m_list->evict_completely(pp);
}
m_cp.destroy();
ctbl.list.destroy();
......
ft/tests/cachetable-evictor-class.cc
View file @ 800921a7
......@@ -94,6 +94,7 @@ class evictor_unit_test {
public:
evictor m_ev;
pair_list m_pl;
cachefile_list m_cf_list;
KIBBUTZ m_kb;
void init();
void destroy();
......@@ -111,13 +112,16 @@ public:
// initialize this class to run tests
void evictor_unit_test::init() {
ZERO_STRUCT(m_pl);
ZERO_STRUCT(m_cf_list);
m_pl.init();
m_cf_list.init();
m_kb = toku_kibbutz_create(1);
}
// destroy class after tests have run
void evictor_unit_test::destroy() {
m_pl.destroy();
m_cf_list.destroy();
toku_kibbutz_destroy(m_kb);
}
......@@ -125,6 +129,7 @@ void evictor_unit_test::destroy() {
void evictor_unit_test::verify_ev_init(long limit) {
assert(m_ev.m_kibbutz == m_kb);
assert(m_ev.m_pl == &m_pl);
assert(m_ev.m_cf_list == &m_cf_list);
assert(m_ev.m_low_size_watermark == limit);
assert(m_ev.m_num_sleepers == 0);
assert(m_ev.m_run_thread == true);
......@@ -161,7 +166,7 @@ void evictor_unit_test::verify_ev_counts() {
long limit = 10;
long expected_m_size_reserved = limit/4;
ZERO_STRUCT(m_ev);
m_ev.init(limit, &m_pl, m_kb, 0);
m_ev.init(limit, &m_pl, &m_cf_list, m_kb, 0);
this->verify_ev_init(limit);
m_ev.add_to_size_current(1);
......@@ -227,7 +232,7 @@ void evictor_unit_test::verify_ev_m_size_reserved() {
long limit = 400;
long expected_m_size_reserved = 100; //limit/4
ZERO_STRUCT(m_ev);
m_ev.init(limit, &m_pl, m_kb, 0);
m_ev.init(limit, &m_pl, &m_cf_list, m_kb, 0);
this->verify_ev_init(limit);
assert(m_ev.m_size_reserved == expected_m_size_reserved);
m_ev.m_num_eviction_thread_runs = 0;
......@@ -250,7 +255,7 @@ void evictor_unit_test::verify_ev_m_size_reserved() {
void evictor_unit_test::verify_ev_handling_cache_pressure() {
long limit = 400;
ZERO_STRUCT(m_ev);
m_ev.init(limit, &m_pl, m_kb, 0);
m_ev.init(limit, &m_pl, &m_cf_list, m_kb, 0);
this->verify_ev_init(limit);
m_ev.m_low_size_watermark = 400;
m_ev.m_low_size_hysteresis = 400;
......
ft/tests/cachetable-partial-fetch.cc
View file @ 800921a7
......@@ -213,6 +213,8 @@ cachetable_test (void) {
// close and reopen cachefile so we can do some simple prefetch tests
toku_cachefile_close(&f1, false, ZERO_LSN);
toku_cachetable_close(&ct);
toku_cachetable_create(&ct, test_limit, ZERO_LSN, NULL_LOGGER);
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
//
// verify that a prefetch of the node will succeed
......
ft/tests/cachetable-simple-close.cc 0 → 100644
View file @ 800921a7
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*
COPYING CONDITIONS NOTICE:
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation, and provided that the
following conditions are met:
* Redistributions of source code must retain this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below).
* Redistributions in binary form must reproduce this COPYING
CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
PATENT MARKING NOTICE (below), and the PATENT RIGHTS
GRANT (below) in the documentation and/or other materials
provided with the distribution.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
COPYRIGHT NOTICE:
TokuDB, Tokutek Fractal Tree Indexing Library.
Copyright (C) 2007-2013 Tokutek, Inc.
DISCLAIMER:
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
UNIVERSITY PATENT NOTICE:
The technology is licensed by the Massachusetts Institute of
Technology, Rutgers State University of New Jersey, and the Research
Foundation of State University of New York at Stony Brook under
United States of America Serial No. 11/760379 and to the patents
and/or patent applications resulting from it.
PATENT MARKING NOTICE:
This software is covered by US Patent No. 8,185,551.
PATENT RIGHTS GRANT:
"THIS IMPLEMENTATION" means the copyrightable works distributed by
Tokutek as part of the Fractal Tree project.
"PATENT CLAIMS" means the claims of patents that are owned or
licensable by Tokutek, both currently or in the future; and that in
the absence of this license would be infringed by THIS
IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
"PATENT CHALLENGE" shall mean a challenge to the validity,
patentability, enforceability and/or non-infringement of any of the
PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
Tokutek hereby grants to you, for the term and geographical scope of
the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to
make, have made, use, offer to sell, sell, import, transfer, and
otherwise run, modify, and propagate the contents of THIS
IMPLEMENTATION, where such license applies only to the PATENT
CLAIMS. This grant does not include claims that would be infringed
only as a consequence of further modifications of THIS
IMPLEMENTATION. If you or your agent or licensee institute or order
or agree to the institution of patent litigation against any entity
(including a cross-claim or counterclaim in a lawsuit) alleging that
THIS IMPLEMENTATION constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any rights
granted to you under this License shall terminate as of the date
such litigation is filed. If you or your agent or exclusive
licensee institute or order or agree to the institution of a PATENT
CHALLENGE, then Tokutek may terminate any rights granted to you
under this License.
*/
#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved."
#include "test.h"
#include "cachetable-test.h"
bool close_called;
bool free_called;
static void close_usr(CACHEFILE UU(cf), int UU(i), void* UU(p), bool UU(b), LSN UU(lsn)) {
close_called = true;
}
static void free_usr(CACHEFILE UU(cf), void* UU(p)) {
free_called = true;
}
static void set_cf_userdata(CACHEFILE f1) {
toku_cachefile_set_userdata(
f1,
NULL,
&dummy_log_fassociate,
&close_usr,
&free_usr,
&dummy_chckpnt_usr,
&dummy_begin,
&dummy_end,
&dummy_note_pin,
&dummy_note_unpin
);
}
bool keep_me;
bool write_me;
bool flush_called;
static UU() void
flush (CACHEFILE f __attribute__((__unused__)),
int UU(fd),
CACHEKEY k __attribute__((__unused__)),
void *v __attribute__((__unused__)),
void **dd __attribute__((__unused__)),
void *e __attribute__((__unused__)),
PAIR_ATTR s __attribute__((__unused__)),
PAIR_ATTR* new_size __attribute__((__unused__)),
bool w __attribute__((__unused__)),
bool keep __attribute__((__unused__)),
bool c __attribute__((__unused__)),
bool UU(is_clone)
)
{
flush_called = true;
if (!keep) keep_me = keep;
if (w) write_me = w;
}
static void
simple_test(bool unlink_on_close) {
const int test_limit = 12;
int r;
CACHETABLE ct;
toku_cachetable_create(&ct, test_limit, ZERO_LSN, NULL_LOGGER);
const char *fname1 = TOKU_TEST_FILENAME;
unlink(fname1);
CACHEFILE f1;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
set_cf_userdata(f1);
// test that if we just open a cachefile and then close it (have no pairs active),
// then it does not get cached
close_called = false;
free_called = false;
toku_cachefile_close(&f1, false, ZERO_LSN);
assert(close_called);
assert(free_called);
// now reopen the cachefile
f1 = NULL;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
set_cf_userdata(f1);
void* v1;
long s1;
CACHETABLE_WRITE_CALLBACK wc = def_write_callback(NULL);
wc.flush_callback = flush;
r = toku_cachetable_get_and_pin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), &v1, &s1, wc, def_fetch, def_pf_req_callback, def_pf_callback, true, NULL);
r = toku_test_cachetable_unpin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), CACHETABLE_DIRTY, make_pair_attr(8));
toku_cachetable_verify(ct);
if (unlink_on_close) {
toku_cachefile_unlink_on_close(f1);
}
close_called = false;
free_called = false;
keep_me = true;
write_me = false;
flush_called = false;
// because we ought to have one pair in the cachetable for this cf,
// close should cache the cf and not free it (unless we unlink on close)
// also, make sure we wrote dirty pair, but we did NOT free PAIR unless
// unlink_on_close was set
toku_cachefile_close(&f1, false, ZERO_LSN);
CACHETABLE_STATUS_S stats;
toku_cachetable_get_status(ct, &stats);
assert(flush_called);
assert(close_called);
assert(write_me);
if (unlink_on_close) {
assert(free_called);
assert(!keep_me);
// pair should NOT still be accounted for
assert(stats.status[CT_SIZE_CURRENT].value.num == 0);
}
else {
assert(keep_me);
assert(!free_called);
// pair should still be accounted for
assert(stats.status[CT_SIZE_CURRENT].value.num == 8);
}
toku_cachetable_close(&ct);
if (!unlink_on_close) {
assert(free_called);
assert(!keep_me);
}
}
// test to verify that a PAIR stays in cache
// after the cachefile undergoes a close and reopen
static void test_pair_stays_in_cache(enum cachetable_dirty dirty) {
const int test_limit = 12;
int r;
CACHETABLE ct;
toku_cachetable_create(&ct, test_limit, ZERO_LSN, NULL_LOGGER);
const char *fname1 = TOKU_TEST_FILENAME;
unlink(fname1);
CACHEFILE f1;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
void* v1;
long s1;
CACHETABLE_WRITE_CALLBACK wc = def_write_callback(NULL);
r = toku_cachetable_get_and_pin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), &v1, &s1, wc, def_fetch, def_pf_req_callback, def_pf_callback, true, NULL);
r = toku_test_cachetable_unpin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), dirty, make_pair_attr(8));
toku_cachefile_close(&f1, false, ZERO_LSN);
// now reopen the cachefile
f1 = NULL;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
// do a maybe_get_and_pin and verify that it succeeds
// therefore proving that the PAIR was cached
// and could be successfully retrieved
r = toku_cachetable_maybe_get_and_pin_clean(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), PL_WRITE_EXPENSIVE, &v1);
assert(r == 0);
r = toku_test_cachetable_unpin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), CACHETABLE_DIRTY, make_pair_attr(8));
toku_cachefile_close(&f1, false, ZERO_LSN);
toku_cachetable_close(&ct);
}
static void test_multiple_cachefiles(bool use_same_hash) {
for (int iter = 0; iter < 3; iter++) {
const int test_limit = 1000;
int r;
CACHETABLE ct;
toku_cachetable_create(&ct, test_limit, ZERO_LSN, NULL_LOGGER);
char fname1[strlen(TOKU_TEST_FILENAME) + sizeof("_1")];
strcpy(fname1, TOKU_TEST_FILENAME);
strncat(fname1, "_1", sizeof("_1"));
char fname2[strlen(TOKU_TEST_FILENAME) + sizeof("_2")];
strcpy(fname2, TOKU_TEST_FILENAME);
strncat(fname2, "_2", sizeof("_2"));
char fname3[strlen(TOKU_TEST_FILENAME) + sizeof("_3")];
strcpy(fname3, TOKU_TEST_FILENAME);
strncat(fname3, "_3", sizeof("_3"));
unlink(fname1);
unlink(fname2);
unlink(fname3);
CACHEFILE f1;
CACHEFILE f2;
CACHEFILE f3;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
r = toku_cachetable_openf(&f2, ct, fname2, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
r = toku_cachetable_openf(&f3, ct, fname3, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
void* v1;
long s1;
void* v2;
long s2;
void* v3;
long s3;
CACHETABLE_WRITE_CALLBACK wc = def_write_callback(NULL);
for (int j = 0; j < 3; j++) {
uint32_t hash = use_same_hash ? 1 : toku_cachetable_hash(f1, make_blocknum(j));
r = toku_cachetable_get_and_pin(f1, make_blocknum(j), hash, &v1, &s1, wc, def_fetch, def_pf_req_callback, def_pf_callback, true, NULL);
r = toku_test_cachetable_unpin(f1, make_blocknum(j), hash, CACHETABLE_CLEAN, make_pair_attr(8));
}
for (int j = 0; j < 3; j++) {
uint32_t hash = use_same_hash ? 1 : toku_cachetable_hash(f2, make_blocknum(j));
r = toku_cachetable_get_and_pin(f2, make_blocknum(j), hash, &v2, &s2, wc, def_fetch, def_pf_req_callback, def_pf_callback, true, NULL);
r = toku_test_cachetable_unpin(f2, make_blocknum(j), hash, CACHETABLE_CLEAN, make_pair_attr(8));
}
for (int j = 0; j < 3; j++) {
uint32_t hash = use_same_hash ? 1 : toku_cachetable_hash(f3, make_blocknum(j));
r = toku_cachetable_get_and_pin(f3, make_blocknum(j), hash, &v3, &s3, wc, def_fetch, def_pf_req_callback, def_pf_callback, true, NULL);
r = toku_test_cachetable_unpin(f3, make_blocknum(j), hash, CACHETABLE_CLEAN, make_pair_attr(8));
}
toku_cachefile_close(&f1, false, ZERO_LSN);
toku_cachefile_close(&f2, false, ZERO_LSN);
toku_cachefile_close(&f3, false, ZERO_LSN);
char* fname_to_open = NULL;
if (iter == 0) {
fname_to_open = fname1;
}
else if (iter == 1) {
fname_to_open = fname2;
}
else if (iter == 2) {
fname_to_open = fname3;
}
// now reopen the cachefile
f1 = NULL;
r = toku_cachetable_openf(&f1, ct, fname_to_open, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
// do a maybe_get_and_pin and verify that it succeeds
// therefore proving that the PAIR was cached
// and could be successfully retrieved
for (int j = 0; j < 3; j++) {
uint32_t hash = use_same_hash ? 1 : toku_cachetable_hash(f1, make_blocknum(j));
r = toku_cachetable_maybe_get_and_pin_clean(f1, make_blocknum(j), hash, PL_WRITE_EXPENSIVE, &v1);
assert(r == 0);
r = toku_test_cachetable_unpin(f1, make_blocknum(j), hash, CACHETABLE_CLEAN, make_pair_attr(8));
}
toku_cachefile_close(&f1, false, ZERO_LSN);
toku_cachetable_close(&ct);
}
}
// test that the evictor works properly with closed cachefiles
static void test_evictor(void) {
const int test_limit = 12;
int r;
CACHETABLE ct;
toku_cachetable_create(&ct, test_limit, ZERO_LSN, NULL_LOGGER);
char fname1[strlen(TOKU_TEST_FILENAME) + sizeof("_1")];
strcpy(fname1, TOKU_TEST_FILENAME);
strncat(fname1, "_1", sizeof("_1"));
char fname2[strlen(TOKU_TEST_FILENAME) + sizeof("_2")];
strcpy(fname2, TOKU_TEST_FILENAME);
strncat(fname2, "_2", sizeof("_2"));
unlink(fname1);
unlink(fname2);
CACHEFILE f1;
CACHEFILE f2;
r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
set_cf_userdata(f1);
r = toku_cachetable_openf(&f2, ct, fname2, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
void* v1;
long s1;
CACHETABLE_WRITE_CALLBACK wc = def_write_callback(NULL);
r = toku_cachetable_get_and_pin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), &v1, &s1, wc, def_fetch, def_pf_req_callback, def_pf_callback, true, NULL);
r = toku_test_cachetable_unpin(f1, make_blocknum(1), toku_cachetable_hash(f1, make_blocknum(1)), CACHETABLE_CLEAN, make_pair_attr(8));
close_called = false;
free_called = false;
toku_cachefile_close(&f1, false, ZERO_LSN);
assert(close_called);
assert(!free_called);
// at this point, we should f1, along with one PAIR, stale in the cachetable
// now let's pin another node, and ensure that it causes an eviction and free of f1
r = toku_cachetable_get_and_pin(f2, make_blocknum(1), toku_cachetable_hash(f2, make_blocknum(1)), &v1, &s1, wc, def_fetch, def_pf_req_callback, def_pf_callback, true, NULL);
r = toku_test_cachetable_unpin(f2, make_blocknum(1), toku_cachetable_hash(f2, make_blocknum(1)), CACHETABLE_CLEAN, make_pair_attr(8));
// now sleep for 2 seconds, and check to see if f1 has been closed
sleep(2);
assert(free_called);
toku_cachefile_close(&f2, false, ZERO_LSN);
toku_cachetable_close(&ct);
}
int
test_main(int argc, const char *argv[]) {
default_parse_args(argc, argv);
test_evictor();
test_multiple_cachefiles(false);
test_multiple_cachefiles(true);
simple_test(false);
simple_test(true);
test_pair_stays_in_cache(CACHETABLE_DIRTY);
test_pair_stays_in_cache(CACHETABLE_CLEAN);
return 0;
}
src/tests/test_stress6.cc
View file @ 800921a7
......@@ -116,6 +116,27 @@ PATENT RIGHTS GRANT:
// update operations.
//
static int remove_and_recreate_me(DB_TXN *UU(txn), ARG arg, void* UU(operation_extra), void *UU(stats_extra)) {
int r;
int db_index = myrandom_r(arg->random_data)%arg->cli->num_DBs;
DB* db = arg->dbp[db_index];
r = (db)->close(db, 0); CKERR(r);
char name[30];
ZERO_ARRAY(name);
get_ith_table_name(name, sizeof(name), db_index);
r = arg->env->dbremove(arg->env, null_txn, name, nullptr, 0);
CKERR(r);
r = db_create(&(arg->dbp[db_index]), arg->env, 0);
assert(r == 0);
// TODO: Need to call before_db_open_hook() and after_db_open_hook()
r = arg->dbp[db_index]->open(arg->dbp[db_index], null_txn, name, nullptr, DB_BTREE, DB_CREATE, 0666);
assert(r == 0);
return 0;
}
static void
stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
//
......
src/tests/threaded_stress_test_helpers.h
View file @ 800921a7
......@@ -1635,27 +1635,6 @@ get_ith_table_name(char *buf, size_t len, int i) {
DB_TXN * const null_txn = 0;
static int UU() remove_and_recreate_me(DB_TXN *UU(txn), ARG arg, void* UU(operation_extra), void *UU(stats_extra)) {
int r;
int db_index = myrandom_r(arg->random_data)%arg->cli->num_DBs;
DB* db = arg->dbp[db_index];
r = (db)->close(db, 0); CKERR(r);
char name[30];
ZERO_ARRAY(name);
get_ith_table_name(name, sizeof(name), db_index);
r = arg->env->dbremove(arg->env, null_txn, name, nullptr, 0);
CKERR(r);
r = db_create(&(arg->dbp[db_index]), arg->env, 0);
assert(r == 0);
// TODO: Need to call before_db_open_hook() and after_db_open_hook()
r = arg->dbp[db_index]->open(arg->dbp[db_index], null_txn, name, nullptr, DB_BTREE, DB_CREATE, 0666);
assert(r == 0);
return 0;
}
// For each line of engine status output, look for lines that contain substrings
// that match any of the strings in the pattern string. The pattern string contains
// 0 or more strings separated by the '|' character, kind of like a regex.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7