Commit 004b6862 authored by Chao Yu's avatar Chao Yu Committed by Jaegeuk Kim

f2fs: use rb-tree to track pending discard commands

Introduce rb-tree based discard cache infrastructure to speed up lookup and
merge operation of discard entry.
Signed-off-by: default avatarChao Yu <yuchao0@huawei.com>
[Jaegeuk Kim: initialize dc to avoid build warning]
Signed-off-by: default avatarJaegeuk Kim <jaegeuk@kernel.org>
parent d40d30c5
...@@ -49,7 +49,7 @@ static struct rb_entry *__lookup_rb_tree_slow(struct rb_root *root, ...@@ -49,7 +49,7 @@ static struct rb_entry *__lookup_rb_tree_slow(struct rb_root *root,
return NULL; return NULL;
} }
static struct rb_entry *__lookup_rb_tree(struct rb_root *root, struct rb_entry *__lookup_rb_tree(struct rb_root *root,
struct rb_entry *cached_re, unsigned int ofs) struct rb_entry *cached_re, unsigned int ofs)
{ {
struct rb_entry *re; struct rb_entry *re;
...@@ -61,7 +61,7 @@ static struct rb_entry *__lookup_rb_tree(struct rb_root *root, ...@@ -61,7 +61,7 @@ static struct rb_entry *__lookup_rb_tree(struct rb_root *root,
return re; return re;
} }
static struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi, struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
struct rb_root *root, struct rb_node **parent, struct rb_root *root, struct rb_node **parent,
unsigned int ofs) unsigned int ofs)
{ {
...@@ -92,13 +92,14 @@ static struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi, ...@@ -92,13 +92,14 @@ static struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
* in order to simpfy the insertion after. * in order to simpfy the insertion after.
* tree must stay unchanged between lookup and insertion. * tree must stay unchanged between lookup and insertion.
*/ */
static struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root, struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root,
struct rb_entry *cached_re, struct rb_entry *cached_re,
unsigned int ofs, unsigned int ofs,
struct rb_entry **prev_entry, struct rb_entry **prev_entry,
struct rb_entry **next_entry, struct rb_entry **next_entry,
struct rb_node ***insert_p, struct rb_node ***insert_p,
struct rb_node **insert_parent) struct rb_node **insert_parent,
bool force)
{ {
struct rb_node **pnode = &root->rb_node; struct rb_node **pnode = &root->rb_node;
struct rb_node *parent = NULL, *tmp_node; struct rb_node *parent = NULL, *tmp_node;
...@@ -145,12 +146,12 @@ static struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root, ...@@ -145,12 +146,12 @@ static struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root,
return NULL; return NULL;
lookup_neighbors: lookup_neighbors:
if (ofs == re->ofs) { if (ofs == re->ofs || force) {
/* lookup prev node for merging backward later */ /* lookup prev node for merging backward later */
tmp_node = rb_prev(&re->rb_node); tmp_node = rb_prev(&re->rb_node);
*prev_entry = rb_entry_safe(tmp_node, struct rb_entry, rb_node); *prev_entry = rb_entry_safe(tmp_node, struct rb_entry, rb_node);
} }
if (ofs == re->ofs + re->len - 1) { if (ofs == re->ofs + re->len - 1 || force) {
/* lookup next node for merging frontward later */ /* lookup next node for merging frontward later */
tmp_node = rb_next(&re->rb_node); tmp_node = rb_next(&re->rb_node);
*next_entry = rb_entry_safe(tmp_node, struct rb_entry, rb_node); *next_entry = rb_entry_safe(tmp_node, struct rb_entry, rb_node);
...@@ -481,7 +482,7 @@ static void f2fs_update_extent_tree_range(struct inode *inode, ...@@ -481,7 +482,7 @@ static void f2fs_update_extent_tree_range(struct inode *inode,
(struct rb_entry *)et->cached_en, fofs, (struct rb_entry *)et->cached_en, fofs,
(struct rb_entry **)&prev_en, (struct rb_entry **)&prev_en,
(struct rb_entry **)&next_en, (struct rb_entry **)&next_en,
&insert_p, &insert_parent); &insert_p, &insert_parent, false);
if (!en) if (!en)
en = next_en; en = next_en;
......
...@@ -194,13 +194,26 @@ enum { ...@@ -194,13 +194,26 @@ enum {
D_DONE, D_DONE,
}; };
struct discard_info {
block_t lstart; /* logical start address */
block_t len; /* length */
block_t start; /* actual start address in dev */
};
struct discard_cmd { struct discard_cmd {
struct rb_node rb_node; /* rb node located in rb-tree */
union {
struct {
block_t lstart; /* logical start address */
block_t len; /* length */
block_t start; /* actual start address in dev */
};
struct discard_info di; /* discard info */
};
struct list_head list; /* command list */ struct list_head list; /* command list */
struct completion wait; /* compleation */ struct completion wait; /* compleation */
struct block_device *bdev; /* bdev */ struct block_device *bdev; /* bdev */
block_t lstart; /* logical start address */
block_t start; /* actual start address in dev */
block_t len; /* length */
int state; /* state */ int state; /* state */
int error; /* bio error */ int error; /* bio error */
}; };
...@@ -217,6 +230,7 @@ struct discard_cmd_control { ...@@ -217,6 +230,7 @@ struct discard_cmd_control {
atomic_t issued_discard; /* # of issued discard */ atomic_t issued_discard; /* # of issued discard */
atomic_t issing_discard; /* # of issing discard */ atomic_t issing_discard; /* # of issing discard */
atomic_t discard_cmd_cnt; /* # of cached cmd count */ atomic_t discard_cmd_cnt; /* # of cached cmd count */
struct rb_root root; /* root of discard rb-tree */
}; };
/* for the list of fsync inodes, used only during recovery */ /* for the list of fsync inodes, used only during recovery */
...@@ -517,6 +531,24 @@ static inline void set_extent_info(struct extent_info *ei, unsigned int fofs, ...@@ -517,6 +531,24 @@ static inline void set_extent_info(struct extent_info *ei, unsigned int fofs,
ei->len = len; ei->len = len;
} }
static inline bool __is_discard_mergeable(struct discard_info *back,
struct discard_info *front)
{
return back->lstart + back->len == front->lstart;
}
static inline bool __is_discard_back_mergeable(struct discard_info *cur,
struct discard_info *back)
{
return __is_discard_mergeable(back, cur);
}
static inline bool __is_discard_front_mergeable(struct discard_info *cur,
struct discard_info *front)
{
return __is_discard_mergeable(cur, front);
}
static inline bool __is_extent_mergeable(struct extent_info *back, static inline bool __is_extent_mergeable(struct extent_info *back,
struct extent_info *front) struct extent_info *front)
{ {
...@@ -2573,6 +2605,16 @@ void f2fs_leave_shrinker(struct f2fs_sb_info *sbi); ...@@ -2573,6 +2605,16 @@ void f2fs_leave_shrinker(struct f2fs_sb_info *sbi);
/* /*
* extent_cache.c * extent_cache.c
*/ */
struct rb_entry *__lookup_rb_tree(struct rb_root *root,
struct rb_entry *cached_re, unsigned int ofs);
struct rb_node **__lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
struct rb_root *root, struct rb_node **parent,
unsigned int ofs);
struct rb_entry *__lookup_rb_tree_ret(struct rb_root *root,
struct rb_entry *cached_re, unsigned int ofs,
struct rb_entry **prev_entry, struct rb_entry **next_entry,
struct rb_node ***insert_p, struct rb_node **insert_parent,
bool force);
unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink); unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink);
bool f2fs_init_extent_tree(struct inode *inode, struct f2fs_extent *i_ext); bool f2fs_init_extent_tree(struct inode *inode, struct f2fs_extent *i_ext);
void f2fs_drop_extent_tree(struct inode *inode); void f2fs_drop_extent_tree(struct inode *inode);
......
...@@ -672,7 +672,7 @@ static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) ...@@ -672,7 +672,7 @@ static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
mutex_unlock(&dirty_i->seglist_lock); mutex_unlock(&dirty_i->seglist_lock);
} }
static void __add_discard_cmd(struct f2fs_sb_info *sbi, static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t lstart, struct block_device *bdev, block_t lstart,
block_t start, block_t len) block_t start, block_t len)
{ {
...@@ -689,18 +689,46 @@ static void __add_discard_cmd(struct f2fs_sb_info *sbi, ...@@ -689,18 +689,46 @@ static void __add_discard_cmd(struct f2fs_sb_info *sbi,
dc->state = D_PREP; dc->state = D_PREP;
dc->error = 0; dc->error = 0;
init_completion(&dc->wait); init_completion(&dc->wait);
mutex_lock(&dcc->cmd_lock);
list_add_tail(&dc->list, pend_list); list_add_tail(&dc->list, pend_list);
mutex_unlock(&dcc->cmd_lock);
atomic_inc(&dcc->discard_cmd_cnt); atomic_inc(&dcc->discard_cmd_cnt);
return dc;
} }
static void __remove_discard_cmd(struct f2fs_sb_info *sbi, struct discard_cmd *dc) static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t lstart,
block_t start, block_t len,
struct rb_node *parent, struct rb_node **p)
{
struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
struct discard_cmd *dc;
dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
rb_link_node(&dc->rb_node, parent, p);
rb_insert_color(&dc->rb_node, &dcc->root);
return dc;
}
static void __detach_discard_cmd(struct discard_cmd_control *dcc,
struct discard_cmd *dc)
{ {
if (dc->state == D_DONE) if (dc->state == D_DONE)
atomic_dec(&(SM_I(sbi)->dcc_info->issing_discard)); atomic_dec(&dcc->issing_discard);
list_del(&dc->list);
rb_erase(&dc->rb_node, &dcc->root);
kmem_cache_free(discard_cmd_slab, dc);
atomic_dec(&dcc->discard_cmd_cnt);
}
static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_cmd *dc)
{
struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
if (dc->error == -EOPNOTSUPP) if (dc->error == -EOPNOTSUPP)
dc->error = 0; dc->error = 0;
...@@ -708,9 +736,7 @@ static void __remove_discard_cmd(struct f2fs_sb_info *sbi, struct discard_cmd *d ...@@ -708,9 +736,7 @@ static void __remove_discard_cmd(struct f2fs_sb_info *sbi, struct discard_cmd *d
if (dc->error) if (dc->error)
f2fs_msg(sbi->sb, KERN_INFO, f2fs_msg(sbi->sb, KERN_INFO,
"Issue discard failed, ret: %d", dc->error); "Issue discard failed, ret: %d", dc->error);
list_del(&dc->list); __detach_discard_cmd(dcc, dc);
kmem_cache_free(discard_cmd_slab, dc);
atomic_dec(&SM_I(sbi)->dcc_info->discard_cmd_cnt);
} }
static void f2fs_submit_discard_endio(struct bio *bio) static void f2fs_submit_discard_endio(struct bio *bio)
...@@ -754,63 +780,179 @@ static void __submit_discard_cmd(struct f2fs_sb_info *sbi, ...@@ -754,63 +780,179 @@ static void __submit_discard_cmd(struct f2fs_sb_info *sbi,
} }
} }
static int __queue_discard_cmd(struct f2fs_sb_info *sbi, static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t blkstart, block_t blklen) struct block_device *bdev, block_t lstart,
block_t start, block_t len,
struct rb_node **insert_p,
struct rb_node *insert_parent)
{ {
block_t lblkstart = blkstart; struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
struct rb_node **p = &dcc->root.rb_node;
struct rb_node *parent = NULL;
struct discard_cmd *dc = NULL;
trace_f2fs_issue_discard(bdev, blkstart, blklen); if (insert_p && insert_parent) {
parent = insert_parent;
p = insert_p;
goto do_insert;
}
if (sbi->s_ndevs) { p = __lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, lstart);
int devi = f2fs_target_device_index(sbi, blkstart); do_insert:
dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, p);
if (!dc)
return NULL;
blkstart -= FDEV(devi).start_blk; return dc;
}
__add_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen);
wake_up(&SM_I(sbi)->dcc_info->discard_wait_queue);
return 0;
} }
static void __punch_discard_cmd(struct f2fs_sb_info *sbi, static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
struct discard_cmd *dc, block_t blkaddr) struct discard_cmd *dc, block_t blkaddr)
{ {
block_t end_block = START_BLOCK(sbi, GET_SEGNO(sbi, blkaddr) + 1); struct discard_info di = dc->di;
bool modified = false;
if (dc->state == D_DONE || dc->lstart + dc->len <= end_block) { if (dc->state == D_DONE || dc->len == 1) {
__remove_discard_cmd(sbi, dc); __remove_discard_cmd(sbi, dc);
return; return;
} }
if (blkaddr - dc->lstart < dc->lstart + dc->len - end_block) { if (blkaddr > di.lstart) {
dc->start += (end_block - dc->lstart);
dc->len -= (end_block - dc->lstart);
dc->lstart = end_block;
} else {
dc->len = blkaddr - dc->lstart; dc->len = blkaddr - dc->lstart;
modified = true;
}
if (blkaddr < di.lstart + di.len - 1) {
if (modified) {
__insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
di.start + blkaddr + 1 - di.lstart,
di.lstart + di.len - 1 - blkaddr,
NULL, NULL);
} else {
dc->lstart++;
dc->len--;
dc->start++;
}
} }
} }
/* This should be covered by global mutex, &sit_i->sentry_lock */ static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) struct block_device *bdev, block_t lstart,
block_t start, block_t len)
{ {
struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
struct list_head *pend_list = &(dcc->discard_pend_list); struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
struct list_head *wait_list = &(dcc->discard_wait_list); struct discard_cmd *dc;
struct discard_cmd *dc, *tmp; struct discard_info di = {0};
struct rb_node **insert_p = NULL, *insert_parent = NULL;
block_t end = lstart + len;
mutex_lock(&dcc->cmd_lock); mutex_lock(&dcc->cmd_lock);
list_for_each_entry_safe(dc, tmp, pend_list, list) { dc = (struct discard_cmd *)__lookup_rb_tree_ret(&dcc->root,
if (dc->lstart <= blkaddr && blkaddr < dc->lstart + dc->len) NULL, lstart,
__punch_discard_cmd(sbi, dc, blkaddr); (struct rb_entry **)&prev_dc,
(struct rb_entry **)&next_dc,
&insert_p, &insert_parent, true);
if (dc)
prev_dc = dc;
if (!prev_dc) {
di.lstart = lstart;
di.len = next_dc ? next_dc->lstart - lstart : len;
di.len = min(di.len, len);
di.start = start;
} }
list_for_each_entry_safe(dc, tmp, wait_list, list) { while (1) {
if (dc->lstart <= blkaddr && blkaddr < dc->lstart + dc->len) { struct rb_node *node;
bool merged = false;
struct discard_cmd *tdc = NULL;
if (prev_dc) {
di.lstart = prev_dc->lstart + prev_dc->len;
if (di.lstart < lstart)
di.lstart = lstart;
if (di.lstart >= end)
break;
if (!next_dc || next_dc->lstart > end)
di.len = end - di.lstart;
else
di.len = next_dc->lstart - di.lstart;
di.start = start + di.lstart - lstart;
}
if (!di.len)
goto next;
if (prev_dc && prev_dc->state == D_PREP &&
prev_dc->bdev == bdev &&
__is_discard_back_mergeable(&di, &prev_dc->di)) {
prev_dc->di.len += di.len;
di = prev_dc->di;
tdc = prev_dc;
merged = true;
}
if (next_dc && next_dc->state == D_PREP &&
next_dc->bdev == bdev &&
__is_discard_front_mergeable(&di, &next_dc->di)) {
next_dc->di.lstart = di.lstart;
next_dc->di.len += di.len;
next_dc->di.start = di.start;
if (tdc)
__remove_discard_cmd(sbi, tdc);
merged = true;
}
if (!merged)
__insert_discard_tree(sbi, bdev, di.lstart, di.start,
di.len, NULL, NULL);
next:
prev_dc = next_dc;
if (!prev_dc)
break;
node = rb_next(&prev_dc->rb_node);
next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
}
mutex_unlock(&dcc->cmd_lock);
}
static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
struct block_device *bdev, block_t blkstart, block_t blklen)
{
block_t lblkstart = blkstart;
trace_f2fs_issue_discard(bdev, blkstart, blklen);
if (sbi->s_ndevs) {
int devi = f2fs_target_device_index(sbi, blkstart);
blkstart -= FDEV(devi).start_blk;
}
__update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
wake_up(&SM_I(sbi)->dcc_info->discard_wait_queue);
return 0;
}
/* This should be covered by global mutex, &sit_i->sentry_lock */
void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
{
struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
struct discard_cmd *dc;
mutex_lock(&dcc->cmd_lock);
dc = (struct discard_cmd *)__lookup_rb_tree(&dcc->root, NULL, blkaddr);
if (dc) {
if (dc->state != D_PREP)
wait_for_completion_io(&dc->wait); wait_for_completion_io(&dc->wait);
__punch_discard_cmd(sbi, dc, blkaddr); __punch_discard_cmd(sbi, dc, blkaddr);
} }
}
mutex_unlock(&dcc->cmd_lock); mutex_unlock(&dcc->cmd_lock);
} }
...@@ -1178,6 +1320,7 @@ static int create_discard_cmd_control(struct f2fs_sb_info *sbi) ...@@ -1178,6 +1320,7 @@ static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
atomic_set(&dcc->discard_cmd_cnt, 0); atomic_set(&dcc->discard_cmd_cnt, 0);
dcc->nr_discards = 0; dcc->nr_discards = 0;
dcc->max_discards = 0; dcc->max_discards = 0;
dcc->root = RB_ROOT;
init_waitqueue_head(&dcc->discard_wait_queue); init_waitqueue_head(&dcc->discard_wait_queue);
SM_I(sbi)->dcc_info = dcc; SM_I(sbi)->dcc_info = dcc;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment