Commit 773ca82f authored by Shaohua Li's avatar Shaohua Li Committed by NeilBrown

raid5: make release_stripe lockless

release_stripe still has big lock contention. We just add the stripe to a llist
without taking device_lock. We let the raid5d thread to do the real stripe
release, which must hold device_lock anyway. In this way, release_stripe
doesn't hold any locks.

The side effect is the released stripes order is changed. But sounds not a big
deal, stripes are never handled in order. And I thought block layer can already
do nice request merge, which means order isn't that important.

I kept the unplug release batch, which is unnecessary with this patch from lock
contention avoid point of view, and actually if we delete it, the stripe_head
release_list and lru can share storage. But the unplug release batch is also
helpful for request merge. We probably can delay wakeup raid5d till unplug, but
I'm still afraid of the case which raid5d is running.
Signed-off-by: default avatarShaohua Li <shli@fusionio.com>
Signed-off-by: default avatarNeilBrown <neilb@suse.de>
parent 260fa034
...@@ -239,12 +239,47 @@ static void __release_stripe(struct r5conf *conf, struct stripe_head *sh) ...@@ -239,12 +239,47 @@ static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
do_release_stripe(conf, sh); do_release_stripe(conf, sh);
} }
/* should hold conf->device_lock already */
static int release_stripe_list(struct r5conf *conf)
{
struct stripe_head *sh;
int count = 0;
struct llist_node *head;
head = llist_del_all(&conf->released_stripes);
while (head) {
sh = llist_entry(head, struct stripe_head, release_list);
head = llist_next(head);
/* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
smp_mb();
clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
/*
* Don't worry the bit is set here, because if the bit is set
* again, the count is always > 1. This is true for
* STRIPE_ON_UNPLUG_LIST bit too.
*/
__release_stripe(conf, sh);
count++;
}
return count;
}
static void release_stripe(struct stripe_head *sh) static void release_stripe(struct stripe_head *sh)
{ {
struct r5conf *conf = sh->raid_conf; struct r5conf *conf = sh->raid_conf;
unsigned long flags; unsigned long flags;
bool wakeup;
if (test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
goto slow_path;
wakeup = llist_add(&sh->release_list, &conf->released_stripes);
if (wakeup)
md_wakeup_thread(conf->mddev->thread);
return;
slow_path:
local_irq_save(flags); local_irq_save(flags);
/* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
do_release_stripe(conf, sh); do_release_stripe(conf, sh);
spin_unlock(&conf->device_lock); spin_unlock(&conf->device_lock);
...@@ -491,7 +526,8 @@ get_active_stripe(struct r5conf *conf, sector_t sector, ...@@ -491,7 +526,8 @@ get_active_stripe(struct r5conf *conf, sector_t sector,
if (atomic_read(&sh->count)) { if (atomic_read(&sh->count)) {
BUG_ON(!list_empty(&sh->lru) BUG_ON(!list_empty(&sh->lru)
&& !test_bit(STRIPE_EXPANDING, &sh->state) && !test_bit(STRIPE_EXPANDING, &sh->state)
&& !test_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)); && !test_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)
&& !test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
} else { } else {
if (!test_bit(STRIPE_HANDLE, &sh->state)) if (!test_bit(STRIPE_HANDLE, &sh->state))
atomic_inc(&conf->active_stripes); atomic_inc(&conf->active_stripes);
...@@ -4127,6 +4163,10 @@ static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) ...@@ -4127,6 +4163,10 @@ static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
*/ */
smp_mb__before_clear_bit(); smp_mb__before_clear_bit();
clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
/*
* STRIPE_ON_RELEASE_LIST could be set here. In that
* case, the count is always > 1 here
*/
__release_stripe(conf, sh); __release_stripe(conf, sh);
cnt++; cnt++;
} }
...@@ -4836,7 +4876,9 @@ static void raid5d(struct md_thread *thread) ...@@ -4836,7 +4876,9 @@ static void raid5d(struct md_thread *thread)
spin_lock_irq(&conf->device_lock); spin_lock_irq(&conf->device_lock);
while (1) { while (1) {
struct bio *bio; struct bio *bio;
int batch_size; int batch_size, released;
released = release_stripe_list(conf);
if ( if (
!list_empty(&conf->bitmap_list)) { !list_empty(&conf->bitmap_list)) {
...@@ -4861,7 +4903,7 @@ static void raid5d(struct md_thread *thread) ...@@ -4861,7 +4903,7 @@ static void raid5d(struct md_thread *thread)
} }
batch_size = handle_active_stripes(conf); batch_size = handle_active_stripes(conf);
if (!batch_size) if (!batch_size && !released)
break; break;
handled += batch_size; handled += batch_size;
...@@ -5176,6 +5218,7 @@ static struct r5conf *setup_conf(struct mddev *mddev) ...@@ -5176,6 +5218,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)
INIT_LIST_HEAD(&conf->delayed_list); INIT_LIST_HEAD(&conf->delayed_list);
INIT_LIST_HEAD(&conf->bitmap_list); INIT_LIST_HEAD(&conf->bitmap_list);
INIT_LIST_HEAD(&conf->inactive_list); INIT_LIST_HEAD(&conf->inactive_list);
init_llist_head(&conf->released_stripes);
atomic_set(&conf->active_stripes, 0); atomic_set(&conf->active_stripes, 0);
atomic_set(&conf->preread_active_stripes, 0); atomic_set(&conf->preread_active_stripes, 0);
atomic_set(&conf->active_aligned_reads, 0); atomic_set(&conf->active_aligned_reads, 0);
......
...@@ -197,6 +197,7 @@ enum reconstruct_states { ...@@ -197,6 +197,7 @@ enum reconstruct_states {
struct stripe_head { struct stripe_head {
struct hlist_node hash; struct hlist_node hash;
struct list_head lru; /* inactive_list or handle_list */ struct list_head lru; /* inactive_list or handle_list */
struct llist_node release_list;
struct r5conf *raid_conf; struct r5conf *raid_conf;
short generation; /* increments with every short generation; /* increments with every
* reshape */ * reshape */
...@@ -321,6 +322,7 @@ enum { ...@@ -321,6 +322,7 @@ enum {
STRIPE_OPS_REQ_PENDING, STRIPE_OPS_REQ_PENDING,
STRIPE_ON_UNPLUG_LIST, STRIPE_ON_UNPLUG_LIST,
STRIPE_DISCARD, STRIPE_DISCARD,
STRIPE_ON_RELEASE_LIST,
}; };
/* /*
...@@ -445,6 +447,7 @@ struct r5conf { ...@@ -445,6 +447,7 @@ struct r5conf {
*/ */
atomic_t active_stripes; atomic_t active_stripes;
struct list_head inactive_list; struct list_head inactive_list;
struct llist_head released_stripes;
wait_queue_head_t wait_for_stripe; wait_queue_head_t wait_for_stripe;
wait_queue_head_t wait_for_overlap; wait_queue_head_t wait_for_overlap;
int inactive_blocked; /* release of inactive stripes blocked, int inactive_blocked; /* release of inactive stripes blocked,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment