Commit 4fb72036 authored by Tejun Heo's avatar Tejun Heo Committed by Jens Axboe

blk-throttle: remove asynchrnous percpu stats allocation mechanism

Because percpu allocator couldn't do non-blocking allocations,
blk-throttle was forced to implement an ad-hoc asynchronous allocation
mechanism for its percpu stats for cases where blkg's (blkcg_gq's) are
allocated from an IO path without sleepable context.

Now that percpu allocator can handle gfp_mask and blkg_policy_data
alloc / free are handled by policy methods, the ad-hoc asynchronous
allocation mechanism can be replaced with direct allocation from
tg_stats_alloc_fn().  Rit it out.

This ensures that an active throtl_grp always has valid non-NULL
->stats_cpu.  Remove checks on it.
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Cc: Vivek Goyal <vgoyal@redhat.com>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent 001bea73
...@@ -144,9 +144,6 @@ struct throtl_grp { ...@@ -144,9 +144,6 @@ struct throtl_grp {
/* Per cpu stats pointer */ /* Per cpu stats pointer */
struct tg_stats_cpu __percpu *stats_cpu; struct tg_stats_cpu __percpu *stats_cpu;
/* List of tgs waiting for per cpu stats memory to be allocated */
struct list_head stats_alloc_node;
}; };
struct throtl_data struct throtl_data
...@@ -168,13 +165,6 @@ struct throtl_data ...@@ -168,13 +165,6 @@ struct throtl_data
struct work_struct dispatch_work; struct work_struct dispatch_work;
}; };
/* list and work item to allocate percpu group stats */
static DEFINE_SPINLOCK(tg_stats_alloc_lock);
static LIST_HEAD(tg_stats_alloc_list);
static void tg_stats_alloc_fn(struct work_struct *);
static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
static void throtl_pending_timer_fn(unsigned long arg); static void throtl_pending_timer_fn(unsigned long arg);
static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd) static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
...@@ -256,53 +246,6 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq) ...@@ -256,53 +246,6 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
} \ } \
} while (0) } while (0)
static void tg_stats_init(struct tg_stats_cpu *tg_stats)
{
blkg_rwstat_init(&tg_stats->service_bytes);
blkg_rwstat_init(&tg_stats->serviced);
}
/*
* Worker for allocating per cpu stat for tgs. This is scheduled on the
* system_wq once there are some groups on the alloc_list waiting for
* allocation.
*/
static void tg_stats_alloc_fn(struct work_struct *work)
{
static struct tg_stats_cpu *stats_cpu; /* this fn is non-reentrant */
struct delayed_work *dwork = to_delayed_work(work);
bool empty = false;
alloc_stats:
if (!stats_cpu) {
int cpu;
stats_cpu = alloc_percpu(struct tg_stats_cpu);
if (!stats_cpu) {
/* allocation failed, try again after some time */
schedule_delayed_work(dwork, msecs_to_jiffies(10));
return;
}
for_each_possible_cpu(cpu)
tg_stats_init(per_cpu_ptr(stats_cpu, cpu));
}
spin_lock_irq(&tg_stats_alloc_lock);
if (!list_empty(&tg_stats_alloc_list)) {
struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list,
struct throtl_grp,
stats_alloc_node);
swap(tg->stats_cpu, stats_cpu);
list_del_init(&tg->stats_alloc_node);
}
empty = list_empty(&tg_stats_alloc_list);
spin_unlock_irq(&tg_stats_alloc_lock);
if (!empty)
goto alloc_stats;
}
static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg) static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg)
{ {
INIT_LIST_HEAD(&qn->node); INIT_LIST_HEAD(&qn->node);
...@@ -405,7 +348,27 @@ static void throtl_service_queue_exit(struct throtl_service_queue *sq) ...@@ -405,7 +348,27 @@ static void throtl_service_queue_exit(struct throtl_service_queue *sq)
static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, int node) static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, int node)
{ {
return kzalloc_node(sizeof(struct throtl_grp), gfp, node); struct throtl_grp *tg;
int cpu;
tg = kzalloc_node(sizeof(*tg), gfp, node);
if (!tg)
return NULL;
tg->stats_cpu = alloc_percpu_gfp(struct tg_stats_cpu, gfp);
if (!tg->stats_cpu) {
kfree(tg);
return NULL;
}
for_each_possible_cpu(cpu) {
struct tg_stats_cpu *stats_cpu = per_cpu_ptr(tg->stats_cpu, cpu);
blkg_rwstat_init(&stats_cpu->service_bytes);
blkg_rwstat_init(&stats_cpu->serviced);
}
return &tg->pd;
} }
static void throtl_pd_init(struct blkcg_gq *blkg) static void throtl_pd_init(struct blkcg_gq *blkg)
...@@ -413,7 +376,6 @@ static void throtl_pd_init(struct blkcg_gq *blkg) ...@@ -413,7 +376,6 @@ static void throtl_pd_init(struct blkcg_gq *blkg)
struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_grp *tg = blkg_to_tg(blkg);
struct throtl_data *td = blkg->q->td; struct throtl_data *td = blkg->q->td;
struct throtl_service_queue *parent_sq; struct throtl_service_queue *parent_sq;
unsigned long flags;
int rw; int rw;
/* /*
...@@ -448,16 +410,6 @@ static void throtl_pd_init(struct blkcg_gq *blkg) ...@@ -448,16 +410,6 @@ static void throtl_pd_init(struct blkcg_gq *blkg)
tg->bps[WRITE] = -1; tg->bps[WRITE] = -1;
tg->iops[READ] = -1; tg->iops[READ] = -1;
tg->iops[WRITE] = -1; tg->iops[WRITE] = -1;
/*
* Ugh... We need to perform per-cpu allocation for tg->stats_cpu
* but percpu allocator can't be called from IO path. Queue tg on
* tg_stats_alloc_list and allocate from work item.
*/
spin_lock_irqsave(&tg_stats_alloc_lock, flags);
list_add(&tg->stats_alloc_node, &tg_stats_alloc_list);
schedule_delayed_work(&tg_stats_alloc_work, 0);
spin_unlock_irqrestore(&tg_stats_alloc_lock, flags);
} }
/* /*
...@@ -487,20 +439,16 @@ static void throtl_pd_online(struct blkcg_gq *blkg) ...@@ -487,20 +439,16 @@ static void throtl_pd_online(struct blkcg_gq *blkg)
static void throtl_pd_exit(struct blkcg_gq *blkg) static void throtl_pd_exit(struct blkcg_gq *blkg)
{ {
struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_grp *tg = blkg_to_tg(blkg);
unsigned long flags;
spin_lock_irqsave(&tg_stats_alloc_lock, flags);
list_del_init(&tg->stats_alloc_node);
spin_unlock_irqrestore(&tg_stats_alloc_lock, flags);
free_percpu(tg->stats_cpu);
throtl_service_queue_exit(&tg->service_queue); throtl_service_queue_exit(&tg->service_queue);
} }
static void throtl_pd_free(struct blkg_policy_data *pd) static void throtl_pd_free(struct blkg_policy_data *pd)
{ {
kfree(pd); struct throtl_grp *tg = pd_to_tg(pd);
free_percpu(tg->stats_cpu);
kfree(tg);
} }
static void throtl_pd_reset_stats(struct blkcg_gq *blkg) static void throtl_pd_reset_stats(struct blkcg_gq *blkg)
...@@ -508,9 +456,6 @@ static void throtl_pd_reset_stats(struct blkcg_gq *blkg) ...@@ -508,9 +456,6 @@ static void throtl_pd_reset_stats(struct blkcg_gq *blkg)
struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_grp *tg = blkg_to_tg(blkg);
int cpu; int cpu;
if (tg->stats_cpu == NULL)
return;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu); struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
...@@ -973,10 +918,6 @@ static void throtl_update_dispatch_stats(struct blkcg_gq *blkg, u64 bytes, ...@@ -973,10 +918,6 @@ static void throtl_update_dispatch_stats(struct blkcg_gq *blkg, u64 bytes,
struct tg_stats_cpu *stats_cpu; struct tg_stats_cpu *stats_cpu;
unsigned long flags; unsigned long flags;
/* If per cpu stats are not allocated yet, don't do any accounting. */
if (tg->stats_cpu == NULL)
return;
/* /*
* Disabling interrupts to provide mutual exclusion between two * Disabling interrupts to provide mutual exclusion between two
* writes on same cpu. It probably is not needed for 64bit. Not * writes on same cpu. It probably is not needed for 64bit. Not
...@@ -1302,9 +1243,6 @@ static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, ...@@ -1302,9 +1243,6 @@ static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
struct blkg_rwstat rwstat = { }, tmp; struct blkg_rwstat rwstat = { }, tmp;
int i, cpu; int i, cpu;
if (tg->stats_cpu == NULL)
return 0;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu); struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment