Commit 3f0abd80 authored by Shaohua Li's avatar Shaohua Li Committed by Jens Axboe

blk-throttle: add downgrade logic

When queue state machine is in LIMIT_MAX state, but a cgroup is below
its low limit for some time, the queue should be downgraded to lower
state as one cgroup's low limit isn't met.
Signed-off-by: default avatarShaohua Li <shli@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent c79892c5
...@@ -140,6 +140,13 @@ struct throtl_grp { ...@@ -140,6 +140,13 @@ struct throtl_grp {
/* Number of bio's dispatched in current slice */ /* Number of bio's dispatched in current slice */
unsigned int io_disp[2]; unsigned int io_disp[2];
unsigned long last_low_overflow_time[2];
uint64_t last_bytes_disp[2];
unsigned int last_io_disp[2];
unsigned long last_check_time;
/* When did we start a new slice */ /* When did we start a new slice */
unsigned long slice_start[2]; unsigned long slice_start[2];
unsigned long slice_end[2]; unsigned long slice_end[2];
...@@ -159,6 +166,9 @@ struct throtl_data ...@@ -159,6 +166,9 @@ struct throtl_data
struct work_struct dispatch_work; struct work_struct dispatch_work;
unsigned int limit_index; unsigned int limit_index;
bool limit_valid[LIMIT_CNT]; bool limit_valid[LIMIT_CNT];
unsigned long low_upgrade_time;
unsigned long low_downgrade_time;
}; };
static void throtl_pending_timer_fn(unsigned long arg); static void throtl_pending_timer_fn(unsigned long arg);
...@@ -898,6 +908,8 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) ...@@ -898,6 +908,8 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
/* Charge the bio to the group */ /* Charge the bio to the group */
tg->bytes_disp[rw] += bio->bi_iter.bi_size; tg->bytes_disp[rw] += bio->bi_iter.bi_size;
tg->io_disp[rw]++; tg->io_disp[rw]++;
tg->last_bytes_disp[rw] += bio->bi_iter.bi_size;
tg->last_io_disp[rw]++;
/* /*
* BIO_THROTTLED is used to prevent the same bio to be throttled * BIO_THROTTLED is used to prevent the same bio to be throttled
...@@ -1527,6 +1539,45 @@ static struct blkcg_policy blkcg_policy_throtl = { ...@@ -1527,6 +1539,45 @@ static struct blkcg_policy blkcg_policy_throtl = {
.pd_free_fn = throtl_pd_free, .pd_free_fn = throtl_pd_free,
}; };
static unsigned long __tg_last_low_overflow_time(struct throtl_grp *tg)
{
unsigned long rtime = jiffies, wtime = jiffies;
if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW])
rtime = tg->last_low_overflow_time[READ];
if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW])
wtime = tg->last_low_overflow_time[WRITE];
return min(rtime, wtime);
}
/* tg should not be an intermediate node */
static unsigned long tg_last_low_overflow_time(struct throtl_grp *tg)
{
struct throtl_service_queue *parent_sq;
struct throtl_grp *parent = tg;
unsigned long ret = __tg_last_low_overflow_time(tg);
while (true) {
parent_sq = parent->service_queue.parent_sq;
parent = sq_to_tg(parent_sq);
if (!parent)
break;
/*
* The parent doesn't have low limit, it always reaches low
* limit. Its overflow time is useless for children
*/
if (!parent->bps[READ][LIMIT_LOW] &&
!parent->iops[READ][LIMIT_LOW] &&
!parent->bps[WRITE][LIMIT_LOW] &&
!parent->iops[WRITE][LIMIT_LOW])
continue;
if (time_after(__tg_last_low_overflow_time(parent), ret))
ret = __tg_last_low_overflow_time(parent);
}
return ret;
}
static bool throtl_tg_can_upgrade(struct throtl_grp *tg) static bool throtl_tg_can_upgrade(struct throtl_grp *tg)
{ {
struct throtl_service_queue *sq = &tg->service_queue; struct throtl_service_queue *sq = &tg->service_queue;
...@@ -1570,6 +1621,9 @@ static bool throtl_can_upgrade(struct throtl_data *td, ...@@ -1570,6 +1621,9 @@ static bool throtl_can_upgrade(struct throtl_data *td,
if (td->limit_index != LIMIT_LOW) if (td->limit_index != LIMIT_LOW)
return false; return false;
if (time_before(jiffies, td->low_downgrade_time + throtl_slice))
return false;
rcu_read_lock(); rcu_read_lock();
blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_grp *tg = blkg_to_tg(blkg);
...@@ -1593,6 +1647,7 @@ static void throtl_upgrade_state(struct throtl_data *td) ...@@ -1593,6 +1647,7 @@ static void throtl_upgrade_state(struct throtl_data *td)
struct blkcg_gq *blkg; struct blkcg_gq *blkg;
td->limit_index = LIMIT_MAX; td->limit_index = LIMIT_MAX;
td->low_upgrade_time = jiffies;
rcu_read_lock(); rcu_read_lock();
blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
struct throtl_grp *tg = blkg_to_tg(blkg); struct throtl_grp *tg = blkg_to_tg(blkg);
...@@ -1608,6 +1663,99 @@ static void throtl_upgrade_state(struct throtl_data *td) ...@@ -1608,6 +1663,99 @@ static void throtl_upgrade_state(struct throtl_data *td)
queue_work(kthrotld_workqueue, &td->dispatch_work); queue_work(kthrotld_workqueue, &td->dispatch_work);
} }
static void throtl_downgrade_state(struct throtl_data *td, int new)
{
td->limit_index = new;
td->low_downgrade_time = jiffies;
}
static bool throtl_tg_can_downgrade(struct throtl_grp *tg)
{
struct throtl_data *td = tg->td;
unsigned long now = jiffies;
/*
* If cgroup is below low limit, consider downgrade and throttle other
* cgroups
*/
if (time_after_eq(now, td->low_upgrade_time + throtl_slice) &&
time_after_eq(now, tg_last_low_overflow_time(tg) + throtl_slice))
return true;
return false;
}
static bool throtl_hierarchy_can_downgrade(struct throtl_grp *tg)
{
while (true) {
if (!throtl_tg_can_downgrade(tg))
return false;
tg = sq_to_tg(tg->service_queue.parent_sq);
if (!tg || !tg_to_blkg(tg)->parent)
break;
}
return true;
}
static void throtl_downgrade_check(struct throtl_grp *tg)
{
uint64_t bps;
unsigned int iops;
unsigned long elapsed_time;
unsigned long now = jiffies;
if (tg->td->limit_index != LIMIT_MAX ||
!tg->td->limit_valid[LIMIT_LOW])
return;
if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children))
return;
if (time_after(tg->last_check_time + throtl_slice, now))
return;
elapsed_time = now - tg->last_check_time;
tg->last_check_time = now;
if (time_before(now, tg_last_low_overflow_time(tg) + throtl_slice))
return;
if (tg->bps[READ][LIMIT_LOW]) {
bps = tg->last_bytes_disp[READ] * HZ;
do_div(bps, elapsed_time);
if (bps >= tg->bps[READ][LIMIT_LOW])
tg->last_low_overflow_time[READ] = now;
}
if (tg->bps[WRITE][LIMIT_LOW]) {
bps = tg->last_bytes_disp[WRITE] * HZ;
do_div(bps, elapsed_time);
if (bps >= tg->bps[WRITE][LIMIT_LOW])
tg->last_low_overflow_time[WRITE] = now;
}
if (tg->iops[READ][LIMIT_LOW]) {
iops = tg->last_io_disp[READ] * HZ / elapsed_time;
if (iops >= tg->iops[READ][LIMIT_LOW])
tg->last_low_overflow_time[READ] = now;
}
if (tg->iops[WRITE][LIMIT_LOW]) {
iops = tg->last_io_disp[WRITE] * HZ / elapsed_time;
if (iops >= tg->iops[WRITE][LIMIT_LOW])
tg->last_low_overflow_time[WRITE] = now;
}
/*
* If cgroup is below low limit, consider downgrade and throttle other
* cgroups
*/
if (throtl_hierarchy_can_downgrade(tg))
throtl_downgrade_state(tg->td, LIMIT_LOW);
tg->last_bytes_disp[READ] = 0;
tg->last_bytes_disp[WRITE] = 0;
tg->last_io_disp[READ] = 0;
tg->last_io_disp[WRITE] = 0;
}
bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
struct bio *bio) struct bio *bio)
{ {
...@@ -1632,12 +1780,16 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, ...@@ -1632,12 +1780,16 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
again: again:
while (true) { while (true) {
if (tg->last_low_overflow_time[rw] == 0)
tg->last_low_overflow_time[rw] = jiffies;
throtl_downgrade_check(tg);
/* throtl is FIFO - if bios are already queued, should queue */ /* throtl is FIFO - if bios are already queued, should queue */
if (sq->nr_queued[rw]) if (sq->nr_queued[rw])
break; break;
/* if above limits, break to queue */ /* if above limits, break to queue */
if (!tg_may_dispatch(tg, bio, NULL)) { if (!tg_may_dispatch(tg, bio, NULL)) {
tg->last_low_overflow_time[rw] = jiffies;
if (throtl_can_upgrade(tg->td, tg)) { if (throtl_can_upgrade(tg->td, tg)) {
throtl_upgrade_state(tg->td); throtl_upgrade_state(tg->td);
goto again; goto again;
...@@ -1681,6 +1833,8 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, ...@@ -1681,6 +1833,8 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
tg->io_disp[rw], tg_iops_limit(tg, rw), tg->io_disp[rw], tg_iops_limit(tg, rw),
sq->nr_queued[READ], sq->nr_queued[WRITE]); sq->nr_queued[READ], sq->nr_queued[WRITE]);
tg->last_low_overflow_time[rw] = jiffies;
bio_associate_current(bio); bio_associate_current(bio);
tg->td->nr_queued[rw]++; tg->td->nr_queued[rw]++;
throtl_add_bio_tg(bio, qn, tg); throtl_add_bio_tg(bio, qn, tg);
...@@ -1791,6 +1945,8 @@ int blk_throtl_init(struct request_queue *q) ...@@ -1791,6 +1945,8 @@ int blk_throtl_init(struct request_queue *q)
td->limit_valid[LIMIT_MAX] = true; td->limit_valid[LIMIT_MAX] = true;
td->limit_index = LIMIT_MAX; td->limit_index = LIMIT_MAX;
td->low_upgrade_time = jiffies;
td->low_downgrade_time = jiffies;
/* activate policy */ /* activate policy */
ret = blkcg_activate_policy(q, &blkcg_policy_throtl); ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
if (ret) if (ret)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment