Commit c05f8525 authored by Omar Sandoval's avatar Omar Sandoval Committed by Jens Axboe

blk-mq-sched: make completed_request() callback more useful

Currently, this callback is called right after put_request() and has no
distinguishable purpose. Instead, let's call it before put_request() as
soon as I/O has completed on the request, before we account it in
blk-stat. With this, Kyber can enable stats when it sees a latency
outlier and make sure the outlier gets accounted.
Signed-off-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent 5b727272
...@@ -82,17 +82,12 @@ blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq, ...@@ -82,17 +82,12 @@ blk_mq_sched_allow_merge(struct request_queue *q, struct request *rq,
return true; return true;
} }
static inline void static inline void blk_mq_sched_completed_request(struct request *rq)
blk_mq_sched_completed_request(struct blk_mq_hw_ctx *hctx, struct request *rq)
{ {
struct elevator_queue *e = hctx->queue->elevator; struct elevator_queue *e = rq->q->elevator;
if (e && e->type->ops.mq.completed_request) if (e && e->type->ops.mq.completed_request)
e->type->ops.mq.completed_request(hctx, rq); e->type->ops.mq.completed_request(rq);
BUG_ON(rq->internal_tag == -1);
blk_mq_put_tag(hctx, hctx->sched_tags, rq->mq_ctx, rq->internal_tag);
} }
static inline void blk_mq_sched_started_request(struct request *rq) static inline void blk_mq_sched_started_request(struct request *rq)
......
...@@ -350,7 +350,7 @@ void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, ...@@ -350,7 +350,7 @@ void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
if (rq->tag != -1) if (rq->tag != -1)
blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
if (sched_tag != -1) if (sched_tag != -1)
blk_mq_sched_completed_request(hctx, rq); blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
blk_mq_sched_restart(hctx); blk_mq_sched_restart(hctx);
blk_queue_exit(q); blk_queue_exit(q);
} }
...@@ -444,6 +444,9 @@ static void __blk_mq_complete_request(struct request *rq) ...@@ -444,6 +444,9 @@ static void __blk_mq_complete_request(struct request *rq)
{ {
struct request_queue *q = rq->q; struct request_queue *q = rq->q;
if (rq->internal_tag != -1)
blk_mq_sched_completed_request(rq);
blk_mq_stat_add(rq); blk_mq_stat_add(rq);
if (!q->softirq_done_fn) if (!q->softirq_done_fn)
......
...@@ -106,7 +106,7 @@ struct elevator_mq_ops { ...@@ -106,7 +106,7 @@ struct elevator_mq_ops {
void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool);
struct request *(*dispatch_request)(struct blk_mq_hw_ctx *); struct request *(*dispatch_request)(struct blk_mq_hw_ctx *);
bool (*has_work)(struct blk_mq_hw_ctx *); bool (*has_work)(struct blk_mq_hw_ctx *);
void (*completed_request)(struct blk_mq_hw_ctx *, struct request *); void (*completed_request)(struct request *);
void (*started_request)(struct request *); void (*started_request)(struct request *);
void (*requeue_request)(struct request *); void (*requeue_request)(struct request *);
struct request *(*former_request)(struct request_queue *, struct request *); struct request *(*former_request)(struct request_queue *, struct request *);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment