Commit 200e86b3 authored by Jens Axboe's avatar Jens Axboe

blk-mq: only apply active queue tag throttling for driver tags

If we have a scheduler attached, we have two sets of tags. We don't
want to apply our active queue throttling for the scheduler side
of tags, that only applies to driver tags since that's the resource
we need to dispatch an IO.
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent 1cf41753
...@@ -90,9 +90,11 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx, ...@@ -90,9 +90,11 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
return atomic_read(&hctx->nr_active) < depth; return atomic_read(&hctx->nr_active) < depth;
} }
static int __blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt) static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
struct sbitmap_queue *bt)
{ {
if (!hctx_may_queue(hctx, bt)) if (!(data->flags & BLK_MQ_REQ_INTERNAL) &&
!hctx_may_queue(data->hctx, bt))
return -1; return -1;
return __sbitmap_queue_get(bt); return __sbitmap_queue_get(bt);
} }
...@@ -118,7 +120,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) ...@@ -118,7 +120,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
tag_offset = tags->nr_reserved_tags; tag_offset = tags->nr_reserved_tags;
} }
tag = __blk_mq_get_tag(data->hctx, bt); tag = __blk_mq_get_tag(data, bt);
if (tag != -1) if (tag != -1)
goto found_tag; goto found_tag;
...@@ -129,7 +131,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) ...@@ -129,7 +131,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
do { do {
prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE); prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE);
tag = __blk_mq_get_tag(data->hctx, bt); tag = __blk_mq_get_tag(data, bt);
if (tag != -1) if (tag != -1)
break; break;
...@@ -144,7 +146,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) ...@@ -144,7 +146,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
* Retry tag allocation after running the hardware queue, * Retry tag allocation after running the hardware queue,
* as running the queue may also have found completions. * as running the queue may also have found completions.
*/ */
tag = __blk_mq_get_tag(data->hctx, bt); tag = __blk_mq_get_tag(data, bt);
if (tag != -1) if (tag != -1)
break; break;
......
...@@ -230,15 +230,14 @@ struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data, ...@@ -230,15 +230,14 @@ struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
rq = tags->static_rqs[tag]; rq = tags->static_rqs[tag];
if (blk_mq_tag_busy(data->hctx)) {
rq->rq_flags = RQF_MQ_INFLIGHT;
atomic_inc(&data->hctx->nr_active);
}
if (data->flags & BLK_MQ_REQ_INTERNAL) { if (data->flags & BLK_MQ_REQ_INTERNAL) {
rq->tag = -1; rq->tag = -1;
rq->internal_tag = tag; rq->internal_tag = tag;
} else { } else {
if (blk_mq_tag_busy(data->hctx)) {
rq->rq_flags = RQF_MQ_INFLIGHT;
atomic_inc(&data->hctx->nr_active);
}
rq->tag = tag; rq->tag = tag;
rq->internal_tag = -1; rq->internal_tag = -1;
} }
...@@ -869,6 +868,10 @@ static bool blk_mq_get_driver_tag(struct request *rq, ...@@ -869,6 +868,10 @@ static bool blk_mq_get_driver_tag(struct request *rq,
rq->tag = blk_mq_get_tag(&data); rq->tag = blk_mq_get_tag(&data);
if (rq->tag >= 0) { if (rq->tag >= 0) {
if (blk_mq_tag_busy(data.hctx)) {
rq->rq_flags |= RQF_MQ_INFLIGHT;
atomic_inc(&data.hctx->nr_active);
}
data.hctx->tags->rqs[rq->tag] = rq; data.hctx->tags->rqs[rq->tag] = rq;
goto done; goto done;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment