Commit cc71a6f4 authored by Jens Axboe's avatar Jens Axboe

blk-mq: abstract out helpers for allocating/freeing tag maps

Prep patch for adding an extra tag map for scheduler requests.
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
Reviewed-by: default avatarBart Van Assche <bart.vanassche@sandisk.com>
Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
parent 4941115b
...@@ -1553,8 +1553,8 @@ static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio) ...@@ -1553,8 +1553,8 @@ static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
return cookie; return cookie;
} }
void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
unsigned int hctx_idx) unsigned int hctx_idx)
{ {
struct page *page; struct page *page;
...@@ -1580,33 +1580,30 @@ void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, ...@@ -1580,33 +1580,30 @@ void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
kmemleak_free(page_address(page)); kmemleak_free(page_address(page));
__free_pages(page, page->private); __free_pages(page, page->private);
} }
}
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
kfree(tags->rqs); kfree(tags->rqs);
tags->rqs = NULL;
blk_mq_free_tags(tags); blk_mq_free_tags(tags);
} }
static size_t order_to_size(unsigned int order) struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
{ unsigned int hctx_idx,
return (size_t)PAGE_SIZE << order; unsigned int nr_tags,
} unsigned int reserved_tags)
struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
unsigned int hctx_idx)
{ {
struct blk_mq_tags *tags; struct blk_mq_tags *tags;
unsigned int i, j, entries_per_page, max_order = 4;
size_t rq_size, left;
tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags, tags = blk_mq_init_tags(nr_tags, reserved_tags,
set->numa_node, set->numa_node,
BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
if (!tags) if (!tags)
return NULL; return NULL;
INIT_LIST_HEAD(&tags->page_list); tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
tags->rqs = kzalloc_node(set->queue_depth * sizeof(struct request *),
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
set->numa_node); set->numa_node);
if (!tags->rqs) { if (!tags->rqs) {
...@@ -1614,15 +1611,31 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set, ...@@ -1614,15 +1611,31 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
return NULL; return NULL;
} }
return tags;
}
static size_t order_to_size(unsigned int order)
{
return (size_t)PAGE_SIZE << order;
}
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
unsigned int hctx_idx, unsigned int depth)
{
unsigned int i, j, entries_per_page, max_order = 4;
size_t rq_size, left;
INIT_LIST_HEAD(&tags->page_list);
/* /*
* rq_size is the size of the request plus driver payload, rounded * rq_size is the size of the request plus driver payload, rounded
* to the cacheline size * to the cacheline size
*/ */
rq_size = round_up(sizeof(struct request) + set->cmd_size, rq_size = round_up(sizeof(struct request) + set->cmd_size,
cache_line_size()); cache_line_size());
left = rq_size * set->queue_depth; left = rq_size * depth;
for (i = 0; i < set->queue_depth; ) { for (i = 0; i < depth; ) {
int this_order = max_order; int this_order = max_order;
struct page *page; struct page *page;
int to_do; int to_do;
...@@ -1656,7 +1669,7 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set, ...@@ -1656,7 +1669,7 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
*/ */
kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
entries_per_page = order_to_size(this_order) / rq_size; entries_per_page = order_to_size(this_order) / rq_size;
to_do = min(entries_per_page, set->queue_depth - i); to_do = min(entries_per_page, depth - i);
left -= to_do * rq_size; left -= to_do * rq_size;
for (j = 0; j < to_do; j++) { for (j = 0; j < to_do; j++) {
tags->rqs[i] = p; tags->rqs[i] = p;
...@@ -1673,11 +1686,11 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set, ...@@ -1673,11 +1686,11 @@ struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
i++; i++;
} }
} }
return tags; return 0;
fail: fail:
blk_mq_free_rq_map(set, tags, hctx_idx); blk_mq_free_rqs(set, tags, hctx_idx);
return NULL; return -ENOMEM;
} }
/* /*
...@@ -1869,6 +1882,33 @@ static void blk_mq_init_cpu_queues(struct request_queue *q, ...@@ -1869,6 +1882,33 @@ static void blk_mq_init_cpu_queues(struct request_queue *q,
} }
} }
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
{
int ret = 0;
set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
set->queue_depth, set->reserved_tags);
if (!set->tags[hctx_idx])
return false;
ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
set->queue_depth);
if (!ret)
return true;
blk_mq_free_rq_map(set->tags[hctx_idx]);
set->tags[hctx_idx] = NULL;
return false;
}
static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
unsigned int hctx_idx)
{
blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
blk_mq_free_rq_map(set->tags[hctx_idx]);
set->tags[hctx_idx] = NULL;
}
static void blk_mq_map_swqueue(struct request_queue *q, static void blk_mq_map_swqueue(struct request_queue *q,
const struct cpumask *online_mask) const struct cpumask *online_mask)
{ {
...@@ -1897,17 +1937,15 @@ static void blk_mq_map_swqueue(struct request_queue *q, ...@@ -1897,17 +1937,15 @@ static void blk_mq_map_swqueue(struct request_queue *q,
hctx_idx = q->mq_map[i]; hctx_idx = q->mq_map[i];
/* unmapped hw queue can be remapped after CPU topo changed */ /* unmapped hw queue can be remapped after CPU topo changed */
if (!set->tags[hctx_idx]) { if (!set->tags[hctx_idx] &&
set->tags[hctx_idx] = blk_mq_init_rq_map(set, hctx_idx); !__blk_mq_alloc_rq_map(set, hctx_idx)) {
/* /*
* If tags initialization fail for some hctx, * If tags initialization fail for some hctx,
* that hctx won't be brought online. In this * that hctx won't be brought online. In this
* case, remap the current ctx to hctx[0] which * case, remap the current ctx to hctx[0] which
* is guaranteed to always have tags allocated * is guaranteed to always have tags allocated
*/ */
if (!set->tags[hctx_idx]) q->mq_map[i] = 0;
q->mq_map[i] = 0;
} }
ctx = per_cpu_ptr(q->queue_ctx, i); ctx = per_cpu_ptr(q->queue_ctx, i);
...@@ -1930,10 +1968,9 @@ static void blk_mq_map_swqueue(struct request_queue *q, ...@@ -1930,10 +1968,9 @@ static void blk_mq_map_swqueue(struct request_queue *q,
* fallback in case of a new remap fails * fallback in case of a new remap fails
* allocation * allocation
*/ */
if (i && set->tags[i]) { if (i && set->tags[i])
blk_mq_free_rq_map(set, set->tags[i], i); blk_mq_free_map_and_requests(set, i);
set->tags[i] = NULL;
}
hctx->tags = NULL; hctx->tags = NULL;
continue; continue;
} }
...@@ -2100,10 +2137,8 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, ...@@ -2100,10 +2137,8 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
struct blk_mq_hw_ctx *hctx = hctxs[j]; struct blk_mq_hw_ctx *hctx = hctxs[j];
if (hctx) { if (hctx) {
if (hctx->tags) { if (hctx->tags)
blk_mq_free_rq_map(set, hctx->tags, j); blk_mq_free_map_and_requests(set, j);
set->tags[j] = NULL;
}
blk_mq_exit_hctx(q, set, hctx, j); blk_mq_exit_hctx(q, set, hctx, j);
free_cpumask_var(hctx->cpumask); free_cpumask_var(hctx->cpumask);
kobject_put(&hctx->kobj); kobject_put(&hctx->kobj);
...@@ -2299,17 +2334,15 @@ static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) ...@@ -2299,17 +2334,15 @@ static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{ {
int i; int i;
for (i = 0; i < set->nr_hw_queues; i++) { for (i = 0; i < set->nr_hw_queues; i++)
set->tags[i] = blk_mq_init_rq_map(set, i); if (!__blk_mq_alloc_rq_map(set, i))
if (!set->tags[i])
goto out_unwind; goto out_unwind;
}
return 0; return 0;
out_unwind: out_unwind:
while (--i >= 0) while (--i >= 0)
blk_mq_free_rq_map(set, set->tags[i], i); blk_mq_free_rq_map(set->tags[i]);
return -ENOMEM; return -ENOMEM;
} }
...@@ -2433,10 +2466,8 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) ...@@ -2433,10 +2466,8 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{ {
int i; int i;
for (i = 0; i < nr_cpu_ids; i++) { for (i = 0; i < nr_cpu_ids; i++)
if (set->tags[i]) blk_mq_free_map_and_requests(set, i);
blk_mq_free_rq_map(set, set->tags[i], i);
}
kfree(set->mq_map); kfree(set->mq_map);
set->mq_map = NULL; set->mq_map = NULL;
......
...@@ -37,17 +37,21 @@ void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list); ...@@ -37,17 +37,21 @@ void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
/* /*
* Internal helpers for allocating/freeing the request map * Internal helpers for allocating/freeing the request map
*/ */
void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
unsigned int hctx_idx); unsigned int hctx_idx);
struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set, void blk_mq_free_rq_map(struct blk_mq_tags *tags);
unsigned int hctx_idx); struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
unsigned int hctx_idx,
unsigned int nr_tags,
unsigned int reserved_tags);
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
unsigned int hctx_idx, unsigned int depth);
/* /*
* Internal helpers for request insertion into sw queues * Internal helpers for request insertion into sw queues
*/ */
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
bool at_head); bool at_head);
/* /*
* CPU hotplug helpers * CPU hotplug helpers
*/ */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment