Commit 0bf36498 authored by Jens Axboe's avatar Jens Axboe

blk-mq: fix false negative out-of-tags condition

The blk-mq tagging tries to maintain some locality between CPUs and
the tags issued. The tags are split into groups of words, and the
words may not be fully populated. When searching for a new free tag,
blk-mq may look at partial words, hence it passes in an offset/size
to find_next_zero_bit(). However, it does that wrong, the size must
always be the full length of the number of tags in that word,
otherwise we'll potentially miss some near the end.

Another issue is when __bt_get() goes from one word set to the next.
It bumps the index, but not the last_tag associated with the
previous index. Bump that to be in the range of the new word.

Finally, clean up __bt_get() and __bt_get_word() a bit and get
rid of the goto in there, and the unnecessary 'wrap' variable.
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent dd22f551
...@@ -142,29 +142,30 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx, ...@@ -142,29 +142,30 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag) static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
{ {
int tag, org_last_tag, end; int tag, org_last_tag = last_tag;
bool wrap = last_tag != 0;
org_last_tag = last_tag; while (1) {
end = bm->depth; tag = find_next_zero_bit(&bm->word, bm->depth, last_tag);
do { if (unlikely(tag >= bm->depth)) {
restart:
tag = find_next_zero_bit(&bm->word, end, last_tag);
if (unlikely(tag >= end)) {
/* /*
* We started with an offset, start from 0 to * We started with an offset, and we didn't reset the
* offset to 0 in a failure case, so start from 0 to
* exhaust the map. * exhaust the map.
*/ */
if (wrap) { if (org_last_tag && last_tag) {
wrap = false; last_tag = org_last_tag = 0;
end = org_last_tag; continue;
last_tag = 0;
goto restart;
} }
return -1; return -1;
} }
if (!test_and_set_bit(tag, &bm->word))
break;
last_tag = tag + 1; last_tag = tag + 1;
} while (test_and_set_bit(tag, &bm->word)); if (last_tag >= bm->depth - 1)
last_tag = 0;
}
return tag; return tag;
} }
...@@ -199,9 +200,17 @@ static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt, ...@@ -199,9 +200,17 @@ static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt,
goto done; goto done;
} }
last_tag = 0; /*
if (++index >= bt->map_nr) * Jump to next index, and reset the last tag to be the
* first tag of that index
*/
index++;
last_tag = (index << bt->bits_per_word);
if (index >= bt->map_nr) {
index = 0; index = 0;
last_tag = 0;
}
} }
*tag_cache = 0; *tag_cache = 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment