Commit dcebd755 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe

block: use bio_for_each_bvec() to compute multi-page bvec count

First it is more efficient to use bio_for_each_bvec() in both
blk_bio_segment_split() and __blk_recalc_rq_segments() to compute how
many multi-page bvecs there are in the bio.

Secondly once bio_for_each_bvec() is used, the bvec may need to be
splitted because its length can be very longer than max segment size,
so we have to split the big bvec into several segments.

Thirdly when splitting multi-page bvec into segments, the max segment
limit may be reached, so the bio split need to be considered under
this situation too.
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent d18d9174
...@@ -161,6 +161,73 @@ static inline unsigned get_max_io_size(struct request_queue *q, ...@@ -161,6 +161,73 @@ static inline unsigned get_max_io_size(struct request_queue *q,
return sectors; return sectors;
} }
static unsigned get_max_segment_size(struct request_queue *q,
unsigned offset)
{
unsigned long mask = queue_segment_boundary(q);
/* default segment boundary mask means no boundary limit */
if (mask == BLK_SEG_BOUNDARY_MASK)
return queue_max_segment_size(q);
return min_t(unsigned long, mask - (mask & offset) + 1,
queue_max_segment_size(q));
}
/*
* Split the bvec @bv into segments, and update all kinds of
* variables.
*/
static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
unsigned *nsegs, unsigned *last_seg_size,
unsigned *front_seg_size, unsigned *sectors)
{
unsigned len = bv->bv_len;
unsigned total_len = 0;
unsigned new_nsegs = 0, seg_size = 0;
/*
* Multi-page bvec may be too big to hold in one segment, so the
* current bvec has to be splitted as multiple segments.
*/
while (len && new_nsegs + *nsegs < queue_max_segments(q)) {
seg_size = get_max_segment_size(q, bv->bv_offset + total_len);
seg_size = min(seg_size, len);
new_nsegs++;
total_len += seg_size;
len -= seg_size;
if ((bv->bv_offset + total_len) & queue_virt_boundary(q))
break;
}
if (!new_nsegs)
return !!len;
/* update front segment size */
if (!*nsegs) {
unsigned first_seg_size;
if (new_nsegs == 1)
first_seg_size = get_max_segment_size(q, bv->bv_offset);
else
first_seg_size = queue_max_segment_size(q);
if (*front_seg_size < first_seg_size)
*front_seg_size = first_seg_size;
}
/* update other varibles */
*last_seg_size = seg_size;
*nsegs += new_nsegs;
if (sectors)
*sectors += total_len >> 9;
/* split in the middle of the bvec if len != 0 */
return !!len;
}
static struct bio *blk_bio_segment_split(struct request_queue *q, static struct bio *blk_bio_segment_split(struct request_queue *q,
struct bio *bio, struct bio *bio,
struct bio_set *bs, struct bio_set *bs,
...@@ -174,7 +241,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, ...@@ -174,7 +241,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
struct bio *new = NULL; struct bio *new = NULL;
const unsigned max_sectors = get_max_io_size(q, bio); const unsigned max_sectors = get_max_io_size(q, bio);
bio_for_each_segment(bv, bio, iter) { bio_for_each_bvec(bv, bio, iter) {
/* /*
* If the queue doesn't support SG gaps and adding this * If the queue doesn't support SG gaps and adding this
* offset would create a gap, disallow it. * offset would create a gap, disallow it.
...@@ -189,8 +256,12 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, ...@@ -189,8 +256,12 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
*/ */
if (nsegs < queue_max_segments(q) && if (nsegs < queue_max_segments(q) &&
sectors < max_sectors) { sectors < max_sectors) {
nsegs++; /* split in the middle of bvec */
sectors = max_sectors; bv.bv_len = (max_sectors - sectors) << 9;
bvec_split_segs(q, &bv, &nsegs,
&seg_size,
&front_seg_size,
&sectors);
} }
goto split; goto split;
} }
...@@ -212,14 +283,12 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, ...@@ -212,14 +283,12 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
if (nsegs == queue_max_segments(q)) if (nsegs == queue_max_segments(q))
goto split; goto split;
if (nsegs == 1 && seg_size > front_seg_size)
front_seg_size = seg_size;
nsegs++;
bvprv = bv; bvprv = bv;
bvprvp = &bvprv; bvprvp = &bvprv;
seg_size = bv.bv_len;
sectors += bv.bv_len >> 9; if (bvec_split_segs(q, &bv, &nsegs, &seg_size,
&front_seg_size, &sectors))
goto split;
} }
...@@ -233,8 +302,6 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, ...@@ -233,8 +302,6 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
bio = new; bio = new;
} }
if (nsegs == 1 && seg_size > front_seg_size)
front_seg_size = seg_size;
bio->bi_seg_front_size = front_seg_size; bio->bi_seg_front_size = front_seg_size;
if (seg_size > bio->bi_seg_back_size) if (seg_size > bio->bi_seg_back_size)
bio->bi_seg_back_size = seg_size; bio->bi_seg_back_size = seg_size;
...@@ -297,6 +364,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, ...@@ -297,6 +364,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
struct bio_vec bv, bvprv = { NULL }; struct bio_vec bv, bvprv = { NULL };
int prev = 0; int prev = 0;
unsigned int seg_size, nr_phys_segs; unsigned int seg_size, nr_phys_segs;
unsigned front_seg_size = bio->bi_seg_front_size;
struct bio *fbio, *bbio; struct bio *fbio, *bbio;
struct bvec_iter iter; struct bvec_iter iter;
...@@ -316,7 +384,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, ...@@ -316,7 +384,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
seg_size = 0; seg_size = 0;
nr_phys_segs = 0; nr_phys_segs = 0;
for_each_bio(bio) { for_each_bio(bio) {
bio_for_each_segment(bv, bio, iter) { bio_for_each_bvec(bv, bio, iter) {
/* /*
* If SG merging is disabled, each bio vector is * If SG merging is disabled, each bio vector is
* a segment * a segment
...@@ -336,20 +404,15 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, ...@@ -336,20 +404,15 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
continue; continue;
} }
new_segment: new_segment:
if (nr_phys_segs == 1 && seg_size >
fbio->bi_seg_front_size)
fbio->bi_seg_front_size = seg_size;
nr_phys_segs++;
bvprv = bv; bvprv = bv;
prev = 1; prev = 1;
seg_size = bv.bv_len; bvec_split_segs(q, &bv, &nr_phys_segs, &seg_size,
&front_seg_size, NULL);
} }
bbio = bio; bbio = bio;
} }
if (nr_phys_segs == 1 && seg_size > fbio->bi_seg_front_size) fbio->bi_seg_front_size = front_seg_size;
fbio->bi_seg_front_size = seg_size;
if (seg_size > bbio->bi_seg_back_size) if (seg_size > bbio->bi_seg_back_size)
bbio->bi_seg_back_size = seg_size; bbio->bi_seg_back_size = seg_size;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment