Commit 52d52d1c authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe

block: only allow contiguous page structs in a bio_vec

We currently have to call nth_page when iterating over pages inside a
bio_vec.  Jens complained a while ago that this is fairly expensive.
To mitigate this we can check that that the actual page structures
are contiguous when adding them to the bio, and just do check pointer
arithmetics later on.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 7321ecbf
...@@ -659,8 +659,13 @@ static inline bool page_is_mergeable(const struct bio_vec *bv, ...@@ -659,8 +659,13 @@ static inline bool page_is_mergeable(const struct bio_vec *bv,
return false; return false;
if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) if (xen_domain() && !xen_biovec_phys_mergeable(bv, page))
return false; return false;
if (same_page && (vec_end_addr & PAGE_MASK) != page_addr)
return false; if ((vec_end_addr & PAGE_MASK) != page_addr) {
if (same_page)
return false;
if (pfn_to_page(PFN_DOWN(vec_end_addr)) + 1 != page)
return false;
}
return true; return true;
} }
......
...@@ -51,11 +51,6 @@ struct bvec_iter_all { ...@@ -51,11 +51,6 @@ struct bvec_iter_all {
unsigned done; unsigned done;
}; };
static inline struct page *bvec_nth_page(struct page *page, int idx)
{
return idx == 0 ? page : nth_page(page, idx);
}
/* /*
* various member access, note that bio_data should of course not be used * various member access, note that bio_data should of course not be used
* on highmem page vectors * on highmem page vectors
...@@ -92,8 +87,8 @@ static inline struct page *bvec_nth_page(struct page *page, int idx) ...@@ -92,8 +87,8 @@ static inline struct page *bvec_nth_page(struct page *page, int idx)
PAGE_SIZE - bvec_iter_offset((bvec), (iter))) PAGE_SIZE - bvec_iter_offset((bvec), (iter)))
#define bvec_iter_page(bvec, iter) \ #define bvec_iter_page(bvec, iter) \
bvec_nth_page(mp_bvec_iter_page((bvec), (iter)), \ (mp_bvec_iter_page((bvec), (iter)) + \
mp_bvec_iter_page_idx((bvec), (iter))) mp_bvec_iter_page_idx((bvec), (iter)))
#define bvec_iter_bvec(bvec, iter) \ #define bvec_iter_bvec(bvec, iter) \
((struct bio_vec) { \ ((struct bio_vec) { \
...@@ -157,7 +152,7 @@ static inline void mp_bvec_next_segment(const struct bio_vec *bvec, ...@@ -157,7 +152,7 @@ static inline void mp_bvec_next_segment(const struct bio_vec *bvec,
struct bio_vec *bv = &iter_all->bv; struct bio_vec *bv = &iter_all->bv;
if (bv->bv_page) { if (bv->bv_page) {
bv->bv_page = nth_page(bv->bv_page, 1); bv->bv_page++;
bv->bv_offset = 0; bv->bv_offset = 0;
} else { } else {
bv->bv_page = bvec->bv_page; bv->bv_page = bvec->bv_page;
...@@ -177,7 +172,7 @@ static inline void mp_bvec_last_segment(const struct bio_vec *bvec, ...@@ -177,7 +172,7 @@ static inline void mp_bvec_last_segment(const struct bio_vec *bvec,
unsigned total = bvec->bv_offset + bvec->bv_len; unsigned total = bvec->bv_offset + bvec->bv_len;
unsigned last_page = (total - 1) / PAGE_SIZE; unsigned last_page = (total - 1) / PAGE_SIZE;
seg->bv_page = bvec_nth_page(bvec->bv_page, last_page); seg->bv_page = bvec->bv_page + last_page;
/* the whole segment is inside the last page */ /* the whole segment is inside the last page */
if (bvec->bv_offset >= last_page * PAGE_SIZE) { if (bvec->bv_offset >= last_page * PAGE_SIZE) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment