Commit 392ddc32 authored by Jens Axboe's avatar Jens Axboe

bio: add support for inlining a number of bio_vecs inside the bio

When we go and allocate a bio for IO, we actually do two allocations.
One for the bio itself, and one for the bi_io_vec that holds the
actual pages we are interested in.

This feature inlines a definable amount of io vecs inside the bio
itself, so we eliminate the bio_vec array allocation for IO's up
to a certain size. It defaults to 4 vecs, which is typically 16k
of IO.
Signed-off-by: default avatarJens Axboe <jens.axboe@oracle.com>
parent bb799ca0
...@@ -31,6 +31,12 @@ ...@@ -31,6 +31,12 @@
DEFINE_TRACE(block_split); DEFINE_TRACE(block_split);
/*
* Test patch to inline a certain number of bi_io_vec's inside the bio
* itself, to shrink a bio data allocation from two mempool calls to one
*/
#define BIO_INLINE_VECS 4
static mempool_t *bio_split_pool __read_mostly; static mempool_t *bio_split_pool __read_mostly;
/* /*
...@@ -241,7 +247,7 @@ void bio_free(struct bio *bio, struct bio_set *bs) ...@@ -241,7 +247,7 @@ void bio_free(struct bio *bio, struct bio_set *bs)
{ {
void *p; void *p;
if (bio->bi_io_vec) if (bio_has_allocated_vec(bio))
bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio)); bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));
if (bio_integrity(bio)) if (bio_integrity(bio))
...@@ -267,6 +273,7 @@ static void bio_fs_destructor(struct bio *bio) ...@@ -267,6 +273,7 @@ static void bio_fs_destructor(struct bio *bio)
static void bio_kmalloc_destructor(struct bio *bio) static void bio_kmalloc_destructor(struct bio *bio)
{ {
if (bio_has_allocated_vec(bio))
kfree(bio->bi_io_vec); kfree(bio->bi_io_vec);
kfree(bio); kfree(bio);
} }
...@@ -314,7 +321,16 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) ...@@ -314,7 +321,16 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
if (likely(nr_iovecs)) { if (likely(nr_iovecs)) {
unsigned long uninitialized_var(idx); unsigned long uninitialized_var(idx);
bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs); if (nr_iovecs <= BIO_INLINE_VECS) {
idx = 0;
bvl = bio->bi_inline_vecs;
nr_iovecs = BIO_INLINE_VECS;
memset(bvl, 0, BIO_INLINE_VECS * sizeof(*bvl));
} else {
bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx,
bs);
nr_iovecs = bvec_nr_vecs(idx);
}
if (unlikely(!bvl)) { if (unlikely(!bvl)) {
if (bs) if (bs)
mempool_free(bio, bs->bio_pool); mempool_free(bio, bs->bio_pool);
...@@ -324,7 +340,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) ...@@ -324,7 +340,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
goto out; goto out;
} }
bio->bi_flags |= idx << BIO_POOL_OFFSET; bio->bi_flags |= idx << BIO_POOL_OFFSET;
bio->bi_max_vecs = bvec_nr_vecs(idx); bio->bi_max_vecs = nr_iovecs;
} }
bio->bi_io_vec = bvl; bio->bi_io_vec = bvl;
} }
...@@ -1525,6 +1541,7 @@ void bioset_free(struct bio_set *bs) ...@@ -1525,6 +1541,7 @@ void bioset_free(struct bio_set *bs)
*/ */
struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
{ {
unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
struct bio_set *bs; struct bio_set *bs;
bs = kzalloc(sizeof(*bs), GFP_KERNEL); bs = kzalloc(sizeof(*bs), GFP_KERNEL);
...@@ -1533,7 +1550,7 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) ...@@ -1533,7 +1550,7 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
bs->front_pad = front_pad; bs->front_pad = front_pad;
bs->bio_slab = bio_find_or_create_slab(front_pad); bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
if (!bs->bio_slab) { if (!bs->bio_slab) {
kfree(bs); kfree(bs);
return NULL; return NULL;
......
...@@ -102,6 +102,13 @@ struct bio { ...@@ -102,6 +102,13 @@ struct bio {
#endif #endif
bio_destructor_t *bi_destructor; /* destructor */ bio_destructor_t *bi_destructor; /* destructor */
/*
* We can inline a number of vecs at the end of the bio, to avoid
* double allocations for a small number of bio_vecs. This member
* MUST obviously be kept at the very end of the bio.
*/
struct bio_vec bi_inline_vecs[0];
}; };
/* /*
...@@ -213,6 +220,11 @@ static inline void *bio_data(struct bio *bio) ...@@ -213,6 +220,11 @@ static inline void *bio_data(struct bio *bio)
return NULL; return NULL;
} }
static inline int bio_has_allocated_vec(struct bio *bio)
{
return bio->bi_io_vec && bio->bi_io_vec != bio->bi_inline_vecs;
}
/* /*
* will die * will die
*/ */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment