Commit 8aa6ba2f authored by Kent Overstreet's avatar Kent Overstreet Committed by Jens Axboe

block: Convert bio_set to mempool_init()

Minor performance improvement by getting rid of pointer indirections
from allocation/freeing fastpaths.
Signed-off-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c1a67fef
...@@ -56,12 +56,12 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, ...@@ -56,12 +56,12 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
struct bio_set *bs = bio->bi_pool; struct bio_set *bs = bio->bi_pool;
unsigned inline_vecs; unsigned inline_vecs;
if (!bs || !bs->bio_integrity_pool) { if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) {
bip = kmalloc(sizeof(struct bio_integrity_payload) + bip = kmalloc(sizeof(struct bio_integrity_payload) +
sizeof(struct bio_vec) * nr_vecs, gfp_mask); sizeof(struct bio_vec) * nr_vecs, gfp_mask);
inline_vecs = nr_vecs; inline_vecs = nr_vecs;
} else { } else {
bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask); bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
inline_vecs = BIP_INLINE_VECS; inline_vecs = BIP_INLINE_VECS;
} }
...@@ -74,7 +74,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, ...@@ -74,7 +74,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
unsigned long idx = 0; unsigned long idx = 0;
bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx, bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
bs->bvec_integrity_pool); &bs->bvec_integrity_pool);
if (!bip->bip_vec) if (!bip->bip_vec)
goto err; goto err;
bip->bip_max_vcnt = bvec_nr_vecs(idx); bip->bip_max_vcnt = bvec_nr_vecs(idx);
...@@ -90,7 +90,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, ...@@ -90,7 +90,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
return bip; return bip;
err: err:
mempool_free(bip, bs->bio_integrity_pool); mempool_free(bip, &bs->bio_integrity_pool);
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
} }
EXPORT_SYMBOL(bio_integrity_alloc); EXPORT_SYMBOL(bio_integrity_alloc);
...@@ -111,10 +111,10 @@ static void bio_integrity_free(struct bio *bio) ...@@ -111,10 +111,10 @@ static void bio_integrity_free(struct bio *bio)
kfree(page_address(bip->bip_vec->bv_page) + kfree(page_address(bip->bip_vec->bv_page) +
bip->bip_vec->bv_offset); bip->bip_vec->bv_offset);
if (bs && bs->bio_integrity_pool) { if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
bvec_free(bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab); bvec_free(&bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab);
mempool_free(bip, bs->bio_integrity_pool); mempool_free(bip, &bs->bio_integrity_pool);
} else { } else {
kfree(bip); kfree(bip);
} }
...@@ -465,16 +465,15 @@ EXPORT_SYMBOL(bio_integrity_clone); ...@@ -465,16 +465,15 @@ EXPORT_SYMBOL(bio_integrity_clone);
int bioset_integrity_create(struct bio_set *bs, int pool_size) int bioset_integrity_create(struct bio_set *bs, int pool_size)
{ {
if (bs->bio_integrity_pool) if (mempool_initialized(&bs->bio_integrity_pool))
return 0; return 0;
bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab); if (mempool_init_slab_pool(&bs->bio_integrity_pool,
if (!bs->bio_integrity_pool) pool_size, bip_slab))
return -1; return -1;
bs->bvec_integrity_pool = biovec_create_pool(pool_size); if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) {
if (!bs->bvec_integrity_pool) { mempool_exit(&bs->bio_integrity_pool);
mempool_destroy(bs->bio_integrity_pool);
return -1; return -1;
} }
...@@ -484,8 +483,8 @@ EXPORT_SYMBOL(bioset_integrity_create); ...@@ -484,8 +483,8 @@ EXPORT_SYMBOL(bioset_integrity_create);
void bioset_integrity_free(struct bio_set *bs) void bioset_integrity_free(struct bio_set *bs)
{ {
mempool_destroy(bs->bio_integrity_pool); mempool_exit(&bs->bio_integrity_pool);
mempool_destroy(bs->bvec_integrity_pool); mempool_exit(&bs->bvec_integrity_pool);
} }
EXPORT_SYMBOL(bioset_integrity_free); EXPORT_SYMBOL(bioset_integrity_free);
......
...@@ -254,7 +254,7 @@ static void bio_free(struct bio *bio) ...@@ -254,7 +254,7 @@ static void bio_free(struct bio *bio)
bio_uninit(bio); bio_uninit(bio);
if (bs) { if (bs) {
bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio)); bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
/* /*
* If we have front padding, adjust the bio pointer before freeing * If we have front padding, adjust the bio pointer before freeing
...@@ -262,7 +262,7 @@ static void bio_free(struct bio *bio) ...@@ -262,7 +262,7 @@ static void bio_free(struct bio *bio)
p = bio; p = bio;
p -= bs->front_pad; p -= bs->front_pad;
mempool_free(p, bs->bio_pool); mempool_free(p, &bs->bio_pool);
} else { } else {
/* Bio was allocated by bio_kmalloc() */ /* Bio was allocated by bio_kmalloc() */
kfree(bio); kfree(bio);
...@@ -454,7 +454,8 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, ...@@ -454,7 +454,8 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
inline_vecs = nr_iovecs; inline_vecs = nr_iovecs;
} else { } else {
/* should not use nobvec bioset for nr_iovecs > 0 */ /* should not use nobvec bioset for nr_iovecs > 0 */
if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0)) if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
nr_iovecs > 0))
return NULL; return NULL;
/* /*
* generic_make_request() converts recursion to iteration; this * generic_make_request() converts recursion to iteration; this
...@@ -483,11 +484,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, ...@@ -483,11 +484,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
bs->rescue_workqueue) bs->rescue_workqueue)
gfp_mask &= ~__GFP_DIRECT_RECLAIM; gfp_mask &= ~__GFP_DIRECT_RECLAIM;
p = mempool_alloc(bs->bio_pool, gfp_mask); p = mempool_alloc(&bs->bio_pool, gfp_mask);
if (!p && gfp_mask != saved_gfp) { if (!p && gfp_mask != saved_gfp) {
punt_bios_to_rescuer(bs); punt_bios_to_rescuer(bs);
gfp_mask = saved_gfp; gfp_mask = saved_gfp;
p = mempool_alloc(bs->bio_pool, gfp_mask); p = mempool_alloc(&bs->bio_pool, gfp_mask);
} }
front_pad = bs->front_pad; front_pad = bs->front_pad;
...@@ -503,11 +504,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, ...@@ -503,11 +504,11 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
if (nr_iovecs > inline_vecs) { if (nr_iovecs > inline_vecs) {
unsigned long idx = 0; unsigned long idx = 0;
bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool); bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
if (!bvl && gfp_mask != saved_gfp) { if (!bvl && gfp_mask != saved_gfp) {
punt_bios_to_rescuer(bs); punt_bios_to_rescuer(bs);
gfp_mask = saved_gfp; gfp_mask = saved_gfp;
bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool); bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
} }
if (unlikely(!bvl)) if (unlikely(!bvl))
...@@ -524,7 +525,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, ...@@ -524,7 +525,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
return bio; return bio;
err_free: err_free:
mempool_free(p, bs->bio_pool); mempool_free(p, &bs->bio_pool);
return NULL; return NULL;
} }
EXPORT_SYMBOL(bio_alloc_bioset); EXPORT_SYMBOL(bio_alloc_bioset);
...@@ -1848,11 +1849,11 @@ EXPORT_SYMBOL_GPL(bio_trim); ...@@ -1848,11 +1849,11 @@ EXPORT_SYMBOL_GPL(bio_trim);
* create memory pools for biovec's in a bio_set. * create memory pools for biovec's in a bio_set.
* use the global biovec slabs created for general use. * use the global biovec slabs created for general use.
*/ */
mempool_t *biovec_create_pool(int pool_entries) int biovec_init_pool(mempool_t *pool, int pool_entries)
{ {
struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX; struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
return mempool_create_slab_pool(pool_entries, bp->slab); return mempool_init_slab_pool(pool, pool_entries, bp->slab);
} }
void bioset_free(struct bio_set *bs) void bioset_free(struct bio_set *bs)
...@@ -1860,8 +1861,8 @@ void bioset_free(struct bio_set *bs) ...@@ -1860,8 +1861,8 @@ void bioset_free(struct bio_set *bs)
if (bs->rescue_workqueue) if (bs->rescue_workqueue)
destroy_workqueue(bs->rescue_workqueue); destroy_workqueue(bs->rescue_workqueue);
mempool_destroy(bs->bio_pool); mempool_exit(&bs->bio_pool);
mempool_destroy(bs->bvec_pool); mempool_exit(&bs->bvec_pool);
bioset_integrity_free(bs); bioset_integrity_free(bs);
bio_put_slab(bs); bio_put_slab(bs);
...@@ -1913,15 +1914,12 @@ struct bio_set *bioset_create(unsigned int pool_size, ...@@ -1913,15 +1914,12 @@ struct bio_set *bioset_create(unsigned int pool_size,
return NULL; return NULL;
} }
bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab); if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
if (!bs->bio_pool)
goto bad; goto bad;
if (flags & BIOSET_NEED_BVECS) { if ((flags & BIOSET_NEED_BVECS) &&
bs->bvec_pool = biovec_create_pool(pool_size); biovec_init_pool(&bs->bvec_pool, pool_size))
if (!bs->bvec_pool) goto bad;
goto bad;
}
if (!(flags & BIOSET_NEED_RESCUER)) if (!(flags & BIOSET_NEED_RESCUER))
return bs; return bs;
......
...@@ -412,7 +412,7 @@ enum { ...@@ -412,7 +412,7 @@ enum {
BIOSET_NEED_RESCUER = BIT(1), BIOSET_NEED_RESCUER = BIT(1),
}; };
extern void bioset_free(struct bio_set *); extern void bioset_free(struct bio_set *);
extern mempool_t *biovec_create_pool(int pool_entries); extern int biovec_init_pool(mempool_t *pool, int pool_entries);
extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *); extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
extern void bio_put(struct bio *); extern void bio_put(struct bio *);
...@@ -722,11 +722,11 @@ struct bio_set { ...@@ -722,11 +722,11 @@ struct bio_set {
struct kmem_cache *bio_slab; struct kmem_cache *bio_slab;
unsigned int front_pad; unsigned int front_pad;
mempool_t *bio_pool; mempool_t bio_pool;
mempool_t *bvec_pool; mempool_t bvec_pool;
#if defined(CONFIG_BLK_DEV_INTEGRITY) #if defined(CONFIG_BLK_DEV_INTEGRITY)
mempool_t *bio_integrity_pool; mempool_t bio_integrity_pool;
mempool_t *bvec_integrity_pool; mempool_t bvec_integrity_pool;
#endif #endif
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment