Commit eb8db831 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe

dm: always defer request allocation to the owner of the request_queue

DM already calls blk_mq_alloc_request on the request_queue of the
underlying device if it is a blk-mq device.  But now that we allow drivers
to allocate additional data and initialize it ahead of time we need to do
the same for all drivers.   Doing so and using the new cmd_size
infrastructure in the block layer greatly simplifies the dm-rq and mpath
code, and should also make arbitrary combinations of SQ and MQ devices
with SQ or MQ device mapper tables easily possible as a further step.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarHannes Reinecke <hare@suse.com>
Reviewed-by: default avatarMike Snitzer <snitzer@redhat.com>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent 4bf58435
...@@ -92,7 +92,6 @@ struct mapped_device { ...@@ -92,7 +92,6 @@ struct mapped_device {
* io objects are allocated from here. * io objects are allocated from here.
*/ */
mempool_t *io_pool; mempool_t *io_pool;
mempool_t *rq_pool;
struct bio_set *bs; struct bio_set *bs;
......
...@@ -92,12 +92,6 @@ struct multipath { ...@@ -92,12 +92,6 @@ struct multipath {
unsigned queue_mode; unsigned queue_mode;
/*
* We must use a mempool of dm_mpath_io structs so that we
* can resubmit bios on error.
*/
mempool_t *mpio_pool;
struct mutex work_mutex; struct mutex work_mutex;
struct work_struct trigger_event; struct work_struct trigger_event;
...@@ -115,8 +109,6 @@ struct dm_mpath_io { ...@@ -115,8 +109,6 @@ struct dm_mpath_io {
typedef int (*action_fn) (struct pgpath *pgpath); typedef int (*action_fn) (struct pgpath *pgpath);
static struct kmem_cache *_mpio_cache;
static struct workqueue_struct *kmultipathd, *kmpath_handlerd; static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
static void trigger_event(struct work_struct *work); static void trigger_event(struct work_struct *work);
static void activate_path(struct work_struct *work); static void activate_path(struct work_struct *work);
...@@ -209,7 +201,6 @@ static struct multipath *alloc_multipath(struct dm_target *ti) ...@@ -209,7 +201,6 @@ static struct multipath *alloc_multipath(struct dm_target *ti)
init_waitqueue_head(&m->pg_init_wait); init_waitqueue_head(&m->pg_init_wait);
mutex_init(&m->work_mutex); mutex_init(&m->work_mutex);
m->mpio_pool = NULL;
m->queue_mode = DM_TYPE_NONE; m->queue_mode = DM_TYPE_NONE;
m->ti = ti; m->ti = ti;
...@@ -229,16 +220,7 @@ static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) ...@@ -229,16 +220,7 @@ static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
else else
m->queue_mode = DM_TYPE_REQUEST_BASED; m->queue_mode = DM_TYPE_REQUEST_BASED;
} } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
if (m->queue_mode == DM_TYPE_REQUEST_BASED) {
unsigned min_ios = dm_get_reserved_rq_based_ios();
m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache);
if (!m->mpio_pool)
return -ENOMEM;
}
else if (m->queue_mode == DM_TYPE_BIO_BASED) {
INIT_WORK(&m->process_queued_bios, process_queued_bios); INIT_WORK(&m->process_queued_bios, process_queued_bios);
/* /*
* bio-based doesn't support any direct scsi_dh management; * bio-based doesn't support any direct scsi_dh management;
...@@ -263,7 +245,6 @@ static void free_multipath(struct multipath *m) ...@@ -263,7 +245,6 @@ static void free_multipath(struct multipath *m)
kfree(m->hw_handler_name); kfree(m->hw_handler_name);
kfree(m->hw_handler_params); kfree(m->hw_handler_params);
mempool_destroy(m->mpio_pool);
kfree(m); kfree(m);
} }
...@@ -272,38 +253,6 @@ static struct dm_mpath_io *get_mpio(union map_info *info) ...@@ -272,38 +253,6 @@ static struct dm_mpath_io *get_mpio(union map_info *info)
return info->ptr; return info->ptr;
} }
static struct dm_mpath_io *set_mpio(struct multipath *m, union map_info *info)
{
struct dm_mpath_io *mpio;
if (!m->mpio_pool) {
/* Use blk-mq pdu memory requested via per_io_data_size */
mpio = get_mpio(info);
memset(mpio, 0, sizeof(*mpio));
return mpio;
}
mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
if (!mpio)
return NULL;
memset(mpio, 0, sizeof(*mpio));
info->ptr = mpio;
return mpio;
}
static void clear_request_fn_mpio(struct multipath *m, union map_info *info)
{
/* Only needed for non blk-mq (.request_fn) multipath */
if (m->mpio_pool) {
struct dm_mpath_io *mpio = info->ptr;
info->ptr = NULL;
mempool_free(mpio, m->mpio_pool);
}
}
static size_t multipath_per_bio_data_size(void) static size_t multipath_per_bio_data_size(void)
{ {
return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details); return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
...@@ -530,16 +479,17 @@ static bool must_push_back_bio(struct multipath *m) ...@@ -530,16 +479,17 @@ static bool must_push_back_bio(struct multipath *m)
/* /*
* Map cloned requests (request-based multipath) * Map cloned requests (request-based multipath)
*/ */
static int __multipath_map(struct dm_target *ti, struct request *clone, static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
union map_info *map_context, union map_info *map_context,
struct request *rq, struct request **__clone) struct request **__clone)
{ {
struct multipath *m = ti->private; struct multipath *m = ti->private;
int r = DM_MAPIO_REQUEUE; int r = DM_MAPIO_REQUEUE;
size_t nr_bytes = clone ? blk_rq_bytes(clone) : blk_rq_bytes(rq); size_t nr_bytes = blk_rq_bytes(rq);
struct pgpath *pgpath; struct pgpath *pgpath;
struct block_device *bdev; struct block_device *bdev;
struct dm_mpath_io *mpio; struct dm_mpath_io *mpio = get_mpio(map_context);
struct request *clone;
/* Do we need to select a new pgpath? */ /* Do we need to select a new pgpath? */
pgpath = lockless_dereference(m->current_pgpath); pgpath = lockless_dereference(m->current_pgpath);
...@@ -556,42 +506,23 @@ static int __multipath_map(struct dm_target *ti, struct request *clone, ...@@ -556,42 +506,23 @@ static int __multipath_map(struct dm_target *ti, struct request *clone,
return r; return r;
} }
mpio = set_mpio(m, map_context); memset(mpio, 0, sizeof(*mpio));
if (!mpio)
/* ENOMEM, requeue */
return r;
mpio->pgpath = pgpath; mpio->pgpath = pgpath;
mpio->nr_bytes = nr_bytes; mpio->nr_bytes = nr_bytes;
bdev = pgpath->path.dev->bdev; bdev = pgpath->path.dev->bdev;
if (clone) { clone = blk_get_request(bdev_get_queue(bdev),
/* rq->cmd_flags | REQ_NOMERGE,
* Old request-based interface: allocated clone is passed in. GFP_ATOMIC);
* Used by: .request_fn stacked on .request_fn path(s). if (IS_ERR(clone)) {
*/ /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
clone->q = bdev_get_queue(bdev); return r;
clone->rq_disk = bdev->bd_disk;
clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
} else {
/*
* blk-mq request-based interface; used by both:
* .request_fn stacked on blk-mq path(s) and
* blk-mq stacked on blk-mq path(s).
*/
clone = blk_mq_alloc_request(bdev_get_queue(bdev),
rq_data_dir(rq), BLK_MQ_REQ_NOWAIT);
if (IS_ERR(clone)) {
/* EBUSY, ENODEV or EWOULDBLOCK: requeue */
clear_request_fn_mpio(m, map_context);
return r;
}
clone->bio = clone->biotail = NULL;
clone->rq_disk = bdev->bd_disk;
clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
*__clone = clone;
} }
clone->bio = clone->biotail = NULL;
clone->rq_disk = bdev->bd_disk;
clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
*__clone = clone;
if (pgpath->pg->ps.type->start_io) if (pgpath->pg->ps.type->start_io)
pgpath->pg->ps.type->start_io(&pgpath->pg->ps, pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
...@@ -600,22 +531,9 @@ static int __multipath_map(struct dm_target *ti, struct request *clone, ...@@ -600,22 +531,9 @@ static int __multipath_map(struct dm_target *ti, struct request *clone,
return DM_MAPIO_REMAPPED; return DM_MAPIO_REMAPPED;
} }
static int multipath_map(struct dm_target *ti, struct request *clone,
union map_info *map_context)
{
return __multipath_map(ti, clone, map_context, NULL, NULL);
}
static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
union map_info *map_context,
struct request **clone)
{
return __multipath_map(ti, NULL, map_context, rq, clone);
}
static void multipath_release_clone(struct request *clone) static void multipath_release_clone(struct request *clone)
{ {
blk_mq_free_request(clone); blk_put_request(clone);
} }
/* /*
...@@ -1187,7 +1105,7 @@ static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv) ...@@ -1187,7 +1105,7 @@ static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
ti->num_write_same_bios = 1; ti->num_write_same_bios = 1;
if (m->queue_mode == DM_TYPE_BIO_BASED) if (m->queue_mode == DM_TYPE_BIO_BASED)
ti->per_io_data_size = multipath_per_bio_data_size(); ti->per_io_data_size = multipath_per_bio_data_size();
else if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED) else
ti->per_io_data_size = sizeof(struct dm_mpath_io); ti->per_io_data_size = sizeof(struct dm_mpath_io);
return 0; return 0;
...@@ -1610,7 +1528,6 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone, ...@@ -1610,7 +1528,6 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone,
if (ps->type->end_io) if (ps->type->end_io)
ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
} }
clear_request_fn_mpio(m, map_context);
return r; return r;
} }
...@@ -2060,7 +1977,6 @@ static struct target_type multipath_target = { ...@@ -2060,7 +1977,6 @@ static struct target_type multipath_target = {
.module = THIS_MODULE, .module = THIS_MODULE,
.ctr = multipath_ctr, .ctr = multipath_ctr,
.dtr = multipath_dtr, .dtr = multipath_dtr,
.map_rq = multipath_map,
.clone_and_map_rq = multipath_clone_and_map, .clone_and_map_rq = multipath_clone_and_map,
.release_clone_rq = multipath_release_clone, .release_clone_rq = multipath_release_clone,
.rq_end_io = multipath_end_io, .rq_end_io = multipath_end_io,
...@@ -2080,11 +1996,6 @@ static int __init dm_multipath_init(void) ...@@ -2080,11 +1996,6 @@ static int __init dm_multipath_init(void)
{ {
int r; int r;
/* allocate a slab for the dm_mpath_ios */
_mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
if (!_mpio_cache)
return -ENOMEM;
r = dm_register_target(&multipath_target); r = dm_register_target(&multipath_target);
if (r < 0) { if (r < 0) {
DMERR("request-based register failed %d", r); DMERR("request-based register failed %d", r);
...@@ -2120,8 +2031,6 @@ static int __init dm_multipath_init(void) ...@@ -2120,8 +2031,6 @@ static int __init dm_multipath_init(void)
bad_alloc_kmultipathd: bad_alloc_kmultipathd:
dm_unregister_target(&multipath_target); dm_unregister_target(&multipath_target);
bad_register_target: bad_register_target:
kmem_cache_destroy(_mpio_cache);
return r; return r;
} }
...@@ -2131,7 +2040,6 @@ static void __exit dm_multipath_exit(void) ...@@ -2131,7 +2040,6 @@ static void __exit dm_multipath_exit(void)
destroy_workqueue(kmultipathd); destroy_workqueue(kmultipathd);
dm_unregister_target(&multipath_target); dm_unregister_target(&multipath_target);
kmem_cache_destroy(_mpio_cache);
} }
module_init(dm_multipath_init); module_init(dm_multipath_init);
......
...@@ -109,28 +109,6 @@ void dm_stop_queue(struct request_queue *q) ...@@ -109,28 +109,6 @@ void dm_stop_queue(struct request_queue *q)
dm_mq_stop_queue(q); dm_mq_stop_queue(q);
} }
static struct dm_rq_target_io *alloc_old_rq_tio(struct mapped_device *md,
gfp_t gfp_mask)
{
return mempool_alloc(md->io_pool, gfp_mask);
}
static void free_old_rq_tio(struct dm_rq_target_io *tio)
{
mempool_free(tio, tio->md->io_pool);
}
static struct request *alloc_old_clone_request(struct mapped_device *md,
gfp_t gfp_mask)
{
return mempool_alloc(md->rq_pool, gfp_mask);
}
static void free_old_clone_request(struct mapped_device *md, struct request *rq)
{
mempool_free(rq, md->rq_pool);
}
/* /*
* Partial completion handling for request-based dm * Partial completion handling for request-based dm
*/ */
...@@ -185,7 +163,7 @@ static void end_clone_bio(struct bio *clone) ...@@ -185,7 +163,7 @@ static void end_clone_bio(struct bio *clone)
static struct dm_rq_target_io *tio_from_request(struct request *rq) static struct dm_rq_target_io *tio_from_request(struct request *rq)
{ {
return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special); return blk_mq_rq_to_pdu(rq);
} }
static void rq_end_stats(struct mapped_device *md, struct request *orig) static void rq_end_stats(struct mapped_device *md, struct request *orig)
...@@ -233,31 +211,6 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue) ...@@ -233,31 +211,6 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
dm_put(md); dm_put(md);
} }
static void free_rq_clone(struct request *clone)
{
struct dm_rq_target_io *tio = clone->end_io_data;
struct mapped_device *md = tio->md;
blk_rq_unprep_clone(clone);
/*
* It is possible for a clone_old_rq() allocated clone to
* get passed in -- it may not yet have a request_queue.
* This is known to occur if the error target replaces
* a multipath target that has a request_fn queue stacked
* on blk-mq queue(s).
*/
if (clone->q && clone->q->mq_ops)
/* stacked on blk-mq queue(s) */
tio->ti->type->release_clone_rq(clone);
else if (!md->queue->mq_ops)
/* request_fn queue stacked on request_fn queue(s) */
free_old_clone_request(md, clone);
if (!md->queue->mq_ops)
free_old_rq_tio(tio);
}
/* /*
* Complete the clone and the original request. * Complete the clone and the original request.
* Must be called without clone's queue lock held, * Must be called without clone's queue lock held,
...@@ -270,7 +223,9 @@ static void dm_end_request(struct request *clone, int error) ...@@ -270,7 +223,9 @@ static void dm_end_request(struct request *clone, int error)
struct mapped_device *md = tio->md; struct mapped_device *md = tio->md;
struct request *rq = tio->orig; struct request *rq = tio->orig;
free_rq_clone(clone); blk_rq_unprep_clone(clone);
tio->ti->type->release_clone_rq(clone);
rq_end_stats(md, rq); rq_end_stats(md, rq);
if (!rq->q->mq_ops) if (!rq->q->mq_ops)
blk_end_request_all(rq, error); blk_end_request_all(rq, error);
...@@ -279,22 +234,6 @@ static void dm_end_request(struct request *clone, int error) ...@@ -279,22 +234,6 @@ static void dm_end_request(struct request *clone, int error)
rq_completed(md, rw, true); rq_completed(md, rw, true);
} }
static void dm_unprep_request(struct request *rq)
{
struct dm_rq_target_io *tio = tio_from_request(rq);
struct request *clone = tio->clone;
if (!rq->q->mq_ops) {
rq->special = NULL;
rq->rq_flags &= ~RQF_DONTPREP;
}
if (clone)
free_rq_clone(clone);
else if (!tio->md->queue->mq_ops)
free_old_rq_tio(tio);
}
/* /*
* Requeue the original request of a clone. * Requeue the original request of a clone.
*/ */
...@@ -333,7 +272,10 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_ ...@@ -333,7 +272,10 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_
int rw = rq_data_dir(rq); int rw = rq_data_dir(rq);
rq_end_stats(md, rq); rq_end_stats(md, rq);
dm_unprep_request(rq); if (tio->clone) {
blk_rq_unprep_clone(tio->clone);
tio->ti->type->release_clone_rq(tio->clone);
}
if (!rq->q->mq_ops) if (!rq->q->mq_ops)
dm_old_requeue_request(rq); dm_old_requeue_request(rq);
...@@ -388,14 +330,11 @@ static void dm_softirq_done(struct request *rq) ...@@ -388,14 +330,11 @@ static void dm_softirq_done(struct request *rq)
if (!clone) { if (!clone) {
rq_end_stats(tio->md, rq); rq_end_stats(tio->md, rq);
rw = rq_data_dir(rq); rw = rq_data_dir(rq);
if (!rq->q->mq_ops) { if (!rq->q->mq_ops)
blk_end_request_all(rq, tio->error); blk_end_request_all(rq, tio->error);
rq_completed(tio->md, rw, false); else
free_old_rq_tio(tio);
} else {
blk_mq_end_request(rq, tio->error); blk_mq_end_request(rq, tio->error);
rq_completed(tio->md, rw, false); rq_completed(tio->md, rw, false);
}
return; return;
} }
...@@ -439,16 +378,6 @@ static void end_clone_request(struct request *clone, int error) ...@@ -439,16 +378,6 @@ static void end_clone_request(struct request *clone, int error)
{ {
struct dm_rq_target_io *tio = clone->end_io_data; struct dm_rq_target_io *tio = clone->end_io_data;
if (!clone->q->mq_ops) {
/*
* For just cleaning up the information of the queue in which
* the clone was dispatched.
* The clone is *NOT* freed actually here because it is alloced
* from dm own mempool (RQF_ALLOCED isn't set).
*/
__blk_put_request(clone->q, clone);
}
/* /*
* Actual request completion is done in a softirq context which doesn't * Actual request completion is done in a softirq context which doesn't
* hold the clone's queue lock. Otherwise, deadlock could occur because: * hold the clone's queue lock. Otherwise, deadlock could occur because:
...@@ -506,28 +435,6 @@ static int setup_clone(struct request *clone, struct request *rq, ...@@ -506,28 +435,6 @@ static int setup_clone(struct request *clone, struct request *rq,
return 0; return 0;
} }
static struct request *clone_old_rq(struct request *rq, struct mapped_device *md,
struct dm_rq_target_io *tio, gfp_t gfp_mask)
{
/*
* Create clone for use with .request_fn request_queue
*/
struct request *clone;
clone = alloc_old_clone_request(md, gfp_mask);
if (!clone)
return NULL;
blk_rq_init(NULL, clone);
if (setup_clone(clone, rq, tio, gfp_mask)) {
/* -ENOMEM */
free_old_clone_request(md, clone);
return NULL;
}
return clone;
}
static void map_tio_request(struct kthread_work *work); static void map_tio_request(struct kthread_work *work);
static void init_tio(struct dm_rq_target_io *tio, struct request *rq, static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
...@@ -549,60 +456,6 @@ static void init_tio(struct dm_rq_target_io *tio, struct request *rq, ...@@ -549,60 +456,6 @@ static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
kthread_init_work(&tio->work, map_tio_request); kthread_init_work(&tio->work, map_tio_request);
} }
static struct dm_rq_target_io *dm_old_prep_tio(struct request *rq,
struct mapped_device *md,
gfp_t gfp_mask)
{
struct dm_rq_target_io *tio;
int srcu_idx;
struct dm_table *table;
tio = alloc_old_rq_tio(md, gfp_mask);
if (!tio)
return NULL;
init_tio(tio, rq, md);
table = dm_get_live_table(md, &srcu_idx);
/*
* Must clone a request if this .request_fn DM device
* is stacked on .request_fn device(s).
*/
if (!dm_table_all_blk_mq_devices(table)) {
if (!clone_old_rq(rq, md, tio, gfp_mask)) {
dm_put_live_table(md, srcu_idx);
free_old_rq_tio(tio);
return NULL;
}
}
dm_put_live_table(md, srcu_idx);
return tio;
}
/*
* Called with the queue lock held.
*/
static int dm_old_prep_fn(struct request_queue *q, struct request *rq)
{
struct mapped_device *md = q->queuedata;
struct dm_rq_target_io *tio;
if (unlikely(rq->special)) {
DMWARN("Already has something in rq->special.");
return BLKPREP_KILL;
}
tio = dm_old_prep_tio(rq, md, GFP_ATOMIC);
if (!tio)
return BLKPREP_DEFER;
rq->special = tio;
rq->rq_flags |= RQF_DONTPREP;
return BLKPREP_OK;
}
/* /*
* Returns: * Returns:
* DM_MAPIO_* : the request has been processed as indicated * DM_MAPIO_* : the request has been processed as indicated
...@@ -617,31 +470,18 @@ static int map_request(struct dm_rq_target_io *tio) ...@@ -617,31 +470,18 @@ static int map_request(struct dm_rq_target_io *tio)
struct request *rq = tio->orig; struct request *rq = tio->orig;
struct request *clone = NULL; struct request *clone = NULL;
if (tio->clone) { r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);
clone = tio->clone;
r = ti->type->map_rq(ti, clone, &tio->info);
if (r == DM_MAPIO_DELAY_REQUEUE)
return DM_MAPIO_REQUEUE; /* .request_fn requeue is always immediate */
} else {
r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);
if (r < 0) {
/* The target wants to complete the I/O */
dm_kill_unmapped_request(rq, r);
return r;
}
if (r == DM_MAPIO_REMAPPED &&
setup_clone(clone, rq, tio, GFP_ATOMIC)) {
/* -ENOMEM */
ti->type->release_clone_rq(clone);
return DM_MAPIO_REQUEUE;
}
}
switch (r) { switch (r) {
case DM_MAPIO_SUBMITTED: case DM_MAPIO_SUBMITTED:
/* The target has taken the I/O to submit by itself later */ /* The target has taken the I/O to submit by itself later */
break; break;
case DM_MAPIO_REMAPPED: case DM_MAPIO_REMAPPED:
if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {
/* -ENOMEM */
ti->type->release_clone_rq(clone);
return DM_MAPIO_REQUEUE;
}
/* The target has remapped the I/O so dispatch it */ /* The target has remapped the I/O so dispatch it */
trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
blk_rq_pos(rq)); blk_rq_pos(rq));
...@@ -700,6 +540,29 @@ static void dm_start_request(struct mapped_device *md, struct request *orig) ...@@ -700,6 +540,29 @@ static void dm_start_request(struct mapped_device *md, struct request *orig)
dm_get(md); dm_get(md);
} }
static int __dm_rq_init_rq(struct mapped_device *md, struct request *rq)
{
struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);
/*
* Must initialize md member of tio, otherwise it won't
* be available in dm_mq_queue_rq.
*/
tio->md = md;
if (md->init_tio_pdu) {
/* target-specific per-io data is immediately after the tio */
tio->info.ptr = tio + 1;
}
return 0;
}
static int dm_rq_init_rq(struct request_queue *q, struct request *rq, gfp_t gfp)
{
return __dm_rq_init_rq(q->rq_alloc_data, rq);
}
static void map_tio_request(struct kthread_work *work) static void map_tio_request(struct kthread_work *work)
{ {
struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work);
...@@ -794,6 +657,7 @@ static void dm_old_request_fn(struct request_queue *q) ...@@ -794,6 +657,7 @@ static void dm_old_request_fn(struct request_queue *q)
dm_start_request(md, rq); dm_start_request(md, rq);
tio = tio_from_request(rq); tio = tio_from_request(rq);
init_tio(tio, rq, md);
/* Establish tio->ti before queuing work (map_tio_request) */ /* Establish tio->ti before queuing work (map_tio_request) */
tio->ti = ti; tio->ti = ti;
kthread_queue_work(&md->kworker, &tio->work); kthread_queue_work(&md->kworker, &tio->work);
...@@ -804,10 +668,22 @@ static void dm_old_request_fn(struct request_queue *q) ...@@ -804,10 +668,22 @@ static void dm_old_request_fn(struct request_queue *q)
/* /*
* Fully initialize a .request_fn request-based queue. * Fully initialize a .request_fn request-based queue.
*/ */
int dm_old_init_request_queue(struct mapped_device *md) int dm_old_init_request_queue(struct mapped_device *md, struct dm_table *t)
{ {
struct dm_target *immutable_tgt;
/* Fully initialize the queue */ /* Fully initialize the queue */
md->queue->cmd_size = sizeof(struct dm_rq_target_io);
md->queue->rq_alloc_data = md;
md->queue->request_fn = dm_old_request_fn; md->queue->request_fn = dm_old_request_fn;
md->queue->init_rq_fn = dm_rq_init_rq;
immutable_tgt = dm_table_get_immutable_target(t);
if (immutable_tgt && immutable_tgt->per_io_data_size) {
/* any target-specific per-io data is immediately after the tio */
md->queue->cmd_size += immutable_tgt->per_io_data_size;
md->init_tio_pdu = true;
}
if (blk_init_allocated_queue(md->queue) < 0) if (blk_init_allocated_queue(md->queue) < 0)
return -EINVAL; return -EINVAL;
...@@ -816,7 +692,6 @@ int dm_old_init_request_queue(struct mapped_device *md) ...@@ -816,7 +692,6 @@ int dm_old_init_request_queue(struct mapped_device *md)
dm_init_normal_md_queue(md); dm_init_normal_md_queue(md);
blk_queue_softirq_done(md->queue, dm_softirq_done); blk_queue_softirq_done(md->queue, dm_softirq_done);
blk_queue_prep_rq(md->queue, dm_old_prep_fn);
/* Initialize the request-based DM worker thread */ /* Initialize the request-based DM worker thread */
kthread_init_worker(&md->kworker); kthread_init_worker(&md->kworker);
...@@ -837,21 +712,7 @@ static int dm_mq_init_request(void *data, struct request *rq, ...@@ -837,21 +712,7 @@ static int dm_mq_init_request(void *data, struct request *rq,
unsigned int hctx_idx, unsigned int request_idx, unsigned int hctx_idx, unsigned int request_idx,
unsigned int numa_node) unsigned int numa_node)
{ {
struct mapped_device *md = data; return __dm_rq_init_rq(data, rq);
struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);
/*
* Must initialize md member of tio, otherwise it won't
* be available in dm_mq_queue_rq.
*/
tio->md = md;
if (md->init_tio_pdu) {
/* target-specific per-io data is immediately after the tio */
tio->info.ptr = tio + 1;
}
return 0;
} }
static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
......
...@@ -48,7 +48,7 @@ struct dm_rq_clone_bio_info { ...@@ -48,7 +48,7 @@ struct dm_rq_clone_bio_info {
bool dm_use_blk_mq_default(void); bool dm_use_blk_mq_default(void);
bool dm_use_blk_mq(struct mapped_device *md); bool dm_use_blk_mq(struct mapped_device *md);
int dm_old_init_request_queue(struct mapped_device *md); int dm_old_init_request_queue(struct mapped_device *md, struct dm_table *t);
int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t); int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t);
void dm_mq_cleanup_mapped_device(struct mapped_device *md); void dm_mq_cleanup_mapped_device(struct mapped_device *md);
......
...@@ -131,12 +131,6 @@ static int io_err_map(struct dm_target *tt, struct bio *bio) ...@@ -131,12 +131,6 @@ static int io_err_map(struct dm_target *tt, struct bio *bio)
return -EIO; return -EIO;
} }
static int io_err_map_rq(struct dm_target *ti, struct request *clone,
union map_info *map_context)
{
return -EIO;
}
static int io_err_clone_and_map_rq(struct dm_target *ti, struct request *rq, static int io_err_clone_and_map_rq(struct dm_target *ti, struct request *rq,
union map_info *map_context, union map_info *map_context,
struct request **clone) struct request **clone)
...@@ -161,7 +155,6 @@ static struct target_type error_target = { ...@@ -161,7 +155,6 @@ static struct target_type error_target = {
.ctr = io_err_ctr, .ctr = io_err_ctr,
.dtr = io_err_dtr, .dtr = io_err_dtr,
.map = io_err_map, .map = io_err_map,
.map_rq = io_err_map_rq,
.clone_and_map_rq = io_err_clone_and_map_rq, .clone_and_map_rq = io_err_clone_and_map_rq,
.release_clone_rq = io_err_release_clone_rq, .release_clone_rq = io_err_release_clone_rq,
.direct_access = io_err_direct_access, .direct_access = io_err_direct_access,
......
...@@ -91,7 +91,6 @@ static int dm_numa_node = DM_NUMA_NODE; ...@@ -91,7 +91,6 @@ static int dm_numa_node = DM_NUMA_NODE;
*/ */
struct dm_md_mempools { struct dm_md_mempools {
mempool_t *io_pool; mempool_t *io_pool;
mempool_t *rq_pool;
struct bio_set *bs; struct bio_set *bs;
}; };
...@@ -1419,7 +1418,6 @@ static void cleanup_mapped_device(struct mapped_device *md) ...@@ -1419,7 +1418,6 @@ static void cleanup_mapped_device(struct mapped_device *md)
if (md->kworker_task) if (md->kworker_task)
kthread_stop(md->kworker_task); kthread_stop(md->kworker_task);
mempool_destroy(md->io_pool); mempool_destroy(md->io_pool);
mempool_destroy(md->rq_pool);
if (md->bs) if (md->bs)
bioset_free(md->bs); bioset_free(md->bs);
...@@ -1595,12 +1593,10 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t) ...@@ -1595,12 +1593,10 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
goto out; goto out;
} }
BUG_ON(!p || md->io_pool || md->rq_pool || md->bs); BUG_ON(!p || md->io_pool || md->bs);
md->io_pool = p->io_pool; md->io_pool = p->io_pool;
p->io_pool = NULL; p->io_pool = NULL;
md->rq_pool = p->rq_pool;
p->rq_pool = NULL;
md->bs = p->bs; md->bs = p->bs;
p->bs = NULL; p->bs = NULL;
...@@ -1777,7 +1773,7 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) ...@@ -1777,7 +1773,7 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
switch (type) { switch (type) {
case DM_TYPE_REQUEST_BASED: case DM_TYPE_REQUEST_BASED:
r = dm_old_init_request_queue(md); r = dm_old_init_request_queue(md, t);
if (r) { if (r) {
DMERR("Cannot initialize queue for request-based mapped device"); DMERR("Cannot initialize queue for request-based mapped device");
return r; return r;
...@@ -2493,7 +2489,6 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t ...@@ -2493,7 +2489,6 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t
unsigned integrity, unsigned per_io_data_size) unsigned integrity, unsigned per_io_data_size)
{ {
struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id); struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
struct kmem_cache *cachep = NULL;
unsigned int pool_size = 0; unsigned int pool_size = 0;
unsigned int front_pad; unsigned int front_pad;
...@@ -2503,20 +2498,16 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t ...@@ -2503,20 +2498,16 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t
switch (type) { switch (type) {
case DM_TYPE_BIO_BASED: case DM_TYPE_BIO_BASED:
case DM_TYPE_DAX_BIO_BASED: case DM_TYPE_DAX_BIO_BASED:
cachep = _io_cache;
pool_size = dm_get_reserved_bio_based_ios(); pool_size = dm_get_reserved_bio_based_ios();
front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
if (!pools->io_pool)
goto out;
break; break;
case DM_TYPE_REQUEST_BASED: case DM_TYPE_REQUEST_BASED:
cachep = _rq_tio_cache;
pool_size = dm_get_reserved_rq_based_ios();
pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
if (!pools->rq_pool)
goto out;
/* fall through to setup remaining rq-based pools */
case DM_TYPE_MQ_REQUEST_BASED: case DM_TYPE_MQ_REQUEST_BASED:
if (!pool_size) pool_size = dm_get_reserved_rq_based_ios();
pool_size = dm_get_reserved_rq_based_ios();
front_pad = offsetof(struct dm_rq_clone_bio_info, clone); front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
/* per_io_data_size is used for blk-mq pdu at queue allocation */ /* per_io_data_size is used for blk-mq pdu at queue allocation */
break; break;
...@@ -2524,12 +2515,6 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t ...@@ -2524,12 +2515,6 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t
BUG(); BUG();
} }
if (cachep) {
pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
if (!pools->io_pool)
goto out;
}
pools->bs = bioset_create_nobvec(pool_size, front_pad); pools->bs = bioset_create_nobvec(pool_size, front_pad);
if (!pools->bs) if (!pools->bs)
goto out; goto out;
...@@ -2551,7 +2536,6 @@ void dm_free_md_mempools(struct dm_md_mempools *pools) ...@@ -2551,7 +2536,6 @@ void dm_free_md_mempools(struct dm_md_mempools *pools)
return; return;
mempool_destroy(pools->io_pool); mempool_destroy(pools->io_pool);
mempool_destroy(pools->rq_pool);
if (pools->bs) if (pools->bs)
bioset_free(pools->bs); bioset_free(pools->bs);
......
...@@ -95,8 +95,7 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t); ...@@ -95,8 +95,7 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t);
/* /*
* To check whether the target type is request-based or not (bio-based). * To check whether the target type is request-based or not (bio-based).
*/ */
#define dm_target_request_based(t) (((t)->type->map_rq != NULL) || \ #define dm_target_request_based(t) ((t)->type->clone_and_map_rq != NULL)
((t)->type->clone_and_map_rq != NULL))
/* /*
* To check whether the target type is a hybrid (capable of being * To check whether the target type is a hybrid (capable of being
......
...@@ -55,8 +55,6 @@ typedef void (*dm_dtr_fn) (struct dm_target *ti); ...@@ -55,8 +55,6 @@ typedef void (*dm_dtr_fn) (struct dm_target *ti);
* = 2: The target wants to push back the io * = 2: The target wants to push back the io
*/ */
typedef int (*dm_map_fn) (struct dm_target *ti, struct bio *bio); typedef int (*dm_map_fn) (struct dm_target *ti, struct bio *bio);
typedef int (*dm_map_request_fn) (struct dm_target *ti, struct request *clone,
union map_info *map_context);
typedef int (*dm_clone_and_map_request_fn) (struct dm_target *ti, typedef int (*dm_clone_and_map_request_fn) (struct dm_target *ti,
struct request *rq, struct request *rq,
union map_info *map_context, union map_info *map_context,
...@@ -163,7 +161,6 @@ struct target_type { ...@@ -163,7 +161,6 @@ struct target_type {
dm_ctr_fn ctr; dm_ctr_fn ctr;
dm_dtr_fn dtr; dm_dtr_fn dtr;
dm_map_fn map; dm_map_fn map;
dm_map_request_fn map_rq;
dm_clone_and_map_request_fn clone_and_map_rq; dm_clone_and_map_request_fn clone_and_map_rq;
dm_release_clone_request_fn release_clone_rq; dm_release_clone_request_fn release_clone_rq;
dm_endio_fn end_io; dm_endio_fn end_io;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment