Commit bfebd1cd authored by Mike Snitzer's avatar Mike Snitzer

dm: add full blk-mq support to request-based DM

Commit e5863d9a ("dm: allocate requests in target when stacking on
blk-mq devices") served as the first step toward fully utilizing blk-mq
in request-based DM -- it enabled stacking an old-style (request_fn)
request_queue ontop of the underlying blk-mq device(s).  That first step
didn't improve performance of DM multipath ontop of fast blk-mq devices
(e.g. NVMe) because the top-level old-style request_queue was severely
limited by the queue_lock.

The second step offered here enables stacking a blk-mq request_queue
ontop of the underlying blk-mq device(s).  This unlocks significant
performance gains on fast blk-mq devices, Keith Busch tested on his NVMe
testbed and offered this really positive news:

 "Just providing a performance update. All my fio tests are getting
  roughly equal performance whether accessed through the raw block
  device or the multipath device mapper (~470k IOPS). I could only push
  ~20% of the raw iops through dm before this conversion, so this latest
  tree is looking really solid from a performance standpoint."
Signed-off-by: default avatarMike Snitzer <snitzer@redhat.com>
Tested-by: default avatarKeith Busch <keith.busch@intel.com>
parent 0ce65797
...@@ -1703,7 +1703,7 @@ static int multipath_busy(struct dm_target *ti) ...@@ -1703,7 +1703,7 @@ static int multipath_busy(struct dm_target *ti)
*---------------------------------------------------------------*/ *---------------------------------------------------------------*/
static struct target_type multipath_target = { static struct target_type multipath_target = {
.name = "multipath", .name = "multipath",
.version = {1, 8, 0}, .version = {1, 9, 0},
.module = THIS_MODULE, .module = THIS_MODULE,
.ctr = multipath_ctr, .ctr = multipath_ctr,
.dtr = multipath_dtr, .dtr = multipath_dtr,
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <linux/mutex.h> #include <linux/mutex.h>
#include <linux/delay.h> #include <linux/delay.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#include <linux/blk-mq.h>
#define DM_MSG_PREFIX "table" #define DM_MSG_PREFIX "table"
...@@ -1695,9 +1696,13 @@ void dm_table_run_md_queue_async(struct dm_table *t) ...@@ -1695,9 +1696,13 @@ void dm_table_run_md_queue_async(struct dm_table *t)
md = dm_table_get_md(t); md = dm_table_get_md(t);
queue = dm_get_md_queue(md); queue = dm_get_md_queue(md);
if (queue) { if (queue) {
spin_lock_irqsave(queue->queue_lock, flags); if (queue->mq_ops)
blk_run_queue_async(queue); blk_mq_run_hw_queues(queue, true);
spin_unlock_irqrestore(queue->queue_lock, flags); else {
spin_lock_irqsave(queue->queue_lock, flags);
blk_run_queue_async(queue);
spin_unlock_irqrestore(queue->queue_lock, flags);
}
} }
} }
EXPORT_SYMBOL(dm_table_run_md_queue_async); EXPORT_SYMBOL(dm_table_run_md_queue_async);
......
...@@ -23,6 +23,7 @@ ...@@ -23,6 +23,7 @@
#include <linux/kthread.h> #include <linux/kthread.h>
#include <linux/ktime.h> #include <linux/ktime.h>
#include <linux/elevator.h> /* for rq_end_sector() */ #include <linux/elevator.h> /* for rq_end_sector() */
#include <linux/blk-mq.h>
#include <trace/events/block.h> #include <trace/events/block.h>
...@@ -224,6 +225,9 @@ struct mapped_device { ...@@ -224,6 +225,9 @@ struct mapped_device {
int last_rq_rw; int last_rq_rw;
sector_t last_rq_pos; sector_t last_rq_pos;
ktime_t last_rq_start_time; ktime_t last_rq_start_time;
/* for blk-mq request-based DM support */
struct blk_mq_tag_set tag_set;
}; };
/* /*
...@@ -1025,6 +1029,11 @@ static void end_clone_bio(struct bio *clone, int error) ...@@ -1025,6 +1029,11 @@ static void end_clone_bio(struct bio *clone, int error)
blk_update_request(tio->orig, 0, nr_bytes); blk_update_request(tio->orig, 0, nr_bytes);
} }
static struct dm_rq_target_io *tio_from_request(struct request *rq)
{
return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special);
}
/* /*
* Don't touch any member of the md after calling this function because * Don't touch any member of the md after calling this function because
* the md may be freed in dm_put() at the end of this function. * the md may be freed in dm_put() at the end of this function.
...@@ -1048,8 +1057,10 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue) ...@@ -1048,8 +1057,10 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
* queue lock again. * queue lock again.
*/ */
if (run_queue) { if (run_queue) {
if (!nr_requests_pending || if (md->queue->mq_ops)
(nr_requests_pending >= md->queue->nr_congestion_on)) blk_mq_run_hw_queues(md->queue, true);
else if (!nr_requests_pending ||
(nr_requests_pending >= md->queue->nr_congestion_on))
blk_run_queue_async(md->queue); blk_run_queue_async(md->queue);
} }
...@@ -1062,13 +1073,17 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue) ...@@ -1062,13 +1073,17 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
static void free_rq_clone(struct request *clone) static void free_rq_clone(struct request *clone)
{ {
struct dm_rq_target_io *tio = clone->end_io_data; struct dm_rq_target_io *tio = clone->end_io_data;
struct mapped_device *md = tio->md;
blk_rq_unprep_clone(clone); blk_rq_unprep_clone(clone);
if (clone->q && clone->q->mq_ops) if (clone->q && clone->q->mq_ops)
tio->ti->type->release_clone_rq(clone); tio->ti->type->release_clone_rq(clone);
else else
free_clone_request(tio->md, clone); free_clone_request(md, clone);
free_rq_tio(tio);
if (!md->queue->mq_ops)
free_rq_tio(tio);
} }
/* /*
...@@ -1097,17 +1112,22 @@ static void dm_end_request(struct request *clone, int error) ...@@ -1097,17 +1112,22 @@ static void dm_end_request(struct request *clone, int error)
} }
free_rq_clone(clone); free_rq_clone(clone);
blk_end_request_all(rq, error); if (!rq->q->mq_ops)
blk_end_request_all(rq, error);
else
blk_mq_end_request(rq, error);
rq_completed(md, rw, true); rq_completed(md, rw, true);
} }
static void dm_unprep_request(struct request *rq) static void dm_unprep_request(struct request *rq)
{ {
struct dm_rq_target_io *tio = rq->special; struct dm_rq_target_io *tio = tio_from_request(rq);
struct request *clone = tio->clone; struct request *clone = tio->clone;
rq->special = NULL; if (!rq->q->mq_ops) {
rq->cmd_flags &= ~REQ_DONTPREP; rq->special = NULL;
rq->cmd_flags &= ~REQ_DONTPREP;
}
if (clone) if (clone)
free_rq_clone(clone); free_rq_clone(clone);
...@@ -1116,18 +1136,29 @@ static void dm_unprep_request(struct request *rq) ...@@ -1116,18 +1136,29 @@ static void dm_unprep_request(struct request *rq)
/* /*
* Requeue the original request of a clone. * Requeue the original request of a clone.
*/ */
static void dm_requeue_unmapped_original_request(struct mapped_device *md, static void old_requeue_request(struct request *rq)
struct request *rq)
{ {
int rw = rq_data_dir(rq);
struct request_queue *q = rq->q; struct request_queue *q = rq->q;
unsigned long flags; unsigned long flags;
dm_unprep_request(rq);
spin_lock_irqsave(q->queue_lock, flags); spin_lock_irqsave(q->queue_lock, flags);
blk_requeue_request(q, rq); blk_requeue_request(q, rq);
spin_unlock_irqrestore(q->queue_lock, flags); spin_unlock_irqrestore(q->queue_lock, flags);
}
static void dm_requeue_unmapped_original_request(struct mapped_device *md,
struct request *rq)
{
int rw = rq_data_dir(rq);
dm_unprep_request(rq);
if (!rq->q->mq_ops)
old_requeue_request(rq);
else {
blk_mq_requeue_request(rq);
blk_mq_kick_requeue_list(rq->q);
}
rq_completed(md, rw, false); rq_completed(md, rw, false);
} }
...@@ -1139,35 +1170,44 @@ static void dm_requeue_unmapped_request(struct request *clone) ...@@ -1139,35 +1170,44 @@ static void dm_requeue_unmapped_request(struct request *clone)
dm_requeue_unmapped_original_request(tio->md, tio->orig); dm_requeue_unmapped_original_request(tio->md, tio->orig);
} }
static void __stop_queue(struct request_queue *q) static void old_stop_queue(struct request_queue *q)
{
blk_stop_queue(q);
}
static void stop_queue(struct request_queue *q)
{ {
unsigned long flags; unsigned long flags;
if (blk_queue_stopped(q))
return;
spin_lock_irqsave(q->queue_lock, flags); spin_lock_irqsave(q->queue_lock, flags);
__stop_queue(q); blk_stop_queue(q);
spin_unlock_irqrestore(q->queue_lock, flags); spin_unlock_irqrestore(q->queue_lock, flags);
} }
static void __start_queue(struct request_queue *q) static void stop_queue(struct request_queue *q)
{ {
if (blk_queue_stopped(q)) if (!q->mq_ops)
blk_start_queue(q); old_stop_queue(q);
else
blk_mq_stop_hw_queues(q);
} }
static void start_queue(struct request_queue *q) static void old_start_queue(struct request_queue *q)
{ {
unsigned long flags; unsigned long flags;
spin_lock_irqsave(q->queue_lock, flags); spin_lock_irqsave(q->queue_lock, flags);
__start_queue(q); if (blk_queue_stopped(q))
blk_start_queue(q);
spin_unlock_irqrestore(q->queue_lock, flags); spin_unlock_irqrestore(q->queue_lock, flags);
} }
static void start_queue(struct request_queue *q)
{
if (!q->mq_ops)
old_start_queue(q);
else
blk_mq_start_stopped_hw_queues(q, true);
}
static void dm_done(struct request *clone, int error, bool mapped) static void dm_done(struct request *clone, int error, bool mapped)
{ {
int r = error; int r = error;
...@@ -1206,13 +1246,20 @@ static void dm_done(struct request *clone, int error, bool mapped) ...@@ -1206,13 +1246,20 @@ static void dm_done(struct request *clone, int error, bool mapped)
static void dm_softirq_done(struct request *rq) static void dm_softirq_done(struct request *rq)
{ {
bool mapped = true; bool mapped = true;
struct dm_rq_target_io *tio = rq->special; struct dm_rq_target_io *tio = tio_from_request(rq);
struct request *clone = tio->clone; struct request *clone = tio->clone;
int rw;
if (!clone) { if (!clone) {
blk_end_request_all(rq, tio->error); rw = rq_data_dir(rq);
rq_completed(tio->md, rq_data_dir(rq), false); if (!rq->q->mq_ops) {
free_rq_tio(tio); blk_end_request_all(rq, tio->error);
rq_completed(tio->md, rw, false);
free_rq_tio(tio);
} else {
blk_mq_end_request(rq, tio->error);
rq_completed(tio->md, rw, false);
}
return; return;
} }
...@@ -1228,7 +1275,7 @@ static void dm_softirq_done(struct request *rq) ...@@ -1228,7 +1275,7 @@ static void dm_softirq_done(struct request *rq)
*/ */
static void dm_complete_request(struct request *rq, int error) static void dm_complete_request(struct request *rq, int error)
{ {
struct dm_rq_target_io *tio = rq->special; struct dm_rq_target_io *tio = tio_from_request(rq);
tio->error = error; tio->error = error;
blk_complete_request(rq); blk_complete_request(rq);
...@@ -1247,7 +1294,7 @@ static void dm_kill_unmapped_request(struct request *rq, int error) ...@@ -1247,7 +1294,7 @@ static void dm_kill_unmapped_request(struct request *rq, int error)
} }
/* /*
* Called with the clone's queue lock held * Called with the clone's queue lock held (for non-blk-mq)
*/ */
static void end_clone_request(struct request *clone, int error) static void end_clone_request(struct request *clone, int error)
{ {
...@@ -1808,6 +1855,18 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md, ...@@ -1808,6 +1855,18 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
static void map_tio_request(struct kthread_work *work); static void map_tio_request(struct kthread_work *work);
static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
struct mapped_device *md)
{
tio->md = md;
tio->ti = NULL;
tio->clone = NULL;
tio->orig = rq;
tio->error = 0;
memset(&tio->info, 0, sizeof(tio->info));
init_kthread_work(&tio->work, map_tio_request);
}
static struct dm_rq_target_io *prep_tio(struct request *rq, static struct dm_rq_target_io *prep_tio(struct request *rq,
struct mapped_device *md, gfp_t gfp_mask) struct mapped_device *md, gfp_t gfp_mask)
{ {
...@@ -1819,13 +1878,7 @@ static struct dm_rq_target_io *prep_tio(struct request *rq, ...@@ -1819,13 +1878,7 @@ static struct dm_rq_target_io *prep_tio(struct request *rq,
if (!tio) if (!tio)
return NULL; return NULL;
tio->md = md; init_tio(tio, rq, md);
tio->ti = NULL;
tio->clone = NULL;
tio->orig = rq;
tio->error = 0;
memset(&tio->info, 0, sizeof(tio->info));
init_kthread_work(&tio->work, map_tio_request);
table = dm_get_live_table(md, &srcu_idx); table = dm_get_live_table(md, &srcu_idx);
if (!dm_table_mq_request_based(table)) { if (!dm_table_mq_request_based(table)) {
...@@ -1869,11 +1922,11 @@ static int dm_prep_fn(struct request_queue *q, struct request *rq) ...@@ -1869,11 +1922,11 @@ static int dm_prep_fn(struct request_queue *q, struct request *rq)
* DM_MAPIO_REQUEUE : the original request needs to be requeued * DM_MAPIO_REQUEUE : the original request needs to be requeued
* < 0 : the request was completed due to failure * < 0 : the request was completed due to failure
*/ */
static int map_request(struct dm_target *ti, struct request *rq, static int map_request(struct dm_rq_target_io *tio, struct request *rq,
struct mapped_device *md) struct mapped_device *md)
{ {
int r; int r;
struct dm_rq_target_io *tio = rq->special; struct dm_target *ti = tio->ti;
struct request *clone = NULL; struct request *clone = NULL;
if (tio->clone) { if (tio->clone) {
...@@ -1888,7 +1941,7 @@ static int map_request(struct dm_target *ti, struct request *rq, ...@@ -1888,7 +1941,7 @@ static int map_request(struct dm_target *ti, struct request *rq,
} }
if (IS_ERR(clone)) if (IS_ERR(clone))
return DM_MAPIO_REQUEUE; return DM_MAPIO_REQUEUE;
if (setup_clone(clone, rq, tio, GFP_KERNEL)) { if (setup_clone(clone, rq, tio, GFP_NOIO)) {
/* -ENOMEM */ /* -ENOMEM */
ti->type->release_clone_rq(clone); ti->type->release_clone_rq(clone);
return DM_MAPIO_REQUEUE; return DM_MAPIO_REQUEUE;
...@@ -1929,13 +1982,16 @@ static void map_tio_request(struct kthread_work *work) ...@@ -1929,13 +1982,16 @@ static void map_tio_request(struct kthread_work *work)
struct request *rq = tio->orig; struct request *rq = tio->orig;
struct mapped_device *md = tio->md; struct mapped_device *md = tio->md;
if (map_request(tio->ti, rq, md) == DM_MAPIO_REQUEUE) if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE)
dm_requeue_unmapped_original_request(md, rq); dm_requeue_unmapped_original_request(md, rq);
} }
static void dm_start_request(struct mapped_device *md, struct request *orig) static void dm_start_request(struct mapped_device *md, struct request *orig)
{ {
blk_start_request(orig); if (!orig->q->mq_ops)
blk_start_request(orig);
else
blk_mq_start_request(orig);
atomic_inc(&md->pending[rq_data_dir(orig)]); atomic_inc(&md->pending[rq_data_dir(orig)]);
if (md->seq_rq_merge_deadline_usecs) { if (md->seq_rq_merge_deadline_usecs) {
...@@ -2045,7 +2101,7 @@ static void dm_request_fn(struct request_queue *q) ...@@ -2045,7 +2101,7 @@ static void dm_request_fn(struct request_queue *q)
dm_start_request(md, rq); dm_start_request(md, rq);
tio = rq->special; tio = tio_from_request(rq);
/* Establish tio->ti before queuing work (map_tio_request) */ /* Establish tio->ti before queuing work (map_tio_request) */
tio->ti = ti; tio->ti = ti;
queue_kthread_work(&md->kworker, &tio->work); queue_kthread_work(&md->kworker, &tio->work);
...@@ -2142,7 +2198,7 @@ static void dm_init_md_queue(struct mapped_device *md) ...@@ -2142,7 +2198,7 @@ static void dm_init_md_queue(struct mapped_device *md)
{ {
/* /*
* Request-based dm devices cannot be stacked on top of bio-based dm * Request-based dm devices cannot be stacked on top of bio-based dm
* devices. The type of this dm device has not been decided yet. * devices. The type of this dm device may not have been decided yet.
* The type is decided at the first table loading time. * The type is decided at the first table loading time.
* To prevent problematic device stacking, clear the queue flag * To prevent problematic device stacking, clear the queue flag
* for request stacking support until then. * for request stacking support until then.
...@@ -2150,7 +2206,15 @@ static void dm_init_md_queue(struct mapped_device *md) ...@@ -2150,7 +2206,15 @@ static void dm_init_md_queue(struct mapped_device *md)
* This queue is new, so no concurrency on the queue_flags. * This queue is new, so no concurrency on the queue_flags.
*/ */
queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue); queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
}
static void dm_init_old_md_queue(struct mapped_device *md)
{
dm_init_md_queue(md);
/*
* Initialize aspects of queue that aren't relevant for blk-mq
*/
md->queue->queuedata = md; md->queue->queuedata = md;
md->queue->backing_dev_info.congested_fn = dm_any_congested; md->queue->backing_dev_info.congested_fn = dm_any_congested;
md->queue->backing_dev_info.congested_data = md; md->queue->backing_dev_info.congested_data = md;
...@@ -2273,6 +2337,7 @@ static void unlock_fs(struct mapped_device *md); ...@@ -2273,6 +2337,7 @@ static void unlock_fs(struct mapped_device *md);
static void free_dev(struct mapped_device *md) static void free_dev(struct mapped_device *md)
{ {
int minor = MINOR(disk_devt(md->disk)); int minor = MINOR(disk_devt(md->disk));
bool using_blk_mq = !!md->queue->mq_ops;
unlock_fs(md); unlock_fs(md);
destroy_workqueue(md->wq); destroy_workqueue(md->wq);
...@@ -2298,6 +2363,8 @@ static void free_dev(struct mapped_device *md) ...@@ -2298,6 +2363,8 @@ static void free_dev(struct mapped_device *md)
del_gendisk(md->disk); del_gendisk(md->disk);
put_disk(md->disk); put_disk(md->disk);
blk_cleanup_queue(md->queue); blk_cleanup_queue(md->queue);
if (using_blk_mq)
blk_mq_free_tag_set(&md->tag_set);
bdput(md->bdev); bdput(md->bdev);
free_minor(minor); free_minor(minor);
...@@ -2457,7 +2524,7 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, ...@@ -2457,7 +2524,7 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
* This must be done before setting the queue restrictions, * This must be done before setting the queue restrictions,
* because request-based dm may be run just after the setting. * because request-based dm may be run just after the setting.
*/ */
if (dm_table_request_based(t) && !blk_queue_stopped(q)) if (dm_table_request_based(t))
stop_queue(q); stop_queue(q);
__bind_mempools(md, t); __bind_mempools(md, t);
...@@ -2539,14 +2606,6 @@ unsigned dm_get_md_type(struct mapped_device *md) ...@@ -2539,14 +2606,6 @@ unsigned dm_get_md_type(struct mapped_device *md)
return md->type; return md->type;
} }
static bool dm_md_type_request_based(struct mapped_device *md)
{
unsigned table_type = dm_get_md_type(md);
return (table_type == DM_TYPE_REQUEST_BASED ||
table_type == DM_TYPE_MQ_REQUEST_BASED);
}
struct target_type *dm_get_immutable_target_type(struct mapped_device *md) struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
{ {
return md->immutable_target_type; return md->immutable_target_type;
...@@ -2563,6 +2622,14 @@ struct queue_limits *dm_get_queue_limits(struct mapped_device *md) ...@@ -2563,6 +2622,14 @@ struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
} }
EXPORT_SYMBOL_GPL(dm_get_queue_limits); EXPORT_SYMBOL_GPL(dm_get_queue_limits);
static void init_rq_based_worker_thread(struct mapped_device *md)
{
/* Initialize the request-based DM worker thread */
init_kthread_worker(&md->kworker);
md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker,
"kdmwork-%s", dm_device_name(md));
}
/* /*
* Fully initialize a request-based queue (->elevator, ->request_fn, etc). * Fully initialize a request-based queue (->elevator, ->request_fn, etc).
*/ */
...@@ -2571,29 +2638,131 @@ static int dm_init_request_based_queue(struct mapped_device *md) ...@@ -2571,29 +2638,131 @@ static int dm_init_request_based_queue(struct mapped_device *md)
struct request_queue *q = NULL; struct request_queue *q = NULL;
if (md->queue->elevator) if (md->queue->elevator)
return 1; return 0;
/* Fully initialize the queue */ /* Fully initialize the queue */
q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL); q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL);
if (!q) if (!q)
return 0; return -EINVAL;
/* disable dm_request_fn's merge heuristic by default */ /* disable dm_request_fn's merge heuristic by default */
md->seq_rq_merge_deadline_usecs = 0; md->seq_rq_merge_deadline_usecs = 0;
md->queue = q; md->queue = q;
dm_init_md_queue(md); dm_init_old_md_queue(md);
blk_queue_softirq_done(md->queue, dm_softirq_done); blk_queue_softirq_done(md->queue, dm_softirq_done);
blk_queue_prep_rq(md->queue, dm_prep_fn); blk_queue_prep_rq(md->queue, dm_prep_fn);
/* Also initialize the request-based DM worker thread */ init_rq_based_worker_thread(md);
init_kthread_worker(&md->kworker);
md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker,
"kdmwork-%s", dm_device_name(md));
elv_register_queue(md->queue); elv_register_queue(md->queue);
return 1; return 0;
}
static int dm_mq_init_request(void *data, struct request *rq,
unsigned int hctx_idx, unsigned int request_idx,
unsigned int numa_node)
{
struct mapped_device *md = data;
struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);
/*
* Must initialize md member of tio, otherwise it won't
* be available in dm_mq_queue_rq.
*/
tio->md = md;
return 0;
}
static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *bd)
{
struct request *rq = bd->rq;
struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);
struct mapped_device *md = tio->md;
int srcu_idx;
struct dm_table *map = dm_get_live_table(md, &srcu_idx);
struct dm_target *ti;
sector_t pos;
/* always use block 0 to find the target for flushes for now */
pos = 0;
if (!(rq->cmd_flags & REQ_FLUSH))
pos = blk_rq_pos(rq);
ti = dm_table_find_target(map, pos);
if (!dm_target_is_valid(ti)) {
dm_put_live_table(md, srcu_idx);
DMERR_LIMIT("request attempted access beyond the end of device");
/*
* Must perform setup, that rq_completed() requires,
* before returning BLK_MQ_RQ_QUEUE_ERROR
*/
dm_start_request(md, rq);
return BLK_MQ_RQ_QUEUE_ERROR;
}
dm_put_live_table(md, srcu_idx);
if (ti->type->busy && ti->type->busy(ti))
return BLK_MQ_RQ_QUEUE_BUSY;
dm_start_request(md, rq);
/* Init tio using md established in .init_request */
init_tio(tio, rq, md);
/* Establish tio->ti before queuing work (map_tio_request) */
tio->ti = ti;
queue_kthread_work(&md->kworker, &tio->work);
return BLK_MQ_RQ_QUEUE_OK;
}
static struct blk_mq_ops dm_mq_ops = {
.queue_rq = dm_mq_queue_rq,
.map_queue = blk_mq_map_queue,
.complete = dm_softirq_done,
.init_request = dm_mq_init_request,
};
static int dm_init_request_based_blk_mq_queue(struct mapped_device *md)
{
struct request_queue *q;
int err;
memset(&md->tag_set, 0, sizeof(md->tag_set));
md->tag_set.ops = &dm_mq_ops;
md->tag_set.queue_depth = BLKDEV_MAX_RQ;
md->tag_set.numa_node = NUMA_NO_NODE;
md->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
md->tag_set.nr_hw_queues = 1;
md->tag_set.cmd_size = sizeof(struct dm_rq_target_io);
md->tag_set.driver_data = md;
err = blk_mq_alloc_tag_set(&md->tag_set);
if (err)
return err;
q = blk_mq_init_allocated_queue(&md->tag_set, md->queue);
if (IS_ERR(q)) {
err = PTR_ERR(q);
goto out_tag_set;
}
md->queue = q;
dm_init_md_queue(md);
/* backfill 'mq' sysfs registration normally done in blk_register_queue */
blk_mq_register_disk(md->disk);
init_rq_based_worker_thread(md);
return 0;
out_tag_set:
blk_mq_free_tag_set(&md->tag_set);
return err;
} }
/* /*
...@@ -2601,15 +2770,29 @@ static int dm_init_request_based_queue(struct mapped_device *md) ...@@ -2601,15 +2770,29 @@ static int dm_init_request_based_queue(struct mapped_device *md)
*/ */
int dm_setup_md_queue(struct mapped_device *md) int dm_setup_md_queue(struct mapped_device *md)
{ {
if (dm_md_type_request_based(md)) { int r;
if (!dm_init_request_based_queue(md)) { unsigned md_type = dm_get_md_type(md);
switch (md_type) {
case DM_TYPE_REQUEST_BASED:
r = dm_init_request_based_queue(md);
if (r) {
DMWARN("Cannot initialize queue for request-based mapped device"); DMWARN("Cannot initialize queue for request-based mapped device");
return -EINVAL; return r;
} }
} else { break;
/* bio-based specific initialization */ case DM_TYPE_MQ_REQUEST_BASED:
r = dm_init_request_based_blk_mq_queue(md);
if (r) {
DMWARN("Cannot initialize queue for request-based blk-mq mapped device");
return r;
}
break;
case DM_TYPE_BIO_BASED:
dm_init_old_md_queue(md);
blk_queue_make_request(md->queue, dm_make_request); blk_queue_make_request(md->queue, dm_make_request);
blk_queue_merge_bvec(md->queue, dm_merge_bvec); blk_queue_merge_bvec(md->queue, dm_merge_bvec);
break;
} }
return 0; return 0;
......
...@@ -267,9 +267,9 @@ enum { ...@@ -267,9 +267,9 @@ enum {
#define DM_DEV_SET_GEOMETRY _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl) #define DM_DEV_SET_GEOMETRY _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl)
#define DM_VERSION_MAJOR 4 #define DM_VERSION_MAJOR 4
#define DM_VERSION_MINOR 30 #define DM_VERSION_MINOR 31
#define DM_VERSION_PATCHLEVEL 0 #define DM_VERSION_PATCHLEVEL 0
#define DM_VERSION_EXTRA "-ioctl (2014-12-22)" #define DM_VERSION_EXTRA "-ioctl (2015-3-12)"
/* Status bits */ /* Status bits */
#define DM_READONLY_FLAG (1 << 0) /* In/Out */ #define DM_READONLY_FLAG (1 << 0) /* In/Out */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment