Commit 7c3fb70f authored by Jens Axboe's avatar Jens Axboe

block: rearrange a few request fields for better cache layout

Move completion related items (like the call single data) near the
end of the struct, instead of mixing them in with the initial
queueing related fields.

Move queuelist below the bio structures. Then we have all
queueing related bits in the first cache line.

This yields a 1.5-2% increase in IOPS for a null_blk test, both for
sync and for high thread count access. Sync test goes form 975K to
992K, 32-thread case from 20.8M to 21.2M IOPS.
Reviewed-by: default avatarBart Van Assche <bart.vanassche@wdc.com>
Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e14575b3
...@@ -270,8 +270,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, ...@@ -270,8 +270,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
struct blk_mq_tags *tags = blk_mq_tags_from_data(data); struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
struct request *rq = tags->static_rqs[tag]; struct request *rq = tags->static_rqs[tag];
rq->rq_flags = 0;
if (data->flags & BLK_MQ_REQ_INTERNAL) { if (data->flags & BLK_MQ_REQ_INTERNAL) {
rq->tag = -1; rq->tag = -1;
rq->internal_tag = tag; rq->internal_tag = tag;
...@@ -285,26 +283,22 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, ...@@ -285,26 +283,22 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
data->hctx->tags->rqs[rq->tag] = rq; data->hctx->tags->rqs[rq->tag] = rq;
} }
INIT_LIST_HEAD(&rq->queuelist);
/* csd/requeue_work/fifo_time is initialized before use */ /* csd/requeue_work/fifo_time is initialized before use */
rq->q = data->q; rq->q = data->q;
rq->mq_ctx = data->ctx; rq->mq_ctx = data->ctx;
rq->rq_flags = 0;
rq->cpu = -1;
rq->cmd_flags = op; rq->cmd_flags = op;
if (data->flags & BLK_MQ_REQ_PREEMPT) if (data->flags & BLK_MQ_REQ_PREEMPT)
rq->rq_flags |= RQF_PREEMPT; rq->rq_flags |= RQF_PREEMPT;
if (blk_queue_io_stat(data->q)) if (blk_queue_io_stat(data->q))
rq->rq_flags |= RQF_IO_STAT; rq->rq_flags |= RQF_IO_STAT;
rq->cpu = -1; INIT_LIST_HEAD(&rq->queuelist);
INIT_HLIST_NODE(&rq->hash); INIT_HLIST_NODE(&rq->hash);
RB_CLEAR_NODE(&rq->rb_node); RB_CLEAR_NODE(&rq->rb_node);
rq->rq_disk = NULL; rq->rq_disk = NULL;
rq->part = NULL; rq->part = NULL;
rq->start_time = jiffies; rq->start_time = jiffies;
#ifdef CONFIG_BLK_CGROUP
rq->rl = NULL;
set_start_time_ns(rq);
rq->io_start_time_ns = 0;
#endif
rq->nr_phys_segments = 0; rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY) #if defined(CONFIG_BLK_DEV_INTEGRITY)
rq->nr_integrity_segments = 0; rq->nr_integrity_segments = 0;
...@@ -321,6 +315,12 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, ...@@ -321,6 +315,12 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
rq->end_io_data = NULL; rq->end_io_data = NULL;
rq->next_rq = NULL; rq->next_rq = NULL;
#ifdef CONFIG_BLK_CGROUP
rq->rl = NULL;
set_start_time_ns(rq);
rq->io_start_time_ns = 0;
#endif
data->ctx->rq_dispatched[op_is_sync(op)]++; data->ctx->rq_dispatched[op_is_sync(op)]++;
return rq; return rq;
} }
......
...@@ -141,12 +141,6 @@ typedef __u32 __bitwise req_flags_t; ...@@ -141,12 +141,6 @@ typedef __u32 __bitwise req_flags_t;
* especially blk_mq_rq_ctx_init() to take care of the added fields. * especially blk_mq_rq_ctx_init() to take care of the added fields.
*/ */
struct request { struct request {
struct list_head queuelist;
union {
call_single_data_t csd;
u64 fifo_time;
};
struct request_queue *q; struct request_queue *q;
struct blk_mq_ctx *mq_ctx; struct blk_mq_ctx *mq_ctx;
...@@ -164,6 +158,8 @@ struct request { ...@@ -164,6 +158,8 @@ struct request {
struct bio *bio; struct bio *bio;
struct bio *biotail; struct bio *biotail;
struct list_head queuelist;
/* /*
* The hash is used inside the scheduler, and killed once the * The hash is used inside the scheduler, and killed once the
* request reaches the dispatch list. The ipi_list is only used * request reaches the dispatch list. The ipi_list is only used
...@@ -211,19 +207,16 @@ struct request { ...@@ -211,19 +207,16 @@ struct request {
struct hd_struct *part; struct hd_struct *part;
unsigned long start_time; unsigned long start_time;
struct blk_issue_stat issue_stat; struct blk_issue_stat issue_stat;
#ifdef CONFIG_BLK_CGROUP
struct request_list *rl; /* rl this rq is alloced from */
unsigned long long start_time_ns;
unsigned long long io_start_time_ns; /* when passed to hardware */
#endif
/* Number of scatter-gather DMA addr+len pairs after /* Number of scatter-gather DMA addr+len pairs after
* physical address coalescing is performed. * physical address coalescing is performed.
*/ */
unsigned short nr_phys_segments; unsigned short nr_phys_segments;
#if defined(CONFIG_BLK_DEV_INTEGRITY) #if defined(CONFIG_BLK_DEV_INTEGRITY)
unsigned short nr_integrity_segments; unsigned short nr_integrity_segments;
#endif #endif
unsigned short write_hint;
unsigned short ioprio; unsigned short ioprio;
unsigned int timeout; unsigned int timeout;
...@@ -232,8 +225,6 @@ struct request { ...@@ -232,8 +225,6 @@ struct request {
unsigned int extra_len; /* length of alignment and padding */ unsigned int extra_len; /* length of alignment and padding */
unsigned short write_hint;
/* /*
* On blk-mq, the lower bits of ->gstate (generation number and * On blk-mq, the lower bits of ->gstate (generation number and
* state) carry the MQ_RQ_* state value and the upper bits the * state) carry the MQ_RQ_* state value and the upper bits the
...@@ -260,6 +251,11 @@ struct request { ...@@ -260,6 +251,11 @@ struct request {
struct list_head timeout_list; struct list_head timeout_list;
union {
call_single_data_t csd;
u64 fifo_time;
};
/* /*
* completion callback. * completion callback.
*/ */
...@@ -268,6 +264,12 @@ struct request { ...@@ -268,6 +264,12 @@ struct request {
/* for bidi */ /* for bidi */
struct request *next_rq; struct request *next_rq;
#ifdef CONFIG_BLK_CGROUP
struct request_list *rl; /* rl this rq is alloced from */
unsigned long long start_time_ns;
unsigned long long io_start_time_ns; /* when passed to hardware */
#endif
}; };
static inline bool blk_rq_is_scsi(struct request *rq) static inline bool blk_rq_is_scsi(struct request *rq)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment