Commit 5a72e899 authored by Jens Axboe's avatar Jens Axboe

block: add a struct io_comp_batch argument to fops->iopoll()

struct io_comp_batch contains a list head and a completion handler, which
will allow completions to more effciently completed batches of IO.

For now, no functional changes in this patch, we just define the
io_comp_batch structure and add the argument to the file_operations iopoll
handler.
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 013a7f95
...@@ -1078,7 +1078,7 @@ EXPORT_SYMBOL(submit_bio); ...@@ -1078,7 +1078,7 @@ EXPORT_SYMBOL(submit_bio);
* Note: the caller must either be the context that submitted @bio, or * Note: the caller must either be the context that submitted @bio, or
* be in a RCU critical section to prevent freeing of @bio. * be in a RCU critical section to prevent freeing of @bio.
*/ */
int bio_poll(struct bio *bio, unsigned int flags) int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags)
{ {
struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct request_queue *q = bio->bi_bdev->bd_disk->queue;
blk_qc_t cookie = READ_ONCE(bio->bi_cookie); blk_qc_t cookie = READ_ONCE(bio->bi_cookie);
...@@ -1096,7 +1096,7 @@ int bio_poll(struct bio *bio, unsigned int flags) ...@@ -1096,7 +1096,7 @@ int bio_poll(struct bio *bio, unsigned int flags)
if (WARN_ON_ONCE(!queue_is_mq(q))) if (WARN_ON_ONCE(!queue_is_mq(q)))
ret = 0; /* not yet implemented, should not happen */ ret = 0; /* not yet implemented, should not happen */
else else
ret = blk_mq_poll(q, cookie, flags); ret = blk_mq_poll(q, cookie, iob, flags);
blk_queue_exit(q); blk_queue_exit(q);
return ret; return ret;
} }
...@@ -1106,7 +1106,8 @@ EXPORT_SYMBOL_GPL(bio_poll); ...@@ -1106,7 +1106,8 @@ EXPORT_SYMBOL_GPL(bio_poll);
* Helper to implement file_operations.iopoll. Requires the bio to be stored * Helper to implement file_operations.iopoll. Requires the bio to be stored
* in iocb->private, and cleared before freeing the bio. * in iocb->private, and cleared before freeing the bio.
*/ */
int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags) int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
unsigned int flags)
{ {
struct bio *bio; struct bio *bio;
int ret = 0; int ret = 0;
...@@ -1134,7 +1135,7 @@ int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags) ...@@ -1134,7 +1135,7 @@ int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags)
rcu_read_lock(); rcu_read_lock();
bio = READ_ONCE(kiocb->private); bio = READ_ONCE(kiocb->private);
if (bio && bio->bi_bdev) if (bio && bio->bi_bdev)
ret = bio_poll(bio, flags); ret = bio_poll(bio, iob, flags);
rcu_read_unlock(); rcu_read_unlock();
return ret; return ret;
......
...@@ -77,7 +77,7 @@ static bool blk_rq_is_poll(struct request *rq) ...@@ -77,7 +77,7 @@ static bool blk_rq_is_poll(struct request *rq)
static void blk_rq_poll_completion(struct request *rq, struct completion *wait) static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
{ {
do { do {
bio_poll(rq->bio, 0); bio_poll(rq->bio, NULL, 0);
cond_resched(); cond_resched();
} while (!completion_done(wait)); } while (!completion_done(wait));
} }
......
...@@ -4174,14 +4174,14 @@ static bool blk_mq_poll_hybrid(struct request_queue *q, blk_qc_t qc) ...@@ -4174,14 +4174,14 @@ static bool blk_mq_poll_hybrid(struct request_queue *q, blk_qc_t qc)
} }
static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie, static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
unsigned int flags) struct io_comp_batch *iob, unsigned int flags)
{ {
struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie); struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie);
long state = get_current_state(); long state = get_current_state();
int ret; int ret;
do { do {
ret = q->mq_ops->poll(hctx); ret = q->mq_ops->poll(hctx, iob);
if (ret > 0) { if (ret > 0) {
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
return ret; return ret;
...@@ -4201,14 +4201,15 @@ static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie, ...@@ -4201,14 +4201,15 @@ static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
return 0; return 0;
} }
int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags) int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, struct io_comp_batch *iob,
unsigned int flags)
{ {
if (!(flags & BLK_POLL_NOSLEEP) && if (!(flags & BLK_POLL_NOSLEEP) &&
q->poll_nsec != BLK_MQ_POLL_CLASSIC) { q->poll_nsec != BLK_MQ_POLL_CLASSIC) {
if (blk_mq_poll_hybrid(q, cookie)) if (blk_mq_poll_hybrid(q, cookie))
return 1; return 1;
} }
return blk_mq_poll_classic(q, cookie, flags); return blk_mq_poll_classic(q, cookie, iob, flags);
} }
unsigned int blk_mq_rq_cpu(struct request *rq) unsigned int blk_mq_rq_cpu(struct request *rq)
......
...@@ -31,7 +31,8 @@ struct blk_mq_ctx { ...@@ -31,7 +31,8 @@ struct blk_mq_ctx {
} ____cacheline_aligned_in_smp; } ____cacheline_aligned_in_smp;
void blk_mq_submit_bio(struct bio *bio); void blk_mq_submit_bio(struct bio *bio);
int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags); int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, struct io_comp_batch *iob,
unsigned int flags);
void blk_mq_exit_queue(struct request_queue *q); void blk_mq_exit_queue(struct request_queue *q);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr); int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
void blk_mq_wake_waiters(struct request_queue *q); void blk_mq_wake_waiters(struct request_queue *q);
......
...@@ -105,7 +105,7 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb, ...@@ -105,7 +105,7 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,
set_current_state(TASK_UNINTERRUPTIBLE); set_current_state(TASK_UNINTERRUPTIBLE);
if (!READ_ONCE(bio.bi_private)) if (!READ_ONCE(bio.bi_private))
break; break;
if (!(iocb->ki_flags & IOCB_HIPRI) || !bio_poll(&bio, 0)) if (!(iocb->ki_flags & IOCB_HIPRI) || !bio_poll(&bio, NULL, 0))
blk_io_schedule(); blk_io_schedule();
} }
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
...@@ -291,7 +291,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ...@@ -291,7 +291,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
if (!READ_ONCE(dio->waiter)) if (!READ_ONCE(dio->waiter))
break; break;
if (!do_poll || !bio_poll(bio, 0)) if (!do_poll || !bio_poll(bio, NULL, 0))
blk_io_schedule(); blk_io_schedule();
} }
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
......
...@@ -1176,7 +1176,7 @@ static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx, ...@@ -1176,7 +1176,7 @@ static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
return ret; return ret;
} }
static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx) static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
{ {
struct rnbd_queue *q = hctx->driver_data; struct rnbd_queue *q = hctx->driver_data;
struct rnbd_clt_dev *dev = q->dev; struct rnbd_clt_dev *dev = q->dev;
......
...@@ -1092,7 +1092,7 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq) ...@@ -1092,7 +1092,7 @@ static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector)); enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
} }
static int nvme_poll(struct blk_mq_hw_ctx *hctx) static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
{ {
struct nvme_queue *nvmeq = hctx->driver_data; struct nvme_queue *nvmeq = hctx->driver_data;
bool found; bool found;
...@@ -1274,7 +1274,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) ...@@ -1274,7 +1274,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
* Did we miss an interrupt? * Did we miss an interrupt?
*/ */
if (test_bit(NVMEQ_POLLED, &nvmeq->flags)) if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
nvme_poll(req->mq_hctx); nvme_poll(req->mq_hctx, NULL);
else else
nvme_poll_irqdisable(nvmeq); nvme_poll_irqdisable(nvmeq);
......
...@@ -2106,7 +2106,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, ...@@ -2106,7 +2106,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
return ret; return ret;
} }
static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx) static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
{ {
struct nvme_rdma_queue *queue = hctx->driver_data; struct nvme_rdma_queue *queue = hctx->driver_data;
......
...@@ -2429,7 +2429,7 @@ static int nvme_tcp_map_queues(struct blk_mq_tag_set *set) ...@@ -2429,7 +2429,7 @@ static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
return 0; return 0;
} }
static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
{ {
struct nvme_tcp_queue *queue = hctx->driver_data; struct nvme_tcp_queue *queue = hctx->driver_data;
struct sock *sk = queue->sock->sk; struct sock *sk = queue->sock->sk;
......
...@@ -1784,7 +1784,7 @@ static void scsi_mq_exit_request(struct blk_mq_tag_set *set, struct request *rq, ...@@ -1784,7 +1784,7 @@ static void scsi_mq_exit_request(struct blk_mq_tag_set *set, struct request *rq,
} }
static int scsi_mq_poll(struct blk_mq_hw_ctx *hctx) static int scsi_mq_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
{ {
struct Scsi_Host *shost = hctx->driver_data; struct Scsi_Host *shost = hctx->driver_data;
......
...@@ -2483,7 +2483,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, ...@@ -2483,7 +2483,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
if (!list_empty(&done)) if (!list_empty(&done))
break; break;
ret = kiocb->ki_filp->f_op->iopoll(kiocb, poll_flags); ret = kiocb->ki_filp->f_op->iopoll(kiocb, NULL, poll_flags);
if (unlikely(ret < 0)) if (unlikely(ret < 0))
return ret; return ret;
else if (ret) else if (ret)
......
...@@ -630,7 +630,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, ...@@ -630,7 +630,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
break; break;
if (!dio->submit.poll_bio || if (!dio->submit.poll_bio ||
!bio_poll(dio->submit.poll_bio, 0)) !bio_poll(dio->submit.poll_bio, NULL, 0))
blk_io_schedule(); blk_io_schedule();
} }
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
......
...@@ -532,7 +532,7 @@ struct blk_mq_ops { ...@@ -532,7 +532,7 @@ struct blk_mq_ops {
/** /**
* @poll: Called to poll for completion of a specific tag. * @poll: Called to poll for completion of a specific tag.
*/ */
int (*poll)(struct blk_mq_hw_ctx *); int (*poll)(struct blk_mq_hw_ctx *, struct io_comp_batch *);
/** /**
* @complete: Mark the request as complete. * @complete: Mark the request as complete.
......
...@@ -569,8 +569,9 @@ blk_status_t errno_to_blk_status(int errno); ...@@ -569,8 +569,9 @@ blk_status_t errno_to_blk_status(int errno);
#define BLK_POLL_ONESHOT (1 << 0) #define BLK_POLL_ONESHOT (1 << 0)
/* do not sleep to wait for the expected completion time */ /* do not sleep to wait for the expected completion time */
#define BLK_POLL_NOSLEEP (1 << 1) #define BLK_POLL_NOSLEEP (1 << 1)
int bio_poll(struct bio *bio, unsigned int flags); int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags); int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
unsigned int flags);
static inline struct request_queue *bdev_get_queue(struct block_device *bdev) static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
{ {
...@@ -1298,6 +1299,14 @@ int fsync_bdev(struct block_device *bdev); ...@@ -1298,6 +1299,14 @@ int fsync_bdev(struct block_device *bdev);
int freeze_bdev(struct block_device *bdev); int freeze_bdev(struct block_device *bdev);
int thaw_bdev(struct block_device *bdev); int thaw_bdev(struct block_device *bdev);
struct io_comp_batch {
struct request *req_list;
bool need_ts;
void (*complete)(struct io_comp_batch *);
};
#define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { }
#define rq_list_add(listptr, rq) do { \ #define rq_list_add(listptr, rq) do { \
(rq)->rq_next = *(listptr); \ (rq)->rq_next = *(listptr); \
*(listptr) = rq; \ *(listptr) = rq; \
......
...@@ -48,6 +48,7 @@ ...@@ -48,6 +48,7 @@
struct backing_dev_info; struct backing_dev_info;
struct bdi_writeback; struct bdi_writeback;
struct bio; struct bio;
struct io_comp_batch;
struct export_operations; struct export_operations;
struct fiemap_extent_info; struct fiemap_extent_info;
struct hd_geometry; struct hd_geometry;
...@@ -2071,7 +2072,8 @@ struct file_operations { ...@@ -2071,7 +2072,8 @@ struct file_operations {
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *); ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
ssize_t (*read_iter) (struct kiocb *, struct iov_iter *); ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
ssize_t (*write_iter) (struct kiocb *, struct iov_iter *); ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
int (*iopoll)(struct kiocb *kiocb, unsigned int flags); int (*iopoll)(struct kiocb *kiocb, struct io_comp_batch *,
unsigned int flags);
int (*iterate) (struct file *, struct dir_context *); int (*iterate) (struct file *, struct dir_context *);
int (*iterate_shared) (struct file *, struct dir_context *); int (*iterate_shared) (struct file *, struct dir_context *);
__poll_t (*poll) (struct file *, struct poll_table_struct *); __poll_t (*poll) (struct file *, struct poll_table_struct *);
......
...@@ -424,7 +424,7 @@ int swap_readpage(struct page *page, bool synchronous) ...@@ -424,7 +424,7 @@ int swap_readpage(struct page *page, bool synchronous)
if (!READ_ONCE(bio->bi_private)) if (!READ_ONCE(bio->bi_private))
break; break;
if (!bio_poll(bio, 0)) if (!bio_poll(bio, NULL, 0))
blk_io_schedule(); blk_io_schedule();
} }
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment