Commit 81214bab authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe

iomap: wire up the iopoll method

Store the request queue the last bio was submitted to in the iocb
private data in addition to the cookie so that we find the right block
device.  Also refactor the common direct I/O bio submission code into a
nice little helper.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>

Modified to use bio_set_polled().
Reviewed-by: default avatarHannes Reinecke <hare@suse.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 0bbb280d
...@@ -1280,6 +1280,7 @@ const struct file_operations gfs2_file_fops = { ...@@ -1280,6 +1280,7 @@ const struct file_operations gfs2_file_fops = {
.llseek = gfs2_llseek, .llseek = gfs2_llseek,
.read_iter = gfs2_file_read_iter, .read_iter = gfs2_file_read_iter,
.write_iter = gfs2_file_write_iter, .write_iter = gfs2_file_write_iter,
.iopoll = iomap_dio_iopoll,
.unlocked_ioctl = gfs2_ioctl, .unlocked_ioctl = gfs2_ioctl,
.mmap = gfs2_mmap, .mmap = gfs2_mmap,
.open = gfs2_open, .open = gfs2_open,
...@@ -1310,6 +1311,7 @@ const struct file_operations gfs2_file_fops_nolock = { ...@@ -1310,6 +1311,7 @@ const struct file_operations gfs2_file_fops_nolock = {
.llseek = gfs2_llseek, .llseek = gfs2_llseek,
.read_iter = gfs2_file_read_iter, .read_iter = gfs2_file_read_iter,
.write_iter = gfs2_file_write_iter, .write_iter = gfs2_file_write_iter,
.iopoll = iomap_dio_iopoll,
.unlocked_ioctl = gfs2_ioctl, .unlocked_ioctl = gfs2_ioctl,
.mmap = gfs2_mmap, .mmap = gfs2_mmap,
.open = gfs2_open, .open = gfs2_open,
......
...@@ -1464,6 +1464,28 @@ struct iomap_dio { ...@@ -1464,6 +1464,28 @@ struct iomap_dio {
}; };
}; };
int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
{
struct request_queue *q = READ_ONCE(kiocb->private);
if (!q)
return 0;
return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
}
EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap,
struct bio *bio)
{
atomic_inc(&dio->ref);
if (dio->iocb->ki_flags & IOCB_HIPRI)
bio_set_polled(bio, dio->iocb);
dio->submit.last_queue = bdev_get_queue(iomap->bdev);
dio->submit.cookie = submit_bio(bio);
}
static ssize_t iomap_dio_complete(struct iomap_dio *dio) static ssize_t iomap_dio_complete(struct iomap_dio *dio)
{ {
struct kiocb *iocb = dio->iocb; struct kiocb *iocb = dio->iocb;
...@@ -1577,7 +1599,7 @@ static void iomap_dio_bio_end_io(struct bio *bio) ...@@ -1577,7 +1599,7 @@ static void iomap_dio_bio_end_io(struct bio *bio)
} }
} }
static blk_qc_t static void
iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,
unsigned len) unsigned len)
{ {
...@@ -1591,15 +1613,10 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, ...@@ -1591,15 +1613,10 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,
bio->bi_private = dio; bio->bi_private = dio;
bio->bi_end_io = iomap_dio_bio_end_io; bio->bi_end_io = iomap_dio_bio_end_io;
if (dio->iocb->ki_flags & IOCB_HIPRI)
flags |= REQ_HIPRI;
get_page(page); get_page(page);
__bio_add_page(bio, page, len, 0); __bio_add_page(bio, page, len, 0);
bio_set_op_attrs(bio, REQ_OP_WRITE, flags); bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
iomap_dio_submit_bio(dio, iomap, bio);
atomic_inc(&dio->ref);
return submit_bio(bio);
} }
static loff_t static loff_t
...@@ -1702,9 +1719,6 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, ...@@ -1702,9 +1719,6 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
bio_set_pages_dirty(bio); bio_set_pages_dirty(bio);
} }
if (dio->iocb->ki_flags & IOCB_HIPRI)
bio->bi_opf |= REQ_HIPRI;
iov_iter_advance(dio->submit.iter, n); iov_iter_advance(dio->submit.iter, n);
dio->size += n; dio->size += n;
...@@ -1712,11 +1726,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, ...@@ -1712,11 +1726,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
copied += n; copied += n;
nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES); nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
iomap_dio_submit_bio(dio, iomap, bio);
atomic_inc(&dio->ref);
dio->submit.last_queue = bdev_get_queue(iomap->bdev);
dio->submit.cookie = submit_bio(bio);
} while (nr_pages); } while (nr_pages);
/* /*
...@@ -1927,6 +1937,9 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, ...@@ -1927,6 +1937,9 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
if (dio->flags & IOMAP_DIO_WRITE_FUA) if (dio->flags & IOMAP_DIO_WRITE_FUA)
dio->flags &= ~IOMAP_DIO_NEED_SYNC; dio->flags &= ~IOMAP_DIO_NEED_SYNC;
WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);
WRITE_ONCE(iocb->private, dio->submit.last_queue);
/* /*
* We are about to drop our additional submission reference, which * We are about to drop our additional submission reference, which
* might be the last reference to the dio. There are three three * might be the last reference to the dio. There are three three
......
...@@ -1203,6 +1203,7 @@ const struct file_operations xfs_file_operations = { ...@@ -1203,6 +1203,7 @@ const struct file_operations xfs_file_operations = {
.write_iter = xfs_file_write_iter, .write_iter = xfs_file_write_iter,
.splice_read = generic_file_splice_read, .splice_read = generic_file_splice_read,
.splice_write = iter_file_splice_write, .splice_write = iter_file_splice_write,
.iopoll = iomap_dio_iopoll,
.unlocked_ioctl = xfs_file_ioctl, .unlocked_ioctl = xfs_file_ioctl,
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
.compat_ioctl = xfs_file_compat_ioctl, .compat_ioctl = xfs_file_compat_ioctl,
......
...@@ -162,6 +162,7 @@ typedef int (iomap_dio_end_io_t)(struct kiocb *iocb, ssize_t ret, ...@@ -162,6 +162,7 @@ typedef int (iomap_dio_end_io_t)(struct kiocb *iocb, ssize_t ret,
unsigned flags); unsigned flags);
ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
const struct iomap_ops *ops, iomap_dio_end_io_t end_io); const struct iomap_ops *ops, iomap_dio_end_io_t end_io);
int iomap_dio_iopoll(struct kiocb *kiocb, bool spin);
#ifdef CONFIG_SWAP #ifdef CONFIG_SWAP
struct file; struct file;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment