Commit 293bc982 authored by Al Viro's avatar Al Viro

new methods: ->read_iter() and ->write_iter()

Beginning to introduce those.  Just the callers for now, and it's
clumsier than it'll eventually become; once we finish converting
aio_read and aio_write instances, the things will get nicer.

For now, these guys are in parallel to ->aio_read() and ->aio_write();
they take iocb and iov_iter, with everything in iov_iter already
validated.  File offset is passed in iocb->ki_pos, iov/nr_segs -
in iov_iter.

Main concerns in that series are stack footprint and ability to
split the damn thing cleanly.

[fix from Peter Ujfalusi <peter.ujfalusi@ti.com> folded]
Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent 7f7f25e8
...@@ -430,6 +430,8 @@ prototypes: ...@@ -430,6 +430,8 @@ prototypes:
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *); ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
ssize_t (*aio_read) (struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_read) (struct kiocb *, const struct iovec *, unsigned long, loff_t);
ssize_t (*aio_write) (struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_write) (struct kiocb *, const struct iovec *, unsigned long, loff_t);
ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
int (*iterate) (struct file *, struct dir_context *); int (*iterate) (struct file *, struct dir_context *);
unsigned int (*poll) (struct file *, struct poll_table_struct *); unsigned int (*poll) (struct file *, struct poll_table_struct *);
long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
......
...@@ -806,6 +806,8 @@ struct file_operations { ...@@ -806,6 +806,8 @@ struct file_operations {
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *); ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
ssize_t (*aio_read) (struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_read) (struct kiocb *, const struct iovec *, unsigned long, loff_t);
ssize_t (*aio_write) (struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_write) (struct kiocb *, const struct iovec *, unsigned long, loff_t);
ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
int (*iterate) (struct file *, struct dir_context *); int (*iterate) (struct file *, struct dir_context *);
unsigned int (*poll) (struct file *, struct poll_table_struct *); unsigned int (*poll) (struct file *, struct poll_table_struct *);
long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
...@@ -836,11 +838,15 @@ otherwise noted. ...@@ -836,11 +838,15 @@ otherwise noted.
read: called by read(2) and related system calls read: called by read(2) and related system calls
aio_read: called by io_submit(2) and other asynchronous I/O operations aio_read: vectored, possibly asynchronous read
read_iter: possibly asynchronous read with iov_iter as destination
write: called by write(2) and related system calls write: called by write(2) and related system calls
aio_write: called by io_submit(2) and other asynchronous I/O operations aio_write: vectored, possibly asynchronous write
write_iter: possibly asynchronous write with iov_iter as source
iterate: called when the VFS needs to read the directory contents iterate: called when the VFS needs to read the directory contents
......
...@@ -1241,6 +1241,7 @@ SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx) ...@@ -1241,6 +1241,7 @@ SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
typedef ssize_t (aio_rw_op)(struct kiocb *, const struct iovec *, typedef ssize_t (aio_rw_op)(struct kiocb *, const struct iovec *,
unsigned long, loff_t); unsigned long, loff_t);
typedef ssize_t (rw_iter_op)(struct kiocb *, struct iov_iter *);
static ssize_t aio_setup_vectored_rw(struct kiocb *kiocb, static ssize_t aio_setup_vectored_rw(struct kiocb *kiocb,
int rw, char __user *buf, int rw, char __user *buf,
...@@ -1298,7 +1299,9 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode, ...@@ -1298,7 +1299,9 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
int rw; int rw;
fmode_t mode; fmode_t mode;
aio_rw_op *rw_op; aio_rw_op *rw_op;
rw_iter_op *iter_op;
struct iovec inline_vec, *iovec = &inline_vec; struct iovec inline_vec, *iovec = &inline_vec;
struct iov_iter iter;
switch (opcode) { switch (opcode) {
case IOCB_CMD_PREAD: case IOCB_CMD_PREAD:
...@@ -1306,6 +1309,7 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode, ...@@ -1306,6 +1309,7 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
mode = FMODE_READ; mode = FMODE_READ;
rw = READ; rw = READ;
rw_op = file->f_op->aio_read; rw_op = file->f_op->aio_read;
iter_op = file->f_op->read_iter;
goto rw_common; goto rw_common;
case IOCB_CMD_PWRITE: case IOCB_CMD_PWRITE:
...@@ -1313,12 +1317,13 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode, ...@@ -1313,12 +1317,13 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
mode = FMODE_WRITE; mode = FMODE_WRITE;
rw = WRITE; rw = WRITE;
rw_op = file->f_op->aio_write; rw_op = file->f_op->aio_write;
iter_op = file->f_op->write_iter;
goto rw_common; goto rw_common;
rw_common: rw_common:
if (unlikely(!(file->f_mode & mode))) if (unlikely(!(file->f_mode & mode)))
return -EBADF; return -EBADF;
if (!rw_op) if (!rw_op && !iter_op)
return -EINVAL; return -EINVAL;
ret = (opcode == IOCB_CMD_PREADV || ret = (opcode == IOCB_CMD_PREADV ||
...@@ -1347,7 +1352,12 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode, ...@@ -1347,7 +1352,12 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
if (rw == WRITE) if (rw == WRITE)
file_start_write(file); file_start_write(file);
ret = rw_op(req, iovec, nr_segs, req->ki_pos); if (iter_op) {
iov_iter_init(&iter, rw, iovec, nr_segs, req->ki_nbytes);
ret = iter_op(req, &iter);
} else {
ret = rw_op(req, iovec, nr_segs, req->ki_pos);
}
if (rw == WRITE) if (rw == WRITE)
file_end_write(file); file_end_write(file);
......
...@@ -175,9 +175,11 @@ struct file *alloc_file(struct path *path, fmode_t mode, ...@@ -175,9 +175,11 @@ struct file *alloc_file(struct path *path, fmode_t mode,
file->f_path = *path; file->f_path = *path;
file->f_inode = path->dentry->d_inode; file->f_inode = path->dentry->d_inode;
file->f_mapping = path->dentry->d_inode->i_mapping; file->f_mapping = path->dentry->d_inode->i_mapping;
if ((mode & FMODE_READ) && likely(fop->read || fop->aio_read)) if ((mode & FMODE_READ) &&
likely(fop->read || fop->aio_read || fop->read_iter))
mode |= FMODE_CAN_READ; mode |= FMODE_CAN_READ;
if ((mode & FMODE_WRITE) && likely(fop->write || fop->aio_write)) if ((mode & FMODE_WRITE) &&
likely(fop->write || fop->aio_write || fop->write_iter))
mode |= FMODE_CAN_WRITE; mode |= FMODE_CAN_WRITE;
file->f_mode = mode; file->f_mode = mode;
file->f_op = fop; file->f_op = fop;
......
...@@ -725,9 +725,11 @@ static int do_dentry_open(struct file *f, ...@@ -725,9 +725,11 @@ static int do_dentry_open(struct file *f,
} }
if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ)
i_readcount_inc(inode); i_readcount_inc(inode);
if ((f->f_mode & FMODE_READ) && likely(f->f_op->read || f->f_op->aio_read)) if ((f->f_mode & FMODE_READ) &&
likely(f->f_op->read || f->f_op->aio_read || f->f_op->read_iter))
f->f_mode |= FMODE_CAN_READ; f->f_mode |= FMODE_CAN_READ;
if ((f->f_mode & FMODE_WRITE) && likely(f->f_op->write || f->f_op->aio_write)) if ((f->f_mode & FMODE_WRITE) &&
likely(f->f_op->write || f->f_op->aio_write || f->f_op->write_iter))
f->f_mode |= FMODE_CAN_WRITE; f->f_mode |= FMODE_CAN_WRITE;
f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC); f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC);
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *); typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *);
typedef ssize_t (*iov_fn_t)(struct kiocb *, const struct iovec *, typedef ssize_t (*iov_fn_t)(struct kiocb *, const struct iovec *,
unsigned long, loff_t); unsigned long, loff_t);
typedef ssize_t (*iter_fn_t)(struct kiocb *, struct iov_iter *);
const struct file_operations generic_ro_fops = { const struct file_operations generic_ro_fops = {
.llseek = generic_file_llseek, .llseek = generic_file_llseek,
...@@ -390,6 +391,27 @@ ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *pp ...@@ -390,6 +391,27 @@ ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *pp
EXPORT_SYMBOL(do_sync_read); EXPORT_SYMBOL(do_sync_read);
ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
{
struct iovec iov = { .iov_base = buf, .iov_len = len };
struct kiocb kiocb;
struct iov_iter iter;
ssize_t ret;
init_sync_kiocb(&kiocb, filp);
kiocb.ki_pos = *ppos;
kiocb.ki_nbytes = len;
iov_iter_init(&iter, READ, &iov, 1, len);
ret = filp->f_op->read_iter(&kiocb, &iter);
if (-EIOCBQUEUED == ret)
ret = wait_on_sync_kiocb(&kiocb);
*ppos = kiocb.ki_pos;
return ret;
}
EXPORT_SYMBOL(new_sync_read);
ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
{ {
ssize_t ret; ssize_t ret;
...@@ -406,8 +428,10 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) ...@@ -406,8 +428,10 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
count = ret; count = ret;
if (file->f_op->read) if (file->f_op->read)
ret = file->f_op->read(file, buf, count, pos); ret = file->f_op->read(file, buf, count, pos);
else else if (file->f_op->aio_read)
ret = do_sync_read(file, buf, count, pos); ret = do_sync_read(file, buf, count, pos);
else
ret = new_sync_read(file, buf, count, pos);
if (ret > 0) { if (ret > 0) {
fsnotify_access(file); fsnotify_access(file);
add_rchar(current, ret); add_rchar(current, ret);
...@@ -439,6 +463,27 @@ ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, lof ...@@ -439,6 +463,27 @@ ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, lof
EXPORT_SYMBOL(do_sync_write); EXPORT_SYMBOL(do_sync_write);
ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
{
struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
struct kiocb kiocb;
struct iov_iter iter;
ssize_t ret;
init_sync_kiocb(&kiocb, filp);
kiocb.ki_pos = *ppos;
kiocb.ki_nbytes = len;
iov_iter_init(&iter, WRITE, &iov, 1, len);
ret = filp->f_op->write_iter(&kiocb, &iter);
if (-EIOCBQUEUED == ret)
ret = wait_on_sync_kiocb(&kiocb);
*ppos = kiocb.ki_pos;
return ret;
}
EXPORT_SYMBOL(new_sync_write);
ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos) ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos)
{ {
mm_segment_t old_fs; mm_segment_t old_fs;
...@@ -455,8 +500,10 @@ ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t ...@@ -455,8 +500,10 @@ ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t
count = MAX_RW_COUNT; count = MAX_RW_COUNT;
if (file->f_op->write) if (file->f_op->write)
ret = file->f_op->write(file, p, count, pos); ret = file->f_op->write(file, p, count, pos);
else else if (file->f_op->aio_write)
ret = do_sync_write(file, p, count, pos); ret = do_sync_write(file, p, count, pos);
else
ret = new_sync_write(file, p, count, pos);
set_fs(old_fs); set_fs(old_fs);
if (ret > 0) { if (ret > 0) {
fsnotify_modify(file); fsnotify_modify(file);
...@@ -483,8 +530,10 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_ ...@@ -483,8 +530,10 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_
file_start_write(file); file_start_write(file);
if (file->f_op->write) if (file->f_op->write)
ret = file->f_op->write(file, buf, count, pos); ret = file->f_op->write(file, buf, count, pos);
else else if (file->f_op->aio_write)
ret = do_sync_write(file, buf, count, pos); ret = do_sync_write(file, buf, count, pos);
else
ret = new_sync_write(file, buf, count, pos);
if (ret > 0) { if (ret > 0) {
fsnotify_modify(file); fsnotify_modify(file);
add_wchar(current, ret); add_wchar(current, ret);
...@@ -601,6 +650,25 @@ unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to) ...@@ -601,6 +650,25 @@ unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
} }
EXPORT_SYMBOL(iov_shorten); EXPORT_SYMBOL(iov_shorten);
static ssize_t do_iter_readv_writev(struct file *filp, int rw, const struct iovec *iov,
unsigned long nr_segs, size_t len, loff_t *ppos, iter_fn_t fn)
{
struct kiocb kiocb;
struct iov_iter iter;
ssize_t ret;
init_sync_kiocb(&kiocb, filp);
kiocb.ki_pos = *ppos;
kiocb.ki_nbytes = len;
iov_iter_init(&iter, rw, iov, nr_segs, len);
ret = fn(&kiocb, &iter);
if (ret == -EIOCBQUEUED)
ret = wait_on_sync_kiocb(&kiocb);
*ppos = kiocb.ki_pos;
return ret;
}
static ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov, static ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn) unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
{ {
...@@ -738,6 +806,7 @@ static ssize_t do_readv_writev(int type, struct file *file, ...@@ -738,6 +806,7 @@ static ssize_t do_readv_writev(int type, struct file *file,
ssize_t ret; ssize_t ret;
io_fn_t fn; io_fn_t fn;
iov_fn_t fnv; iov_fn_t fnv;
iter_fn_t iter_fn;
ret = rw_copy_check_uvector(type, uvector, nr_segs, ret = rw_copy_check_uvector(type, uvector, nr_segs,
ARRAY_SIZE(iovstack), iovstack, &iov); ARRAY_SIZE(iovstack), iovstack, &iov);
...@@ -753,13 +822,18 @@ static ssize_t do_readv_writev(int type, struct file *file, ...@@ -753,13 +822,18 @@ static ssize_t do_readv_writev(int type, struct file *file,
if (type == READ) { if (type == READ) {
fn = file->f_op->read; fn = file->f_op->read;
fnv = file->f_op->aio_read; fnv = file->f_op->aio_read;
iter_fn = file->f_op->read_iter;
} else { } else {
fn = (io_fn_t)file->f_op->write; fn = (io_fn_t)file->f_op->write;
fnv = file->f_op->aio_write; fnv = file->f_op->aio_write;
iter_fn = file->f_op->write_iter;
file_start_write(file); file_start_write(file);
} }
if (fnv) if (iter_fn)
ret = do_iter_readv_writev(file, type, iov, nr_segs, tot_len,
pos, iter_fn);
else if (fnv)
ret = do_sync_readv_writev(file, iov, nr_segs, tot_len, ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
pos, fnv); pos, fnv);
else else
...@@ -912,6 +986,7 @@ static ssize_t compat_do_readv_writev(int type, struct file *file, ...@@ -912,6 +986,7 @@ static ssize_t compat_do_readv_writev(int type, struct file *file,
ssize_t ret; ssize_t ret;
io_fn_t fn; io_fn_t fn;
iov_fn_t fnv; iov_fn_t fnv;
iter_fn_t iter_fn;
ret = compat_rw_copy_check_uvector(type, uvector, nr_segs, ret = compat_rw_copy_check_uvector(type, uvector, nr_segs,
UIO_FASTIOV, iovstack, &iov); UIO_FASTIOV, iovstack, &iov);
...@@ -927,13 +1002,18 @@ static ssize_t compat_do_readv_writev(int type, struct file *file, ...@@ -927,13 +1002,18 @@ static ssize_t compat_do_readv_writev(int type, struct file *file,
if (type == READ) { if (type == READ) {
fn = file->f_op->read; fn = file->f_op->read;
fnv = file->f_op->aio_read; fnv = file->f_op->aio_read;
iter_fn = file->f_op->read_iter;
} else { } else {
fn = (io_fn_t)file->f_op->write; fn = (io_fn_t)file->f_op->write;
fnv = file->f_op->aio_write; fnv = file->f_op->aio_write;
iter_fn = file->f_op->write_iter;
file_start_write(file); file_start_write(file);
} }
if (fnv) if (iter_fn)
ret = do_iter_readv_writev(file, type, iov, nr_segs, tot_len,
pos, iter_fn);
else if (fnv)
ret = do_sync_readv_writev(file, iov, nr_segs, tot_len, ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
pos, fnv); pos, fnv);
else else
......
...@@ -1451,6 +1451,8 @@ struct block_device_operations; ...@@ -1451,6 +1451,8 @@ struct block_device_operations;
#define HAVE_COMPAT_IOCTL 1 #define HAVE_COMPAT_IOCTL 1
#define HAVE_UNLOCKED_IOCTL 1 #define HAVE_UNLOCKED_IOCTL 1
struct iov_iter;
struct file_operations { struct file_operations {
struct module *owner; struct module *owner;
loff_t (*llseek) (struct file *, loff_t, int); loff_t (*llseek) (struct file *, loff_t, int);
...@@ -1458,6 +1460,8 @@ struct file_operations { ...@@ -1458,6 +1460,8 @@ struct file_operations {
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *); ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
ssize_t (*aio_read) (struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_read) (struct kiocb *, const struct iovec *, unsigned long, loff_t);
ssize_t (*aio_write) (struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_write) (struct kiocb *, const struct iovec *, unsigned long, loff_t);
ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
int (*iterate) (struct file *, struct dir_context *); int (*iterate) (struct file *, struct dir_context *);
unsigned int (*poll) (struct file *, struct poll_table_struct *); unsigned int (*poll) (struct file *, struct poll_table_struct *);
long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
...@@ -2415,6 +2419,8 @@ extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *, loff ...@@ -2415,6 +2419,8 @@ extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *, loff
extern ssize_t generic_perform_write(struct file *, struct iov_iter *, loff_t); extern ssize_t generic_perform_write(struct file *, struct iov_iter *, loff_t);
extern ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos); extern ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos);
extern ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos); extern ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos);
extern ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos);
extern ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos);
/* fs/block_dev.c */ /* fs/block_dev.c */
extern ssize_t blkdev_aio_write(struct kiocb *iocb, const struct iovec *iov, extern ssize_t blkdev_aio_write(struct kiocb *iocb, const struct iovec *iov,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment