Commit 487c20b0 authored by Linus Torvalds's avatar Linus Torvalds

iov: improve copy_iovec_from_user() code generation

Use the same pattern as the compat version of this code does: instead of
copying the whole array to a kernel buffer and then having a separate
phase of verifying it, just do it one entry at a time, verifying as you
go.

On Jens' /dev/zero readv() test this improves performance by ~6%.

[ This was obviously triggered by Jens' ITER_UBUF updates series ]
Reported-and-tested-by: default avatarJens Axboe <axboe@kernel.dk>
Link: https://lore.kernel.org/all/de35d11d-bce7-e976-7372-1f2caf417103@kernel.dk/Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b9dff219
...@@ -1735,18 +1735,35 @@ static __noclone int copy_compat_iovec_from_user(struct iovec *iov, ...@@ -1735,18 +1735,35 @@ static __noclone int copy_compat_iovec_from_user(struct iovec *iov,
} }
static int copy_iovec_from_user(struct iovec *iov, static int copy_iovec_from_user(struct iovec *iov,
const struct iovec __user *uvec, unsigned long nr_segs) const struct iovec __user *uiov, unsigned long nr_segs)
{ {
unsigned long seg; int ret = -EFAULT;
if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec))) if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
return -EFAULT; return -EFAULT;
for (seg = 0; seg < nr_segs; seg++) {
if ((ssize_t)iov[seg].iov_len < 0)
return -EINVAL;
}
return 0; do {
void __user *buf;
ssize_t len;
unsafe_get_user(len, &uiov->iov_len, uaccess_end);
unsafe_get_user(buf, &uiov->iov_base, uaccess_end);
/* check for size_t not fitting in ssize_t .. */
if (unlikely(len < 0)) {
ret = -EINVAL;
goto uaccess_end;
}
iov->iov_base = buf;
iov->iov_len = len;
uiov++; iov++;
} while (--nr_segs);
ret = 0;
uaccess_end:
user_access_end();
return ret;
} }
struct iovec *iovec_from_user(const struct iovec __user *uvec, struct iovec *iovec_from_user(const struct iovec __user *uvec,
...@@ -1771,7 +1788,7 @@ struct iovec *iovec_from_user(const struct iovec __user *uvec, ...@@ -1771,7 +1788,7 @@ struct iovec *iovec_from_user(const struct iovec __user *uvec,
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
} }
if (compat) if (unlikely(compat))
ret = copy_compat_iovec_from_user(iov, uvec, nr_segs); ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
else else
ret = copy_iovec_from_user(iov, uvec, nr_segs); ret = copy_iovec_from_user(iov, uvec, nr_segs);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment