Commit b9dff219 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'iter-ubuf.2-2023-04-21' of git://git.kernel.dk/linux

Pull ITER_UBUF updates from Jens Axboe:
 "This turns singe vector imports into ITER_UBUF, rather than
  ITER_IOVEC.

  The former is more trivial to iterate and advance, and hence a bit
  more efficient. From some very unscientific testing, ~60% of all iovec
  imports are single vector"

* tag 'iter-ubuf.2-2023-04-21' of git://git.kernel.dk/linux:
  iov_iter: Mark copy_compat_iovec_from_user() noinline
  iov_iter: import single vector iovecs as ITER_UBUF
  iov_iter: convert import_single_range() to ITER_UBUF
  iov_iter: overlay struct iovec and ubuf/len
  iov_iter: set nr_segs = 1 for ITER_UBUF
  iov_iter: remove iov_iter_iovec()
  iov_iter: add iter_iov_addr() and iter_iov_len() helpers
  ALSA: pcm: check for user backed iterator, not specific iterator type
  IB/qib: check for user backed iterator, not specific iterator type
  IB/hfi1: check for user backed iterator, not specific iterator type
  iov_iter: add iter_iovec() helper
  block: ensure bio_alloc_map_data() deals with ITER_UBUF correctly
parents d88867a2 50f9a76e
...@@ -29,10 +29,11 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, ...@@ -29,10 +29,11 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,
bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask);
if (!bmd) if (!bmd)
return NULL; return NULL;
memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);
bmd->iter = *data; bmd->iter = *data;
if (iter_is_iovec(data)) if (iter_is_iovec(data)) {
bmd->iter.iov = bmd->iov; memcpy(bmd->iov, iter_iov(data), sizeof(struct iovec) * data->nr_segs);
bmd->iter.__iov = bmd->iov;
}
return bmd; return bmd;
} }
......
...@@ -267,6 +267,8 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from) ...@@ -267,6 +267,8 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
if (!HFI1_CAP_IS_KSET(SDMA)) if (!HFI1_CAP_IS_KSET(SDMA))
return -EINVAL; return -EINVAL;
if (!from->user_backed)
return -EINVAL;
idx = srcu_read_lock(&fd->pq_srcu); idx = srcu_read_lock(&fd->pq_srcu);
pq = srcu_dereference(fd->pq, &fd->pq_srcu); pq = srcu_dereference(fd->pq, &fd->pq_srcu);
if (!cq || !pq) { if (!cq || !pq) {
...@@ -274,11 +276,6 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from) ...@@ -274,11 +276,6 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
return -EIO; return -EIO;
} }
if (!iter_is_iovec(from) || !dim) {
srcu_read_unlock(&fd->pq_srcu, idx);
return -EINVAL;
}
trace_hfi1_sdma_request(fd->dd, fd->uctxt->ctxt, fd->subctxt, dim); trace_hfi1_sdma_request(fd->dd, fd->uctxt->ctxt, fd->subctxt, dim);
if (atomic_read(&pq->n_reqs) == pq->n_max_reqs) { if (atomic_read(&pq->n_reqs) == pq->n_max_reqs) {
...@@ -287,11 +284,12 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from) ...@@ -287,11 +284,12 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
} }
while (dim) { while (dim) {
const struct iovec *iov = iter_iov(from);
int ret; int ret;
unsigned long count = 0; unsigned long count = 0;
ret = hfi1_user_sdma_process_request( ret = hfi1_user_sdma_process_request(
fd, (struct iovec *)(from->iov + done), fd, (struct iovec *)(iov + done),
dim, &count); dim, &count);
if (ret) { if (ret) {
reqs = ret; reqs = ret;
......
...@@ -2245,10 +2245,10 @@ static ssize_t qib_write_iter(struct kiocb *iocb, struct iov_iter *from) ...@@ -2245,10 +2245,10 @@ static ssize_t qib_write_iter(struct kiocb *iocb, struct iov_iter *from)
struct qib_ctxtdata *rcd = ctxt_fp(iocb->ki_filp); struct qib_ctxtdata *rcd = ctxt_fp(iocb->ki_filp);
struct qib_user_sdma_queue *pq = fp->pq; struct qib_user_sdma_queue *pq = fp->pq;
if (!iter_is_iovec(from) || !from->nr_segs || !pq) if (!from->user_backed || !from->nr_segs || !pq)
return -EINVAL; return -EINVAL;
return qib_user_sdma_writev(rcd, pq, from->iov, from->nr_segs); return qib_user_sdma_writev(rcd, pq, iter_iov(from), from->nr_segs);
} }
static struct class *qib_class; static struct class *qib_class;
......
...@@ -1486,7 +1486,8 @@ static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, ...@@ -1486,7 +1486,8 @@ static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
skb->truesize += skb->data_len; skb->truesize += skb->data_len;
for (i = 1; i < it->nr_segs; i++) { for (i = 1; i < it->nr_segs; i++) {
size_t fragsz = it->iov[i].iov_len; const struct iovec *iov = iter_iov(it);
size_t fragsz = iov->iov_len;
struct page *page; struct page *page;
void *frag; void *frag;
......
...@@ -665,7 +665,7 @@ vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls) ...@@ -665,7 +665,7 @@ vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
{ {
int sgl_count = 0; int sgl_count = 0;
if (!iter || !iter->iov) { if (!iter || !iter_iov(iter)) {
pr_err("%s: iter->iov is NULL, but expected bytes: %zu" pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
" present\n", __func__, bytes); " present\n", __func__, bytes);
return -EINVAL; return -EINVAL;
......
...@@ -3730,10 +3730,15 @@ static int check_direct_read(struct btrfs_fs_info *fs_info, ...@@ -3730,10 +3730,15 @@ static int check_direct_read(struct btrfs_fs_info *fs_info,
if (!iter_is_iovec(iter)) if (!iter_is_iovec(iter))
return 0; return 0;
for (seg = 0; seg < iter->nr_segs; seg++) for (seg = 0; seg < iter->nr_segs; seg++) {
for (i = seg + 1; i < iter->nr_segs; i++) for (i = seg + 1; i < iter->nr_segs; i++) {
if (iter->iov[seg].iov_base == iter->iov[i].iov_base) const struct iovec *iov1 = iter_iov(iter) + seg;
const struct iovec *iov2 = iter_iov(iter) + i;
if (iov1->iov_base == iov2->iov_base)
return -EINVAL; return -EINVAL;
}
}
return 0; return 0;
} }
......
...@@ -1419,7 +1419,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) ...@@ -1419,7 +1419,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii) static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
{ {
return (unsigned long)ii->iov->iov_base + ii->iov_offset; return (unsigned long)iter_iov(ii)->iov_base + ii->iov_offset;
} }
static inline size_t fuse_get_frag_size(const struct iov_iter *ii, static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
......
...@@ -749,15 +749,14 @@ static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter, ...@@ -749,15 +749,14 @@ static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
return -EOPNOTSUPP; return -EOPNOTSUPP;
while (iov_iter_count(iter)) { while (iov_iter_count(iter)) {
struct iovec iovec = iov_iter_iovec(iter);
ssize_t nr; ssize_t nr;
if (type == READ) { if (type == READ) {
nr = filp->f_op->read(filp, iovec.iov_base, nr = filp->f_op->read(filp, iter_iov_addr(iter),
iovec.iov_len, ppos); iter_iov_len(iter), ppos);
} else { } else {
nr = filp->f_op->write(filp, iovec.iov_base, nr = filp->f_op->write(filp, iter_iov_addr(iter),
iovec.iov_len, ppos); iter_iov_len(iter), ppos);
} }
if (nr < 0) { if (nr < 0) {
...@@ -766,7 +765,7 @@ static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter, ...@@ -766,7 +765,7 @@ static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
break; break;
} }
ret += nr; ret += nr;
if (nr != iovec.iov_len) if (nr != iter_iov_len(iter))
break; break;
iov_iter_advance(iter, nr); iov_iter_advance(iter, nr);
} }
......
...@@ -49,15 +49,36 @@ struct iov_iter { ...@@ -49,15 +49,36 @@ struct iov_iter {
size_t iov_offset; size_t iov_offset;
int last_offset; int last_offset;
}; };
size_t count; /*
* Hack alert: overlay ubuf_iovec with iovec + count, so
* that the members resolve correctly regardless of the type
* of iterator used. This means that you can use:
*
* &iter->__ubuf_iovec or iter->__iov
*
* interchangably for the user_backed cases, hence simplifying
* some of the cases that need to deal with both.
*/
union { union {
const struct iovec *iov; /*
* This really should be a const, but we cannot do that without
* also modifying any of the zero-filling iter init functions.
* Leave it non-const for now, but it should be treated as such.
*/
struct iovec __ubuf_iovec;
struct {
union {
/* use iter_iov() to get the current vec */
const struct iovec *__iov;
const struct kvec *kvec; const struct kvec *kvec;
const struct bio_vec *bvec; const struct bio_vec *bvec;
struct xarray *xarray; struct xarray *xarray;
struct pipe_inode_info *pipe; struct pipe_inode_info *pipe;
void __user *ubuf; void __user *ubuf;
}; };
size_t count;
};
};
union { union {
unsigned long nr_segs; unsigned long nr_segs;
struct { struct {
...@@ -68,6 +89,16 @@ struct iov_iter { ...@@ -68,6 +89,16 @@ struct iov_iter {
}; };
}; };
static inline const struct iovec *iter_iov(const struct iov_iter *iter)
{
if (iter->iter_type == ITER_UBUF)
return (const struct iovec *) &iter->__ubuf_iovec;
return iter->__iov;
}
#define iter_iov_addr(iter) (iter_iov(iter)->iov_base + (iter)->iov_offset)
#define iter_iov_len(iter) (iter_iov(iter)->iov_len - (iter)->iov_offset)
static inline enum iter_type iov_iter_type(const struct iov_iter *i) static inline enum iter_type iov_iter_type(const struct iov_iter *i)
{ {
return i->iter_type; return i->iter_type;
...@@ -143,15 +174,6 @@ static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs) ...@@ -143,15 +174,6 @@ static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
return ret; return ret;
} }
static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
{
return (struct iovec) {
.iov_base = iter->iov->iov_base + iter->iov_offset,
.iov_len = min(iter->count,
iter->iov->iov_len - iter->iov_offset),
};
}
size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t copy_page_from_iter_atomic(struct page *page, unsigned offset,
size_t bytes, struct iov_iter *i); size_t bytes, struct iov_iter *i);
void iov_iter_advance(struct iov_iter *i, size_t bytes); void iov_iter_advance(struct iov_iter *i, size_t bytes);
...@@ -359,7 +381,8 @@ static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction, ...@@ -359,7 +381,8 @@ static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction,
.user_backed = true, .user_backed = true,
.data_source = direction, .data_source = direction,
.ubuf = buf, .ubuf = buf,
.count = count .count = count,
.nr_segs = 1
}; };
} }
/* Flags for iov_iter_get/extract_pages*() */ /* Flags for iov_iter_get/extract_pages*() */
......
...@@ -184,8 +184,8 @@ static int io_setup_async_msg(struct io_kiocb *req, ...@@ -184,8 +184,8 @@ static int io_setup_async_msg(struct io_kiocb *req,
async_msg->msg.msg_name = &async_msg->addr; async_msg->msg.msg_name = &async_msg->addr;
/* if were using fast_iov, set it to the new one */ /* if were using fast_iov, set it to the new one */
if (iter_is_iovec(&kmsg->msg.msg_iter) && !kmsg->free_iov) { if (iter_is_iovec(&kmsg->msg.msg_iter) && !kmsg->free_iov) {
size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov; size_t fast_idx = iter_iov(&kmsg->msg.msg_iter) - kmsg->fast_iov;
async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx]; async_msg->msg.msg_iter.__iov = &async_msg->fast_iov[fast_idx];
} }
return -EAGAIN; return -EAGAIN;
......
...@@ -447,26 +447,25 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter) ...@@ -447,26 +447,25 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)
ppos = io_kiocb_ppos(kiocb); ppos = io_kiocb_ppos(kiocb);
while (iov_iter_count(iter)) { while (iov_iter_count(iter)) {
struct iovec iovec; void __user *addr;
size_t len;
ssize_t nr; ssize_t nr;
if (iter_is_ubuf(iter)) { if (iter_is_ubuf(iter)) {
iovec.iov_base = iter->ubuf + iter->iov_offset; addr = iter->ubuf + iter->iov_offset;
iovec.iov_len = iov_iter_count(iter); len = iov_iter_count(iter);
} else if (!iov_iter_is_bvec(iter)) { } else if (!iov_iter_is_bvec(iter)) {
iovec = iov_iter_iovec(iter); addr = iter_iov_addr(iter);
len = iter_iov_len(iter);
} else { } else {
iovec.iov_base = u64_to_user_ptr(rw->addr); addr = u64_to_user_ptr(rw->addr);
iovec.iov_len = rw->len; len = rw->len;
} }
if (ddir == READ) { if (ddir == READ)
nr = file->f_op->read(file, iovec.iov_base, nr = file->f_op->read(file, addr, len, ppos);
iovec.iov_len, ppos); else
} else { nr = file->f_op->write(file, addr, len, ppos);
nr = file->f_op->write(file, iovec.iov_base,
iovec.iov_len, ppos);
}
if (nr < 0) { if (nr < 0) {
if (!ret) if (!ret)
...@@ -482,7 +481,7 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter) ...@@ -482,7 +481,7 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)
if (!rw->len) if (!rw->len)
break; break;
} }
if (nr != iovec.iov_len) if (nr != len)
break; break;
} }
...@@ -503,10 +502,10 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, ...@@ -503,10 +502,10 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
if (!iovec) { if (!iovec) {
unsigned iov_off = 0; unsigned iov_off = 0;
io->s.iter.iov = io->s.fast_iov; io->s.iter.__iov = io->s.fast_iov;
if (iter->iov != fast_iov) { if (iter->__iov != fast_iov) {
iov_off = iter->iov - fast_iov; iov_off = iter_iov(iter) - fast_iov;
io->s.iter.iov += iov_off; io->s.iter.__iov += iov_off;
} }
if (io->s.fast_iov != fast_iov) if (io->s.fast_iov != fast_iov)
memcpy(io->s.fast_iov + iov_off, fast_iov + iov_off, memcpy(io->s.fast_iov + iov_off, fast_iov + iov_off,
......
...@@ -126,13 +126,13 @@ __out: \ ...@@ -126,13 +126,13 @@ __out: \
iterate_buf(i, n, base, len, off, \ iterate_buf(i, n, base, len, off, \
i->ubuf, (I)) \ i->ubuf, (I)) \
} else if (likely(iter_is_iovec(i))) { \ } else if (likely(iter_is_iovec(i))) { \
const struct iovec *iov = i->iov; \ const struct iovec *iov = iter_iov(i); \
void __user *base; \ void __user *base; \
size_t len; \ size_t len; \
iterate_iovec(i, n, base, len, off, \ iterate_iovec(i, n, base, len, off, \
iov, (I)) \ iov, (I)) \
i->nr_segs -= iov - i->iov; \ i->nr_segs -= iov - iter_iov(i); \
i->iov = iov; \ i->__iov = iov; \
} else if (iov_iter_is_bvec(i)) { \ } else if (iov_iter_is_bvec(i)) { \
const struct bio_vec *bvec = i->bvec; \ const struct bio_vec *bvec = i->bvec; \
void *base; \ void *base; \
...@@ -355,7 +355,7 @@ size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) ...@@ -355,7 +355,7 @@ size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)
size_t skip; size_t skip;
size -= count; size -= count;
for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
size_t len = min(count, p->iov_len - skip); size_t len = min(count, p->iov_len - skip);
size_t ret; size_t ret;
...@@ -398,7 +398,7 @@ size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) ...@@ -398,7 +398,7 @@ size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)
size_t skip; size_t skip;
size -= count; size -= count;
for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
size_t len = min(count, p->iov_len - skip); size_t len = min(count, p->iov_len - skip);
size_t ret; size_t ret;
...@@ -425,7 +425,7 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, ...@@ -425,7 +425,7 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction,
.nofault = false, .nofault = false,
.user_backed = true, .user_backed = true,
.data_source = direction, .data_source = direction,
.iov = iov, .__iov = iov,
.nr_segs = nr_segs, .nr_segs = nr_segs,
.iov_offset = 0, .iov_offset = 0,
.count = count .count = count
...@@ -876,14 +876,14 @@ static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) ...@@ -876,14 +876,14 @@ static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)
i->count -= size; i->count -= size;
size += i->iov_offset; // from beginning of current segment size += i->iov_offset; // from beginning of current segment
for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) { for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) {
if (likely(size < iov->iov_len)) if (likely(size < iov->iov_len))
break; break;
size -= iov->iov_len; size -= iov->iov_len;
} }
i->iov_offset = size; i->iov_offset = size;
i->nr_segs -= iov - i->iov; i->nr_segs -= iov - iter_iov(i);
i->iov = iov; i->__iov = iov;
} }
void iov_iter_advance(struct iov_iter *i, size_t size) void iov_iter_advance(struct iov_iter *i, size_t size)
...@@ -958,12 +958,12 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll) ...@@ -958,12 +958,12 @@ void iov_iter_revert(struct iov_iter *i, size_t unroll)
unroll -= n; unroll -= n;
} }
} else { /* same logics for iovec and kvec */ } else { /* same logics for iovec and kvec */
const struct iovec *iov = i->iov; const struct iovec *iov = iter_iov(i);
while (1) { while (1) {
size_t n = (--iov)->iov_len; size_t n = (--iov)->iov_len;
i->nr_segs++; i->nr_segs++;
if (unroll <= n) { if (unroll <= n) {
i->iov = iov; i->__iov = iov;
i->iov_offset = n - unroll; i->iov_offset = n - unroll;
return; return;
} }
...@@ -980,7 +980,7 @@ size_t iov_iter_single_seg_count(const struct iov_iter *i) ...@@ -980,7 +980,7 @@ size_t iov_iter_single_seg_count(const struct iov_iter *i)
{ {
if (i->nr_segs > 1) { if (i->nr_segs > 1) {
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
return min(i->count, i->iov->iov_len - i->iov_offset); return min(i->count, iter_iov(i)->iov_len - i->iov_offset);
if (iov_iter_is_bvec(i)) if (iov_iter_is_bvec(i))
return min(i->count, i->bvec->bv_len - i->iov_offset); return min(i->count, i->bvec->bv_len - i->iov_offset);
} }
...@@ -1095,13 +1095,14 @@ static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask, ...@@ -1095,13 +1095,14 @@ static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
unsigned k; unsigned k;
for (k = 0; k < i->nr_segs; k++, skip = 0) { for (k = 0; k < i->nr_segs; k++, skip = 0) {
size_t len = i->iov[k].iov_len - skip; const struct iovec *iov = iter_iov(i) + k;
size_t len = iov->iov_len - skip;
if (len > size) if (len > size)
len = size; len = size;
if (len & len_mask) if (len & len_mask)
return false; return false;
if ((unsigned long)(i->iov[k].iov_base + skip) & addr_mask) if ((unsigned long)(iov->iov_base + skip) & addr_mask)
return false; return false;
size -= len; size -= len;
...@@ -1194,9 +1195,10 @@ static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) ...@@ -1194,9 +1195,10 @@ static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
unsigned k; unsigned k;
for (k = 0; k < i->nr_segs; k++, skip = 0) { for (k = 0; k < i->nr_segs; k++, skip = 0) {
size_t len = i->iov[k].iov_len - skip; const struct iovec *iov = iter_iov(i) + k;
size_t len = iov->iov_len - skip;
if (len) { if (len) {
res |= (unsigned long)i->iov[k].iov_base + skip; res |= (unsigned long)iov->iov_base + skip;
if (len > size) if (len > size)
len = size; len = size;
res |= len; res |= len;
...@@ -1273,14 +1275,15 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i) ...@@ -1273,14 +1275,15 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
return ~0U; return ~0U;
for (k = 0; k < i->nr_segs; k++) { for (k = 0; k < i->nr_segs; k++) {
if (i->iov[k].iov_len) { const struct iovec *iov = iter_iov(i) + k;
unsigned long base = (unsigned long)i->iov[k].iov_base; if (iov->iov_len) {
unsigned long base = (unsigned long)iov->iov_base;
if (v) // if not the first one if (v) // if not the first one
res |= base | v; // this start | previous end res |= base | v; // this start | previous end
v = base + i->iov[k].iov_len; v = base + iov->iov_len;
if (size <= i->iov[k].iov_len) if (size <= iov->iov_len)
break; break;
size -= i->iov[k].iov_len; size -= iov->iov_len;
} }
} }
return res; return res;
...@@ -1396,13 +1399,14 @@ static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size) ...@@ -1396,13 +1399,14 @@ static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)
return (unsigned long)i->ubuf + i->iov_offset; return (unsigned long)i->ubuf + i->iov_offset;
for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {
size_t len = i->iov[k].iov_len - skip; const struct iovec *iov = iter_iov(i) + k;
size_t len = iov->iov_len - skip;
if (unlikely(!len)) if (unlikely(!len))
continue; continue;
if (*size > len) if (*size > len)
*size = len; *size = len;
return (unsigned long)i->iov[k].iov_base + skip; return (unsigned long)iov->iov_base + skip;
} }
BUG(); // if it had been empty, we wouldn't get called BUG(); // if it had been empty, we wouldn't get called
} }
...@@ -1614,7 +1618,7 @@ static int iov_npages(const struct iov_iter *i, int maxpages) ...@@ -1614,7 +1618,7 @@ static int iov_npages(const struct iov_iter *i, int maxpages)
const struct iovec *p; const struct iovec *p;
int npages = 0; int npages = 0;
for (p = i->iov; size; skip = 0, p++) { for (p = iter_iov(i); size; skip = 0, p++) {
unsigned offs = offset_in_page(p->iov_base + skip); unsigned offs = offset_in_page(p->iov_base + skip);
size_t len = min(p->iov_len - skip, size); size_t len = min(p->iov_len - skip, size);
...@@ -1691,14 +1695,14 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) ...@@ -1691,14 +1695,14 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
flags); flags);
else if (iov_iter_is_kvec(new) || iter_is_iovec(new)) else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
/* iovec and kvec have identical layout */ /* iovec and kvec have identical layout */
return new->iov = kmemdup(new->iov, return new->__iov = kmemdup(new->__iov,
new->nr_segs * sizeof(struct iovec), new->nr_segs * sizeof(struct iovec),
flags); flags);
return NULL; return NULL;
} }
EXPORT_SYMBOL(dup_iter); EXPORT_SYMBOL(dup_iter);
static int copy_compat_iovec_from_user(struct iovec *iov, static __noclone int copy_compat_iovec_from_user(struct iovec *iov,
const struct iovec __user *uvec, unsigned long nr_segs) const struct iovec __user *uvec, unsigned long nr_segs)
{ {
const struct compat_iovec __user *uiov = const struct compat_iovec __user *uiov =
...@@ -1780,6 +1784,30 @@ struct iovec *iovec_from_user(const struct iovec __user *uvec, ...@@ -1780,6 +1784,30 @@ struct iovec *iovec_from_user(const struct iovec __user *uvec,
return iov; return iov;
} }
/*
* Single segment iovec supplied by the user, import it as ITER_UBUF.
*/
static ssize_t __import_iovec_ubuf(int type, const struct iovec __user *uvec,
struct iovec **iovp, struct iov_iter *i,
bool compat)
{
struct iovec *iov = *iovp;
ssize_t ret;
if (compat)
ret = copy_compat_iovec_from_user(iov, uvec, 1);
else
ret = copy_iovec_from_user(iov, uvec, 1);
if (unlikely(ret))
return ret;
ret = import_ubuf(type, iov->iov_base, iov->iov_len, i);
if (unlikely(ret))
return ret;
*iovp = NULL;
return i->count;
}
ssize_t __import_iovec(int type, const struct iovec __user *uvec, ssize_t __import_iovec(int type, const struct iovec __user *uvec,
unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
struct iov_iter *i, bool compat) struct iov_iter *i, bool compat)
...@@ -1788,6 +1816,9 @@ ssize_t __import_iovec(int type, const struct iovec __user *uvec, ...@@ -1788,6 +1816,9 @@ ssize_t __import_iovec(int type, const struct iovec __user *uvec,
unsigned long seg; unsigned long seg;
struct iovec *iov; struct iovec *iov;
if (nr_segs == 1)
return __import_iovec_ubuf(type, uvec, iovp, i, compat);
iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat); iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
if (IS_ERR(iov)) { if (IS_ERR(iov)) {
*iovp = NULL; *iovp = NULL;
...@@ -1866,9 +1897,7 @@ int import_single_range(int rw, void __user *buf, size_t len, ...@@ -1866,9 +1897,7 @@ int import_single_range(int rw, void __user *buf, size_t len,
if (unlikely(!access_ok(buf, len))) if (unlikely(!access_ok(buf, len)))
return -EFAULT; return -EFAULT;
iov->iov_base = buf; iov_iter_ubuf(i, rw, buf, len);
iov->iov_len = len;
iov_iter_init(i, rw, iov, 1, len);
return 0; return 0;
} }
EXPORT_SYMBOL(import_single_range); EXPORT_SYMBOL(import_single_range);
...@@ -1918,7 +1947,7 @@ void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) ...@@ -1918,7 +1947,7 @@ void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
if (iov_iter_is_bvec(i)) if (iov_iter_is_bvec(i))
i->bvec -= state->nr_segs - i->nr_segs; i->bvec -= state->nr_segs - i->nr_segs;
else else
i->iov -= state->nr_segs - i->nr_segs; i->__iov -= state->nr_segs - i->nr_segs;
i->nr_segs = state->nr_segs; i->nr_segs = state->nr_segs;
} }
......
...@@ -1456,7 +1456,7 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, ...@@ -1456,7 +1456,7 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
size_t, vlen, int, behavior, unsigned int, flags) size_t, vlen, int, behavior, unsigned int, flags)
{ {
ssize_t ret; ssize_t ret;
struct iovec iovstack[UIO_FASTIOV], iovec; struct iovec iovstack[UIO_FASTIOV];
struct iovec *iov = iovstack; struct iovec *iov = iovstack;
struct iov_iter iter; struct iov_iter iter;
struct task_struct *task; struct task_struct *task;
...@@ -1503,12 +1503,11 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, ...@@ -1503,12 +1503,11 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
total_len = iov_iter_count(&iter); total_len = iov_iter_count(&iter);
while (iov_iter_count(&iter)) { while (iov_iter_count(&iter)) {
iovec = iov_iter_iovec(&iter); ret = do_madvise(mm, (unsigned long)iter_iov_addr(&iter),
ret = do_madvise(mm, (unsigned long)iovec.iov_base, iter_iov_len(&iter), behavior);
iovec.iov_len, behavior);
if (ret < 0) if (ret < 0)
break; break;
iov_iter_advance(&iter, iovec.iov_len); iov_iter_advance(&iter, iter_iov_len(&iter));
} }
ret = (total_len - iov_iter_count(&iter)) ? : ret; ret = (total_len - iov_iter_count(&iter)) ? : ret;
......
...@@ -3521,6 +3521,7 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) ...@@ -3521,6 +3521,7 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to)
unsigned long i; unsigned long i;
void __user **bufs; void __user **bufs;
snd_pcm_uframes_t frames; snd_pcm_uframes_t frames;
const struct iovec *iov = iter_iov(to);
pcm_file = iocb->ki_filp->private_data; pcm_file = iocb->ki_filp->private_data;
substream = pcm_file->substream; substream = pcm_file->substream;
...@@ -3530,18 +3531,20 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) ...@@ -3530,18 +3531,20 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to)
if (runtime->state == SNDRV_PCM_STATE_OPEN || if (runtime->state == SNDRV_PCM_STATE_OPEN ||
runtime->state == SNDRV_PCM_STATE_DISCONNECTED) runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
return -EBADFD; return -EBADFD;
if (!iter_is_iovec(to)) if (!to->user_backed)
return -EINVAL; return -EINVAL;
if (to->nr_segs > 1024 || to->nr_segs != runtime->channels) if (to->nr_segs > 1024 || to->nr_segs != runtime->channels)
return -EINVAL; return -EINVAL;
if (!frame_aligned(runtime, to->iov->iov_len)) if (!frame_aligned(runtime, iov->iov_len))
return -EINVAL; return -EINVAL;
frames = bytes_to_samples(runtime, to->iov->iov_len); frames = bytes_to_samples(runtime, iov->iov_len);
bufs = kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); bufs = kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL);
if (bufs == NULL) if (bufs == NULL)
return -ENOMEM; return -ENOMEM;
for (i = 0; i < to->nr_segs; ++i) for (i = 0; i < to->nr_segs; ++i) {
bufs[i] = to->iov[i].iov_base; bufs[i] = iov->iov_base;
iov++;
}
result = snd_pcm_lib_readv(substream, bufs, frames); result = snd_pcm_lib_readv(substream, bufs, frames);
if (result > 0) if (result > 0)
result = frames_to_bytes(runtime, result); result = frames_to_bytes(runtime, result);
...@@ -3558,6 +3561,7 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) ...@@ -3558,6 +3561,7 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from)
unsigned long i; unsigned long i;
void __user **bufs; void __user **bufs;
snd_pcm_uframes_t frames; snd_pcm_uframes_t frames;
const struct iovec *iov = iter_iov(from);
pcm_file = iocb->ki_filp->private_data; pcm_file = iocb->ki_filp->private_data;
substream = pcm_file->substream; substream = pcm_file->substream;
...@@ -3567,17 +3571,19 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) ...@@ -3567,17 +3571,19 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from)
if (runtime->state == SNDRV_PCM_STATE_OPEN || if (runtime->state == SNDRV_PCM_STATE_OPEN ||
runtime->state == SNDRV_PCM_STATE_DISCONNECTED) runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
return -EBADFD; return -EBADFD;
if (!iter_is_iovec(from)) if (!from->user_backed)
return -EINVAL; return -EINVAL;
if (from->nr_segs > 128 || from->nr_segs != runtime->channels || if (from->nr_segs > 128 || from->nr_segs != runtime->channels ||
!frame_aligned(runtime, from->iov->iov_len)) !frame_aligned(runtime, iov->iov_len))
return -EINVAL; return -EINVAL;
frames = bytes_to_samples(runtime, from->iov->iov_len); frames = bytes_to_samples(runtime, iov->iov_len);
bufs = kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); bufs = kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL);
if (bufs == NULL) if (bufs == NULL)
return -ENOMEM; return -ENOMEM;
for (i = 0; i < from->nr_segs; ++i) for (i = 0; i < from->nr_segs; ++i) {
bufs[i] = from->iov[i].iov_base; bufs[i] = iov->iov_base;
iov++;
}
result = snd_pcm_lib_writev(substream, bufs, frames); result = snd_pcm_lib_writev(substream, bufs, frames);
if (result > 0) if (result > 0)
result = frames_to_bytes(runtime, result); result = frames_to_bytes(runtime, result);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment