Skip to content

L
linux
Commit c7f36857 authored by Al Viro's avatar Al Viro
Browse files
Options

Merge branch 'iov_iter' into for-davem-2

parents d6b00fec aa583096
Hide whitespace changes
Inline Side-by-side
Showing with 426 additions and 638 deletions
include/linux/uio.h
View file @ c7f36857
...@@ -31,6 +31,7 @@ struct iov_iter { ...@@ -31,6 +31,7 @@ struct iov_iter {
size_t count; size_t count;
union { union {
const struct iovec *iov; const struct iovec *iov;
const struct kvec *kvec;
const struct bio_vec *bvec; const struct bio_vec *bvec;
}; };
unsigned long nr_segs; unsigned long nr_segs;
...@@ -82,10 +83,13 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, ...@@ -82,10 +83,13 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i); struct iov_iter *i);
size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i); size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i);
size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i); size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
size_t iov_iter_zero(size_t bytes, struct iov_iter *); size_t iov_iter_zero(size_t bytes, struct iov_iter *);
unsigned long iov_iter_alignment(const struct iov_iter *i); unsigned long iov_iter_alignment(const struct iov_iter *i);
void iov_iter_init(struct iov_iter *i, int direction, const struct iovec *iov, void iov_iter_init(struct iov_iter *i, int direction, const struct iovec *iov,
unsigned long nr_segs, size_t count); unsigned long nr_segs, size_t count);
void iov_iter_kvec(struct iov_iter *i, int direction, const struct kvec *iov,
unsigned long nr_segs, size_t count);
ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages, ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages,
size_t maxsize, unsigned maxpages, size_t *start); size_t maxsize, unsigned maxpages, size_t *start);
ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages, ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages,
...@@ -123,6 +127,8 @@ static inline void iov_iter_reexpand(struct iov_iter *i, size_t count) ...@@ -123,6 +127,8 @@ static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
{ {
i->count = count; i->count = count;
} }
size_t csum_and_copy_to_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
int memcpy_fromiovec(unsigned char *kdata, struct iovec *iov, int len); int memcpy_fromiovec(unsigned char *kdata, struct iovec *iov, int len);
int memcpy_toiovec(struct iovec *iov, unsigned char *kdata, int len); int memcpy_toiovec(struct iovec *iov, unsigned char *kdata, int len);
......
mm/iov_iter.c
View file @ c7f36857
...@@ -3,95 +3,136 @@ ...@@ -3,95 +3,136 @@
#include <linux/pagemap.h> #include <linux/pagemap.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/vmalloc.h> #include <linux/vmalloc.h>
#include <net/checksum.h>
static size_t copy_to_iter_iovec(void *from, size_t bytes, struct iov_iter *i)
{ #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
size_t skip, copy, left, wanted; size_t left; \
const struct iovec *iov; size_t wanted = n; \
char __user *buf; __p = i->iov; \
__v.iov_len = min(n, __p->iov_len - skip); \
if (unlikely(bytes > i->count)) if (likely(__v.iov_len)) { \
bytes = i->count; __v.iov_base = __p->iov_base + skip; \
left = (STEP); \
if (unlikely(!bytes)) __v.iov_len -= left; \
return 0; skip += __v.iov_len; \
n -= __v.iov_len; \
wanted = bytes; } else { \
iov = i->iov; left = 0; \
skip = i->iov_offset; } \
buf = iov->iov_base + skip; while (unlikely(!left && n)) { \
copy = min(bytes, iov->iov_len - skip); __p++; \
__v.iov_len = min(n, __p->iov_len); \
left = __copy_to_user(buf, from, copy); if (unlikely(!__v.iov_len)) \
copy -= left; continue; \
skip += copy; __v.iov_base = __p->iov_base; \
from += copy; left = (STEP); \
bytes -= copy; __v.iov_len -= left; \
while (unlikely(!left && bytes)) { skip = __v.iov_len; \
iov++; n -= __v.iov_len; \
buf = iov->iov_base; } \
copy = min(bytes, iov->iov_len); n = wanted - n; \
left = __copy_to_user(buf, from, copy); }
copy -= left;
skip = copy; #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
from += copy; size_t wanted = n; \
bytes -= copy; __p = i->kvec; \
} __v.iov_len = min(n, __p->iov_len - skip); \
if (likely(__v.iov_len)) { \
if (skip == iov->iov_len) { __v.iov_base = __p->iov_base + skip; \
iov++; (void)(STEP); \
skip = 0; skip += __v.iov_len; \
} n -= __v.iov_len; \
i->count -= wanted - bytes; } \
i->nr_segs -= iov - i->iov; while (unlikely(n)) { \
i->iov = iov; __p++; \
i->iov_offset = skip; __v.iov_len = min(n, __p->iov_len); \
return wanted - bytes; if (unlikely(!__v.iov_len)) \
} continue; \
__v.iov_base = __p->iov_base; \
static size_t copy_from_iter_iovec(void *to, size_t bytes, struct iov_iter *i) (void)(STEP); \
{ skip = __v.iov_len; \
size_t skip, copy, left, wanted; n -= __v.iov_len; \
const struct iovec *iov; } \
char __user *buf; n = wanted; \
}
if (unlikely(bytes > i->count))
bytes = i->count; #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
size_t wanted = n; \
if (unlikely(!bytes)) __p = i->bvec; \
return 0; __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
if (likely(__v.bv_len)) { \
wanted = bytes; __v.bv_page = __p->bv_page; \
iov = i->iov; __v.bv_offset = __p->bv_offset + skip; \
skip = i->iov_offset; (void)(STEP); \
buf = iov->iov_base + skip; skip += __v.bv_len; \
copy = min(bytes, iov->iov_len - skip); n -= __v.bv_len; \
} \
left = __copy_from_user(to, buf, copy); while (unlikely(n)) { \
copy -= left; __p++; \
skip += copy; __v.bv_len = min_t(size_t, n, __p->bv_len); \
to += copy; if (unlikely(!__v.bv_len)) \
bytes -= copy; continue; \
while (unlikely(!left && bytes)) { __v.bv_page = __p->bv_page; \
iov++; __v.bv_offset = __p->bv_offset; \
buf = iov->iov_base; (void)(STEP); \
copy = min(bytes, iov->iov_len); skip = __v.bv_len; \
left = __copy_from_user(to, buf, copy); n -= __v.bv_len; \
copy -= left; } \
skip = copy; n = wanted; \
to += copy; }
bytes -= copy;
} #define iterate_all_kinds(i, n, v, I, B, K) { \
size_t skip = i->iov_offset; \
if (skip == iov->iov_len) { if (unlikely(i->type & ITER_BVEC)) { \
iov++; const struct bio_vec *bvec; \
skip = 0; struct bio_vec v; \
} iterate_bvec(i, n, v, bvec, skip, (B)) \
i->count -= wanted - bytes; } else if (unlikely(i->type & ITER_KVEC)) { \
i->nr_segs -= iov - i->iov; const struct kvec *kvec; \
i->iov = iov; struct kvec v; \
i->iov_offset = skip; iterate_kvec(i, n, v, kvec, skip, (K)) \
return wanted - bytes; } else { \
const struct iovec *iov; \
struct iovec v; \
iterate_iovec(i, n, v, iov, skip, (I)) \
} \
}
#define iterate_and_advance(i, n, v, I, B, K) { \
size_t skip = i->iov_offset; \
if (unlikely(i->type & ITER_BVEC)) { \
const struct bio_vec *bvec; \
struct bio_vec v; \
iterate_bvec(i, n, v, bvec, skip, (B)) \
if (skip == bvec->bv_len) { \
bvec++; \
skip = 0; \
} \
i->nr_segs -= bvec - i->bvec; \
i->bvec = bvec; \
} else if (unlikely(i->type & ITER_KVEC)) { \
const struct kvec *kvec; \
struct kvec v; \
iterate_kvec(i, n, v, kvec, skip, (K)) \
if (skip == kvec->iov_len) { \
kvec++; \
skip = 0; \
} \
i->nr_segs -= kvec - i->kvec; \
i->kvec = kvec; \
} else { \
const struct iovec *iov; \
struct iovec v; \
iterate_iovec(i, n, v, iov, skip, (I)) \
if (skip == iov->iov_len) { \
iov++; \
skip = 0; \
} \
i->nr_segs -= iov - i->iov; \
i->iov = iov; \
} \
i->count -= n; \
i->iov_offset = skip; \
} }
static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
...@@ -256,134 +297,6 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t ...@@ -256,134 +297,6 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
return wanted - bytes; return wanted - bytes;
} }
static size_t zero_iovec(size_t bytes, struct iov_iter *i)
{
size_t skip, copy, left, wanted;
const struct iovec *iov;
char __user *buf;
if (unlikely(bytes > i->count))
bytes = i->count;
if (unlikely(!bytes))
return 0;
wanted = bytes;
iov = i->iov;
skip = i->iov_offset;
buf = iov->iov_base + skip;
copy = min(bytes, iov->iov_len - skip);
left = __clear_user(buf, copy);
copy -= left;
skip += copy;
bytes -= copy;
while (unlikely(!left && bytes)) {
iov++;
buf = iov->iov_base;
copy = min(bytes, iov->iov_len);
left = __clear_user(buf, copy);
copy -= left;
skip = copy;
bytes -= copy;
}
if (skip == iov->iov_len) {
iov++;
skip = 0;
}
i->count -= wanted - bytes;
i->nr_segs -= iov - i->iov;
i->iov = iov;
i->iov_offset = skip;
return wanted - bytes;
}
static size_t __iovec_copy_from_user_inatomic(char *vaddr,
const struct iovec *iov, size_t base, size_t bytes)
{
size_t copied = 0, left = 0;
while (bytes) {
char __user *buf = iov->iov_base + base;
int copy = min(bytes, iov->iov_len - base);
base = 0;
left = __copy_from_user_inatomic(vaddr, buf, copy);
copied += copy;
bytes -= copy;
vaddr += copy;
iov++;
if (unlikely(left))
break;
}
return copied - left;
}
/*
* Copy as much as we can into the page and return the number of bytes which
* were successfully copied. If a fault is encountered then return the number of
* bytes which were copied.
*/
static size_t copy_from_user_atomic_iovec(struct page *page,
struct iov_iter *i, unsigned long offset, size_t bytes)
{
char *kaddr;
size_t copied;
kaddr = kmap_atomic(page);
if (likely(i->nr_segs == 1)) {
int left;
char __user *buf = i->iov->iov_base + i->iov_offset;
left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
copied = bytes - left;
} else {
copied = __iovec_copy_from_user_inatomic(kaddr + offset,
i->iov, i->iov_offset, bytes);
}
kunmap_atomic(kaddr);
return copied;
}
static void advance_iovec(struct iov_iter *i, size_t bytes)
{
BUG_ON(i->count < bytes);
if (likely(i->nr_segs == 1)) {
i->iov_offset += bytes;
i->count -= bytes;
} else {
const struct iovec *iov = i->iov;
size_t base = i->iov_offset;
unsigned long nr_segs = i->nr_segs;
/*
* The !iov->iov_len check ensures we skip over unlikely
* zero-length segments (without overruning the iovec).
*/
while (bytes || unlikely(i->count && !iov->iov_len)) {
int copy;
copy = min(bytes, iov->iov_len - base);
BUG_ON(!i->count || i->count < copy);
i->count -= copy;
bytes -= copy;
base += copy;
if (iov->iov_len == base) {
iov++;
nr_segs--;
base = 0;
}
}
i->iov = iov;
i->iov_offset = base;
i->nr_segs = nr_segs;
}
}
/* /*
* Fault in the first iovec of the given iov_iter, to a maximum length * Fault in the first iovec of the given iov_iter, to a maximum length
* of bytes. Returns 0 on success, or non-zero if the memory could not be * of bytes. Returns 0 on success, or non-zero if the memory could not be
...@@ -395,7 +308,7 @@ static void advance_iovec(struct iov_iter *i, size_t bytes) ...@@ -395,7 +308,7 @@ static void advance_iovec(struct iov_iter *i, size_t bytes)
*/ */
int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
{ {
if (!(i->type & ITER_BVEC)) { if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
char __user *buf = i->iov->iov_base + i->iov_offset; char __user *buf = i->iov->iov_base + i->iov_offset;
bytes = min(bytes, i->iov->iov_len - i->iov_offset); bytes = min(bytes, i->iov->iov_len - i->iov_offset);
return fault_in_pages_readable(buf, bytes); return fault_in_pages_readable(buf, bytes);
...@@ -404,136 +317,25 @@ int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) ...@@ -404,136 +317,25 @@ int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
} }
EXPORT_SYMBOL(iov_iter_fault_in_readable); EXPORT_SYMBOL(iov_iter_fault_in_readable);
static unsigned long alignment_iovec(const struct iov_iter *i)
{
const struct iovec *iov = i->iov;
unsigned long res;
size_t size = i->count;
size_t n;
if (!size)
return 0;
res = (unsigned long)iov->iov_base + i->iov_offset;
n = iov->iov_len - i->iov_offset;
if (n >= size)
return res | size;
size -= n;
res |= n;
while (size > (++iov)->iov_len) {
res |= (unsigned long)iov->iov_base | iov->iov_len;
size -= iov->iov_len;
}
res |= (unsigned long)iov->iov_base | size;
return res;
}
void iov_iter_init(struct iov_iter *i, int direction, void iov_iter_init(struct iov_iter *i, int direction,
const struct iovec *iov, unsigned long nr_segs, const struct iovec *iov, unsigned long nr_segs,
size_t count) size_t count)
{ {
/* It will get better. Eventually... */ /* It will get better. Eventually... */
if (segment_eq(get_fs(), KERNEL_DS)) if (segment_eq(get_fs(), KERNEL_DS)) {
direction |= ITER_KVEC; direction |= ITER_KVEC;
i->type = direction; i->type = direction;
i->iov = iov; i->kvec = (struct kvec *)iov;
} else {
i->type = direction;
i->iov = iov;
}
i->nr_segs = nr_segs; i->nr_segs = nr_segs;
i->iov_offset = 0; i->iov_offset = 0;
i->count = count; i->count = count;
} }
EXPORT_SYMBOL(iov_iter_init); EXPORT_SYMBOL(iov_iter_init);
static ssize_t get_pages_iovec(struct iov_iter *i,
struct page **pages, size_t maxsize, unsigned maxpages,
size_t *start)
{
size_t offset = i->iov_offset;
const struct iovec *iov = i->iov;
size_t len;
unsigned long addr;
int n;
int res;
len = iov->iov_len - offset;
if (len > i->count)
len = i->count;
if (len > maxsize)
len = maxsize;
addr = (unsigned long)iov->iov_base + offset;
len += *start = addr & (PAGE_SIZE - 1);
if (len > maxpages * PAGE_SIZE)
len = maxpages * PAGE_SIZE;
addr &= ~(PAGE_SIZE - 1);
n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
if (unlikely(res < 0))
return res;
return (res == n ? len : res * PAGE_SIZE) - *start;
}
static ssize_t get_pages_alloc_iovec(struct iov_iter *i,
struct page ***pages, size_t maxsize,
size_t *start)
{
size_t offset = i->iov_offset;
const struct iovec *iov = i->iov;
size_t len;
unsigned long addr;
void *p;
int n;
int res;
len = iov->iov_len - offset;
if (len > i->count)
len = i->count;
if (len > maxsize)
len = maxsize;
addr = (unsigned long)iov->iov_base + offset;
len += *start = addr & (PAGE_SIZE - 1);
addr &= ~(PAGE_SIZE - 1);
n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
if (!p)
p = vmalloc(n * sizeof(struct page *));
if (!p)
return -ENOMEM;
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
if (unlikely(res < 0)) {
kvfree(p);
return res;
}
*pages = p;
return (res == n ? len : res * PAGE_SIZE) - *start;
}
static int iov_iter_npages_iovec(const struct iov_iter *i, int maxpages)
{
size_t offset = i->iov_offset;
size_t size = i->count;
const struct iovec *iov = i->iov;
int npages = 0;
int n;
for (n = 0; size && n < i->nr_segs; n++, iov++) {
unsigned long addr = (unsigned long)iov->iov_base + offset;
size_t len = iov->iov_len - offset;
offset = 0;
if (unlikely(!len)) /* empty segment */
continue;
if (len > size)
len = size;
npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
- addr / PAGE_SIZE;
if (npages >= maxpages) /* don't bother going further */
return maxpages;
size -= len;
offset = 0;
}
return min(npages, maxpages);
}
static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
{ {
char *from = kmap_atomic(page); char *from = kmap_atomic(page);
...@@ -555,293 +357,78 @@ static void memzero_page(struct page *page, size_t offset, size_t len) ...@@ -555,293 +357,78 @@ static void memzero_page(struct page *page, size_t offset, size_t len)
kunmap_atomic(addr); kunmap_atomic(addr);
} }
static size_t copy_to_iter_bvec(void *from, size_t bytes, struct iov_iter *i) size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i)
{ {
size_t skip, copy, wanted; char *from = addr;
const struct bio_vec *bvec;
if (unlikely(bytes > i->count)) if (unlikely(bytes > i->count))
bytes = i->count; bytes = i->count;
if (unlikely(!bytes)) if (unlikely(!bytes))
return 0; return 0;
wanted = bytes; iterate_and_advance(i, bytes, v,
bvec = i->bvec; __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
skip = i->iov_offset; v.iov_len),
copy = min_t(size_t, bytes, bvec->bv_len - skip); memcpy_to_page(v.bv_page, v.bv_offset,
(from += v.bv_len) - v.bv_len, v.bv_len),
memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
)
memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy); return bytes;
skip += copy;
from += copy;
bytes -= copy;
while (bytes) {
bvec++;
copy = min(bytes, (size_t)bvec->bv_len);
memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, copy);
skip = copy;
from += copy;
bytes -= copy;
}
if (skip == bvec->bv_len) {
bvec++;
skip = 0;
}
i->count -= wanted - bytes;
i->nr_segs -= bvec - i->bvec;
i->bvec = bvec;
i->iov_offset = skip;
return wanted - bytes;
} }
EXPORT_SYMBOL(copy_to_iter);
static size_t copy_from_iter_bvec(void *to, size_t bytes, struct iov_iter *i) size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{ {
size_t skip, copy, wanted; char *to = addr;
const struct bio_vec *bvec;
if (unlikely(bytes > i->count)) if (unlikely(bytes > i->count))
bytes = i->count; bytes = i->count;
if (unlikely(!bytes)) if (unlikely(!bytes))
return 0; return 0;
wanted = bytes; iterate_and_advance(i, bytes, v,
bvec = i->bvec; __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
skip = i->iov_offset; v.iov_len),
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
copy = min(bytes, bvec->bv_len - skip); v.bv_offset, v.bv_len),
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
)
memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy); return bytes;
to += copy;
skip += copy;
bytes -= copy;
while (bytes) {
bvec++;
copy = min(bytes, (size_t)bvec->bv_len);
memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy);
skip = copy;
to += copy;
bytes -= copy;
}
if (skip == bvec->bv_len) {
bvec++;
skip = 0;
}
i->count -= wanted;
i->nr_segs -= bvec - i->bvec;
i->bvec = bvec;
i->iov_offset = skip;
return wanted;
}
static size_t copy_page_to_iter_bvec(struct page *page, size_t offset,
size_t bytes, struct iov_iter *i)
{
void *kaddr = kmap_atomic(page);
size_t wanted = copy_to_iter_bvec(kaddr + offset, bytes, i);
kunmap_atomic(kaddr);
return wanted;
}
static size_t copy_page_from_iter_bvec(struct page *page, size_t offset,
size_t bytes, struct iov_iter *i)
{
void *kaddr = kmap_atomic(page);
size_t wanted = copy_from_iter_bvec(kaddr + offset, bytes, i);
kunmap_atomic(kaddr);
return wanted;
} }
EXPORT_SYMBOL(copy_from_iter);
static size_t zero_bvec(size_t bytes, struct iov_iter *i) size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
{ {
size_t skip, copy, wanted; char *to = addr;
const struct bio_vec *bvec;
if (unlikely(bytes > i->count)) if (unlikely(bytes > i->count))
bytes = i->count; bytes = i->count;
if (unlikely(!bytes)) if (unlikely(!bytes))
return 0; return 0;
wanted = bytes; iterate_and_advance(i, bytes, v,
bvec = i->bvec; __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
skip = i->iov_offset; v.iov_base, v.iov_len),
copy = min_t(size_t, bytes, bvec->bv_len - skip); memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
v.bv_offset, v.bv_len),
memzero_page(bvec->bv_page, skip + bvec->bv_offset, copy); memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
skip += copy; )
bytes -= copy;
while (bytes) {
bvec++;
copy = min(bytes, (size_t)bvec->bv_len);
memzero_page(bvec->bv_page, bvec->bv_offset, copy);
skip = copy;
bytes -= copy;
}
if (skip == bvec->bv_len) {
bvec++;
skip = 0;
}
i->count -= wanted - bytes;
i->nr_segs -= bvec - i->bvec;
i->bvec = bvec;
i->iov_offset = skip;
return wanted - bytes;
}
static size_t copy_from_user_bvec(struct page *page,
struct iov_iter *i, unsigned long offset, size_t bytes)
{
char *kaddr;
size_t left;
const struct bio_vec *bvec;
size_t base = i->iov_offset;
kaddr = kmap_atomic(page);
for (left = bytes, bvec = i->bvec; left; bvec++, base = 0) {
size_t copy = min(left, bvec->bv_len - base);
if (!bvec->bv_len)
continue;
memcpy_from_page(kaddr + offset, bvec->bv_page,
bvec->bv_offset + base, copy);
offset += copy;
left -= copy;
}
kunmap_atomic(kaddr);
return bytes; return bytes;
} }
EXPORT_SYMBOL(copy_from_iter_nocache);
static void advance_bvec(struct iov_iter *i, size_t bytes)
{
BUG_ON(i->count < bytes);
if (likely(i->nr_segs == 1)) {
i->iov_offset += bytes;
i->count -= bytes;
} else {
const struct bio_vec *bvec = i->bvec;
size_t base = i->iov_offset;
unsigned long nr_segs = i->nr_segs;
/*
* The !iov->iov_len check ensures we skip over unlikely
* zero-length segments (without overruning the iovec).
*/
while (bytes || unlikely(i->count && !bvec->bv_len)) {
int copy;
copy = min(bytes, bvec->bv_len - base);
BUG_ON(!i->count || i->count < copy);
i->count -= copy;
bytes -= copy;
base += copy;
if (bvec->bv_len == base) {
bvec++;
nr_segs--;
base = 0;
}
}
i->bvec = bvec;
i->iov_offset = base;
i->nr_segs = nr_segs;
}
}
static unsigned long alignment_bvec(const struct iov_iter *i)
{
const struct bio_vec *bvec = i->bvec;
unsigned long res;
size_t size = i->count;
size_t n;
if (!size)
return 0;
res = bvec->bv_offset + i->iov_offset;
n = bvec->bv_len - i->iov_offset;
if (n >= size)
return res | size;
size -= n;
res |= n;
while (size > (++bvec)->bv_len) {
res |= bvec->bv_offset | bvec->bv_len;
size -= bvec->bv_len;
}
res |= bvec->bv_offset | size;
return res;
}
static ssize_t get_pages_bvec(struct iov_iter *i,
struct page **pages, size_t maxsize, unsigned maxpages,
size_t *start)
{
const struct bio_vec *bvec = i->bvec;
size_t len = bvec->bv_len - i->iov_offset;
if (len > i->count)
len = i->count;
if (len > maxsize)
len = maxsize;
/* can't be more than PAGE_SIZE */
*start = bvec->bv_offset + i->iov_offset;
get_page(*pages = bvec->bv_page);
return len;
}
static ssize_t get_pages_alloc_bvec(struct iov_iter *i,
struct page ***pages, size_t maxsize,
size_t *start)
{
const struct bio_vec *bvec = i->bvec;
size_t len = bvec->bv_len - i->iov_offset;
if (len > i->count)
len = i->count;
if (len > maxsize)
len = maxsize;
*start = bvec->bv_offset + i->iov_offset;
*pages = kmalloc(sizeof(struct page *), GFP_KERNEL);
if (!*pages)
return -ENOMEM;
get_page(**pages = bvec->bv_page);
return len;
}
static int iov_iter_npages_bvec(const struct iov_iter *i, int maxpages)
{
size_t offset = i->iov_offset;
size_t size = i->count;
const struct bio_vec *bvec = i->bvec;
int npages = 0;
int n;
for (n = 0; size && n < i->nr_segs; n++, bvec++) {
size_t len = bvec->bv_len - offset;
offset = 0;
if (unlikely(!len)) /* empty segment */
continue;
if (len > size)
len = size;
npages++;
if (npages >= maxpages) /* don't bother going further */
return maxpages;
size -= len;
offset = 0;
}
return min(npages, maxpages);
}
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i) struct iov_iter *i)
{ {
if (i->type & ITER_BVEC) if (i->type & (ITER_BVEC|ITER_KVEC)) {
return copy_page_to_iter_bvec(page, offset, bytes, i); void *kaddr = kmap_atomic(page);
else size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
kunmap_atomic(kaddr);
return wanted;
} else
return copy_page_to_iter_iovec(page, offset, bytes, i); return copy_page_to_iter_iovec(page, offset, bytes, i);
} }
EXPORT_SYMBOL(copy_page_to_iter); EXPORT_SYMBOL(copy_page_to_iter);
...@@ -849,57 +436,53 @@ EXPORT_SYMBOL(copy_page_to_iter); ...@@ -849,57 +436,53 @@ EXPORT_SYMBOL(copy_page_to_iter);
size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i) struct iov_iter *i)
{ {
if (i->type & ITER_BVEC) if (i->type & (ITER_BVEC|ITER_KVEC)) {
return copy_page_from_iter_bvec(page, offset, bytes, i); void *kaddr = kmap_atomic(page);
else size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
kunmap_atomic(kaddr);
return wanted;
} else
return copy_page_from_iter_iovec(page, offset, bytes, i); return copy_page_from_iter_iovec(page, offset, bytes, i);
} }
EXPORT_SYMBOL(copy_page_from_iter); EXPORT_SYMBOL(copy_page_from_iter);
size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i) size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
{ {
if (i->type & ITER_BVEC) if (unlikely(bytes > i->count))
return copy_to_iter_bvec(addr, bytes, i); bytes = i->count;
else
return copy_to_iter_iovec(addr, bytes, i);
}
EXPORT_SYMBOL(copy_to_iter);
size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) if (unlikely(!bytes))
{ return 0;
if (i->type & ITER_BVEC)
return copy_from_iter_bvec(addr, bytes, i);
else
return copy_from_iter_iovec(addr, bytes, i);
}
EXPORT_SYMBOL(copy_from_iter);
size_t iov_iter_zero(size_t bytes, struct iov_iter *i) iterate_and_advance(i, bytes, v,
{ __clear_user(v.iov_base, v.iov_len),
if (i->type & ITER_BVEC) { memzero_page(v.bv_page, v.bv_offset, v.bv_len),
return zero_bvec(bytes, i); memset(v.iov_base, 0, v.iov_len)
} else { )
return zero_iovec(bytes, i);
} return bytes;
} }
EXPORT_SYMBOL(iov_iter_zero); EXPORT_SYMBOL(iov_iter_zero);
size_t iov_iter_copy_from_user_atomic(struct page *page, size_t iov_iter_copy_from_user_atomic(struct page *page,
struct iov_iter *i, unsigned long offset, size_t bytes) struct iov_iter *i, unsigned long offset, size_t bytes)
{ {
if (i->type & ITER_BVEC) char *kaddr = kmap_atomic(page), *p = kaddr + offset;
return copy_from_user_bvec(page, i, offset, bytes); iterate_all_kinds(i, bytes, v,
else __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
return copy_from_user_atomic_iovec(page, i, offset, bytes); v.iov_base, v.iov_len),
memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
v.bv_offset, v.bv_len),
memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
)
kunmap_atomic(kaddr);
return bytes;
} }
EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
void iov_iter_advance(struct iov_iter *i, size_t size) void iov_iter_advance(struct iov_iter *i, size_t size)
{ {
if (i->type & ITER_BVEC) iterate_and_advance(i, size, v, 0, 0, 0)
advance_bvec(i, size);
else
advance_iovec(i, size);
} }
EXPORT_SYMBOL(iov_iter_advance); EXPORT_SYMBOL(iov_iter_advance);
...@@ -917,12 +500,33 @@ size_t iov_iter_single_seg_count(const struct iov_iter *i) ...@@ -917,12 +500,33 @@ size_t iov_iter_single_seg_count(const struct iov_iter *i)
} }
EXPORT_SYMBOL(iov_iter_single_seg_count); EXPORT_SYMBOL(iov_iter_single_seg_count);
void iov_iter_kvec(struct iov_iter *i, int direction,
const struct kvec *iov, unsigned long nr_segs,
size_t count)
{
BUG_ON(!(direction & ITER_KVEC));
i->type = direction;
i->kvec = (struct kvec *)iov;
i->nr_segs = nr_segs;
i->iov_offset = 0;
i->count = count;
}
EXPORT_SYMBOL(iov_iter_kvec);
unsigned long iov_iter_alignment(const struct iov_iter *i) unsigned long iov_iter_alignment(const struct iov_iter *i)
{ {
if (i->type & ITER_BVEC) unsigned long res = 0;
return alignment_bvec(i); size_t size = i->count;
else
return alignment_iovec(i); if (!size)
return 0;
iterate_all_kinds(i, size, v,
(res |= (unsigned long)v.iov_base | v.iov_len, 0),
res |= v.bv_offset | v.bv_len,
res |= (unsigned long)v.iov_base | v.iov_len
)
return res;
} }
EXPORT_SYMBOL(iov_iter_alignment); EXPORT_SYMBOL(iov_iter_alignment);
...@@ -930,29 +534,207 @@ ssize_t iov_iter_get_pages(struct iov_iter *i, ...@@ -930,29 +534,207 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
struct page **pages, size_t maxsize, unsigned maxpages, struct page **pages, size_t maxsize, unsigned maxpages,
size_t *start) size_t *start)
{ {
if (i->type & ITER_BVEC) if (maxsize > i->count)
return get_pages_bvec(i, pages, maxsize, maxpages, start); maxsize = i->count;
else
return get_pages_iovec(i, pages, maxsize, maxpages, start); if (!maxsize)
return 0;
iterate_all_kinds(i, maxsize, v, ({
unsigned long addr = (unsigned long)v.iov_base;
size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
int n;
int res;
if (len > maxpages * PAGE_SIZE)
len = maxpages * PAGE_SIZE;
addr &= ~(PAGE_SIZE - 1);
n = DIV_ROUND_UP(len, PAGE_SIZE);
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
if (unlikely(res < 0))
return res;
return (res == n ? len : res * PAGE_SIZE) - *start;
0;}),({
/* can't be more than PAGE_SIZE */
*start = v.bv_offset;
get_page(*pages = v.bv_page);
return v.bv_len;
}),({
return -EFAULT;
})
)
return 0;
} }
EXPORT_SYMBOL(iov_iter_get_pages); EXPORT_SYMBOL(iov_iter_get_pages);
static struct page **get_pages_array(size_t n)
{
struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
if (!p)
p = vmalloc(n * sizeof(struct page *));
return p;
}
ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
struct page ***pages, size_t maxsize, struct page ***pages, size_t maxsize,
size_t *start) size_t *start)
{ {
if (i->type & ITER_BVEC) struct page **p;
return get_pages_alloc_bvec(i, pages, maxsize, start);
else if (maxsize > i->count)
return get_pages_alloc_iovec(i, pages, maxsize, start); maxsize = i->count;
if (!maxsize)
return 0;
iterate_all_kinds(i, maxsize, v, ({
unsigned long addr = (unsigned long)v.iov_base;
size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
int n;
int res;
addr &= ~(PAGE_SIZE - 1);
n = DIV_ROUND_UP(len, PAGE_SIZE);
p = get_pages_array(n);
if (!p)
return -ENOMEM;
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
if (unlikely(res < 0)) {
kvfree(p);
return res;
}
*pages = p;
return (res == n ? len : res * PAGE_SIZE) - *start;
0;}),({
/* can't be more than PAGE_SIZE */
*start = v.bv_offset;
*pages = p = get_pages_array(1);
if (!p)
return -ENOMEM;
get_page(*p = v.bv_page);
return v.bv_len;
}),({
return -EFAULT;
})
)
return 0;
} }
EXPORT_SYMBOL(iov_iter_get_pages_alloc); EXPORT_SYMBOL(iov_iter_get_pages_alloc);
size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
struct iov_iter *i)
{
char *to = addr;
__wsum sum, next;
size_t off = 0;
if (unlikely(bytes > i->count))
bytes = i->count;
if (unlikely(!bytes))
return 0;
sum = *csum;
iterate_and_advance(i, bytes, v, ({
int err = 0;
next = csum_and_copy_from_user(v.iov_base,
(to += v.iov_len) - v.iov_len,
v.iov_len, 0, &err);
if (!err) {
sum = csum_block_add(sum, next, off);
off += v.iov_len;
}
err ? v.iov_len : 0;
}), ({
char *p = kmap_atomic(v.bv_page);
next = csum_partial_copy_nocheck(p + v.bv_offset,
(to += v.bv_len) - v.bv_len,
v.bv_len, 0);
kunmap_atomic(p);
sum = csum_block_add(sum, next, off);
off += v.bv_len;
}),({
next = csum_partial_copy_nocheck(v.iov_base,
(to += v.iov_len) - v.iov_len,
v.iov_len, 0);
sum = csum_block_add(sum, next, off);
off += v.iov_len;
})
)
*csum = sum;
return bytes;
}
EXPORT_SYMBOL(csum_and_copy_from_iter);
size_t csum_and_copy_to_iter(void *addr, size_t bytes, __wsum *csum,
struct iov_iter *i)
{
char *from = addr;
__wsum sum, next;
size_t off = 0;
if (unlikely(bytes > i->count))
bytes = i->count;
if (unlikely(!bytes))
return 0;
sum = *csum;
iterate_and_advance(i, bytes, v, ({
int err = 0;
next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
v.iov_base,
v.iov_len, 0, &err);
if (!err) {
sum = csum_block_add(sum, next, off);
off += v.iov_len;
}
err ? v.iov_len : 0;
}), ({
char *p = kmap_atomic(v.bv_page);
next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
p + v.bv_offset,
v.bv_len, 0);
kunmap_atomic(p);
sum = csum_block_add(sum, next, off);
off += v.bv_len;
}),({
next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
v.iov_base,
v.iov_len, 0);
sum = csum_block_add(sum, next, off);
off += v.iov_len;
})
)
*csum = sum;
return bytes;
}
EXPORT_SYMBOL(csum_and_copy_to_iter);
int iov_iter_npages(const struct iov_iter *i, int maxpages) int iov_iter_npages(const struct iov_iter *i, int maxpages)
{ {
if (i->type & ITER_BVEC) size_t size = i->count;
return iov_iter_npages_bvec(i, maxpages); int npages = 0;
else
return iov_iter_npages_iovec(i, maxpages); if (!size)
return 0;
iterate_all_kinds(i, size, v, ({
unsigned long p = (unsigned long)v.iov_base;
npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
- p / PAGE_SIZE;
if (npages >= maxpages)
return maxpages;
0;}),({
npages++;
if (npages >= maxpages)
return maxpages;
}),({
unsigned long p = (unsigned long)v.iov_base;
npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
- p / PAGE_SIZE;
if (npages >= maxpages)
return maxpages;
})
)
return npages;
} }
EXPORT_SYMBOL(iov_iter_npages); EXPORT_SYMBOL(iov_iter_npages);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7