Commit 451c0ba9 authored by Al Viro's avatar Al Viro

unify the rest of iov_iter_get_pages()/iov_iter_get_pages_alloc() guts

same as for pipes and xarrays; after that iov_iter_get_pages() becomes
a wrapper for __iov_iter_get_pages_alloc().
Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent 68fe506f
...@@ -1448,19 +1448,18 @@ static struct page *first_bvec_segment(const struct iov_iter *i, ...@@ -1448,19 +1448,18 @@ static struct page *first_bvec_segment(const struct iov_iter *i,
return page; return page;
} }
ssize_t iov_iter_get_pages(struct iov_iter *i, static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
struct page **pages, size_t maxsize, unsigned maxpages, struct page ***pages, size_t maxsize,
size_t *start) unsigned int maxpages, size_t *start)
{ {
int n, res; int n, res;
if (maxsize > i->count) if (maxsize > i->count)
maxsize = i->count; maxsize = i->count;
if (!maxsize || !maxpages) if (!maxsize)
return 0; return 0;
if (maxsize > MAX_RW_COUNT) if (maxsize > MAX_RW_COUNT)
maxsize = MAX_RW_COUNT; maxsize = MAX_RW_COUNT;
BUG_ON(!pages);
if (likely(user_backed_iter(i))) { if (likely(user_backed_iter(i))) {
unsigned int gup_flags = 0; unsigned int gup_flags = 0;
...@@ -1477,83 +1476,52 @@ ssize_t iov_iter_get_pages(struct iov_iter *i, ...@@ -1477,83 +1476,52 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
if (n > maxpages) if (n > maxpages)
n = maxpages; n = maxpages;
res = get_user_pages_fast(addr, n, gup_flags, pages); if (!*pages) {
*pages = get_pages_array(n);
if (!*pages)
return -ENOMEM;
}
res = get_user_pages_fast(addr, n, gup_flags, *pages);
if (unlikely(res <= 0)) if (unlikely(res <= 0))
return res; return res;
return min_t(size_t, maxsize, res * PAGE_SIZE - *start); return min_t(size_t, maxsize, res * PAGE_SIZE - *start);
} }
if (iov_iter_is_bvec(i)) { if (iov_iter_is_bvec(i)) {
struct page **p;
struct page *page; struct page *page;
page = first_bvec_segment(i, &maxsize, start); page = first_bvec_segment(i, &maxsize, start);
n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
if (n > maxpages) if (n > maxpages)
n = maxpages; n = maxpages;
p = *pages;
if (!p) {
*pages = p = get_pages_array(n);
if (!p)
return -ENOMEM;
}
for (int k = 0; k < n; k++) for (int k = 0; k < n; k++)
get_page(*pages++ = page++); get_page(*p++ = page++);
return min_t(size_t, maxsize, n * PAGE_SIZE - *start); return min_t(size_t, maxsize, n * PAGE_SIZE - *start);
} }
if (iov_iter_is_pipe(i)) if (iov_iter_is_pipe(i))
return pipe_get_pages(i, &pages, maxsize, maxpages, start); return pipe_get_pages(i, pages, maxsize, maxpages, start);
if (iov_iter_is_xarray(i)) if (iov_iter_is_xarray(i))
return iter_xarray_get_pages(i, &pages, maxsize, maxpages, start); return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);
return -EFAULT; return -EFAULT;
} }
EXPORT_SYMBOL(iov_iter_get_pages);
static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i, ssize_t iov_iter_get_pages(struct iov_iter *i,
struct page ***pages, size_t maxsize, struct page **pages, size_t maxsize, unsigned maxpages,
size_t *start) size_t *start)
{ {
struct page **p; if (!maxpages)
int n, res;
if (maxsize > i->count)
maxsize = i->count;
if (!maxsize)
return 0; return 0;
if (maxsize > MAX_RW_COUNT) BUG_ON(!pages);
maxsize = MAX_RW_COUNT;
if (likely(user_backed_iter(i))) {
unsigned int gup_flags = 0;
unsigned long addr;
if (iov_iter_rw(i) != WRITE)
gup_flags |= FOLL_WRITE;
if (i->nofault)
gup_flags |= FOLL_NOFAULT;
addr = first_iovec_segment(i, &maxsize);
*start = addr % PAGE_SIZE;
addr &= PAGE_MASK;
n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
*pages = p = get_pages_array(n);
if (!p)
return -ENOMEM;
res = get_user_pages_fast(addr, n, gup_flags, p);
if (unlikely(res <= 0))
return res;
return min_t(size_t, maxsize, res * PAGE_SIZE - *start);
}
if (iov_iter_is_bvec(i)) {
struct page *page;
page = first_bvec_segment(i, &maxsize, start); return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start);
n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
*pages = p = get_pages_array(n);
if (!p)
return -ENOMEM;
for (int k = 0; k < n; k++)
get_page(*p++ = page++);
return min_t(size_t, maxsize, n * PAGE_SIZE - *start);
}
if (iov_iter_is_pipe(i))
return pipe_get_pages(i, pages, maxsize, ~0U, start);
if (iov_iter_is_xarray(i))
return iter_xarray_get_pages(i, pages, maxsize, ~0U, start);
return -EFAULT;
} }
EXPORT_SYMBOL(iov_iter_get_pages);
ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
struct page ***pages, size_t maxsize, struct page ***pages, size_t maxsize,
...@@ -1563,7 +1531,7 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, ...@@ -1563,7 +1531,7 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
*pages = NULL; *pages = NULL;
len = __iov_iter_get_pages_alloc(i, pages, maxsize, start); len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start);
if (len <= 0) { if (len <= 0) {
kvfree(*pages); kvfree(*pages);
*pages = NULL; *pages = NULL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment