Commit 0fd71a56 authored by Andrea Arcangeli's avatar Andrea Arcangeli Committed by Linus Torvalds

mm: gup: add __get_user_pages_unlocked to customize gup_flags

Some callers (like KVM) may want to set the gup_flags like FOLL_HWPOSION
to get a proper -EHWPOSION retval instead of -EFAULT to take a more
appropriate action if get_user_pages runs into a memory failure.
Signed-off-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Reviewed-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andres Lagar-Cavilla <andreslc@google.com>
Cc: Peter Feiner <pfeiner@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent f0818f47
...@@ -1265,6 +1265,10 @@ long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm, ...@@ -1265,6 +1265,10 @@ long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
int write, int force, struct page **pages, int write, int force, struct page **pages,
int *locked); int *locked);
long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
int write, int force, struct page **pages,
unsigned int gup_flags);
long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm, long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
int write, int force, struct page **pages); int write, int force, struct page **pages);
......
...@@ -582,9 +582,9 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, ...@@ -582,9 +582,9 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
int write, int force, int write, int force,
struct page **pages, struct page **pages,
struct vm_area_struct **vmas, struct vm_area_struct **vmas,
int *locked, bool notify_drop) int *locked, bool notify_drop,
unsigned int flags)
{ {
int flags = FOLL_TOUCH;
long ret, pages_done; long ret, pages_done;
bool lock_dropped; bool lock_dropped;
...@@ -698,10 +698,36 @@ long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm, ...@@ -698,10 +698,36 @@ long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
int *locked) int *locked)
{ {
return __get_user_pages_locked(tsk, mm, start, nr_pages, write, force, return __get_user_pages_locked(tsk, mm, start, nr_pages, write, force,
pages, NULL, locked, true); pages, NULL, locked, true, FOLL_TOUCH);
} }
EXPORT_SYMBOL(get_user_pages_locked); EXPORT_SYMBOL(get_user_pages_locked);
/*
* Same as get_user_pages_unlocked(...., FOLL_TOUCH) but it allows to
* pass additional gup_flags as last parameter (like FOLL_HWPOISON).
*
* NOTE: here FOLL_TOUCH is not set implicitly and must be set by the
* caller if required (just like with __get_user_pages). "FOLL_GET",
* "FOLL_WRITE" and "FOLL_FORCE" are set implicitly as needed
* according to the parameters "pages", "write", "force"
* respectively.
*/
__always_inline long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
int write, int force, struct page **pages,
unsigned int gup_flags)
{
long ret;
int locked = 1;
down_read(&mm->mmap_sem);
ret = __get_user_pages_locked(tsk, mm, start, nr_pages, write, force,
pages, NULL, &locked, false, gup_flags);
if (locked)
up_read(&mm->mmap_sem);
return ret;
}
EXPORT_SYMBOL(__get_user_pages_unlocked);
/* /*
* get_user_pages_unlocked() is suitable to replace the form: * get_user_pages_unlocked() is suitable to replace the form:
* *
...@@ -723,14 +749,8 @@ long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm, ...@@ -723,14 +749,8 @@ long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
int write, int force, struct page **pages) int write, int force, struct page **pages)
{ {
long ret; return __get_user_pages_unlocked(tsk, mm, start, nr_pages, write,
int locked = 1; force, pages, FOLL_TOUCH);
down_read(&mm->mmap_sem);
ret = __get_user_pages_locked(tsk, mm, start, nr_pages, write, force,
pages, NULL, &locked, false);
if (locked)
up_read(&mm->mmap_sem);
return ret;
} }
EXPORT_SYMBOL(get_user_pages_unlocked); EXPORT_SYMBOL(get_user_pages_unlocked);
...@@ -794,7 +814,7 @@ long get_user_pages(struct task_struct *tsk, struct mm_struct *mm, ...@@ -794,7 +814,7 @@ long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
int force, struct page **pages, struct vm_area_struct **vmas) int force, struct page **pages, struct vm_area_struct **vmas)
{ {
return __get_user_pages_locked(tsk, mm, start, nr_pages, write, force, return __get_user_pages_locked(tsk, mm, start, nr_pages, write, force,
pages, vmas, NULL, false); pages, vmas, NULL, false, FOLL_TOUCH);
} }
EXPORT_SYMBOL(get_user_pages); EXPORT_SYMBOL(get_user_pages);
......
...@@ -224,9 +224,10 @@ long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm, ...@@ -224,9 +224,10 @@ long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
} }
EXPORT_SYMBOL(get_user_pages_locked); EXPORT_SYMBOL(get_user_pages_locked);
long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm, long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
int write, int force, struct page **pages) int write, int force, struct page **pages,
unsigned int gup_flags)
{ {
long ret; long ret;
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
...@@ -235,6 +236,15 @@ long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm, ...@@ -235,6 +236,15 @@ long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
return ret; return ret;
} }
EXPORT_SYMBOL(__get_user_pages_unlocked);
long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
int write, int force, struct page **pages)
{
return __get_user_pages_unlocked(tsk, mm, start, nr_pages, write,
force, pages, 0);
}
EXPORT_SYMBOL(get_user_pages_unlocked); EXPORT_SYMBOL(get_user_pages_unlocked);
/** /**
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment