Commit 18600332 authored by Michal Hocko's avatar Michal Hocko Committed by Linus Torvalds

mm: make PR_SET_THP_DISABLE immediately active

PR_SET_THP_DISABLE has a rather subtle semantic.  It doesn't affect any
existing mapping because it only updated mm->def_flags which is a
template for new mappings.

The mappings created after prctl(PR_SET_THP_DISABLE) have VM_NOHUGEPAGE
flag set.  This can be quite surprising for all those applications which
do not do prctl(); fork() & exec() and want to control their own THP
behavior.

Another usecase when the immediate semantic of the prctl might be useful
is a combination of pre- and post-copy migration of containers with
CRIU.  In this case CRIU populates a part of a memory region with data
that was saved during the pre-copy stage.  Afterwards, the region is
registered with userfaultfd and CRIU expects to get page faults for the
parts of the region that were not yet populated.  However, khugepaged
collapses the pages and the expected page faults do not occur.

In more general case, the prctl(PR_SET_THP_DISABLE) could be used as a
temporary mechanism for enabling/disabling THP process wide.

Implementation wise, a new MMF_DISABLE_THP flag is added.  This flag is
tested when decision whether to use huge pages is taken either during
page fault of at the time of THP collapse.

It should be noted, that the new implementation makes PR_SET_THP_DISABLE
master override to any per-VMA setting, which was not the case
previously.

Fixes: a0715cc2 ("mm, thp: add VM_INIT_DEF_MASK and PRCTL_THP_DISABLE")
Link: http://lkml.kernel.org/r/1496415802-30944-1-git-send-email-rppt@linux.vnet.ibm.comSigned-off-by: default avatarMichal Hocko <mhocko@suse.com>
Signed-off-by: default avatarMike Rapoport <rppt@linux.vnet.ibm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Pavel Emelyanov <xemul@virtuozzo.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b6bb9811
...@@ -92,6 +92,7 @@ extern bool is_vma_temporary_stack(struct vm_area_struct *vma); ...@@ -92,6 +92,7 @@ extern bool is_vma_temporary_stack(struct vm_area_struct *vma);
(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \ (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \
((__vma)->vm_flags & VM_HUGEPAGE))) && \ ((__vma)->vm_flags & VM_HUGEPAGE))) && \
!((__vma)->vm_flags & VM_NOHUGEPAGE) && \ !((__vma)->vm_flags & VM_NOHUGEPAGE) && \
!test_bit(MMF_DISABLE_THP, &(__vma)->vm_mm->flags) && \
!is_vma_temporary_stack(__vma)) !is_vma_temporary_stack(__vma))
#define transparent_hugepage_use_zero_page() \ #define transparent_hugepage_use_zero_page() \
(transparent_hugepage_flags & \ (transparent_hugepage_flags & \
......
...@@ -48,7 +48,8 @@ static inline int khugepaged_enter(struct vm_area_struct *vma, ...@@ -48,7 +48,8 @@ static inline int khugepaged_enter(struct vm_area_struct *vma,
if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags)) if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags))
if ((khugepaged_always() || if ((khugepaged_always() ||
(khugepaged_req_madv() && (vm_flags & VM_HUGEPAGE))) && (khugepaged_req_madv() && (vm_flags & VM_HUGEPAGE))) &&
!(vm_flags & VM_NOHUGEPAGE)) !(vm_flags & VM_NOHUGEPAGE) &&
!test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
if (__khugepaged_enter(vma->vm_mm)) if (__khugepaged_enter(vma->vm_mm))
return -ENOMEM; return -ENOMEM;
return 0; return 0;
......
...@@ -68,7 +68,10 @@ static inline int get_dumpable(struct mm_struct *mm) ...@@ -68,7 +68,10 @@ static inline int get_dumpable(struct mm_struct *mm)
#define MMF_OOM_SKIP 21 /* mm is of no interest for the OOM killer */ #define MMF_OOM_SKIP 21 /* mm is of no interest for the OOM killer */
#define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */ #define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */
#define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */ #define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */
#define MMF_DISABLE_THP 24 /* disable THP for all VMAs */
#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP)
#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK) #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
MMF_DISABLE_THP_MASK)
#endif /* _LINUX_SCHED_COREDUMP_H */ #endif /* _LINUX_SCHED_COREDUMP_H */
...@@ -2360,7 +2360,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, ...@@ -2360,7 +2360,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
case PR_GET_THP_DISABLE: case PR_GET_THP_DISABLE:
if (arg2 || arg3 || arg4 || arg5) if (arg2 || arg3 || arg4 || arg5)
return -EINVAL; return -EINVAL;
error = !!(me->mm->def_flags & VM_NOHUGEPAGE); error = !!test_bit(MMF_DISABLE_THP, &me->mm->flags);
break; break;
case PR_SET_THP_DISABLE: case PR_SET_THP_DISABLE:
if (arg3 || arg4 || arg5) if (arg3 || arg4 || arg5)
...@@ -2368,9 +2368,9 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, ...@@ -2368,9 +2368,9 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
if (down_write_killable(&me->mm->mmap_sem)) if (down_write_killable(&me->mm->mmap_sem))
return -EINTR; return -EINTR;
if (arg2) if (arg2)
me->mm->def_flags |= VM_NOHUGEPAGE; set_bit(MMF_DISABLE_THP, &me->mm->flags);
else else
me->mm->def_flags &= ~VM_NOHUGEPAGE; clear_bit(MMF_DISABLE_THP, &me->mm->flags);
up_write(&me->mm->mmap_sem); up_write(&me->mm->mmap_sem);
break; break;
case PR_MPX_ENABLE_MANAGEMENT: case PR_MPX_ENABLE_MANAGEMENT:
......
...@@ -816,7 +816,8 @@ khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) ...@@ -816,7 +816,8 @@ khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
static bool hugepage_vma_check(struct vm_area_struct *vma) static bool hugepage_vma_check(struct vm_area_struct *vma)
{ {
if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) || if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
(vma->vm_flags & VM_NOHUGEPAGE)) (vma->vm_flags & VM_NOHUGEPAGE) ||
test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
return false; return false;
if (shmem_file(vma->vm_file)) { if (shmem_file(vma->vm_file)) {
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
......
...@@ -1977,10 +1977,12 @@ static int shmem_fault(struct vm_fault *vmf) ...@@ -1977,10 +1977,12 @@ static int shmem_fault(struct vm_fault *vmf)
} }
sgp = SGP_CACHE; sgp = SGP_CACHE;
if (vma->vm_flags & VM_HUGEPAGE)
sgp = SGP_HUGE; if ((vma->vm_flags & VM_NOHUGEPAGE) ||
else if (vma->vm_flags & VM_NOHUGEPAGE) test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
sgp = SGP_NOHUGE; sgp = SGP_NOHUGE;
else if (vma->vm_flags & VM_HUGEPAGE)
sgp = SGP_HUGE;
error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
gfp, vma, vmf, &ret); gfp, vma, vmf, &ret);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment