Commit 974e6d66 authored by Huang Ying's avatar Huang Ying Committed by Linus Torvalds

mm, hugetlbfs: pass fault address to cow handler

This is to take better advantage of the general huge page copying
optimization.  Where, the target subpage will be copied last to avoid
the cache lines of target subpage to be evicted when copying other
subpages.  This works better if the address of the target subpage is
available when copying huge page.  So hugetlbfs page fault handlers are
changed to pass that information to hugetlb_cow().  This will benefit
workloads which don't access the begin of the hugetlbfs huge page after
the page fault under heavy cache contention.

Link: http://lkml.kernel.org/r/20180524005851.4079-5-ying.huang@intel.comSigned-off-by: default avatar"Huang, Ying" <ying.huang@intel.com>
Reviewed-by: default avatarMike Kravetz <mike.kravetz@oracle.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Shaohua Li <shli@fb.com>
Cc: Christopher Lameter <cl@linux.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: Punit Agrawal <punit.agrawal@arm.com>
Cc: Anshuman Khandual <khandual@linux.vnet.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5b7a1d40
...@@ -3509,7 +3509,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3509,7 +3509,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
* Keep the pte_same checks anyway to make transition from the mutex easier. * Keep the pte_same checks anyway to make transition from the mutex easier.
*/ */
static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long haddr, pte_t *ptep, unsigned long address, pte_t *ptep,
struct page *pagecache_page, spinlock_t *ptl) struct page *pagecache_page, spinlock_t *ptl)
{ {
pte_t pte; pte_t pte;
...@@ -3518,6 +3518,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3518,6 +3518,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
int ret = 0, outside_reserve = 0; int ret = 0, outside_reserve = 0;
unsigned long mmun_start; /* For mmu_notifiers */ unsigned long mmun_start; /* For mmu_notifiers */
unsigned long mmun_end; /* For mmu_notifiers */ unsigned long mmun_end; /* For mmu_notifiers */
unsigned long haddr = address & huge_page_mask(h);
pte = huge_ptep_get(ptep); pte = huge_ptep_get(ptep);
old_page = pte_page(pte); old_page = pte_page(pte);
...@@ -3592,7 +3593,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3592,7 +3593,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
goto out_release_all; goto out_release_all;
} }
copy_user_huge_page(new_page, old_page, haddr, vma, copy_user_huge_page(new_page, old_page, address, vma,
pages_per_huge_page(h)); pages_per_huge_page(h));
__SetPageUptodate(new_page); __SetPageUptodate(new_page);
set_page_huge_active(new_page); set_page_huge_active(new_page);
...@@ -3826,7 +3827,7 @@ static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3826,7 +3827,7 @@ static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
hugetlb_count_add(pages_per_huge_page(h), mm); hugetlb_count_add(pages_per_huge_page(h), mm);
if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
/* Optimization, do the COW without a second fault */ /* Optimization, do the COW without a second fault */
ret = hugetlb_cow(mm, vma, haddr, ptep, page, ptl); ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
} }
spin_unlock(ptl); spin_unlock(ptl);
...@@ -3980,7 +3981,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3980,7 +3981,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
if (flags & FAULT_FLAG_WRITE) { if (flags & FAULT_FLAG_WRITE) {
if (!huge_pte_write(entry)) { if (!huge_pte_write(entry)) {
ret = hugetlb_cow(mm, vma, haddr, ptep, ret = hugetlb_cow(mm, vma, address, ptep,
pagecache_page, ptl); pagecache_page, ptl);
goto out_put_page; goto out_put_page;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment