Commit 5c0a85fa authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

mm: make faultaround produce old ptes

Currently, faultaround code produces young pte.  This can screw up
vmscan behaviour[1], as it makes vmscan think that these pages are hot
and not push them out on first round.

During sparse file access faultaround gets more pages mapped and all of
them are young.  Under memory pressure, this makes vmscan swap out anon
pages instead, or to drop other page cache pages which otherwise stay
resident.

Modify faultaround to produce old ptes, so they can easily be reclaimed
under memory pressure.

This can to some extend defeat the purpose of faultaround on machines
without hardware accessed bit as it will not help us with reducing the
number of minor page faults.

We may want to disable faultaround on such machines altogether, but
that's subject for separate patchset.

Minchan:
 "I tested 512M mmap sequential word read test on non-HW access bit
  system (i.e., ARM) and confirmed it doesn't increase minor fault any
  more.

  old: 4096 fault_around
  minor fault: 131291
  elapsed time: 6747645 usec

  new: 65536 fault_around
  minor fault: 131291
  elapsed time: 6709263 usec

  0.56% benefit"

[1] https://lkml.kernel.org/r/1460992636-711-1-git-send-email-vinmenon@codeaurora.org

Link: http://lkml.kernel.org/r/1463488366-47723-1-git-send-email-kirill.shutemov@linux.intel.comSigned-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarMinchan Kim <minchan@kernel.org>
Tested-by: default avatarMinchan Kim <minchan@kernel.org>
Acked-by: default avatarRik van Riel <riel@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Vinayak Menon <vinmenon@codeaurora.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4b50bcc7
...@@ -596,7 +596,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) ...@@ -596,7 +596,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
} }
void do_set_pte(struct vm_area_struct *vma, unsigned long address, void do_set_pte(struct vm_area_struct *vma, unsigned long address,
struct page *page, pte_t *pte, bool write, bool anon); struct page *page, pte_t *pte, bool write, bool anon, bool old);
#endif #endif
/* /*
......
...@@ -2191,7 +2191,7 @@ void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf) ...@@ -2191,7 +2191,7 @@ void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
if (file->f_ra.mmap_miss > 0) if (file->f_ra.mmap_miss > 0)
file->f_ra.mmap_miss--; file->f_ra.mmap_miss--;
addr = address + (page->index - vmf->pgoff) * PAGE_SIZE; addr = address + (page->index - vmf->pgoff) * PAGE_SIZE;
do_set_pte(vma, addr, page, pte, false, false); do_set_pte(vma, addr, page, pte, false, false, true);
unlock_page(page); unlock_page(page);
goto next; goto next;
unlock: unlock:
......
...@@ -2876,7 +2876,7 @@ static int __do_fault(struct vm_area_struct *vma, unsigned long address, ...@@ -2876,7 +2876,7 @@ static int __do_fault(struct vm_area_struct *vma, unsigned long address,
* vm_ops->map_pages. * vm_ops->map_pages.
*/ */
void do_set_pte(struct vm_area_struct *vma, unsigned long address, void do_set_pte(struct vm_area_struct *vma, unsigned long address,
struct page *page, pte_t *pte, bool write, bool anon) struct page *page, pte_t *pte, bool write, bool anon, bool old)
{ {
pte_t entry; pte_t entry;
...@@ -2884,6 +2884,8 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address, ...@@ -2884,6 +2884,8 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address,
entry = mk_pte(page, vma->vm_page_prot); entry = mk_pte(page, vma->vm_page_prot);
if (write) if (write)
entry = maybe_mkwrite(pte_mkdirty(entry), vma); entry = maybe_mkwrite(pte_mkdirty(entry), vma);
if (old)
entry = pte_mkold(entry);
if (anon) { if (anon) {
inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
page_add_new_anon_rmap(page, vma, address, false); page_add_new_anon_rmap(page, vma, address, false);
...@@ -3021,9 +3023,20 @@ static int do_read_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3021,9 +3023,20 @@ static int do_read_fault(struct mm_struct *mm, struct vm_area_struct *vma,
*/ */
if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) { if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
pte = pte_offset_map_lock(mm, pmd, address, &ptl); pte = pte_offset_map_lock(mm, pmd, address, &ptl);
do_fault_around(vma, address, pte, pgoff, flags);
if (!pte_same(*pte, orig_pte)) if (!pte_same(*pte, orig_pte))
goto unlock_out; goto unlock_out;
do_fault_around(vma, address, pte, pgoff, flags);
/* Check if the fault is handled by faultaround */
if (!pte_same(*pte, orig_pte)) {
/*
* Faultaround produce old pte, but the pte we've
* handler fault for should be young.
*/
pte_t entry = pte_mkyoung(*pte);
if (ptep_set_access_flags(vma, address, pte, entry, 0))
update_mmu_cache(vma, address, pte);
goto unlock_out;
}
pte_unmap_unlock(pte, ptl); pte_unmap_unlock(pte, ptl);
} }
...@@ -3038,7 +3051,7 @@ static int do_read_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3038,7 +3051,7 @@ static int do_read_fault(struct mm_struct *mm, struct vm_area_struct *vma,
put_page(fault_page); put_page(fault_page);
return ret; return ret;
} }
do_set_pte(vma, address, fault_page, pte, false, false); do_set_pte(vma, address, fault_page, pte, false, false, false);
unlock_page(fault_page); unlock_page(fault_page);
unlock_out: unlock_out:
pte_unmap_unlock(pte, ptl); pte_unmap_unlock(pte, ptl);
...@@ -3090,7 +3103,7 @@ static int do_cow_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3090,7 +3103,7 @@ static int do_cow_fault(struct mm_struct *mm, struct vm_area_struct *vma,
} }
goto uncharge_out; goto uncharge_out;
} }
do_set_pte(vma, address, new_page, pte, true, true); do_set_pte(vma, address, new_page, pte, true, true, false);
mem_cgroup_commit_charge(new_page, memcg, false, false); mem_cgroup_commit_charge(new_page, memcg, false, false);
lru_cache_add_active_or_unevictable(new_page, vma); lru_cache_add_active_or_unevictable(new_page, vma);
pte_unmap_unlock(pte, ptl); pte_unmap_unlock(pte, ptl);
...@@ -3147,7 +3160,7 @@ static int do_shared_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -3147,7 +3160,7 @@ static int do_shared_fault(struct mm_struct *mm, struct vm_area_struct *vma,
put_page(fault_page); put_page(fault_page);
return ret; return ret;
} }
do_set_pte(vma, address, fault_page, pte, true, false); do_set_pte(vma, address, fault_page, pte, true, false, false);
pte_unmap_unlock(pte, ptl); pte_unmap_unlock(pte, ptl);
if (set_page_dirty(fault_page)) if (set_page_dirty(fault_page))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment