Commit 802a3a92 authored by Shaohua Li's avatar Shaohua Li Committed by Linus Torvalds

mm: reclaim MADV_FREE pages

When memory pressure is high, we free MADV_FREE pages.  If the pages are
not dirty in pte, the pages could be freed immediately.  Otherwise we
can't reclaim them.  We put the pages back to anonumous LRU list (by
setting SwapBacked flag) and the pages will be reclaimed in normal
swapout way.

We use normal page reclaim policy.  Since MADV_FREE pages are put into
inactive file list, such pages and inactive file pages are reclaimed
according to their age.  This is expected, because we don't want to
reclaim too many MADV_FREE pages before used once pages.

Based on Minchan's original patch

[minchan@kernel.org: clean up lazyfree page handling]
  Link: http://lkml.kernel.org/r/20170303025237.GB3503@bbox
Link: http://lkml.kernel.org/r/14b8eb1d3f6bf6cc492833f183ac8c304e560484.1487965799.git.shli@fb.comSigned-off-by: default avatarShaohua Li <shli@fb.com>
Signed-off-by: default avatarMinchan Kim <minchan@kernel.org>
Acked-by: default avatarMinchan Kim <minchan@kernel.org>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarHillf Danton <hillf.zj@alibaba-inc.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent f7ad2a6c
...@@ -298,6 +298,6 @@ static inline int page_mkclean(struct page *page) ...@@ -298,6 +298,6 @@ static inline int page_mkclean(struct page *page)
#define SWAP_AGAIN 1 #define SWAP_AGAIN 1
#define SWAP_FAIL 2 #define SWAP_FAIL 2
#define SWAP_MLOCK 3 #define SWAP_MLOCK 3
#define SWAP_LZFREE 4 #define SWAP_DIRTY 4
#endif /* _LINUX_RMAP_H */ #endif /* _LINUX_RMAP_H */
...@@ -1572,6 +1572,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, ...@@ -1572,6 +1572,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
set_pmd_at(mm, addr, pmd, orig_pmd); set_pmd_at(mm, addr, pmd, orig_pmd);
tlb_remove_pmd_tlb_entry(tlb, pmd, addr); tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
} }
mark_page_lazyfree(page);
ret = true; ret = true;
out: out:
spin_unlock(ptl); spin_unlock(ptl);
......
...@@ -413,6 +413,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, ...@@ -413,6 +413,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
set_pte_at(mm, addr, pte, ptent); set_pte_at(mm, addr, pte, ptent);
tlb_remove_tlb_entry(tlb, pte, addr); tlb_remove_tlb_entry(tlb, pte, addr);
} }
mark_page_lazyfree(page);
} }
out: out:
if (nr_swap) { if (nr_swap) {
......
...@@ -1288,11 +1288,6 @@ void page_remove_rmap(struct page *page, bool compound) ...@@ -1288,11 +1288,6 @@ void page_remove_rmap(struct page *page, bool compound)
*/ */
} }
struct rmap_private {
enum ttu_flags flags;
int lazyfreed;
};
/* /*
* @arg: enum ttu_flags will be passed to this argument * @arg: enum ttu_flags will be passed to this argument
*/ */
...@@ -1308,8 +1303,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1308,8 +1303,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
pte_t pteval; pte_t pteval;
struct page *subpage; struct page *subpage;
int ret = SWAP_AGAIN; int ret = SWAP_AGAIN;
struct rmap_private *rp = arg; enum ttu_flags flags = (enum ttu_flags)arg;
enum ttu_flags flags = rp->flags;
/* munlock has nothing to gain from examining un-locked vmas */ /* munlock has nothing to gain from examining un-locked vmas */
if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED)) if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
...@@ -1427,11 +1421,21 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1427,11 +1421,21 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
VM_BUG_ON_PAGE(!PageSwapCache(page) && PageSwapBacked(page), VM_BUG_ON_PAGE(!PageSwapCache(page) && PageSwapBacked(page),
page); page);
if (!PageDirty(page)) { /* MADV_FREE page check */
/* It's a freeable page by MADV_FREE */ if (!PageSwapBacked(page)) {
dec_mm_counter(mm, MM_ANONPAGES); if (!PageDirty(page)) {
rp->lazyfreed++; dec_mm_counter(mm, MM_ANONPAGES);
goto discard; goto discard;
}
/*
* If the page was redirtied, it cannot be
* discarded. Remap the page to page table.
*/
set_pte_at(mm, address, pvmw.pte, pteval);
ret = SWAP_DIRTY;
page_vma_mapped_walk_done(&pvmw);
break;
} }
if (swap_duplicate(entry) < 0) { if (swap_duplicate(entry) < 0) {
...@@ -1499,18 +1503,15 @@ static int page_mapcount_is_zero(struct page *page) ...@@ -1499,18 +1503,15 @@ static int page_mapcount_is_zero(struct page *page)
* SWAP_AGAIN - we missed a mapping, try again later * SWAP_AGAIN - we missed a mapping, try again later
* SWAP_FAIL - the page is unswappable * SWAP_FAIL - the page is unswappable
* SWAP_MLOCK - page is mlocked. * SWAP_MLOCK - page is mlocked.
* SWAP_DIRTY - page is dirty MADV_FREE page
*/ */
int try_to_unmap(struct page *page, enum ttu_flags flags) int try_to_unmap(struct page *page, enum ttu_flags flags)
{ {
int ret; int ret;
struct rmap_private rp = {
.flags = flags,
.lazyfreed = 0,
};
struct rmap_walk_control rwc = { struct rmap_walk_control rwc = {
.rmap_one = try_to_unmap_one, .rmap_one = try_to_unmap_one,
.arg = &rp, .arg = (void *)flags,
.done = page_mapcount_is_zero, .done = page_mapcount_is_zero,
.anon_lock = page_lock_anon_vma_read, .anon_lock = page_lock_anon_vma_read,
}; };
...@@ -1531,11 +1532,8 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) ...@@ -1531,11 +1532,8 @@ int try_to_unmap(struct page *page, enum ttu_flags flags)
else else
ret = rmap_walk(page, &rwc); ret = rmap_walk(page, &rwc);
if (ret != SWAP_MLOCK && !page_mapcount(page)) { if (ret != SWAP_MLOCK && !page_mapcount(page))
ret = SWAP_SUCCESS; ret = SWAP_SUCCESS;
if (rp.lazyfreed && !PageDirty(page))
ret = SWAP_LZFREE;
}
return ret; return ret;
} }
...@@ -1562,14 +1560,10 @@ static int page_not_mapped(struct page *page) ...@@ -1562,14 +1560,10 @@ static int page_not_mapped(struct page *page)
int try_to_munlock(struct page *page) int try_to_munlock(struct page *page)
{ {
int ret; int ret;
struct rmap_private rp = {
.flags = TTU_MUNLOCK,
.lazyfreed = 0,
};
struct rmap_walk_control rwc = { struct rmap_walk_control rwc = {
.rmap_one = try_to_unmap_one, .rmap_one = try_to_unmap_one,
.arg = &rp, .arg = (void *)TTU_MUNLOCK,
.done = page_not_mapped, .done = page_not_mapped,
.anon_lock = page_lock_anon_vma_read, .anon_lock = page_lock_anon_vma_read,
......
...@@ -906,7 +906,8 @@ static void page_check_dirty_writeback(struct page *page, ...@@ -906,7 +906,8 @@ static void page_check_dirty_writeback(struct page *page,
* Anonymous pages are not handled by flushers and must be written * Anonymous pages are not handled by flushers and must be written
* from reclaim context. Do not stall reclaim based on them * from reclaim context. Do not stall reclaim based on them
*/ */
if (!page_is_file_cache(page)) { if (!page_is_file_cache(page) ||
(PageAnon(page) && !PageSwapBacked(page))) {
*dirty = false; *dirty = false;
*writeback = false; *writeback = false;
return; return;
...@@ -987,7 +988,8 @@ static unsigned long shrink_page_list(struct list_head *page_list, ...@@ -987,7 +988,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
goto keep_locked; goto keep_locked;
/* Double the slab pressure for mapped and swapcache pages */ /* Double the slab pressure for mapped and swapcache pages */
if (page_mapped(page) || PageSwapCache(page)) if ((page_mapped(page) || PageSwapCache(page)) &&
!(PageAnon(page) && !PageSwapBacked(page)))
sc->nr_scanned++; sc->nr_scanned++;
may_enter_fs = (sc->gfp_mask & __GFP_FS) || may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
...@@ -1113,8 +1115,10 @@ static unsigned long shrink_page_list(struct list_head *page_list, ...@@ -1113,8 +1115,10 @@ static unsigned long shrink_page_list(struct list_head *page_list,
/* /*
* Anonymous process memory has backing store? * Anonymous process memory has backing store?
* Try to allocate it some swap space here. * Try to allocate it some swap space here.
* Lazyfree page could be freed directly
*/ */
if (PageAnon(page) && !PageSwapCache(page)) { if (PageAnon(page) && PageSwapBacked(page) &&
!PageSwapCache(page)) {
if (!(sc->gfp_mask & __GFP_IO)) if (!(sc->gfp_mask & __GFP_IO))
goto keep_locked; goto keep_locked;
if (!add_to_swap(page, page_list)) if (!add_to_swap(page, page_list))
...@@ -1135,9 +1139,12 @@ static unsigned long shrink_page_list(struct list_head *page_list, ...@@ -1135,9 +1139,12 @@ static unsigned long shrink_page_list(struct list_head *page_list,
* The page is mapped into the page tables of one or more * The page is mapped into the page tables of one or more
* processes. Try to unmap it here. * processes. Try to unmap it here.
*/ */
if (page_mapped(page) && mapping) { if (page_mapped(page)) {
switch (ret = try_to_unmap(page, switch (ret = try_to_unmap(page,
ttu_flags | TTU_BATCH_FLUSH)) { ttu_flags | TTU_BATCH_FLUSH)) {
case SWAP_DIRTY:
SetPageSwapBacked(page);
/* fall through */
case SWAP_FAIL: case SWAP_FAIL:
nr_unmap_fail++; nr_unmap_fail++;
goto activate_locked; goto activate_locked;
...@@ -1145,8 +1152,6 @@ static unsigned long shrink_page_list(struct list_head *page_list, ...@@ -1145,8 +1152,6 @@ static unsigned long shrink_page_list(struct list_head *page_list,
goto keep_locked; goto keep_locked;
case SWAP_MLOCK: case SWAP_MLOCK:
goto cull_mlocked; goto cull_mlocked;
case SWAP_LZFREE:
goto lazyfree;
case SWAP_SUCCESS: case SWAP_SUCCESS:
; /* try to free the page below */ ; /* try to free the page below */
} }
...@@ -1258,10 +1263,18 @@ static unsigned long shrink_page_list(struct list_head *page_list, ...@@ -1258,10 +1263,18 @@ static unsigned long shrink_page_list(struct list_head *page_list,
} }
} }
lazyfree: if (PageAnon(page) && !PageSwapBacked(page)) {
if (!mapping || !__remove_mapping(mapping, page, true)) /* follow __remove_mapping for reference */
goto keep_locked; if (!page_ref_freeze(page, 1))
goto keep_locked;
if (PageDirty(page)) {
page_ref_unfreeze(page, 1);
goto keep_locked;
}
count_vm_event(PGLAZYFREED);
} else if (!mapping || !__remove_mapping(mapping, page, true))
goto keep_locked;
/* /*
* At this point, we have no other references and there is * At this point, we have no other references and there is
* no way to pick any more up (removed from LRU, removed * no way to pick any more up (removed from LRU, removed
...@@ -1271,9 +1284,6 @@ static unsigned long shrink_page_list(struct list_head *page_list, ...@@ -1271,9 +1284,6 @@ static unsigned long shrink_page_list(struct list_head *page_list,
*/ */
__ClearPageLocked(page); __ClearPageLocked(page);
free_it: free_it:
if (ret == SWAP_LZFREE)
count_vm_event(PGLAZYFREED);
nr_reclaimed++; nr_reclaimed++;
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment