Commit 013339df authored by Shakeel Butt's avatar Shakeel Butt Committed by Linus Torvalds

mm/rmap: always do TTU_IGNORE_ACCESS

Since commit 369ea824 ("mm/rmap: update to new mmu_notifier semantic
v2"), the code to check the secondary MMU's page table access bit is
broken for !(TTU_IGNORE_ACCESS) because the page is unmapped from the
secondary MMU's page table before the check.  More specifically for those
secondary MMUs which unmap the memory in
mmu_notifier_invalidate_range_start() like kvm.

However memory reclaim is the only user of !(TTU_IGNORE_ACCESS) or the
absence of TTU_IGNORE_ACCESS and it explicitly performs the page table
access check before trying to unmap the page.  So, at worst the reclaim
will miss accesses in a very short window if we remove page table access
check in unmapping code.

There is an unintented consequence of !(TTU_IGNORE_ACCESS) for the memcg
reclaim.  From memcg reclaim the page_referenced() only account the
accesses from the processes which are in the same memcg of the target page
but the unmapping code is considering accesses from all the processes, so,
decreasing the effectiveness of memcg reclaim.

The simplest solution is to always assume TTU_IGNORE_ACCESS in unmapping
code.

Link: https://lkml.kernel.org/r/20201104231928.1494083-1-shakeelb@google.com
Fixes: 369ea824 ("mm/rmap: update to new mmu_notifier semantic v2")
Signed-off-by: default avatarShakeel Butt <shakeelb@google.com>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jerome Glisse <jglisse@redhat.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent eefbfa7f
...@@ -91,7 +91,6 @@ enum ttu_flags { ...@@ -91,7 +91,6 @@ enum ttu_flags {
TTU_SPLIT_HUGE_PMD = 0x4, /* split huge PMD if any */ TTU_SPLIT_HUGE_PMD = 0x4, /* split huge PMD if any */
TTU_IGNORE_MLOCK = 0x8, /* ignore mlock */ TTU_IGNORE_MLOCK = 0x8, /* ignore mlock */
TTU_IGNORE_ACCESS = 0x10, /* don't age */
TTU_IGNORE_HWPOISON = 0x20, /* corrupted page is recoverable */ TTU_IGNORE_HWPOISON = 0x20, /* corrupted page is recoverable */
TTU_BATCH_FLUSH = 0x40, /* Batch TLB flushes where possible TTU_BATCH_FLUSH = 0x40, /* Batch TLB flushes where possible
* and caller guarantees they will * and caller guarantees they will
......
...@@ -2321,7 +2321,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, ...@@ -2321,7 +2321,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma,
static void unmap_page(struct page *page) static void unmap_page(struct page *page)
{ {
enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS | enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK |
TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD; TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD;
bool unmap_success; bool unmap_success;
......
...@@ -989,7 +989,7 @@ static int get_hwpoison_page(struct page *page) ...@@ -989,7 +989,7 @@ static int get_hwpoison_page(struct page *page)
static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
int flags, struct page **hpagep) int flags, struct page **hpagep)
{ {
enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; enum ttu_flags ttu = TTU_IGNORE_MLOCK;
struct address_space *mapping; struct address_space *mapping;
LIST_HEAD(tokill); LIST_HEAD(tokill);
bool unmap_success = true; bool unmap_success = true;
......
...@@ -1304,7 +1304,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) ...@@ -1304,7 +1304,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
if (WARN_ON(PageLRU(page))) if (WARN_ON(PageLRU(page)))
isolate_lru_page(page); isolate_lru_page(page);
if (page_mapped(page)) if (page_mapped(page))
try_to_unmap(page, TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS); try_to_unmap(page, TTU_IGNORE_MLOCK);
continue; continue;
} }
......
...@@ -1122,8 +1122,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, ...@@ -1122,8 +1122,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
/* Establish migration ptes */ /* Establish migration ptes */
VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma, VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
page); page);
try_to_unmap(page, try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK);
TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
page_was_mapped = 1; page_was_mapped = 1;
} }
...@@ -1329,8 +1328,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, ...@@ -1329,8 +1328,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
if (page_mapped(hpage)) { if (page_mapped(hpage)) {
bool mapping_locked = false; bool mapping_locked = false;
enum ttu_flags ttu = TTU_MIGRATION|TTU_IGNORE_MLOCK| enum ttu_flags ttu = TTU_MIGRATION|TTU_IGNORE_MLOCK;
TTU_IGNORE_ACCESS;
if (!PageAnon(hpage)) { if (!PageAnon(hpage)) {
/* /*
...@@ -2688,7 +2686,7 @@ static void migrate_vma_prepare(struct migrate_vma *migrate) ...@@ -2688,7 +2686,7 @@ static void migrate_vma_prepare(struct migrate_vma *migrate)
*/ */
static void migrate_vma_unmap(struct migrate_vma *migrate) static void migrate_vma_unmap(struct migrate_vma *migrate)
{ {
int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS; int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK;
const unsigned long npages = migrate->npages; const unsigned long npages = migrate->npages;
const unsigned long start = migrate->start; const unsigned long start = migrate->start;
unsigned long addr, i, restore = 0; unsigned long addr, i, restore = 0;
......
...@@ -1533,15 +1533,6 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1533,15 +1533,6 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
goto discard; goto discard;
} }
if (!(flags & TTU_IGNORE_ACCESS)) {
if (ptep_clear_flush_young_notify(vma, address,
pvmw.pte)) {
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
}
/* Nuke the page table entry. */ /* Nuke the page table entry. */
flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
if (should_defer_flush(mm, flags)) { if (should_defer_flush(mm, flags)) {
......
...@@ -1072,7 +1072,6 @@ static void page_check_dirty_writeback(struct page *page, ...@@ -1072,7 +1072,6 @@ static void page_check_dirty_writeback(struct page *page,
static unsigned int shrink_page_list(struct list_head *page_list, static unsigned int shrink_page_list(struct list_head *page_list,
struct pglist_data *pgdat, struct pglist_data *pgdat,
struct scan_control *sc, struct scan_control *sc,
enum ttu_flags ttu_flags,
struct reclaim_stat *stat, struct reclaim_stat *stat,
bool ignore_references) bool ignore_references)
{ {
...@@ -1297,7 +1296,7 @@ static unsigned int shrink_page_list(struct list_head *page_list, ...@@ -1297,7 +1296,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
* processes. Try to unmap it here. * processes. Try to unmap it here.
*/ */
if (page_mapped(page)) { if (page_mapped(page)) {
enum ttu_flags flags = ttu_flags | TTU_BATCH_FLUSH; enum ttu_flags flags = TTU_BATCH_FLUSH;
bool was_swapbacked = PageSwapBacked(page); bool was_swapbacked = PageSwapBacked(page);
if (unlikely(PageTransHuge(page))) if (unlikely(PageTransHuge(page)))
...@@ -1514,7 +1513,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, ...@@ -1514,7 +1513,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
} }
nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc, nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
TTU_IGNORE_ACCESS, &stat, true); &stat, true);
list_splice(&clean_pages, page_list); list_splice(&clean_pages, page_list);
mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
-(long)nr_reclaimed); -(long)nr_reclaimed);
...@@ -1958,8 +1957,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, ...@@ -1958,8 +1957,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
if (nr_taken == 0) if (nr_taken == 0)
return 0; return 0;
nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0, nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);
&stat, false);
spin_lock_irq(&pgdat->lru_lock); spin_lock_irq(&pgdat->lru_lock);
...@@ -2131,8 +2129,7 @@ unsigned long reclaim_pages(struct list_head *page_list) ...@@ -2131,8 +2129,7 @@ unsigned long reclaim_pages(struct list_head *page_list)
nr_reclaimed += shrink_page_list(&node_page_list, nr_reclaimed += shrink_page_list(&node_page_list,
NODE_DATA(nid), NODE_DATA(nid),
&sc, 0, &sc, &dummy_stat, false);
&dummy_stat, false);
while (!list_empty(&node_page_list)) { while (!list_empty(&node_page_list)) {
page = lru_to_page(&node_page_list); page = lru_to_page(&node_page_list);
list_del(&page->lru); list_del(&page->lru);
...@@ -2145,8 +2142,7 @@ unsigned long reclaim_pages(struct list_head *page_list) ...@@ -2145,8 +2142,7 @@ unsigned long reclaim_pages(struct list_head *page_list)
if (!list_empty(&node_page_list)) { if (!list_empty(&node_page_list)) {
nr_reclaimed += shrink_page_list(&node_page_list, nr_reclaimed += shrink_page_list(&node_page_list,
NODE_DATA(nid), NODE_DATA(nid),
&sc, 0, &sc, &dummy_stat, false);
&dummy_stat, false);
while (!list_empty(&node_page_list)) { while (!list_empty(&node_page_list)) {
page = lru_to_page(&node_page_list); page = lru_to_page(&node_page_list);
list_del(&page->lru); list_del(&page->lru);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment