Commit 3852f676 authored by Joonsoo Kim's avatar Joonsoo Kim Committed by Linus Torvalds

mm/swapcache: support to handle the shadow entries

Workingset detection for anonymous page will be implemented in the
following patch and it requires to store the shadow entries into the
swapcache.  This patch implements an infrastructure to store the shadow
entry in the swapcache.
Signed-off-by: default avatarJoonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Link: http://lkml.kernel.org/r/1595490560-15117-5-git-send-email-iamjoonsoo.kim@lge.comSigned-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 170b04b7
...@@ -414,9 +414,13 @@ extern struct address_space *swapper_spaces[]; ...@@ -414,9 +414,13 @@ extern struct address_space *swapper_spaces[];
extern unsigned long total_swapcache_pages(void); extern unsigned long total_swapcache_pages(void);
extern void show_swap_cache_info(void); extern void show_swap_cache_info(void);
extern int add_to_swap(struct page *page); extern int add_to_swap(struct page *page);
extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t); extern int add_to_swap_cache(struct page *page, swp_entry_t entry,
extern void __delete_from_swap_cache(struct page *, swp_entry_t entry); gfp_t gfp, void **shadowp);
extern void __delete_from_swap_cache(struct page *page,
swp_entry_t entry, void *shadow);
extern void delete_from_swap_cache(struct page *); extern void delete_from_swap_cache(struct page *);
extern void clear_shadow_from_swap_cache(int type, unsigned long begin,
unsigned long end);
extern void free_page_and_swap_cache(struct page *); extern void free_page_and_swap_cache(struct page *);
extern void free_pages_and_swap_cache(struct page **, int); extern void free_pages_and_swap_cache(struct page **, int);
extern struct page *lookup_swap_cache(swp_entry_t entry, extern struct page *lookup_swap_cache(swp_entry_t entry,
...@@ -570,13 +574,13 @@ static inline int add_to_swap(struct page *page) ...@@ -570,13 +574,13 @@ static inline int add_to_swap(struct page *page)
} }
static inline int add_to_swap_cache(struct page *page, swp_entry_t entry, static inline int add_to_swap_cache(struct page *page, swp_entry_t entry,
gfp_t gfp_mask) gfp_t gfp_mask, void **shadowp)
{ {
return -1; return -1;
} }
static inline void __delete_from_swap_cache(struct page *page, static inline void __delete_from_swap_cache(struct page *page,
swp_entry_t entry) swp_entry_t entry, void *shadow)
{ {
} }
...@@ -584,6 +588,11 @@ static inline void delete_from_swap_cache(struct page *page) ...@@ -584,6 +588,11 @@ static inline void delete_from_swap_cache(struct page *page)
{ {
} }
static inline void clear_shadow_from_swap_cache(int type, unsigned long begin,
unsigned long end)
{
}
static inline int page_swapcount(struct page *page) static inline int page_swapcount(struct page *page)
{ {
return 0; return 0;
......
...@@ -1434,7 +1434,8 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) ...@@ -1434,7 +1434,8 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
list_add(&info->swaplist, &shmem_swaplist); list_add(&info->swaplist, &shmem_swaplist);
if (add_to_swap_cache(page, swap, if (add_to_swap_cache(page, swap,
__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN) == 0) { __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
NULL) == 0) {
spin_lock_irq(&info->lock); spin_lock_irq(&info->lock);
shmem_recalc_inode(inode); shmem_recalc_inode(inode);
info->swapped++; info->swapped++;
......
...@@ -110,12 +110,14 @@ void show_swap_cache_info(void) ...@@ -110,12 +110,14 @@ void show_swap_cache_info(void)
* add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
* but sets SwapCache flag and private instead of mapping and index. * but sets SwapCache flag and private instead of mapping and index.
*/ */
int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) int add_to_swap_cache(struct page *page, swp_entry_t entry,
gfp_t gfp, void **shadowp)
{ {
struct address_space *address_space = swap_address_space(entry); struct address_space *address_space = swap_address_space(entry);
pgoff_t idx = swp_offset(entry); pgoff_t idx = swp_offset(entry);
XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page)); XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page));
unsigned long i, nr = hpage_nr_pages(page); unsigned long i, nr = hpage_nr_pages(page);
void *old;
VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(PageSwapCache(page), page); VM_BUG_ON_PAGE(PageSwapCache(page), page);
...@@ -125,16 +127,25 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) ...@@ -125,16 +127,25 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp)
SetPageSwapCache(page); SetPageSwapCache(page);
do { do {
unsigned long nr_shadows = 0;
xas_lock_irq(&xas); xas_lock_irq(&xas);
xas_create_range(&xas); xas_create_range(&xas);
if (xas_error(&xas)) if (xas_error(&xas))
goto unlock; goto unlock;
for (i = 0; i < nr; i++) { for (i = 0; i < nr; i++) {
VM_BUG_ON_PAGE(xas.xa_index != idx + i, page); VM_BUG_ON_PAGE(xas.xa_index != idx + i, page);
old = xas_load(&xas);
if (xa_is_value(old)) {
nr_shadows++;
if (shadowp)
*shadowp = old;
}
set_page_private(page + i, entry.val + i); set_page_private(page + i, entry.val + i);
xas_store(&xas, page); xas_store(&xas, page);
xas_next(&xas); xas_next(&xas);
} }
address_space->nrexceptional -= nr_shadows;
address_space->nrpages += nr; address_space->nrpages += nr;
__mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
ADD_CACHE_INFO(add_total, nr); ADD_CACHE_INFO(add_total, nr);
...@@ -154,7 +165,8 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) ...@@ -154,7 +165,8 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp)
* This must be called only on pages that have * This must be called only on pages that have
* been verified to be in the swap cache. * been verified to be in the swap cache.
*/ */
void __delete_from_swap_cache(struct page *page, swp_entry_t entry) void __delete_from_swap_cache(struct page *page,
swp_entry_t entry, void *shadow)
{ {
struct address_space *address_space = swap_address_space(entry); struct address_space *address_space = swap_address_space(entry);
int i, nr = hpage_nr_pages(page); int i, nr = hpage_nr_pages(page);
...@@ -166,12 +178,14 @@ void __delete_from_swap_cache(struct page *page, swp_entry_t entry) ...@@ -166,12 +178,14 @@ void __delete_from_swap_cache(struct page *page, swp_entry_t entry)
VM_BUG_ON_PAGE(PageWriteback(page), page); VM_BUG_ON_PAGE(PageWriteback(page), page);
for (i = 0; i < nr; i++) { for (i = 0; i < nr; i++) {
void *entry = xas_store(&xas, NULL); void *entry = xas_store(&xas, shadow);
VM_BUG_ON_PAGE(entry != page, entry); VM_BUG_ON_PAGE(entry != page, entry);
set_page_private(page + i, 0); set_page_private(page + i, 0);
xas_next(&xas); xas_next(&xas);
} }
ClearPageSwapCache(page); ClearPageSwapCache(page);
if (shadow)
address_space->nrexceptional += nr;
address_space->nrpages -= nr; address_space->nrpages -= nr;
__mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
ADD_CACHE_INFO(del_total, nr); ADD_CACHE_INFO(del_total, nr);
...@@ -208,7 +222,7 @@ int add_to_swap(struct page *page) ...@@ -208,7 +222,7 @@ int add_to_swap(struct page *page)
* Add it to the swap cache. * Add it to the swap cache.
*/ */
err = add_to_swap_cache(page, entry, err = add_to_swap_cache(page, entry,
__GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN); __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL);
if (err) if (err)
/* /*
* add_to_swap_cache() doesn't return -EEXIST, so we can safely * add_to_swap_cache() doesn't return -EEXIST, so we can safely
...@@ -246,13 +260,44 @@ void delete_from_swap_cache(struct page *page) ...@@ -246,13 +260,44 @@ void delete_from_swap_cache(struct page *page)
struct address_space *address_space = swap_address_space(entry); struct address_space *address_space = swap_address_space(entry);
xa_lock_irq(&address_space->i_pages); xa_lock_irq(&address_space->i_pages);
__delete_from_swap_cache(page, entry); __delete_from_swap_cache(page, entry, NULL);
xa_unlock_irq(&address_space->i_pages); xa_unlock_irq(&address_space->i_pages);
put_swap_page(page, entry); put_swap_page(page, entry);
page_ref_sub(page, hpage_nr_pages(page)); page_ref_sub(page, hpage_nr_pages(page));
} }
void clear_shadow_from_swap_cache(int type, unsigned long begin,
unsigned long end)
{
unsigned long curr = begin;
void *old;
for (;;) {
unsigned long nr_shadows = 0;
swp_entry_t entry = swp_entry(type, curr);
struct address_space *address_space = swap_address_space(entry);
XA_STATE(xas, &address_space->i_pages, curr);
xa_lock_irq(&address_space->i_pages);
xas_for_each(&xas, old, end) {
if (!xa_is_value(old))
continue;
xas_store(&xas, NULL);
nr_shadows++;
}
address_space->nrexceptional -= nr_shadows;
xa_unlock_irq(&address_space->i_pages);
/* search the next swapcache until we meet end */
curr >>= SWAP_ADDRESS_SPACE_SHIFT;
curr++;
curr <<= SWAP_ADDRESS_SPACE_SHIFT;
if (curr > end)
break;
}
}
/* /*
* If we are the only user, then try to free up the swap cache. * If we are the only user, then try to free up the swap cache.
* *
...@@ -429,7 +474,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, ...@@ -429,7 +474,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
__SetPageSwapBacked(page); __SetPageSwapBacked(page);
/* May fail (-ENOMEM) if XArray node allocation failed. */ /* May fail (-ENOMEM) if XArray node allocation failed. */
if (add_to_swap_cache(page, entry, gfp_mask & GFP_RECLAIM_MASK)) { if (add_to_swap_cache(page, entry, gfp_mask & GFP_RECLAIM_MASK, NULL)) {
put_swap_page(page, entry); put_swap_page(page, entry);
goto fail_unlock; goto fail_unlock;
} }
......
...@@ -696,6 +696,7 @@ static void add_to_avail_list(struct swap_info_struct *p) ...@@ -696,6 +696,7 @@ static void add_to_avail_list(struct swap_info_struct *p)
static void swap_range_free(struct swap_info_struct *si, unsigned long offset, static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
unsigned int nr_entries) unsigned int nr_entries)
{ {
unsigned long begin = offset;
unsigned long end = offset + nr_entries - 1; unsigned long end = offset + nr_entries - 1;
void (*swap_slot_free_notify)(struct block_device *, unsigned long); void (*swap_slot_free_notify)(struct block_device *, unsigned long);
...@@ -721,6 +722,7 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset, ...@@ -721,6 +722,7 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
swap_slot_free_notify(si->bdev, offset); swap_slot_free_notify(si->bdev, offset);
offset++; offset++;
} }
clear_shadow_from_swap_cache(si->type, begin, end);
} }
static void set_cluster_next(struct swap_info_struct *si, unsigned long next) static void set_cluster_next(struct swap_info_struct *si, unsigned long next)
......
...@@ -896,7 +896,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, ...@@ -896,7 +896,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
if (PageSwapCache(page)) { if (PageSwapCache(page)) {
swp_entry_t swap = { .val = page_private(page) }; swp_entry_t swap = { .val = page_private(page) };
mem_cgroup_swapout(page, swap); mem_cgroup_swapout(page, swap);
__delete_from_swap_cache(page, swap); __delete_from_swap_cache(page, swap, NULL);
xa_unlock_irqrestore(&mapping->i_pages, flags); xa_unlock_irqrestore(&mapping->i_pages, flags);
put_swap_page(page, swap); put_swap_page(page, swap);
workingset_eviction(page, target_memcg); workingset_eviction(page, target_memcg);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment