Commit 89b5fae5 authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

mm: vmscan: distinguish global reclaim from global LRU scanning

The traditional zone reclaim code is scanning the per-zone LRU lists
during direct reclaim and kswapd, and the per-zone per-memory cgroup LRU
lists when reclaiming on behalf of a memory cgroup limit.

Subsequent patches will convert the traditional reclaim code to reclaim
exclusively from the per-memory cgroup LRU lists.  As a result, using
the predicate for which LRU list is scanned will no longer be
appropriate to tell global reclaim from limit reclaim.

This patch adds a global_reclaim() predicate to tell direct/kswapd
reclaim from memory cgroup limit reclaim and substitutes it in all
places where currently scanning_global_lru() is used for that.
Signed-off-by: default avatarJohannes Weiner <jweiner@redhat.com>
Reviewed-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Reviewed-by: default avatarMichal Hocko <mhocko@suse.cz>
Reviewed-by: default avatarKirill A. Shutemov <kirill@shutemov.name>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: Ying Han <yinghan@google.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9f3a0d09
...@@ -153,9 +153,25 @@ static LIST_HEAD(shrinker_list); ...@@ -153,9 +153,25 @@ static LIST_HEAD(shrinker_list);
static DECLARE_RWSEM(shrinker_rwsem); static DECLARE_RWSEM(shrinker_rwsem);
#ifdef CONFIG_CGROUP_MEM_RES_CTLR #ifdef CONFIG_CGROUP_MEM_RES_CTLR
#define scanning_global_lru(sc) (!(sc)->mem_cgroup) static bool global_reclaim(struct scan_control *sc)
{
return !sc->mem_cgroup;
}
static bool scanning_global_lru(struct scan_control *sc)
{
return !sc->mem_cgroup;
}
#else #else
#define scanning_global_lru(sc) (1) static bool global_reclaim(struct scan_control *sc)
{
return true;
}
static bool scanning_global_lru(struct scan_control *sc)
{
return true;
}
#endif #endif
static struct zone_reclaim_stat *get_reclaim_stat(struct zone *zone, static struct zone_reclaim_stat *get_reclaim_stat(struct zone *zone,
...@@ -994,7 +1010,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, ...@@ -994,7 +1010,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
* back off and wait for congestion to clear because further reclaim * back off and wait for congestion to clear because further reclaim
* will encounter the same problem * will encounter the same problem
*/ */
if (nr_dirty && nr_dirty == nr_congested && scanning_global_lru(sc)) if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
zone_set_flag(zone, ZONE_CONGESTED); zone_set_flag(zone, ZONE_CONGESTED);
free_hot_cold_page_list(&free_pages, 1); free_hot_cold_page_list(&free_pages, 1);
...@@ -1313,7 +1329,7 @@ static int too_many_isolated(struct zone *zone, int file, ...@@ -1313,7 +1329,7 @@ static int too_many_isolated(struct zone *zone, int file,
if (current_is_kswapd()) if (current_is_kswapd())
return 0; return 0;
if (!scanning_global_lru(sc)) if (!global_reclaim(sc))
return 0; return 0;
if (file) { if (file) {
...@@ -1491,6 +1507,12 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone, ...@@ -1491,6 +1507,12 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone,
if (scanning_global_lru(sc)) { if (scanning_global_lru(sc)) {
nr_taken = isolate_pages_global(nr_to_scan, &page_list, nr_taken = isolate_pages_global(nr_to_scan, &page_list,
&nr_scanned, sc->order, reclaim_mode, zone, 0, file); &nr_scanned, sc->order, reclaim_mode, zone, 0, file);
} else {
nr_taken = mem_cgroup_isolate_pages(nr_to_scan, &page_list,
&nr_scanned, sc->order, reclaim_mode, zone,
sc->mem_cgroup, 0, file);
}
if (global_reclaim(sc)) {
zone->pages_scanned += nr_scanned; zone->pages_scanned += nr_scanned;
if (current_is_kswapd()) if (current_is_kswapd())
__count_zone_vm_events(PGSCAN_KSWAPD, zone, __count_zone_vm_events(PGSCAN_KSWAPD, zone,
...@@ -1498,14 +1520,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone, ...@@ -1498,14 +1520,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone,
else else
__count_zone_vm_events(PGSCAN_DIRECT, zone, __count_zone_vm_events(PGSCAN_DIRECT, zone,
nr_scanned); nr_scanned);
} else {
nr_taken = mem_cgroup_isolate_pages(nr_to_scan, &page_list,
&nr_scanned, sc->order, reclaim_mode, zone,
sc->mem_cgroup, 0, file);
/*
* mem_cgroup_isolate_pages() keeps track of
* scanned pages on its own.
*/
} }
if (nr_taken == 0) { if (nr_taken == 0) {
...@@ -1646,18 +1660,16 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone, ...@@ -1646,18 +1660,16 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
&pgscanned, sc->order, &pgscanned, sc->order,
reclaim_mode, zone, reclaim_mode, zone,
1, file); 1, file);
zone->pages_scanned += pgscanned;
} else { } else {
nr_taken = mem_cgroup_isolate_pages(nr_pages, &l_hold, nr_taken = mem_cgroup_isolate_pages(nr_pages, &l_hold,
&pgscanned, sc->order, &pgscanned, sc->order,
reclaim_mode, zone, reclaim_mode, zone,
sc->mem_cgroup, 1, file); sc->mem_cgroup, 1, file);
/*
* mem_cgroup_isolate_pages() keeps track of
* scanned pages on its own.
*/
} }
if (global_reclaim(sc))
zone->pages_scanned += pgscanned;
reclaim_stat->recent_scanned[file] += nr_taken; reclaim_stat->recent_scanned[file] += nr_taken;
__count_zone_vm_events(PGREFILL, zone, pgscanned); __count_zone_vm_events(PGREFILL, zone, pgscanned);
...@@ -1827,7 +1839,7 @@ static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, ...@@ -1827,7 +1839,7 @@ static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
static int vmscan_swappiness(struct scan_control *sc) static int vmscan_swappiness(struct scan_control *sc)
{ {
if (scanning_global_lru(sc)) if (global_reclaim(sc))
return vm_swappiness; return vm_swappiness;
return mem_cgroup_swappiness(sc->mem_cgroup); return mem_cgroup_swappiness(sc->mem_cgroup);
} }
...@@ -1862,9 +1874,9 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, ...@@ -1862,9 +1874,9 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc,
* latencies, so it's better to scan a minimum amount there as * latencies, so it's better to scan a minimum amount there as
* well. * well.
*/ */
if (scanning_global_lru(sc) && current_is_kswapd()) if (current_is_kswapd())
force_scan = true; force_scan = true;
if (!scanning_global_lru(sc)) if (!global_reclaim(sc))
force_scan = true; force_scan = true;
/* If we have no swap space, do not bother scanning anon pages. */ /* If we have no swap space, do not bother scanning anon pages. */
...@@ -1881,7 +1893,7 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, ...@@ -1881,7 +1893,7 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc,
file = zone_nr_lru_pages(zone, sc, LRU_ACTIVE_FILE) + file = zone_nr_lru_pages(zone, sc, LRU_ACTIVE_FILE) +
zone_nr_lru_pages(zone, sc, LRU_INACTIVE_FILE); zone_nr_lru_pages(zone, sc, LRU_INACTIVE_FILE);
if (scanning_global_lru(sc)) { if (global_reclaim(sc)) {
free = zone_page_state(zone, NR_FREE_PAGES); free = zone_page_state(zone, NR_FREE_PAGES);
/* If we have very few page cache pages, /* If we have very few page cache pages,
force-scan anon pages. */ force-scan anon pages. */
...@@ -2114,7 +2126,7 @@ static bool shrink_zones(int priority, struct zonelist *zonelist, ...@@ -2114,7 +2126,7 @@ static bool shrink_zones(int priority, struct zonelist *zonelist,
* Take care memory controller reclaiming has small influence * Take care memory controller reclaiming has small influence
* to global LRU. * to global LRU.
*/ */
if (scanning_global_lru(sc)) { if (global_reclaim(sc)) {
if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
continue; continue;
if (zone->all_unreclaimable && priority != DEF_PRIORITY) if (zone->all_unreclaimable && priority != DEF_PRIORITY)
...@@ -2212,7 +2224,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, ...@@ -2212,7 +2224,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
get_mems_allowed(); get_mems_allowed();
delayacct_freepages_start(); delayacct_freepages_start();
if (scanning_global_lru(sc)) if (global_reclaim(sc))
count_vm_event(ALLOCSTALL); count_vm_event(ALLOCSTALL);
for (priority = DEF_PRIORITY; priority >= 0; priority--) { for (priority = DEF_PRIORITY; priority >= 0; priority--) {
...@@ -2226,7 +2238,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, ...@@ -2226,7 +2238,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
* Don't shrink slabs when reclaiming memory from * Don't shrink slabs when reclaiming memory from
* over limit cgroups * over limit cgroups
*/ */
if (scanning_global_lru(sc)) { if (global_reclaim(sc)) {
unsigned long lru_pages = 0; unsigned long lru_pages = 0;
for_each_zone_zonelist(zone, z, zonelist, for_each_zone_zonelist(zone, z, zonelist,
gfp_zone(sc->gfp_mask)) { gfp_zone(sc->gfp_mask)) {
...@@ -2288,7 +2300,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, ...@@ -2288,7 +2300,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
return 0; return 0;
/* top priority shrink_zones still had more to do? don't OOM, then */ /* top priority shrink_zones still had more to do? don't OOM, then */
if (scanning_global_lru(sc) && !all_unreclaimable(zonelist, sc)) if (global_reclaim(sc) && !all_unreclaimable(zonelist, sc))
return 1; return 1;
return 0; return 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment