Commit 5564e88b authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

memcg: condense page_cgroup-to-page lookup points

The per-cgroup LRU lists string up 'struct page_cgroup's.  To get from
those structures to the page they represent, a lookup is required.
Currently, the lookup is done through a direct pointer in struct
page_cgroup, so a lot of functions down the callchain do this lookup by
themselves instead of receiving the page pointer from their callers.

The next patch removes this pointer, however, and the lookup is no longer
that straight-forward.  In preparation for that, this patch only leaves
the non-optional lookups when coming directly from the LRU list and passes
the page down the stack.
Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <balbir@linux.vnet.ibm.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent de3638d9
...@@ -1077,9 +1077,11 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, ...@@ -1077,9 +1077,11 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
if (scan >= nr_to_scan) if (scan >= nr_to_scan)
break; break;
page = pc->page;
if (unlikely(!PageCgroupUsed(pc))) if (unlikely(!PageCgroupUsed(pc)))
continue; continue;
page = pc->page;
if (unlikely(!PageLRU(page))) if (unlikely(!PageLRU(page)))
continue; continue;
...@@ -2108,6 +2110,7 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) ...@@ -2108,6 +2110,7 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
} }
static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
struct page *page,
struct page_cgroup *pc, struct page_cgroup *pc,
enum charge_type ctype, enum charge_type ctype,
int page_size) int page_size)
...@@ -2154,7 +2157,7 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, ...@@ -2154,7 +2157,7 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
* Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
* if they exceeds softlimit. * if they exceeds softlimit.
*/ */
memcg_check_events(mem, pc->page); memcg_check_events(mem, page);
} }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
...@@ -2201,6 +2204,7 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail) ...@@ -2201,6 +2204,7 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
/** /**
* mem_cgroup_move_account - move account of the page * mem_cgroup_move_account - move account of the page
* @page: the page
* @pc: page_cgroup of the page. * @pc: page_cgroup of the page.
* @from: mem_cgroup which the page is moved from. * @from: mem_cgroup which the page is moved from.
* @to: mem_cgroup which the page is moved to. @from != @to. * @to: mem_cgroup which the page is moved to. @from != @to.
...@@ -2216,7 +2220,7 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail) ...@@ -2216,7 +2220,7 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
* true, this function does "uncharge" from old cgroup, but it doesn't if * true, this function does "uncharge" from old cgroup, but it doesn't if
* @uncharge is false, so a caller should do "uncharge". * @uncharge is false, so a caller should do "uncharge".
*/ */
static int mem_cgroup_move_account(struct page_cgroup *pc, static int mem_cgroup_move_account(struct page *page, struct page_cgroup *pc,
struct mem_cgroup *from, struct mem_cgroup *to, struct mem_cgroup *from, struct mem_cgroup *to,
bool uncharge, int charge_size) bool uncharge, int charge_size)
{ {
...@@ -2225,7 +2229,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, ...@@ -2225,7 +2229,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
int ret; int ret;
VM_BUG_ON(from == to); VM_BUG_ON(from == to);
VM_BUG_ON(PageLRU(pc->page)); VM_BUG_ON(PageLRU(page));
/* /*
* The page is isolated from LRU. So, collapse function * The page is isolated from LRU. So, collapse function
* will not handle this page. But page splitting can happen. * will not handle this page. But page splitting can happen.
...@@ -2233,7 +2237,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, ...@@ -2233,7 +2237,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
* hold it. * hold it.
*/ */
ret = -EBUSY; ret = -EBUSY;
if (charge_size > PAGE_SIZE && !PageTransHuge(pc->page)) if (charge_size > PAGE_SIZE && !PageTransHuge(page))
goto out; goto out;
lock_page_cgroup(pc); lock_page_cgroup(pc);
...@@ -2273,8 +2277,8 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, ...@@ -2273,8 +2277,8 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
/* /*
* check events * check events
*/ */
memcg_check_events(to, pc->page); memcg_check_events(to, page);
memcg_check_events(from, pc->page); memcg_check_events(from, page);
out: out:
return ret; return ret;
} }
...@@ -2283,11 +2287,11 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, ...@@ -2283,11 +2287,11 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
* move charges to its parent. * move charges to its parent.
*/ */
static int mem_cgroup_move_parent(struct page_cgroup *pc, static int mem_cgroup_move_parent(struct page *page,
struct page_cgroup *pc,
struct mem_cgroup *child, struct mem_cgroup *child,
gfp_t gfp_mask) gfp_t gfp_mask)
{ {
struct page *page = pc->page;
struct cgroup *cg = child->css.cgroup; struct cgroup *cg = child->css.cgroup;
struct cgroup *pcg = cg->parent; struct cgroup *pcg = cg->parent;
struct mem_cgroup *parent; struct mem_cgroup *parent;
...@@ -2317,7 +2321,7 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc, ...@@ -2317,7 +2321,7 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc,
if (page_size > PAGE_SIZE) if (page_size > PAGE_SIZE)
flags = compound_lock_irqsave(page); flags = compound_lock_irqsave(page);
ret = mem_cgroup_move_account(pc, child, parent, true, page_size); ret = mem_cgroup_move_account(page, pc, child, parent, true, page_size);
if (ret) if (ret)
mem_cgroup_cancel_charge(parent, page_size); mem_cgroup_cancel_charge(parent, page_size);
...@@ -2363,7 +2367,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, ...@@ -2363,7 +2367,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
if (ret || !mem) if (ret || !mem)
return ret; return ret;
__mem_cgroup_commit_charge(mem, pc, ctype, page_size); __mem_cgroup_commit_charge(mem, page, pc, ctype, page_size);
return 0; return 0;
} }
...@@ -2501,7 +2505,7 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, ...@@ -2501,7 +2505,7 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
cgroup_exclude_rmdir(&ptr->css); cgroup_exclude_rmdir(&ptr->css);
pc = lookup_page_cgroup(page); pc = lookup_page_cgroup(page);
mem_cgroup_lru_del_before_commit_swapcache(page); mem_cgroup_lru_del_before_commit_swapcache(page);
__mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE); __mem_cgroup_commit_charge(ptr, page, pc, ctype, PAGE_SIZE);
mem_cgroup_lru_add_after_commit_swapcache(page); mem_cgroup_lru_add_after_commit_swapcache(page);
/* /*
* Now swap is on-memory. This means this page may be * Now swap is on-memory. This means this page may be
...@@ -2956,7 +2960,7 @@ int mem_cgroup_prepare_migration(struct page *page, ...@@ -2956,7 +2960,7 @@ int mem_cgroup_prepare_migration(struct page *page,
ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
else else
ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
__mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE); __mem_cgroup_commit_charge(mem, page, pc, ctype, PAGE_SIZE);
return ret; return ret;
} }
...@@ -3323,6 +3327,8 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, ...@@ -3323,6 +3327,8 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
loop += 256; loop += 256;
busy = NULL; busy = NULL;
while (loop--) { while (loop--) {
struct page *page;
ret = 0; ret = 0;
spin_lock_irqsave(&zone->lru_lock, flags); spin_lock_irqsave(&zone->lru_lock, flags);
if (list_empty(list)) { if (list_empty(list)) {
...@@ -3338,7 +3344,9 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, ...@@ -3338,7 +3344,9 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
} }
spin_unlock_irqrestore(&zone->lru_lock, flags); spin_unlock_irqrestore(&zone->lru_lock, flags);
ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL); page = pc->page;
ret = mem_cgroup_move_parent(page, pc, mem, GFP_KERNEL);
if (ret == -ENOMEM) if (ret == -ENOMEM)
break; break;
...@@ -4956,7 +4964,7 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, ...@@ -4956,7 +4964,7 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
if (isolate_lru_page(page)) if (isolate_lru_page(page))
goto put; goto put;
pc = lookup_page_cgroup(page); pc = lookup_page_cgroup(page);
if (!mem_cgroup_move_account(pc, if (!mem_cgroup_move_account(page, pc,
mc.from, mc.to, false, PAGE_SIZE)) { mc.from, mc.to, false, PAGE_SIZE)) {
mc.precharge--; mc.precharge--;
/* we uncharge from mc.from later. */ /* we uncharge from mc.from later. */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment