Commit a840cda6 authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

mm: memcontrol: do not acquire page_cgroup lock for kmem pages

Kmem page charging and uncharging is serialized by means of exclusive
access to the page.  Do not take the page_cgroup lock and don't set
pc->flags atomically.
Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by: default avatarMichal Hocko <mhocko@suse.cz>
Acked-by: default avatarVladimir Davydov <vdavydov@parallels.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 9a2385ee
...@@ -3407,12 +3407,13 @@ void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, ...@@ -3407,12 +3407,13 @@ void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
memcg_uncharge_kmem(memcg, PAGE_SIZE << order); memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
return; return;
} }
/*
* The page is freshly allocated and not visible to any
* outside callers yet. Set up pc non-atomically.
*/
pc = lookup_page_cgroup(page); pc = lookup_page_cgroup(page);
lock_page_cgroup(pc);
pc->mem_cgroup = memcg; pc->mem_cgroup = memcg;
SetPageCgroupUsed(pc); pc->flags = PCG_USED;
unlock_page_cgroup(pc);
} }
void __memcg_kmem_uncharge_pages(struct page *page, int order) void __memcg_kmem_uncharge_pages(struct page *page, int order)
...@@ -3422,19 +3423,11 @@ void __memcg_kmem_uncharge_pages(struct page *page, int order) ...@@ -3422,19 +3423,11 @@ void __memcg_kmem_uncharge_pages(struct page *page, int order)
pc = lookup_page_cgroup(page); pc = lookup_page_cgroup(page);
/*
* Fast unlocked return. Theoretically might have changed, have to
* check again after locking.
*/
if (!PageCgroupUsed(pc)) if (!PageCgroupUsed(pc))
return; return;
lock_page_cgroup(pc); memcg = pc->mem_cgroup;
if (PageCgroupUsed(pc)) { pc->flags = 0;
memcg = pc->mem_cgroup;
ClearPageCgroupUsed(pc);
}
unlock_page_cgroup(pc);
/* /*
* We trust that only if there is a memcg associated with the page, it * We trust that only if there is a memcg associated with the page, it
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment