Commit 854ffa8d authored by Daisuke Nishimura's avatar Daisuke Nishimura Committed by Linus Torvalds

memcg: improve performance in moving charge

Try to reduce overheads in moving charge by:

- Instead of calling res_counter_uncharge() against the old cgroup in
  __mem_cgroup_move_account() everytime, call res_counter_uncharge() at the end
  of task migration once.
- removed css_get(&to->css) from __mem_cgroup_move_account() because callers
  should have already called css_get(). And removed css_put(&to->css) too,
  which was called by callers of move_account on success of move_account.
- Instead of calling __mem_cgroup_try_charge(), i.e. res_counter_charge(),
  repeatedly, call res_counter_charge(PAGE_SIZE * count) in can_attach() if
  possible.
- Instead of calling css_get()/css_put() repeatedly, make use of coalesce
  __css_get()/__css_put() if possible.

These changes reduces the overhead from 1.7sec to 0.6sec to move charges
of 1G anonymous memory in my test environment.
Signed-off-by: default avatarDaisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Paul Menage <menage@google.com>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4ffef5fe
...@@ -253,6 +253,7 @@ static struct move_charge_struct { ...@@ -253,6 +253,7 @@ static struct move_charge_struct {
struct mem_cgroup *from; struct mem_cgroup *from;
struct mem_cgroup *to; struct mem_cgroup *to;
unsigned long precharge; unsigned long precharge;
unsigned long moved_charge;
} mc; } mc;
/* /*
...@@ -1536,14 +1537,23 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm, ...@@ -1536,14 +1537,23 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
* This function is for that and do uncharge, put css's refcnt. * This function is for that and do uncharge, put css's refcnt.
* gotten by try_charge(). * gotten by try_charge().
*/ */
static void mem_cgroup_cancel_charge(struct mem_cgroup *mem) static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
unsigned long count)
{ {
if (!mem_cgroup_is_root(mem)) { if (!mem_cgroup_is_root(mem)) {
res_counter_uncharge(&mem->res, PAGE_SIZE); res_counter_uncharge(&mem->res, PAGE_SIZE * count);
if (do_swap_account) if (do_swap_account)
res_counter_uncharge(&mem->memsw, PAGE_SIZE); res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));
WARN_ON_ONCE(count > INT_MAX);
__css_put(&mem->css, (int)count);
} }
css_put(&mem->css); /* we don't need css_put for root */
}
static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)
{
__mem_cgroup_cancel_charge(mem, 1);
} }
/* /*
...@@ -1646,17 +1656,20 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, ...@@ -1646,17 +1656,20 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
* @pc: page_cgroup of the page. * @pc: page_cgroup of the page.
* @from: mem_cgroup which the page is moved from. * @from: mem_cgroup which the page is moved from.
* @to: mem_cgroup which the page is moved to. @from != @to. * @to: mem_cgroup which the page is moved to. @from != @to.
* @uncharge: whether we should call uncharge and css_put against @from.
* *
* The caller must confirm following. * The caller must confirm following.
* - page is not on LRU (isolate_page() is useful.) * - page is not on LRU (isolate_page() is useful.)
* - the pc is locked, used, and ->mem_cgroup points to @from. * - the pc is locked, used, and ->mem_cgroup points to @from.
* *
* This function does "uncharge" from old cgroup but doesn't do "charge" to * This function doesn't do "charge" nor css_get to new cgroup. It should be
* new cgroup. It should be done by a caller. * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
* true, this function does "uncharge" from old cgroup, but it doesn't if
* @uncharge is false, so a caller should do "uncharge".
*/ */
static void __mem_cgroup_move_account(struct page_cgroup *pc, static void __mem_cgroup_move_account(struct page_cgroup *pc,
struct mem_cgroup *from, struct mem_cgroup *to) struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
{ {
struct page *page; struct page *page;
int cpu; int cpu;
...@@ -1669,10 +1682,6 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc, ...@@ -1669,10 +1682,6 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc,
VM_BUG_ON(!PageCgroupUsed(pc)); VM_BUG_ON(!PageCgroupUsed(pc));
VM_BUG_ON(pc->mem_cgroup != from); VM_BUG_ON(pc->mem_cgroup != from);
if (!mem_cgroup_is_root(from))
res_counter_uncharge(&from->res, PAGE_SIZE);
mem_cgroup_charge_statistics(from, pc, false);
page = pc->page; page = pc->page;
if (page_mapped(page) && !PageAnon(page)) { if (page_mapped(page) && !PageAnon(page)) {
cpu = smp_processor_id(); cpu = smp_processor_id();
...@@ -1688,12 +1697,12 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc, ...@@ -1688,12 +1697,12 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc,
__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_FILE_MAPPED, __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_FILE_MAPPED,
1); 1);
} }
mem_cgroup_charge_statistics(from, pc, false);
if (uncharge)
/* This is not "cancel", but cancel_charge does all we need. */
mem_cgroup_cancel_charge(from);
if (do_swap_account && !mem_cgroup_is_root(from)) /* caller should have done css_get */
res_counter_uncharge(&from->memsw, PAGE_SIZE);
css_put(&from->css);
css_get(&to->css);
pc->mem_cgroup = to; pc->mem_cgroup = to;
mem_cgroup_charge_statistics(to, pc, true); mem_cgroup_charge_statistics(to, pc, true);
/* /*
...@@ -1710,12 +1719,12 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc, ...@@ -1710,12 +1719,12 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc,
* __mem_cgroup_move_account() * __mem_cgroup_move_account()
*/ */
static int mem_cgroup_move_account(struct page_cgroup *pc, static int mem_cgroup_move_account(struct page_cgroup *pc,
struct mem_cgroup *from, struct mem_cgroup *to) struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
{ {
int ret = -EINVAL; int ret = -EINVAL;
lock_page_cgroup(pc); lock_page_cgroup(pc);
if (PageCgroupUsed(pc) && pc->mem_cgroup == from) { if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
__mem_cgroup_move_account(pc, from, to); __mem_cgroup_move_account(pc, from, to, uncharge);
ret = 0; ret = 0;
} }
unlock_page_cgroup(pc); unlock_page_cgroup(pc);
...@@ -1751,11 +1760,9 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc, ...@@ -1751,11 +1760,9 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc,
if (ret || !parent) if (ret || !parent)
goto put_back; goto put_back;
ret = mem_cgroup_move_account(pc, child, parent); ret = mem_cgroup_move_account(pc, child, parent, true);
if (!ret) if (ret)
css_put(&parent->css); /* drop extra refcnt by try_charge() */ mem_cgroup_cancel_charge(parent);
else
mem_cgroup_cancel_charge(parent); /* does css_put */
put_back: put_back:
putback_lru_page(page); putback_lru_page(page);
put: put:
...@@ -3438,16 +3445,58 @@ static int mem_cgroup_populate(struct cgroup_subsys *ss, ...@@ -3438,16 +3445,58 @@ static int mem_cgroup_populate(struct cgroup_subsys *ss,
} }
/* Handlers for move charge at task migration. */ /* Handlers for move charge at task migration. */
static int mem_cgroup_do_precharge(void) #define PRECHARGE_COUNT_AT_ONCE 256
static int mem_cgroup_do_precharge(unsigned long count)
{ {
int ret = -ENOMEM; int ret = 0;
int batch_count = PRECHARGE_COUNT_AT_ONCE;
struct mem_cgroup *mem = mc.to; struct mem_cgroup *mem = mc.to;
ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false, NULL); if (mem_cgroup_is_root(mem)) {
mc.precharge += count;
/* we don't need css_get for root */
return ret;
}
/* try to charge at once */
if (count > 1) {
struct res_counter *dummy;
/*
* "mem" cannot be under rmdir() because we've already checked
* by cgroup_lock_live_cgroup() that it is not removed and we
* are still under the same cgroup_mutex. So we can postpone
* css_get().
*/
if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
goto one_by_one;
if (do_swap_account && res_counter_charge(&mem->memsw,
PAGE_SIZE * count, &dummy)) {
res_counter_uncharge(&mem->res, PAGE_SIZE * count);
goto one_by_one;
}
mc.precharge += count;
VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));
WARN_ON_ONCE(count > INT_MAX);
__css_get(&mem->css, (int)count);
return ret;
}
one_by_one:
/* fall back to one by one charge */
while (count--) {
if (signal_pending(current)) {
ret = -EINTR;
break;
}
if (!batch_count--) {
batch_count = PRECHARGE_COUNT_AT_ONCE;
cond_resched();
}
ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem,
false, NULL);
if (ret || !mem) if (ret || !mem)
/* mem_cgroup_clear_mc() will do uncharge later */
return -ENOMEM; return -ENOMEM;
mc.precharge++; mc.precharge++;
}
return ret; return ret;
} }
...@@ -3570,34 +3619,25 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) ...@@ -3570,34 +3619,25 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
return precharge; return precharge;
} }
#define PRECHARGE_AT_ONCE 256
static int mem_cgroup_precharge_mc(struct mm_struct *mm) static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{ {
int ret = 0; return mem_cgroup_do_precharge(mem_cgroup_count_precharge(mm));
int count = PRECHARGE_AT_ONCE;
unsigned long precharge = mem_cgroup_count_precharge(mm);
while (!ret && precharge--) {
if (signal_pending(current)) {
ret = -EINTR;
break;
}
if (!count--) {
count = PRECHARGE_AT_ONCE;
cond_resched();
}
ret = mem_cgroup_do_precharge();
}
return ret;
} }
static void mem_cgroup_clear_mc(void) static void mem_cgroup_clear_mc(void)
{ {
/* we must uncharge all the leftover precharges from mc.to */ /* we must uncharge all the leftover precharges from mc.to */
while (mc.precharge) { if (mc.precharge) {
mem_cgroup_cancel_charge(mc.to); __mem_cgroup_cancel_charge(mc.to, mc.precharge);
mc.precharge--; mc.precharge = 0;
}
/*
* we didn't uncharge from mc.from at mem_cgroup_move_account(), so
* we must uncharge here.
*/
if (mc.moved_charge) {
__mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
mc.moved_charge = 0;
} }
mc.from = NULL; mc.from = NULL;
mc.to = NULL; mc.to = NULL;
...@@ -3625,9 +3665,11 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss, ...@@ -3625,9 +3665,11 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
VM_BUG_ON(mc.from); VM_BUG_ON(mc.from);
VM_BUG_ON(mc.to); VM_BUG_ON(mc.to);
VM_BUG_ON(mc.precharge); VM_BUG_ON(mc.precharge);
VM_BUG_ON(mc.moved_charge);
mc.from = from; mc.from = from;
mc.to = mem; mc.to = mem;
mc.precharge = 0; mc.precharge = 0;
mc.moved_charge = 0;
ret = mem_cgroup_precharge_mc(mm); ret = mem_cgroup_precharge_mc(mm);
if (ret) if (ret)
...@@ -3674,9 +3716,11 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, ...@@ -3674,9 +3716,11 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
if (isolate_lru_page(page)) if (isolate_lru_page(page))
goto put; goto put;
pc = lookup_page_cgroup(page); pc = lookup_page_cgroup(page);
if (!mem_cgroup_move_account(pc, mc.from, mc.to)) { if (!mem_cgroup_move_account(pc,
css_put(&mc.to->css); mc.from, mc.to, false)) {
mc.precharge--; mc.precharge--;
/* we uncharge from mc.from later. */
mc.moved_charge++;
} }
putback_lru_page(page); putback_lru_page(page);
put: /* is_target_pte_for_mc() gets the page */ put: /* is_target_pte_for_mc() gets the page */
...@@ -3696,7 +3740,7 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, ...@@ -3696,7 +3740,7 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
* charges to mc.to if we have failed in charge once in attach() * charges to mc.to if we have failed in charge once in attach()
* phase. * phase.
*/ */
ret = mem_cgroup_do_precharge(); ret = mem_cgroup_do_precharge(1);
if (!ret) if (!ret)
goto retry; goto retry;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment