Commit fae91d6d authored by Kirill Tkhai's avatar Kirill Tkhai Committed by Linus Torvalds

mm/list_lru.c: set bit in memcg shrinker bitmap on first list_lru item appearance

Introduce set_shrinker_bit() function to set shrinker-related bit in
memcg shrinker bitmap, and set the bit after the first item is added and
in case of reparenting destroyed memcg's items.

This will allow next patch to make shrinkers be called only, in case of
they have charged objects at the moment, and to improve shrink_slab()
performance.

[ktkhai@virtuozzo.com: v9]
  Link: http://lkml.kernel.org/r/153112557572.4097.17315791419810749985.stgit@localhost.localdomain
Link: http://lkml.kernel.org/r/153063065671.1818.15914674956134687268.stgit@localhost.localdomainSigned-off-by: default avatarKirill Tkhai <ktkhai@virtuozzo.com>
Acked-by: default avatarVladimir Davydov <vdavydov.dev@gmail.com>
Tested-by: default avatarShakeel Butt <shakeelb@google.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Josef Bacik <jbacik@fb.com>
Cc: Li RongQing <lirongqing@baidu.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Matthias Kaehlcke <mka@chromium.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Philippe Ombredanne <pombredanne@nexb.com>
Cc: Roman Gushchin <guro@fb.com>
Cc: Sahitya Tummala <stummala@codeaurora.org>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Waiman Long <longman@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent dfd2f10c
...@@ -1286,6 +1286,8 @@ static inline int memcg_cache_id(struct mem_cgroup *memcg) ...@@ -1286,6 +1286,8 @@ static inline int memcg_cache_id(struct mem_cgroup *memcg)
extern int memcg_expand_shrinker_maps(int new_id); extern int memcg_expand_shrinker_maps(int new_id);
extern void memcg_set_shrinker_bit(struct mem_cgroup *memcg,
int nid, int shrinker_id);
#else #else
#define for_each_memcg_cache_index(_idx) \ #define for_each_memcg_cache_index(_idx) \
for (; NULL; ) for (; NULL; )
...@@ -1308,6 +1310,8 @@ static inline void memcg_put_cache_ids(void) ...@@ -1308,6 +1310,8 @@ static inline void memcg_put_cache_ids(void)
{ {
} }
static inline void memcg_set_shrinker_bit(struct mem_cgroup *memcg,
int nid, int shrinker_id) { }
#endif /* CONFIG_MEMCG_KMEM */ #endif /* CONFIG_MEMCG_KMEM */
#endif /* _LINUX_MEMCONTROL_H */ #endif /* _LINUX_MEMCONTROL_H */
...@@ -30,6 +30,11 @@ static void list_lru_unregister(struct list_lru *lru) ...@@ -30,6 +30,11 @@ static void list_lru_unregister(struct list_lru *lru)
mutex_unlock(&list_lrus_mutex); mutex_unlock(&list_lrus_mutex);
} }
static int lru_shrinker_id(struct list_lru *lru)
{
return lru->shrinker_id;
}
static inline bool list_lru_memcg_aware(struct list_lru *lru) static inline bool list_lru_memcg_aware(struct list_lru *lru)
{ {
/* /*
...@@ -93,6 +98,11 @@ static void list_lru_unregister(struct list_lru *lru) ...@@ -93,6 +98,11 @@ static void list_lru_unregister(struct list_lru *lru)
{ {
} }
static int lru_shrinker_id(struct list_lru *lru)
{
return -1;
}
static inline bool list_lru_memcg_aware(struct list_lru *lru) static inline bool list_lru_memcg_aware(struct list_lru *lru)
{ {
return false; return false;
...@@ -118,13 +128,17 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item) ...@@ -118,13 +128,17 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item)
{ {
int nid = page_to_nid(virt_to_page(item)); int nid = page_to_nid(virt_to_page(item));
struct list_lru_node *nlru = &lru->node[nid]; struct list_lru_node *nlru = &lru->node[nid];
struct mem_cgroup *memcg;
struct list_lru_one *l; struct list_lru_one *l;
spin_lock(&nlru->lock); spin_lock(&nlru->lock);
if (list_empty(item)) { if (list_empty(item)) {
l = list_lru_from_kmem(nlru, item, NULL); l = list_lru_from_kmem(nlru, item, &memcg);
list_add_tail(item, &l->list); list_add_tail(item, &l->list);
l->nr_items++; /* Set shrinker bit if the first element was added */
if (!l->nr_items++)
memcg_set_shrinker_bit(memcg, nid,
lru_shrinker_id(lru));
nlru->nr_items++; nlru->nr_items++;
spin_unlock(&nlru->lock); spin_unlock(&nlru->lock);
return true; return true;
...@@ -507,6 +521,7 @@ static void memcg_drain_list_lru_node(struct list_lru *lru, int nid, ...@@ -507,6 +521,7 @@ static void memcg_drain_list_lru_node(struct list_lru *lru, int nid,
struct list_lru_node *nlru = &lru->node[nid]; struct list_lru_node *nlru = &lru->node[nid];
int dst_idx = dst_memcg->kmemcg_id; int dst_idx = dst_memcg->kmemcg_id;
struct list_lru_one *src, *dst; struct list_lru_one *src, *dst;
bool set;
/* /*
* Since list_lru_{add,del} may be called under an IRQ-safe lock, * Since list_lru_{add,del} may be called under an IRQ-safe lock,
...@@ -518,7 +533,10 @@ static void memcg_drain_list_lru_node(struct list_lru *lru, int nid, ...@@ -518,7 +533,10 @@ static void memcg_drain_list_lru_node(struct list_lru *lru, int nid,
dst = list_lru_from_memcg_idx(nlru, dst_idx); dst = list_lru_from_memcg_idx(nlru, dst_idx);
list_splice_init(&src->list, &dst->list); list_splice_init(&src->list, &dst->list);
set = (!dst->nr_items && src->nr_items);
dst->nr_items += src->nr_items; dst->nr_items += src->nr_items;
if (set)
memcg_set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru));
src->nr_items = 0; src->nr_items = 0;
spin_unlock_irq(&nlru->lock); spin_unlock_irq(&nlru->lock);
......
...@@ -422,6 +422,19 @@ int memcg_expand_shrinker_maps(int new_id) ...@@ -422,6 +422,19 @@ int memcg_expand_shrinker_maps(int new_id)
mutex_unlock(&memcg_shrinker_map_mutex); mutex_unlock(&memcg_shrinker_map_mutex);
return ret; return ret;
} }
void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
{
if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
struct memcg_shrinker_map *map;
rcu_read_lock();
map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
set_bit(shrinker_id, map->map);
rcu_read_unlock();
}
}
#else /* CONFIG_MEMCG_KMEM */ #else /* CONFIG_MEMCG_KMEM */
static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg) static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment