Commit 60d3fd32 authored by Vladimir Davydov's avatar Vladimir Davydov Committed by Linus Torvalds

list_lru: introduce per-memcg lists

There are several FS shrinkers, including super_block::s_shrink, that
keep reclaimable objects in the list_lru structure.  Hence to turn them
to memcg-aware shrinkers, it is enough to make list_lru per-memcg.

This patch does the trick.  It adds an array of lru lists to the
list_lru_node structure (per-node part of the list_lru), one for each
kmem-active memcg, and dispatches every item addition or removal to the
list corresponding to the memcg which the item is accounted to.  So now
the list_lru structure is not just per node, but per node and per memcg.

Not all list_lrus need this feature, so this patch also adds a new
method, list_lru_init_memcg, which initializes a list_lru as memcg
aware.  Otherwise (i.e.  if initialized with old list_lru_init), the
list_lru won't have per memcg lists.

Just like per memcg caches arrays, the arrays of per-memcg lists are
indexed by memcg_cache_id, so we must grow them whenever
memcg_nr_cache_ids is increased.  So we introduce a callback,
memcg_update_all_list_lrus, invoked by memcg_alloc_cache_id if the id
space is full.

The locking is implemented in a manner similar to lruvecs, i.e.  we have
one lock per node that protects all lists (both global and per cgroup) on
the node.
Signed-off-by: default avatarVladimir Davydov <vdavydov@parallels.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Greg Thelen <gthelen@google.com>
Cc: Glauber Costa <glommer@gmail.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c0a5b560
...@@ -11,6 +11,8 @@ ...@@ -11,6 +11,8 @@
#include <linux/nodemask.h> #include <linux/nodemask.h>
#include <linux/shrinker.h> #include <linux/shrinker.h>
struct mem_cgroup;
/* list_lru_walk_cb has to always return one of those */ /* list_lru_walk_cb has to always return one of those */
enum lru_status { enum lru_status {
LRU_REMOVED, /* item removed from list */ LRU_REMOVED, /* item removed from list */
...@@ -22,11 +24,26 @@ enum lru_status { ...@@ -22,11 +24,26 @@ enum lru_status {
internally, but has to return locked. */ internally, but has to return locked. */
}; };
struct list_lru_node { struct list_lru_one {
spinlock_t lock;
struct list_head list; struct list_head list;
/* kept as signed so we can catch imbalance bugs */ /* kept as signed so we can catch imbalance bugs */
long nr_items; long nr_items;
};
struct list_lru_memcg {
/* array of per cgroup lists, indexed by memcg_cache_id */
struct list_lru_one *lru[0];
};
struct list_lru_node {
/* protects all lists on the node, including per cgroup */
spinlock_t lock;
/* global list, used for the root cgroup in cgroup aware lrus */
struct list_lru_one lru;
#ifdef CONFIG_MEMCG_KMEM
/* for cgroup aware lrus points to per cgroup lists, otherwise NULL */
struct list_lru_memcg *memcg_lrus;
#endif
} ____cacheline_aligned_in_smp; } ____cacheline_aligned_in_smp;
struct list_lru { struct list_lru {
...@@ -37,11 +54,14 @@ struct list_lru { ...@@ -37,11 +54,14 @@ struct list_lru {
}; };
void list_lru_destroy(struct list_lru *lru); void list_lru_destroy(struct list_lru *lru);
int list_lru_init_key(struct list_lru *lru, struct lock_class_key *key); int __list_lru_init(struct list_lru *lru, bool memcg_aware,
static inline int list_lru_init(struct list_lru *lru) struct lock_class_key *key);
{
return list_lru_init_key(lru, NULL); #define list_lru_init(lru) __list_lru_init((lru), false, NULL)
} #define list_lru_init_key(lru, key) __list_lru_init((lru), false, (key))
#define list_lru_init_memcg(lru) __list_lru_init((lru), true, NULL)
int memcg_update_all_list_lrus(int num_memcgs);
/** /**
* list_lru_add: add an element to the lru list's tail * list_lru_add: add an element to the lru list's tail
...@@ -75,20 +95,23 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item); ...@@ -75,20 +95,23 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item);
bool list_lru_del(struct list_lru *lru, struct list_head *item); bool list_lru_del(struct list_lru *lru, struct list_head *item);
/** /**
* list_lru_count_node: return the number of objects currently held by @lru * list_lru_count_one: return the number of objects currently held by @lru
* @lru: the lru pointer. * @lru: the lru pointer.
* @nid: the node id to count from. * @nid: the node id to count from.
* @memcg: the cgroup to count from.
* *
* Always return a non-negative number, 0 for empty lists. There is no * Always return a non-negative number, 0 for empty lists. There is no
* guarantee that the list is not updated while the count is being computed. * guarantee that the list is not updated while the count is being computed.
* Callers that want such a guarantee need to provide an outer lock. * Callers that want such a guarantee need to provide an outer lock.
*/ */
unsigned long list_lru_count_one(struct list_lru *lru,
int nid, struct mem_cgroup *memcg);
unsigned long list_lru_count_node(struct list_lru *lru, int nid); unsigned long list_lru_count_node(struct list_lru *lru, int nid);
static inline unsigned long list_lru_shrink_count(struct list_lru *lru, static inline unsigned long list_lru_shrink_count(struct list_lru *lru,
struct shrink_control *sc) struct shrink_control *sc)
{ {
return list_lru_count_node(lru, sc->nid); return list_lru_count_one(lru, sc->nid, sc->memcg);
} }
static inline unsigned long list_lru_count(struct list_lru *lru) static inline unsigned long list_lru_count(struct list_lru *lru)
...@@ -105,9 +128,10 @@ static inline unsigned long list_lru_count(struct list_lru *lru) ...@@ -105,9 +128,10 @@ static inline unsigned long list_lru_count(struct list_lru *lru)
typedef enum lru_status typedef enum lru_status
(*list_lru_walk_cb)(struct list_head *item, spinlock_t *lock, void *cb_arg); (*list_lru_walk_cb)(struct list_head *item, spinlock_t *lock, void *cb_arg);
/** /**
* list_lru_walk_node: walk a list_lru, isolating and disposing freeable items. * list_lru_walk_one: walk a list_lru, isolating and disposing freeable items.
* @lru: the lru pointer. * @lru: the lru pointer.
* @nid: the node id to scan from. * @nid: the node id to scan from.
* @memcg: the cgroup to scan from.
* @isolate: callback function that is resposible for deciding what to do with * @isolate: callback function that is resposible for deciding what to do with
* the item currently being scanned * the item currently being scanned
* @cb_arg: opaque type that will be passed to @isolate * @cb_arg: opaque type that will be passed to @isolate
...@@ -125,6 +149,10 @@ typedef enum lru_status ...@@ -125,6 +149,10 @@ typedef enum lru_status
* *
* Return value: the number of objects effectively removed from the LRU. * Return value: the number of objects effectively removed from the LRU.
*/ */
unsigned long list_lru_walk_one(struct list_lru *lru,
int nid, struct mem_cgroup *memcg,
list_lru_walk_cb isolate, void *cb_arg,
unsigned long *nr_to_walk);
unsigned long list_lru_walk_node(struct list_lru *lru, int nid, unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
list_lru_walk_cb isolate, void *cb_arg, list_lru_walk_cb isolate, void *cb_arg,
unsigned long *nr_to_walk); unsigned long *nr_to_walk);
...@@ -133,7 +161,7 @@ static inline unsigned long ...@@ -133,7 +161,7 @@ static inline unsigned long
list_lru_shrink_walk(struct list_lru *lru, struct shrink_control *sc, list_lru_shrink_walk(struct list_lru *lru, struct shrink_control *sc,
list_lru_walk_cb isolate, void *cb_arg) list_lru_walk_cb isolate, void *cb_arg)
{ {
return list_lru_walk_node(lru, sc->nid, isolate, cb_arg, return list_lru_walk_one(lru, sc->nid, sc->memcg, isolate, cb_arg,
&sc->nr_to_scan); &sc->nr_to_scan);
} }
......
...@@ -439,6 +439,8 @@ int memcg_cache_id(struct mem_cgroup *memcg); ...@@ -439,6 +439,8 @@ int memcg_cache_id(struct mem_cgroup *memcg);
struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep); struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep);
void __memcg_kmem_put_cache(struct kmem_cache *cachep); void __memcg_kmem_put_cache(struct kmem_cache *cachep);
struct mem_cgroup *__mem_cgroup_from_kmem(void *ptr);
int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp,
unsigned long nr_pages); unsigned long nr_pages);
void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned long nr_pages); void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned long nr_pages);
...@@ -535,6 +537,13 @@ static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep) ...@@ -535,6 +537,13 @@ static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
if (memcg_kmem_enabled()) if (memcg_kmem_enabled())
__memcg_kmem_put_cache(cachep); __memcg_kmem_put_cache(cachep);
} }
static __always_inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr)
{
if (!memcg_kmem_enabled())
return NULL;
return __mem_cgroup_from_kmem(ptr);
}
#else #else
#define for_each_memcg_cache_index(_idx) \ #define for_each_memcg_cache_index(_idx) \
for (; NULL; ) for (; NULL; )
...@@ -586,6 +595,11 @@ memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) ...@@ -586,6 +595,11 @@ memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
static inline void memcg_kmem_put_cache(struct kmem_cache *cachep) static inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
{ {
} }
static inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr)
{
return NULL;
}
#endif /* CONFIG_MEMCG_KMEM */ #endif /* CONFIG_MEMCG_KMEM */
#endif /* _LINUX_MEMCONTROL_H */ #endif /* _LINUX_MEMCONTROL_H */
This diff is collapsed.
...@@ -2571,6 +2571,8 @@ static int memcg_alloc_cache_id(void) ...@@ -2571,6 +2571,8 @@ static int memcg_alloc_cache_id(void)
size = MEMCG_CACHES_MAX_SIZE; size = MEMCG_CACHES_MAX_SIZE;
err = memcg_update_all_caches(size); err = memcg_update_all_caches(size);
if (!err)
err = memcg_update_all_list_lrus(size);
if (!err) if (!err)
memcg_nr_cache_ids = size; memcg_nr_cache_ids = size;
...@@ -2765,6 +2767,24 @@ void __memcg_kmem_uncharge_pages(struct page *page, int order) ...@@ -2765,6 +2767,24 @@ void __memcg_kmem_uncharge_pages(struct page *page, int order)
memcg_uncharge_kmem(memcg, 1 << order); memcg_uncharge_kmem(memcg, 1 << order);
page->mem_cgroup = NULL; page->mem_cgroup = NULL;
} }
struct mem_cgroup *__mem_cgroup_from_kmem(void *ptr)
{
struct mem_cgroup *memcg = NULL;
struct kmem_cache *cachep;
struct page *page;
page = virt_to_head_page(ptr);
if (PageSlab(page)) {
cachep = page->slab_cache;
if (!is_root_cache(cachep))
memcg = cachep->memcg_params->memcg;
} else
/* page allocated by alloc_kmem_pages */
memcg = page->mem_cgroup;
return memcg;
}
#endif /* CONFIG_MEMCG_KMEM */ #endif /* CONFIG_MEMCG_KMEM */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment