Commit 78869146 authored by Oliver Glitta's avatar Oliver Glitta Committed by Linus Torvalds

mm/slub: use stackdepot to save stack trace in objects

Many stack traces are similar so there are many similar arrays.
Stackdepot saves each unique stack only once.

Replace field addrs in struct track with depot_stack_handle_t handle.  Use
stackdepot to save stack trace.

The benefits are smaller memory overhead and possibility to aggregate
per-cache statistics in the future using the stackdepot handle instead of
matching stacks manually.

[rdunlap@infradead.org: rename save_stack_trace()]
  Link: https://lkml.kernel.org/r/20210513051920.29320-1-rdunlap@infradead.org
[vbabka@suse.cz: fix lockdep splat]
  Link: https://lkml.kernel.org/r/20210516195150.26740-1-vbabka@suse.czLink: https://lkml.kernel.org/r/20210414163434.4376-1-glittao@gmail.comSigned-off-by: default avatarOliver Glitta <glittao@gmail.com>
Signed-off-by: default avatarRandy Dunlap <rdunlap@infradead.org>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 113616ec
...@@ -1847,6 +1847,7 @@ config SLUB_DEBUG ...@@ -1847,6 +1847,7 @@ config SLUB_DEBUG
default y default y
bool "Enable SLUB debugging support" if EXPERT bool "Enable SLUB debugging support" if EXPERT
depends on SLUB && SYSFS depends on SLUB && SYSFS
select STACKDEPOT if STACKTRACE_SUPPORT
help help
SLUB has extensive debug support features. Disabling these can SLUB has extensive debug support features. Disabling these can
result in significant savings in code size. This also disables result in significant savings in code size. This also disables
......
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include <linux/cpuset.h> #include <linux/cpuset.h>
#include <linux/mempolicy.h> #include <linux/mempolicy.h>
#include <linux/ctype.h> #include <linux/ctype.h>
#include <linux/stackdepot.h>
#include <linux/debugobjects.h> #include <linux/debugobjects.h>
#include <linux/kallsyms.h> #include <linux/kallsyms.h>
#include <linux/kfence.h> #include <linux/kfence.h>
...@@ -220,8 +221,8 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) ...@@ -220,8 +221,8 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
#define TRACK_ADDRS_COUNT 16 #define TRACK_ADDRS_COUNT 16
struct track { struct track {
unsigned long addr; /* Called from address */ unsigned long addr; /* Called from address */
#ifdef CONFIG_STACKTRACE #ifdef CONFIG_STACKDEPOT
unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */ depot_stack_handle_t handle;
#endif #endif
int cpu; /* Was running on cpu */ int cpu; /* Was running on cpu */
int pid; /* Pid context */ int pid; /* Pid context */
...@@ -625,22 +626,27 @@ static struct track *get_track(struct kmem_cache *s, void *object, ...@@ -625,22 +626,27 @@ static struct track *get_track(struct kmem_cache *s, void *object,
return kasan_reset_tag(p + alloc); return kasan_reset_tag(p + alloc);
} }
#ifdef CONFIG_STACKDEPOT
static depot_stack_handle_t save_stack_depot_trace(gfp_t flags)
{
unsigned long entries[TRACK_ADDRS_COUNT];
depot_stack_handle_t handle;
unsigned int nr_entries;
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 4);
handle = stack_depot_save(entries, nr_entries, flags);
return handle;
}
#endif
static void set_track(struct kmem_cache *s, void *object, static void set_track(struct kmem_cache *s, void *object,
enum track_item alloc, unsigned long addr) enum track_item alloc, unsigned long addr)
{ {
struct track *p = get_track(s, object, alloc); struct track *p = get_track(s, object, alloc);
if (addr) { if (addr) {
#ifdef CONFIG_STACKTRACE #ifdef CONFIG_STACKDEPOT
unsigned int nr_entries; p->handle = save_stack_depot_trace(GFP_NOWAIT);
metadata_access_enable();
nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
TRACK_ADDRS_COUNT, 3);
metadata_access_disable();
if (nr_entries < TRACK_ADDRS_COUNT)
p->addrs[nr_entries] = 0;
#endif #endif
p->addr = addr; p->addr = addr;
p->cpu = smp_processor_id(); p->cpu = smp_processor_id();
...@@ -667,14 +673,19 @@ static void print_track(const char *s, struct track *t, unsigned long pr_time) ...@@ -667,14 +673,19 @@ static void print_track(const char *s, struct track *t, unsigned long pr_time)
pr_err("%s in %pS age=%lu cpu=%u pid=%d\n", pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid); s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
#ifdef CONFIG_STACKTRACE #ifdef CONFIG_STACKDEPOT
{ {
int i; depot_stack_handle_t handle;
for (i = 0; i < TRACK_ADDRS_COUNT; i++) unsigned long *entries;
if (t->addrs[i]) unsigned int nr_entries;
pr_err("\t%pS\n", (void *)t->addrs[i]);
else handle = READ_ONCE(t->handle);
break; if (!handle) {
pr_err("object allocation/free stack trace missing\n");
} else {
nr_entries = stack_depot_fetch(handle, &entries);
stack_trace_print(entries, nr_entries, 0);
}
} }
#endif #endif
} }
...@@ -4048,18 +4059,26 @@ void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page) ...@@ -4048,18 +4059,26 @@ void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
objp = fixup_red_left(s, objp); objp = fixup_red_left(s, objp);
trackp = get_track(s, objp, TRACK_ALLOC); trackp = get_track(s, objp, TRACK_ALLOC);
kpp->kp_ret = (void *)trackp->addr; kpp->kp_ret = (void *)trackp->addr;
#ifdef CONFIG_STACKTRACE #ifdef CONFIG_STACKDEPOT
for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) { {
kpp->kp_stack[i] = (void *)trackp->addrs[i]; depot_stack_handle_t handle;
if (!kpp->kp_stack[i]) unsigned long *entries;
break; unsigned int nr_entries;
handle = READ_ONCE(trackp->handle);
if (handle) {
nr_entries = stack_depot_fetch(handle, &entries);
for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
kpp->kp_stack[i] = (void *)entries[i];
} }
trackp = get_track(s, objp, TRACK_FREE); trackp = get_track(s, objp, TRACK_FREE);
for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) { handle = READ_ONCE(trackp->handle);
kpp->kp_free_stack[i] = (void *)trackp->addrs[i]; if (handle) {
if (!kpp->kp_free_stack[i]) nr_entries = stack_depot_fetch(handle, &entries);
break; for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
kpp->kp_free_stack[i] = (void *)entries[i];
}
} }
#endif #endif
#endif #endif
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment