Commit f227f0fa authored by Shakeel Butt's avatar Shakeel Butt Committed by Linus Torvalds

slub: fix unreclaimable slab stat for bulk free

SLUB uses page allocator for higher order allocations and update
unreclaimable slab stat for such allocations.  At the moment, the bulk
free for SLUB does not share code with normal free code path for these
type of allocations and have missed the stat update.  So, fix the stat
update by common code.  The user visible impact of the bug is the
potential of inconsistent unreclaimable slab stat visible through
meminfo and vmstat.

Link: https://lkml.kernel.org/r/20210728155354.3440560-1-shakeelb@google.com
Fixes: 6a486c0a ("mm, sl[ou]b: improve memory accounting")
Signed-off-by: default avatarShakeel Butt <shakeelb@google.com>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarRoman Gushchin <guro@fb.com>
Reviewed-by: default avatarMuchun Song <songmuchun@bytedance.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b5916c02
...@@ -3236,6 +3236,16 @@ struct detached_freelist { ...@@ -3236,6 +3236,16 @@ struct detached_freelist {
struct kmem_cache *s; struct kmem_cache *s;
}; };
static inline void free_nonslab_page(struct page *page)
{
unsigned int order = compound_order(page);
VM_BUG_ON_PAGE(!PageCompound(page), page);
kfree_hook(page_address(page));
mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, -(PAGE_SIZE << order));
__free_pages(page, order);
}
/* /*
* This function progressively scans the array with free objects (with * This function progressively scans the array with free objects (with
* a limited look ahead) and extract objects belonging to the same * a limited look ahead) and extract objects belonging to the same
...@@ -3272,9 +3282,7 @@ int build_detached_freelist(struct kmem_cache *s, size_t size, ...@@ -3272,9 +3282,7 @@ int build_detached_freelist(struct kmem_cache *s, size_t size,
if (!s) { if (!s) {
/* Handle kalloc'ed objects */ /* Handle kalloc'ed objects */
if (unlikely(!PageSlab(page))) { if (unlikely(!PageSlab(page))) {
BUG_ON(!PageCompound(page)); free_nonslab_page(page);
kfree_hook(object);
__free_pages(page, compound_order(page));
p[size] = NULL; /* mark object processed */ p[size] = NULL; /* mark object processed */
return size; return size;
} }
...@@ -4250,13 +4258,7 @@ void kfree(const void *x) ...@@ -4250,13 +4258,7 @@ void kfree(const void *x)
page = virt_to_head_page(x); page = virt_to_head_page(x);
if (unlikely(!PageSlab(page))) { if (unlikely(!PageSlab(page))) {
unsigned int order = compound_order(page); free_nonslab_page(page);
BUG_ON(!PageCompound(page));
kfree_hook(object);
mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
-(PAGE_SIZE << order));
__free_pages(page, order);
return; return;
} }
slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_); slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment