Commit 42c0faac authored by Vlastimil Babka's avatar Vlastimil Babka

mm/slab: Convert kmem_getpages() and kmem_freepages() to struct slab

These functions sit at the boundary to page allocator. Also use folio
internally to avoid extra compound_head() when dealing with page flags.
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarRoman Gushchin <guro@fb.com>
Reviewed-by: default avatarHyeonggon Yoo <42.hyeyoo@gmail.com>
Tested-by: default avatarHyeonggon Yoo <42.hyeyoo@gmail.com>
parent c2092c12
...@@ -1367,57 +1367,60 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) ...@@ -1367,57 +1367,60 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
* did not request dmaable memory, we might get it, but that * did not request dmaable memory, we might get it, but that
* would be relatively rare and ignorable. * would be relatively rare and ignorable.
*/ */
static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, static struct slab *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
int nodeid) int nodeid)
{ {
struct page *page; struct folio *folio;
struct slab *slab;
flags |= cachep->allocflags; flags |= cachep->allocflags;
page = __alloc_pages_node(nodeid, flags, cachep->gfporder); folio = (struct folio *) __alloc_pages_node(nodeid, flags, cachep->gfporder);
if (!page) { if (!folio) {
slab_out_of_memory(cachep, flags, nodeid); slab_out_of_memory(cachep, flags, nodeid);
return NULL; return NULL;
} }
account_slab(page_slab(page), cachep->gfporder, cachep, flags); slab = folio_slab(folio);
__SetPageSlab(page);
account_slab(slab, cachep->gfporder, cachep, flags);
__folio_set_slab(folio);
/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */ /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
if (sk_memalloc_socks() && page_is_pfmemalloc(page)) if (sk_memalloc_socks() && page_is_pfmemalloc(folio_page(folio, 0)))
SetPageSlabPfmemalloc(page); slab_set_pfmemalloc(slab);
return page; return slab;
} }
/* /*
* Interface to system's page release. * Interface to system's page release.
*/ */
static void kmem_freepages(struct kmem_cache *cachep, struct page *page) static void kmem_freepages(struct kmem_cache *cachep, struct slab *slab)
{ {
int order = cachep->gfporder; int order = cachep->gfporder;
struct folio *folio = slab_folio(slab);
BUG_ON(!PageSlab(page)); BUG_ON(!folio_test_slab(folio));
__ClearPageSlabPfmemalloc(page); __slab_clear_pfmemalloc(slab);
__ClearPageSlab(page); __folio_clear_slab(folio);
page_mapcount_reset(page); page_mapcount_reset(folio_page(folio, 0));
/* In union with page->mapping where page allocator expects NULL */ folio->mapping = NULL;
page->slab_cache = NULL;
if (current->reclaim_state) if (current->reclaim_state)
current->reclaim_state->reclaimed_slab += 1 << order; current->reclaim_state->reclaimed_slab += 1 << order;
unaccount_slab(page_slab(page), order, cachep); unaccount_slab(slab, order, cachep);
__free_pages(page, order); __free_pages(folio_page(folio, 0), order);
} }
static void kmem_rcu_free(struct rcu_head *head) static void kmem_rcu_free(struct rcu_head *head)
{ {
struct kmem_cache *cachep; struct kmem_cache *cachep;
struct page *page; struct slab *slab;
page = container_of(head, struct page, rcu_head); slab = container_of(head, struct slab, rcu_head);
cachep = page->slab_cache; cachep = slab->slab_cache;
kmem_freepages(cachep, page); kmem_freepages(cachep, slab);
} }
#if DEBUG #if DEBUG
...@@ -1624,7 +1627,7 @@ static void slab_destroy(struct kmem_cache *cachep, struct page *page) ...@@ -1624,7 +1627,7 @@ static void slab_destroy(struct kmem_cache *cachep, struct page *page)
if (unlikely(cachep->flags & SLAB_TYPESAFE_BY_RCU)) if (unlikely(cachep->flags & SLAB_TYPESAFE_BY_RCU))
call_rcu(&page->rcu_head, kmem_rcu_free); call_rcu(&page->rcu_head, kmem_rcu_free);
else else
kmem_freepages(cachep, page); kmem_freepages(cachep, page_slab(page));
/* /*
* From now on, we don't use freelist * From now on, we don't use freelist
...@@ -2578,7 +2581,7 @@ static struct page *cache_grow_begin(struct kmem_cache *cachep, ...@@ -2578,7 +2581,7 @@ static struct page *cache_grow_begin(struct kmem_cache *cachep,
* Get mem for the objs. Attempt to allocate a physical page from * Get mem for the objs. Attempt to allocate a physical page from
* 'nodeid'. * 'nodeid'.
*/ */
page = kmem_getpages(cachep, local_flags, nodeid); page = slab_page(kmem_getpages(cachep, local_flags, nodeid));
if (!page) if (!page)
goto failed; goto failed;
...@@ -2620,7 +2623,7 @@ static struct page *cache_grow_begin(struct kmem_cache *cachep, ...@@ -2620,7 +2623,7 @@ static struct page *cache_grow_begin(struct kmem_cache *cachep,
return page; return page;
opps1: opps1:
kmem_freepages(cachep, page); kmem_freepages(cachep, page_slab(page));
failed: failed:
if (gfpflags_allow_blocking(local_flags)) if (gfpflags_allow_blocking(local_flags))
local_irq_disable(); local_irq_disable();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment