Commit bb14c2c7 authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds

mm: rename and move get/set_freepage_migratetype

The pair of get/set_freepage_migratetype() functions are used to cache
pageblock migratetype for a page put on a pcplist, so that it does not
have to be retrieved again when the page is put on a free list (e.g.
when pcplists become full).  Historically it was also assumed that the
value is accurate for pages on freelists (as the functions' names
unfortunately suggest), but that cannot be guaranteed without affecting
various allocator fast paths.  It is in fact not needed and all such
uses have been removed.

The last remaining (but pointless) usage related to pages of freelists
is in move_freepages(), which this patch removes.

To prevent further confusion, rename the functions to
get/set_pcppage_migratetype() and expand their description.  Since all
the users are now in mm/page_alloc.c, move the functions there from the
shared header.
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Acked-by: default avatarJoonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Minchan Kim <minchan@kernel.org>
Acked-by: default avatarMichal Nazarewicz <mina86@mina86.com>
Cc: Laura Abbott <lauraa@codeaurora.org>
Reviewed-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Seungho Park <seungho1.park@lge.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Acked-by: default avatarMel Gorman <mgorman@techsingularity.net>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent aa016d14
...@@ -309,18 +309,6 @@ struct inode; ...@@ -309,18 +309,6 @@ struct inode;
#define page_private(page) ((page)->private) #define page_private(page) ((page)->private)
#define set_page_private(page, v) ((page)->private = (v)) #define set_page_private(page, v) ((page)->private = (v))
/* It's valid only if the page is free path or free_list */
static inline void set_freepage_migratetype(struct page *page, int migratetype)
{
page->index = migratetype;
}
/* It's valid only if the page is free path or free_list */
static inline int get_freepage_migratetype(struct page *page)
{
return page->index;
}
/* /*
* FIXME: take this include out, include page-flags.h in * FIXME: take this include out, include page-flags.h in
* files which need it (119 of them) * files which need it (119 of them)
......
...@@ -125,6 +125,24 @@ unsigned long dirty_balance_reserve __read_mostly; ...@@ -125,6 +125,24 @@ unsigned long dirty_balance_reserve __read_mostly;
int percpu_pagelist_fraction; int percpu_pagelist_fraction;
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
/*
* A cached value of the page's pageblock's migratetype, used when the page is
* put on a pcplist. Used to avoid the pageblock migratetype lookup when
* freeing from pcplists in most cases, at the cost of possibly becoming stale.
* Also the migratetype set in the page does not necessarily match the pcplist
* index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
* other index - this ensures that it will be put on the correct CMA freelist.
*/
static inline int get_pcppage_migratetype(struct page *page)
{
return page->index;
}
static inline void set_pcppage_migratetype(struct page *page, int migratetype)
{
page->index = migratetype;
}
#ifdef CONFIG_PM_SLEEP #ifdef CONFIG_PM_SLEEP
/* /*
* The following functions are used by the suspend/hibernate code to temporarily * The following functions are used by the suspend/hibernate code to temporarily
...@@ -789,7 +807,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, ...@@ -789,7 +807,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
/* must delete as __free_one_page list manipulates */ /* must delete as __free_one_page list manipulates */
list_del(&page->lru); list_del(&page->lru);
mt = get_freepage_migratetype(page); mt = get_pcppage_migratetype(page);
/* MIGRATE_ISOLATE page should not go to pcplists */ /* MIGRATE_ISOLATE page should not go to pcplists */
VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
/* Pageblock could have been isolated meanwhile */ /* Pageblock could have been isolated meanwhile */
...@@ -956,7 +974,6 @@ static void __free_pages_ok(struct page *page, unsigned int order) ...@@ -956,7 +974,6 @@ static void __free_pages_ok(struct page *page, unsigned int order)
migratetype = get_pfnblock_migratetype(page, pfn); migratetype = get_pfnblock_migratetype(page, pfn);
local_irq_save(flags); local_irq_save(flags);
__count_vm_events(PGFREE, 1 << order); __count_vm_events(PGFREE, 1 << order);
set_freepage_migratetype(page, migratetype);
free_one_page(page_zone(page), page, pfn, order, migratetype); free_one_page(page_zone(page), page, pfn, order, migratetype);
local_irq_restore(flags); local_irq_restore(flags);
} }
...@@ -1384,7 +1401,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, ...@@ -1384,7 +1401,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
rmv_page_order(page); rmv_page_order(page);
area->nr_free--; area->nr_free--;
expand(zone, page, order, current_order, area, migratetype); expand(zone, page, order, current_order, area, migratetype);
set_freepage_migratetype(page, migratetype); set_pcppage_migratetype(page, migratetype);
return page; return page;
} }
...@@ -1461,7 +1478,6 @@ int move_freepages(struct zone *zone, ...@@ -1461,7 +1478,6 @@ int move_freepages(struct zone *zone,
order = page_order(page); order = page_order(page);
list_move(&page->lru, list_move(&page->lru,
&zone->free_area[order].free_list[migratetype]); &zone->free_area[order].free_list[migratetype]);
set_freepage_migratetype(page, migratetype);
page += 1 << order; page += 1 << order;
pages_moved += 1 << order; pages_moved += 1 << order;
} }
...@@ -1631,14 +1647,13 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) ...@@ -1631,14 +1647,13 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
expand(zone, page, order, current_order, area, expand(zone, page, order, current_order, area,
start_migratetype); start_migratetype);
/* /*
* The freepage_migratetype may differ from pageblock's * The pcppage_migratetype may differ from pageblock's
* migratetype depending on the decisions in * migratetype depending on the decisions in
* try_to_steal_freepages(). This is OK as long as it * find_suitable_fallback(). This is OK as long as it does not
* does not differ for MIGRATE_CMA pageblocks. For CMA * differ for MIGRATE_CMA pageblocks. Those can be used as
* we need to make sure unallocated pages flushed from * fallback only via special __rmqueue_cma_fallback() function
* pcp lists are returned to the correct freelist.
*/ */
set_freepage_migratetype(page, start_migratetype); set_pcppage_migratetype(page, start_migratetype);
trace_mm_page_alloc_extfrag(page, order, current_order, trace_mm_page_alloc_extfrag(page, order, current_order,
start_migratetype, fallback_mt); start_migratetype, fallback_mt);
...@@ -1714,7 +1729,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, ...@@ -1714,7 +1729,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
else else
list_add_tail(&page->lru, list); list_add_tail(&page->lru, list);
list = &page->lru; list = &page->lru;
if (is_migrate_cma(get_freepage_migratetype(page))) if (is_migrate_cma(get_pcppage_migratetype(page)))
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
-(1 << order)); -(1 << order));
} }
...@@ -1911,7 +1926,7 @@ void free_hot_cold_page(struct page *page, bool cold) ...@@ -1911,7 +1926,7 @@ void free_hot_cold_page(struct page *page, bool cold)
return; return;
migratetype = get_pfnblock_migratetype(page, pfn); migratetype = get_pfnblock_migratetype(page, pfn);
set_freepage_migratetype(page, migratetype); set_pcppage_migratetype(page, migratetype);
local_irq_save(flags); local_irq_save(flags);
__count_vm_event(PGFREE); __count_vm_event(PGFREE);
...@@ -2116,7 +2131,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, ...@@ -2116,7 +2131,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
if (!page) if (!page)
goto failed; goto failed;
__mod_zone_freepage_state(zone, -(1 << order), __mod_zone_freepage_state(zone, -(1 << order),
get_freepage_migratetype(page)); get_pcppage_migratetype(page));
} }
__mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment