Commit 194159fb authored by Minchan Kim's avatar Minchan Kim Committed by Linus Torvalds

mm: remove MIGRATE_ISOLATE check in hotpath

Several functions test MIGRATE_ISOLATE and some of those are hotpath but
MIGRATE_ISOLATE is used only if we enable CONFIG_MEMORY_ISOLATION(ie,
CMA, memory-hotplug and memory-failure) which are not common config
option.  So let's not add unnecessary overhead and code when we don't
enable CONFIG_MEMORY_ISOLATION.
Signed-off-by: default avatarMinchan Kim <minchan@kernel.org>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Acked-by: default avatarMichal Nazarewicz <mina86@mina86.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c60514b6
...@@ -57,7 +57,9 @@ enum { ...@@ -57,7 +57,9 @@ enum {
*/ */
MIGRATE_CMA, MIGRATE_CMA,
#endif #endif
#ifdef CONFIG_MEMORY_ISOLATION
MIGRATE_ISOLATE, /* can't allocate from here */ MIGRATE_ISOLATE, /* can't allocate from here */
#endif
MIGRATE_TYPES MIGRATE_TYPES
}; };
......
#ifndef __LINUX_PAGEISOLATION_H #ifndef __LINUX_PAGEISOLATION_H
#define __LINUX_PAGEISOLATION_H #define __LINUX_PAGEISOLATION_H
#ifdef CONFIG_MEMORY_ISOLATION
static inline bool is_migrate_isolate_page(struct page *page)
{
return get_pageblock_migratetype(page) == MIGRATE_ISOLATE;
}
static inline bool is_migrate_isolate(int migratetype)
{
return migratetype == MIGRATE_ISOLATE;
}
#else
static inline bool is_migrate_isolate_page(struct page *page)
{
return false;
}
static inline bool is_migrate_isolate(int migratetype)
{
return false;
}
#endif
bool has_unmovable_pages(struct zone *zone, struct page *page, int count, bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
bool skip_hwpoisoned_pages); bool skip_hwpoisoned_pages);
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
#include <linux/sysctl.h> #include <linux/sysctl.h>
#include <linux/sysfs.h> #include <linux/sysfs.h>
#include <linux/balloon_compaction.h> #include <linux/balloon_compaction.h>
#include <linux/page-isolation.h>
#include "internal.h" #include "internal.h"
#ifdef CONFIG_COMPACTION #ifdef CONFIG_COMPACTION
...@@ -215,7 +216,10 @@ static bool suitable_migration_target(struct page *page) ...@@ -215,7 +216,10 @@ static bool suitable_migration_target(struct page *page)
int migratetype = get_pageblock_migratetype(page); int migratetype = get_pageblock_migratetype(page);
/* Don't interfere with memory hot-remove or the min_free_kbytes blocks */ /* Don't interfere with memory hot-remove or the min_free_kbytes blocks */
if (migratetype == MIGRATE_ISOLATE || migratetype == MIGRATE_RESERVE) if (migratetype == MIGRATE_RESERVE)
return false;
if (is_migrate_isolate(migratetype))
return false; return false;
/* If the page is a large free page, then allow migration */ /* If the page is a large free page, then allow migration */
......
...@@ -673,7 +673,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, ...@@ -673,7 +673,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
__free_one_page(page, zone, 0, mt); __free_one_page(page, zone, 0, mt);
trace_mm_page_pcpu_drain(page, 0, mt); trace_mm_page_pcpu_drain(page, 0, mt);
if (likely(get_pageblock_migratetype(page) != MIGRATE_ISOLATE)) { if (likely(!is_migrate_isolate_page(page))) {
__mod_zone_page_state(zone, NR_FREE_PAGES, 1); __mod_zone_page_state(zone, NR_FREE_PAGES, 1);
if (is_migrate_cma(mt)) if (is_migrate_cma(mt))
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1); __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
...@@ -691,7 +691,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order, ...@@ -691,7 +691,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
zone->pages_scanned = 0; zone->pages_scanned = 0;
__free_one_page(page, zone, order, migratetype); __free_one_page(page, zone, order, migratetype);
if (unlikely(migratetype != MIGRATE_ISOLATE)) if (unlikely(!is_migrate_isolate(migratetype)))
__mod_zone_freepage_state(zone, 1 << order, migratetype); __mod_zone_freepage_state(zone, 1 << order, migratetype);
spin_unlock(&zone->lock); spin_unlock(&zone->lock);
} }
...@@ -923,7 +923,9 @@ static int fallbacks[MIGRATE_TYPES][4] = { ...@@ -923,7 +923,9 @@ static int fallbacks[MIGRATE_TYPES][4] = {
[MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE }, [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
#endif #endif
[MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */ [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */
#ifdef CONFIG_MEMORY_ISOLATION
[MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */ [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */
#endif
}; };
/* /*
...@@ -1149,7 +1151,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, ...@@ -1149,7 +1151,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
list_add_tail(&page->lru, list); list_add_tail(&page->lru, list);
if (IS_ENABLED(CONFIG_CMA)) { if (IS_ENABLED(CONFIG_CMA)) {
mt = get_pageblock_migratetype(page); mt = get_pageblock_migratetype(page);
if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE) if (!is_migrate_cma(mt) && !is_migrate_isolate(mt))
mt = migratetype; mt = migratetype;
} }
set_freepage_migratetype(page, mt); set_freepage_migratetype(page, mt);
...@@ -1333,7 +1335,7 @@ void free_hot_cold_page(struct page *page, int cold) ...@@ -1333,7 +1335,7 @@ void free_hot_cold_page(struct page *page, int cold)
* excessively into the page allocator * excessively into the page allocator
*/ */
if (migratetype >= MIGRATE_PCPTYPES) { if (migratetype >= MIGRATE_PCPTYPES) {
if (unlikely(migratetype == MIGRATE_ISOLATE)) { if (unlikely(is_migrate_isolate(migratetype))) {
free_one_page(zone, page, 0, migratetype); free_one_page(zone, page, 0, migratetype);
goto out; goto out;
} }
...@@ -1407,7 +1409,7 @@ static int __isolate_free_page(struct page *page, unsigned int order) ...@@ -1407,7 +1409,7 @@ static int __isolate_free_page(struct page *page, unsigned int order)
zone = page_zone(page); zone = page_zone(page);
mt = get_pageblock_migratetype(page); mt = get_pageblock_migratetype(page);
if (mt != MIGRATE_ISOLATE) { if (!is_migrate_isolate(mt)) {
/* Obey watermarks as if the page was being allocated */ /* Obey watermarks as if the page was being allocated */
watermark = low_wmark_pages(zone) + (1 << order); watermark = low_wmark_pages(zone) + (1 << order);
if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
...@@ -1426,7 +1428,7 @@ static int __isolate_free_page(struct page *page, unsigned int order) ...@@ -1426,7 +1428,7 @@ static int __isolate_free_page(struct page *page, unsigned int order)
struct page *endpage = page + (1 << order) - 1; struct page *endpage = page + (1 << order) - 1;
for (; page < endpage; page += pageblock_nr_pages) { for (; page < endpage; page += pageblock_nr_pages) {
int mt = get_pageblock_migratetype(page); int mt = get_pageblock_migratetype(page);
if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt)) if (!is_migrate_isolate(mt) && !is_migrate_cma(mt))
set_pageblock_migratetype(page, set_pageblock_migratetype(page,
MIGRATE_MOVABLE); MIGRATE_MOVABLE);
} }
...@@ -2904,7 +2906,9 @@ static void show_migration_types(unsigned char type) ...@@ -2904,7 +2906,9 @@ static void show_migration_types(unsigned char type)
#ifdef CONFIG_CMA #ifdef CONFIG_CMA
[MIGRATE_CMA] = 'C', [MIGRATE_CMA] = 'C',
#endif #endif
#ifdef CONFIG_MEMORY_ISOLATION
[MIGRATE_ISOLATE] = 'I', [MIGRATE_ISOLATE] = 'I',
#endif
}; };
char tmp[MIGRATE_TYPES + 1]; char tmp[MIGRATE_TYPES + 1];
char *p = tmp; char *p = tmp;
......
...@@ -628,7 +628,9 @@ static char * const migratetype_names[MIGRATE_TYPES] = { ...@@ -628,7 +628,9 @@ static char * const migratetype_names[MIGRATE_TYPES] = {
#ifdef CONFIG_CMA #ifdef CONFIG_CMA
"CMA", "CMA",
#endif #endif
#ifdef CONFIG_MEMORY_ISOLATION
"Isolate", "Isolate",
#endif
}; };
static void *frag_start(struct seq_file *m, loff_t *pos) static void *frag_start(struct seq_file *m, loff_t *pos)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment