Commit ee6f509c authored by Minchan Kim's avatar Minchan Kim Committed by Linus Torvalds

mm: factor out memory isolate functions

mm/page_alloc.c has some memory isolation functions but they are used only
when we enable CONFIG_{CMA|MEMORY_HOTPLUG|MEMORY_FAILURE}.  So let's make
it configurable by new CONFIG_MEMORY_ISOLATION so that it can reduce
binary size and we can check it simple by CONFIG_MEMORY_ISOLATION, not if
defined CONFIG_{CMA|MEMORY_HOTPLUG|MEMORY_FAILURE}.
Signed-off-by: default avatarMinchan Kim <minchan@kernel.org>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Acked-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Hocko <mhocko@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 876aafbf
...@@ -196,6 +196,7 @@ config CMA ...@@ -196,6 +196,7 @@ config CMA
bool "Contiguous Memory Allocator (EXPERIMENTAL)" bool "Contiguous Memory Allocator (EXPERIMENTAL)"
depends on HAVE_DMA_CONTIGUOUS && HAVE_MEMBLOCK && EXPERIMENTAL depends on HAVE_DMA_CONTIGUOUS && HAVE_MEMBLOCK && EXPERIMENTAL
select MIGRATION select MIGRATION
select MEMORY_ISOLATION
help help
This enables the Contiguous Memory Allocator which allows drivers This enables the Contiguous Memory Allocator which allows drivers
to allocate big physically-contiguous blocks of memory for use with to allocate big physically-contiguous blocks of memory for use with
......
#ifndef __LINUX_PAGEISOLATION_H #ifndef __LINUX_PAGEISOLATION_H
#define __LINUX_PAGEISOLATION_H #define __LINUX_PAGEISOLATION_H
bool has_unmovable_pages(struct zone *zone, struct page *page, int count);
void set_pageblock_migratetype(struct page *page, int migratetype);
int move_freepages_block(struct zone *zone, struct page *page,
int migratetype);
/* /*
* Changes migrate type in [start_pfn, end_pfn) to be MIGRATE_ISOLATE. * Changes migrate type in [start_pfn, end_pfn) to be MIGRATE_ISOLATE.
* If specified range includes migrate types other than MOVABLE or CMA, * If specified range includes migrate types other than MOVABLE or CMA,
...@@ -10,7 +15,7 @@ ...@@ -10,7 +15,7 @@
* free all pages in the range. test_page_isolated() can be used for * free all pages in the range. test_page_isolated() can be used for
* test it. * test it.
*/ */
extern int int
start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
unsigned migratetype); unsigned migratetype);
...@@ -18,7 +23,7 @@ start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, ...@@ -18,7 +23,7 @@ start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
* Changes MIGRATE_ISOLATE to MIGRATE_MOVABLE. * Changes MIGRATE_ISOLATE to MIGRATE_MOVABLE.
* target range is [start_pfn, end_pfn) * target range is [start_pfn, end_pfn)
*/ */
extern int int
undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
unsigned migratetype); unsigned migratetype);
...@@ -30,8 +35,8 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn); ...@@ -30,8 +35,8 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn);
/* /*
* Internal functions. Changes pageblock's migrate type. * Internal functions. Changes pageblock's migrate type.
*/ */
extern int set_migratetype_isolate(struct page *page); int set_migratetype_isolate(struct page *page);
extern void unset_migratetype_isolate(struct page *page, unsigned migratetype); void unset_migratetype_isolate(struct page *page, unsigned migratetype);
#endif #endif
...@@ -140,9 +140,13 @@ config ARCH_DISCARD_MEMBLOCK ...@@ -140,9 +140,13 @@ config ARCH_DISCARD_MEMBLOCK
config NO_BOOTMEM config NO_BOOTMEM
boolean boolean
config MEMORY_ISOLATION
boolean
# eventually, we can have this option just 'select SPARSEMEM' # eventually, we can have this option just 'select SPARSEMEM'
config MEMORY_HOTPLUG config MEMORY_HOTPLUG
bool "Allow for memory hot-add" bool "Allow for memory hot-add"
select MEMORY_ISOLATION
depends on SPARSEMEM || X86_64_ACPI_NUMA depends on SPARSEMEM || X86_64_ACPI_NUMA
depends on HOTPLUG && ARCH_ENABLE_MEMORY_HOTPLUG depends on HOTPLUG && ARCH_ENABLE_MEMORY_HOTPLUG
depends on (IA64 || X86 || PPC_BOOK3S_64 || SUPERH || S390) depends on (IA64 || X86 || PPC_BOOK3S_64 || SUPERH || S390)
...@@ -272,6 +276,7 @@ config MEMORY_FAILURE ...@@ -272,6 +276,7 @@ config MEMORY_FAILURE
depends on MMU depends on MMU
depends on ARCH_SUPPORTS_MEMORY_FAILURE depends on ARCH_SUPPORTS_MEMORY_FAILURE
bool "Enable recovery from hardware memory errors" bool "Enable recovery from hardware memory errors"
select MEMORY_ISOLATION
help help
Enables code to recover from some memory failures on systems Enables code to recover from some memory failures on systems
with MCA recovery. This allows a system to continue running with MCA recovery. This allows a system to continue running
......
...@@ -15,8 +15,8 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ ...@@ -15,8 +15,8 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \
maccess.o page_alloc.o page-writeback.o \ maccess.o page_alloc.o page-writeback.o \
readahead.o swap.o truncate.o vmscan.o shmem.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \
prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \ prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \
page_isolation.o mm_init.o mmu_context.o percpu.o \ mm_init.o mmu_context.o percpu.o slab_common.o \
compaction.o slab_common.o $(mmu-y) compaction.o $(mmu-y)
obj-y += init-mm.o obj-y += init-mm.o
...@@ -56,3 +56,4 @@ obj-$(CONFIG_HWPOISON_INJECT) += hwpoison-inject.o ...@@ -56,3 +56,4 @@ obj-$(CONFIG_HWPOISON_INJECT) += hwpoison-inject.o
obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o
obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak-test.o obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak-test.o
obj-$(CONFIG_CLEANCACHE) += cleancache.o obj-$(CONFIG_CLEANCACHE) += cleancache.o
obj-$(CONFIG_MEMORY_ISOLATION) += page_isolation.o
...@@ -51,7 +51,6 @@ ...@@ -51,7 +51,6 @@
#include <linux/page_cgroup.h> #include <linux/page_cgroup.h>
#include <linux/debugobjects.h> #include <linux/debugobjects.h>
#include <linux/kmemleak.h> #include <linux/kmemleak.h>
#include <linux/memory.h>
#include <linux/compaction.h> #include <linux/compaction.h>
#include <trace/events/kmem.h> #include <trace/events/kmem.h>
#include <linux/ftrace_event.h> #include <linux/ftrace_event.h>
...@@ -219,7 +218,7 @@ EXPORT_SYMBOL(nr_online_nodes); ...@@ -219,7 +218,7 @@ EXPORT_SYMBOL(nr_online_nodes);
int page_group_by_mobility_disabled __read_mostly; int page_group_by_mobility_disabled __read_mostly;
static void set_pageblock_migratetype(struct page *page, int migratetype) void set_pageblock_migratetype(struct page *page, int migratetype)
{ {
if (unlikely(page_group_by_mobility_disabled)) if (unlikely(page_group_by_mobility_disabled))
...@@ -954,7 +953,7 @@ static int move_freepages(struct zone *zone, ...@@ -954,7 +953,7 @@ static int move_freepages(struct zone *zone,
return pages_moved; return pages_moved;
} }
static int move_freepages_block(struct zone *zone, struct page *page, int move_freepages_block(struct zone *zone, struct page *page,
int migratetype) int migratetype)
{ {
unsigned long start_pfn, end_pfn; unsigned long start_pfn, end_pfn;
...@@ -5463,8 +5462,7 @@ void set_pageblock_flags_group(struct page *page, unsigned long flags, ...@@ -5463,8 +5462,7 @@ void set_pageblock_flags_group(struct page *page, unsigned long flags,
* MIGRATE_MOVABLE block might include unmovable pages. It means you can't * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
* expect this function should be exact. * expect this function should be exact.
*/ */
static bool bool has_unmovable_pages(struct zone *zone, struct page *page, int count)
__has_unmovable_pages(struct zone *zone, struct page *page, int count)
{ {
unsigned long pfn, iter, found; unsigned long pfn, iter, found;
int mt; int mt;
...@@ -5541,77 +5539,7 @@ bool is_pageblock_removable_nolock(struct page *page) ...@@ -5541,77 +5539,7 @@ bool is_pageblock_removable_nolock(struct page *page)
zone->zone_start_pfn + zone->spanned_pages <= pfn) zone->zone_start_pfn + zone->spanned_pages <= pfn)
return false; return false;
return !__has_unmovable_pages(zone, page, 0); return !has_unmovable_pages(zone, page, 0);
}
int set_migratetype_isolate(struct page *page)
{
struct zone *zone;
unsigned long flags, pfn;
struct memory_isolate_notify arg;
int notifier_ret;
int ret = -EBUSY;
zone = page_zone(page);
spin_lock_irqsave(&zone->lock, flags);
pfn = page_to_pfn(page);
arg.start_pfn = pfn;
arg.nr_pages = pageblock_nr_pages;
arg.pages_found = 0;
/*
* It may be possible to isolate a pageblock even if the
* migratetype is not MIGRATE_MOVABLE. The memory isolation
* notifier chain is used by balloon drivers to return the
* number of pages in a range that are held by the balloon
* driver to shrink memory. If all the pages are accounted for
* by balloons, are free, or on the LRU, isolation can continue.
* Later, for example, when memory hotplug notifier runs, these
* pages reported as "can be isolated" should be isolated(freed)
* by the balloon driver through the memory notifier chain.
*/
notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
notifier_ret = notifier_to_errno(notifier_ret);
if (notifier_ret)
goto out;
/*
* FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
* We just check MOVABLE pages.
*/
if (!__has_unmovable_pages(zone, page, arg.pages_found))
ret = 0;
/*
* Unmovable means "not-on-lru" pages. If Unmovable pages are
* larger than removable-by-driver pages reported by notifier,
* we'll fail.
*/
out:
if (!ret) {
set_pageblock_migratetype(page, MIGRATE_ISOLATE);
move_freepages_block(zone, page, MIGRATE_ISOLATE);
}
spin_unlock_irqrestore(&zone->lock, flags);
if (!ret)
drain_all_pages();
return ret;
}
void unset_migratetype_isolate(struct page *page, unsigned migratetype)
{
struct zone *zone;
unsigned long flags;
zone = page_zone(page);
spin_lock_irqsave(&zone->lock, flags);
if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
goto out;
set_pageblock_migratetype(page, migratetype);
move_freepages_block(zone, page, migratetype);
out:
spin_unlock_irqrestore(&zone->lock, flags);
} }
#ifdef CONFIG_CMA #ifdef CONFIG_CMA
......
...@@ -5,8 +5,79 @@ ...@@ -5,8 +5,79 @@
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/page-isolation.h> #include <linux/page-isolation.h>
#include <linux/pageblock-flags.h> #include <linux/pageblock-flags.h>
#include <linux/memory.h>
#include "internal.h" #include "internal.h"
int set_migratetype_isolate(struct page *page)
{
struct zone *zone;
unsigned long flags, pfn;
struct memory_isolate_notify arg;
int notifier_ret;
int ret = -EBUSY;
zone = page_zone(page);
spin_lock_irqsave(&zone->lock, flags);
pfn = page_to_pfn(page);
arg.start_pfn = pfn;
arg.nr_pages = pageblock_nr_pages;
arg.pages_found = 0;
/*
* It may be possible to isolate a pageblock even if the
* migratetype is not MIGRATE_MOVABLE. The memory isolation
* notifier chain is used by balloon drivers to return the
* number of pages in a range that are held by the balloon
* driver to shrink memory. If all the pages are accounted for
* by balloons, are free, or on the LRU, isolation can continue.
* Later, for example, when memory hotplug notifier runs, these
* pages reported as "can be isolated" should be isolated(freed)
* by the balloon driver through the memory notifier chain.
*/
notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
notifier_ret = notifier_to_errno(notifier_ret);
if (notifier_ret)
goto out;
/*
* FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
* We just check MOVABLE pages.
*/
if (!has_unmovable_pages(zone, page, arg.pages_found))
ret = 0;
/*
* immobile means "not-on-lru" paes. If immobile is larger than
* removable-by-driver pages reported by notifier, we'll fail.
*/
out:
if (!ret) {
set_pageblock_migratetype(page, MIGRATE_ISOLATE);
move_freepages_block(zone, page, MIGRATE_ISOLATE);
}
spin_unlock_irqrestore(&zone->lock, flags);
if (!ret)
drain_all_pages();
return ret;
}
void unset_migratetype_isolate(struct page *page, unsigned migratetype)
{
struct zone *zone;
unsigned long flags;
zone = page_zone(page);
spin_lock_irqsave(&zone->lock, flags);
if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
goto out;
set_pageblock_migratetype(page, migratetype);
move_freepages_block(zone, page, migratetype);
out:
spin_unlock_irqrestore(&zone->lock, flags);
}
static inline struct page * static inline struct page *
__first_valid_page(unsigned long pfn, unsigned long nr_pages) __first_valid_page(unsigned long pfn, unsigned long nr_pages)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment