Commit e46a2879 authored by Minchan Kim's avatar Minchan Kim Committed by Linus Torvalds

CMA: migrate mlocked pages

Presently CMA cannot migrate mlocked pages so it ends up failing to allocate
contiguous memory space.

This patch makes mlocked pages be migrated out.  Of course, it can affect
realtime processes but in CMA usecase, contiguous memory allocation failing
is far worse than access latency to an mlocked page being variable while
CMA is running.  If someone wants to make the system realtime, he shouldn't
enable CMA because stalls can still happen at random times.

[akpm@linux-foundation.org: tweak comment text, per Mel]
Signed-off-by: default avatarMinchan Kim <minchan@kernel.org>
Acked-by: default avatarMel Gorman <mgorman@suse.de>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 7a71932d
...@@ -218,6 +218,8 @@ struct lruvec { ...@@ -218,6 +218,8 @@ struct lruvec {
#define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2) #define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)
/* Isolate for asynchronous migration */ /* Isolate for asynchronous migration */
#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4) #define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)
/* Isolate unevictable pages */
#define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8)
/* LRU Isolation modes. */ /* LRU Isolation modes. */
typedef unsigned __bitwise__ isolate_mode_t; typedef unsigned __bitwise__ isolate_mode_t;
......
...@@ -461,6 +461,7 @@ static bool too_many_isolated(struct zone *zone) ...@@ -461,6 +461,7 @@ static bool too_many_isolated(struct zone *zone)
* @cc: Compaction control structure. * @cc: Compaction control structure.
* @low_pfn: The first PFN of the range. * @low_pfn: The first PFN of the range.
* @end_pfn: The one-past-the-last PFN of the range. * @end_pfn: The one-past-the-last PFN of the range.
* @unevictable: true if it allows to isolate unevictable pages
* *
* Isolate all pages that can be migrated from the range specified by * Isolate all pages that can be migrated from the range specified by
* [low_pfn, end_pfn). Returns zero if there is a fatal signal * [low_pfn, end_pfn). Returns zero if there is a fatal signal
...@@ -476,7 +477,7 @@ static bool too_many_isolated(struct zone *zone) ...@@ -476,7 +477,7 @@ static bool too_many_isolated(struct zone *zone)
*/ */
unsigned long unsigned long
isolate_migratepages_range(struct zone *zone, struct compact_control *cc, isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
unsigned long low_pfn, unsigned long end_pfn) unsigned long low_pfn, unsigned long end_pfn, bool unevictable)
{ {
unsigned long last_pageblock_nr = 0, pageblock_nr; unsigned long last_pageblock_nr = 0, pageblock_nr;
unsigned long nr_scanned = 0, nr_isolated = 0; unsigned long nr_scanned = 0, nr_isolated = 0;
...@@ -602,6 +603,9 @@ isolate_migratepages_range(struct zone *zone, struct compact_control *cc, ...@@ -602,6 +603,9 @@ isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
if (!cc->sync) if (!cc->sync)
mode |= ISOLATE_ASYNC_MIGRATE; mode |= ISOLATE_ASYNC_MIGRATE;
if (unevictable)
mode |= ISOLATE_UNEVICTABLE;
lruvec = mem_cgroup_page_lruvec(page, zone); lruvec = mem_cgroup_page_lruvec(page, zone);
/* Try isolate the page */ /* Try isolate the page */
...@@ -807,7 +811,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, ...@@ -807,7 +811,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
} }
/* Perform the isolation */ /* Perform the isolation */
low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn); low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn, false);
if (!low_pfn || cc->contended) if (!low_pfn || cc->contended)
return ISOLATE_ABORT; return ISOLATE_ABORT;
......
...@@ -138,7 +138,7 @@ isolate_freepages_range(struct compact_control *cc, ...@@ -138,7 +138,7 @@ isolate_freepages_range(struct compact_control *cc,
unsigned long start_pfn, unsigned long end_pfn); unsigned long start_pfn, unsigned long end_pfn);
unsigned long unsigned long
isolate_migratepages_range(struct zone *zone, struct compact_control *cc, isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
unsigned long low_pfn, unsigned long end_pfn); unsigned long low_pfn, unsigned long end_pfn, bool unevictable);
#endif #endif
......
...@@ -5690,7 +5690,7 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, ...@@ -5690,7 +5690,7 @@ static int __alloc_contig_migrate_range(struct compact_control *cc,
if (list_empty(&cc->migratepages)) { if (list_empty(&cc->migratepages)) {
cc->nr_migratepages = 0; cc->nr_migratepages = 0;
pfn = isolate_migratepages_range(cc->zone, cc, pfn = isolate_migratepages_range(cc->zone, cc,
pfn, end); pfn, end, true);
if (!pfn) { if (!pfn) {
ret = -EINTR; ret = -EINTR;
break; break;
......
...@@ -1009,8 +1009,8 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode) ...@@ -1009,8 +1009,8 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode)
if (!PageLRU(page)) if (!PageLRU(page))
return ret; return ret;
/* Do not give back unevictable pages for compaction */ /* Compaction should not handle unevictable pages but CMA can do so */
if (PageUnevictable(page)) if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
return ret; return ret;
ret = -EBUSY; ret = -EBUSY;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment