Commit 748446bb authored by Mel Gorman's avatar Mel Gorman Committed by Linus Torvalds

mm: compaction: memory compaction core

This patch is the core of a mechanism which compacts memory in a zone by
relocating movable pages towards the end of the zone.

A single compaction run involves a migration scanner and a free scanner.
Both scanners operate on pageblock-sized areas in the zone.  The migration
scanner starts at the bottom of the zone and searches for all movable
pages within each area, isolating them onto a private list called
migratelist.  The free scanner starts at the top of the zone and searches
for suitable areas and consumes the free pages within making them
available for the migration scanner.  The pages isolated for migration are
then migrated to the newly isolated free pages.

[aarcange@redhat.com: Fix unsafe optimisation]
[mel@csn.ul.ie: do not schedule work on other CPUs for compaction]
Signed-off-by: default avatarMel Gorman <mel@csn.ul.ie>
Acked-by: default avatarRik van Riel <riel@redhat.com>
Reviewed-by: default avatarMinchan Kim <minchan.kim@gmail.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c175a0ce
#ifndef _LINUX_COMPACTION_H
#define _LINUX_COMPACTION_H
/* Return values for compact_zone() */
#define COMPACT_CONTINUE 0
#define COMPACT_PARTIAL 1
#define COMPACT_COMPLETE 2
#endif /* _LINUX_COMPACTION_H */
...@@ -19,6 +19,7 @@ extern int fail_migrate_page(struct address_space *, ...@@ -19,6 +19,7 @@ extern int fail_migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *);
extern int migrate_prep(void); extern int migrate_prep(void);
extern int migrate_prep_local(void);
extern int migrate_vmas(struct mm_struct *mm, extern int migrate_vmas(struct mm_struct *mm,
const nodemask_t *from, const nodemask_t *to, const nodemask_t *from, const nodemask_t *to,
unsigned long flags); unsigned long flags);
...@@ -30,6 +31,7 @@ static inline int migrate_pages(struct list_head *l, new_page_t x, ...@@ -30,6 +31,7 @@ static inline int migrate_pages(struct list_head *l, new_page_t x,
unsigned long private, int offlining) { return -ENOSYS; } unsigned long private, int offlining) { return -ENOSYS; }
static inline int migrate_prep(void) { return -ENOSYS; } static inline int migrate_prep(void) { return -ENOSYS; }
static inline int migrate_prep_local(void) { return -ENOSYS; }
static inline int migrate_vmas(struct mm_struct *mm, static inline int migrate_vmas(struct mm_struct *mm,
const nodemask_t *from, const nodemask_t *to, const nodemask_t *from, const nodemask_t *to,
......
...@@ -337,6 +337,7 @@ void put_page(struct page *page); ...@@ -337,6 +337,7 @@ void put_page(struct page *page);
void put_pages_list(struct list_head *pages); void put_pages_list(struct list_head *pages);
void split_page(struct page *page, unsigned int order); void split_page(struct page *page, unsigned int order);
int split_free_page(struct page *page);
/* /*
* Compound pages have a destructor function. Provide a * Compound pages have a destructor function. Provide a
......
...@@ -152,6 +152,7 @@ enum { ...@@ -152,6 +152,7 @@ enum {
}; };
#define SWAP_CLUSTER_MAX 32 #define SWAP_CLUSTER_MAX 32
#define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX
#define SWAP_MAP_MAX 0x3e /* Max duplication count, in first swap_map */ #define SWAP_MAP_MAX 0x3e /* Max duplication count, in first swap_map */
#define SWAP_MAP_BAD 0x3f /* Note pageblock is bad, in first swap_map */ #define SWAP_MAP_BAD 0x3f /* Note pageblock is bad, in first swap_map */
......
...@@ -43,6 +43,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, ...@@ -43,6 +43,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY, KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY,
KSWAPD_SKIP_CONGESTION_WAIT, KSWAPD_SKIP_CONGESTION_WAIT,
PAGEOUTRUN, ALLOCSTALL, PGROTATED, PAGEOUTRUN, ALLOCSTALL, PGROTATED,
#ifdef CONFIG_COMPACTION
COMPACTBLOCKS, COMPACTPAGES, COMPACTPAGEFAILED,
#endif
#ifdef CONFIG_HUGETLB_PAGE #ifdef CONFIG_HUGETLB_PAGE
HTLB_BUDDY_PGALLOC, HTLB_BUDDY_PGALLOC_FAIL, HTLB_BUDDY_PGALLOC, HTLB_BUDDY_PGALLOC_FAIL,
#endif #endif
......
...@@ -23,6 +23,7 @@ obj-$(CONFIG_NUMA) += mempolicy.o ...@@ -23,6 +23,7 @@ obj-$(CONFIG_NUMA) += mempolicy.o
obj-$(CONFIG_SPARSEMEM) += sparse.o obj-$(CONFIG_SPARSEMEM) += sparse.o
obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
obj-$(CONFIG_SLOB) += slob.o obj-$(CONFIG_SLOB) += slob.o
obj-$(CONFIG_COMPACTION) += compaction.o
obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
obj-$(CONFIG_KSM) += ksm.o obj-$(CONFIG_KSM) += ksm.o
obj-$(CONFIG_PAGE_POISONING) += debug-pagealloc.o obj-$(CONFIG_PAGE_POISONING) += debug-pagealloc.o
......
This diff is collapsed.
...@@ -40,7 +40,8 @@ ...@@ -40,7 +40,8 @@
/* /*
* migrate_prep() needs to be called before we start compiling a list of pages * migrate_prep() needs to be called before we start compiling a list of pages
* to be migrated using isolate_lru_page(). * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
* undesirable, use migrate_prep_local()
*/ */
int migrate_prep(void) int migrate_prep(void)
{ {
...@@ -55,6 +56,14 @@ int migrate_prep(void) ...@@ -55,6 +56,14 @@ int migrate_prep(void)
return 0; return 0;
} }
/* Do the necessary work of migrate_prep but not if it involves other CPUs */
int migrate_prep_local(void)
{
lru_add_drain();
return 0;
}
/* /*
* Add isolated pages on the list back to the LRU under page lock * Add isolated pages on the list back to the LRU under page lock
* to avoid leaking evictable pages back onto unevictable list. * to avoid leaking evictable pages back onto unevictable list.
......
...@@ -1207,6 +1207,51 @@ void split_page(struct page *page, unsigned int order) ...@@ -1207,6 +1207,51 @@ void split_page(struct page *page, unsigned int order)
set_page_refcounted(page + i); set_page_refcounted(page + i);
} }
/*
* Similar to split_page except the page is already free. As this is only
* being used for migration, the migratetype of the block also changes.
* As this is called with interrupts disabled, the caller is responsible
* for calling arch_alloc_page() and kernel_map_page() after interrupts
* are enabled.
*
* Note: this is probably too low level an operation for use in drivers.
* Please consult with lkml before using this in your driver.
*/
int split_free_page(struct page *page)
{
unsigned int order;
unsigned long watermark;
struct zone *zone;
BUG_ON(!PageBuddy(page));
zone = page_zone(page);
order = page_order(page);
/* Obey watermarks as if the page was being allocated */
watermark = low_wmark_pages(zone) + (1 << order);
if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
return 0;
/* Remove page from free list */
list_del(&page->lru);
zone->free_area[order].nr_free--;
rmv_page_order(page);
__mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order));
/* Split into individual pages */
set_page_refcounted(page);
split_page(page, order);
if (order >= pageblock_order - 1) {
struct page *endpage = page + (1 << order) - 1;
for (; page < endpage; page += pageblock_nr_pages)
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
}
return 1 << order;
}
/* /*
* Really, prep_compound_page() should be called from __rmqueue_bulk(). But * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
* we cheat by calling it from here, in the order > 0 path. Saves a branch * we cheat by calling it from here, in the order > 0 path. Saves a branch
......
...@@ -766,6 +766,13 @@ static const char * const vmstat_text[] = { ...@@ -766,6 +766,13 @@ static const char * const vmstat_text[] = {
"allocstall", "allocstall",
"pgrotated", "pgrotated",
#ifdef CONFIG_COMPACTION
"compact_blocks_moved",
"compact_pages_moved",
"compact_pagemigrate_failed",
#endif
#ifdef CONFIG_HUGETLB_PAGE #ifdef CONFIG_HUGETLB_PAGE
"htlb_buddy_alloc_success", "htlb_buddy_alloc_success",
"htlb_buddy_alloc_fail", "htlb_buddy_alloc_fail",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment