Commit 62b61f61 authored by Hugh Dickins's avatar Hugh Dickins Committed by Linus Torvalds

ksm: memory hotremove migration only

The previous patch enables page migration of ksm pages, but that soon gets
into trouble: not surprising, since we're using the ksm page lock to lock
operations on its stable_node, but page migration switches the page whose
lock is to be used for that.  Another layer of locking would fix it, but
do we need that yet?

Do we actually need page migration of ksm pages?  Yes, memory hotremove
needs to offline sections of memory: and since we stopped allocating ksm
pages with GFP_HIGHUSER, they will tend to be GFP_HIGHUSER_MOVABLE
candidates for migration.

But KSM is currently unconscious of NUMA issues, happily merging pages
from different NUMA nodes: at present the rule must be, not to use
MADV_MERGEABLE where you care about NUMA.  So no, NUMA page migration of
ksm pages does not make sense yet.

So, to complete support for ksm swapping we need to make hotremove safe.
ksm_memory_callback() take ksm_thread_mutex when MEM_GOING_OFFLINE and
release it when MEM_OFFLINE or MEM_CANCEL_OFFLINE.  But if mapped pages
are freed before migration reaches them, stable_nodes may be left still
pointing to struct pages which have been removed from the system: the
stable_node needs to identify a page by pfn rather than page pointer, then
it can safely prune them when MEM_OFFLINE.

And make NUMA migration skip PageKsm pages where it skips PageReserved.
But it's only when we reach unmap_and_move() that the page lock is taken
and we can be sure that raised pagecount has prevented a PageAnon from
being upgraded: so add offlining arg to migrate_pages(), to migrate ksm
page when offlining (has sufficient locking) but reject it otherwise.
Signed-off-by: default avatarHugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Chris Wright <chrisw@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent e9995ef9
...@@ -12,7 +12,8 @@ typedef struct page *new_page_t(struct page *, unsigned long private, int **); ...@@ -12,7 +12,8 @@ typedef struct page *new_page_t(struct page *, unsigned long private, int **);
extern int putback_lru_pages(struct list_head *l); extern int putback_lru_pages(struct list_head *l);
extern int migrate_page(struct address_space *, extern int migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *);
extern int migrate_pages(struct list_head *l, new_page_t x, unsigned long); extern int migrate_pages(struct list_head *l, new_page_t x,
unsigned long private, int offlining);
extern int fail_migrate_page(struct address_space *, extern int fail_migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *);
...@@ -26,10 +27,7 @@ extern int migrate_vmas(struct mm_struct *mm, ...@@ -26,10 +27,7 @@ extern int migrate_vmas(struct mm_struct *mm,
static inline int putback_lru_pages(struct list_head *l) { return 0; } static inline int putback_lru_pages(struct list_head *l) { return 0; }
static inline int migrate_pages(struct list_head *l, new_page_t x, static inline int migrate_pages(struct list_head *l, new_page_t x,
unsigned long private) { return -ENOSYS; } unsigned long private, int offlining) { return -ENOSYS; }
static inline int migrate_pages_to(struct list_head *pagelist,
struct vm_area_struct *vma, int dest) { return 0; }
static inline int migrate_prep(void) { return -ENOSYS; } static inline int migrate_prep(void) { return -ENOSYS; }
......
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
#include <linux/wait.h> #include <linux/wait.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/rbtree.h> #include <linux/rbtree.h>
#include <linux/memory.h>
#include <linux/mmu_notifier.h> #include <linux/mmu_notifier.h>
#include <linux/swap.h> #include <linux/swap.h>
#include <linux/ksm.h> #include <linux/ksm.h>
...@@ -108,14 +109,14 @@ struct ksm_scan { ...@@ -108,14 +109,14 @@ struct ksm_scan {
/** /**
* struct stable_node - node of the stable rbtree * struct stable_node - node of the stable rbtree
* @page: pointer to struct page of the ksm page
* @node: rb node of this ksm page in the stable tree * @node: rb node of this ksm page in the stable tree
* @hlist: hlist head of rmap_items using this ksm page * @hlist: hlist head of rmap_items using this ksm page
* @kpfn: page frame number of this ksm page
*/ */
struct stable_node { struct stable_node {
struct page *page;
struct rb_node node; struct rb_node node;
struct hlist_head hlist; struct hlist_head hlist;
unsigned long kpfn;
}; };
/** /**
...@@ -515,7 +516,7 @@ static struct page *get_ksm_page(struct stable_node *stable_node) ...@@ -515,7 +516,7 @@ static struct page *get_ksm_page(struct stable_node *stable_node)
struct page *page; struct page *page;
void *expected_mapping; void *expected_mapping;
page = stable_node->page; page = pfn_to_page(stable_node->kpfn);
expected_mapping = (void *)stable_node + expected_mapping = (void *)stable_node +
(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM); (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
rcu_read_lock(); rcu_read_lock();
...@@ -973,7 +974,7 @@ static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, ...@@ -973,7 +974,7 @@ static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,
* This function returns the stable tree node of identical content if found, * This function returns the stable tree node of identical content if found,
* NULL otherwise. * NULL otherwise.
*/ */
static struct stable_node *stable_tree_search(struct page *page) static struct page *stable_tree_search(struct page *page)
{ {
struct rb_node *node = root_stable_tree.rb_node; struct rb_node *node = root_stable_tree.rb_node;
struct stable_node *stable_node; struct stable_node *stable_node;
...@@ -981,7 +982,7 @@ static struct stable_node *stable_tree_search(struct page *page) ...@@ -981,7 +982,7 @@ static struct stable_node *stable_tree_search(struct page *page)
stable_node = page_stable_node(page); stable_node = page_stable_node(page);
if (stable_node) { /* ksm page forked */ if (stable_node) { /* ksm page forked */
get_page(page); get_page(page);
return stable_node; return page;
} }
while (node) { while (node) {
...@@ -1003,7 +1004,7 @@ static struct stable_node *stable_tree_search(struct page *page) ...@@ -1003,7 +1004,7 @@ static struct stable_node *stable_tree_search(struct page *page)
put_page(tree_page); put_page(tree_page);
node = node->rb_right; node = node->rb_right;
} else } else
return stable_node; return tree_page;
} }
return NULL; return NULL;
...@@ -1059,7 +1060,7 @@ static struct stable_node *stable_tree_insert(struct page *kpage) ...@@ -1059,7 +1060,7 @@ static struct stable_node *stable_tree_insert(struct page *kpage)
INIT_HLIST_HEAD(&stable_node->hlist); INIT_HLIST_HEAD(&stable_node->hlist);
stable_node->page = kpage; stable_node->kpfn = page_to_pfn(kpage);
set_page_stable_node(kpage, stable_node); set_page_stable_node(kpage, stable_node);
return stable_node; return stable_node;
...@@ -1170,9 +1171,8 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) ...@@ -1170,9 +1171,8 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
remove_rmap_item_from_tree(rmap_item); remove_rmap_item_from_tree(rmap_item);
/* We first start with searching the page inside the stable tree */ /* We first start with searching the page inside the stable tree */
stable_node = stable_tree_search(page); kpage = stable_tree_search(page);
if (stable_node) { if (kpage) {
kpage = stable_node->page;
err = try_to_merge_with_ksm_page(rmap_item, page, kpage); err = try_to_merge_with_ksm_page(rmap_item, page, kpage);
if (!err) { if (!err) {
/* /*
...@@ -1180,7 +1180,7 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) ...@@ -1180,7 +1180,7 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
* add its rmap_item to the stable tree. * add its rmap_item to the stable tree.
*/ */
lock_page(kpage); lock_page(kpage);
stable_tree_append(rmap_item, stable_node); stable_tree_append(rmap_item, page_stable_node(kpage));
unlock_page(kpage); unlock_page(kpage);
} }
put_page(kpage); put_page(kpage);
...@@ -1715,12 +1715,63 @@ void ksm_migrate_page(struct page *newpage, struct page *oldpage) ...@@ -1715,12 +1715,63 @@ void ksm_migrate_page(struct page *newpage, struct page *oldpage)
stable_node = page_stable_node(newpage); stable_node = page_stable_node(newpage);
if (stable_node) { if (stable_node) {
VM_BUG_ON(stable_node->page != oldpage); VM_BUG_ON(stable_node->kpfn != page_to_pfn(oldpage));
stable_node->page = newpage; stable_node->kpfn = page_to_pfn(newpage);
} }
} }
#endif /* CONFIG_MIGRATION */ #endif /* CONFIG_MIGRATION */
#ifdef CONFIG_MEMORY_HOTREMOVE
static struct stable_node *ksm_check_stable_tree(unsigned long start_pfn,
unsigned long end_pfn)
{
struct rb_node *node;
for (node = rb_first(&root_stable_tree); node; node = rb_next(node)) {
struct stable_node *stable_node;
stable_node = rb_entry(node, struct stable_node, node);
if (stable_node->kpfn >= start_pfn &&
stable_node->kpfn < end_pfn)
return stable_node;
}
return NULL;
}
static int ksm_memory_callback(struct notifier_block *self,
unsigned long action, void *arg)
{
struct memory_notify *mn = arg;
struct stable_node *stable_node;
switch (action) {
case MEM_GOING_OFFLINE:
/*
* Keep it very simple for now: just lock out ksmd and
* MADV_UNMERGEABLE while any memory is going offline.
*/
mutex_lock(&ksm_thread_mutex);
break;
case MEM_OFFLINE:
/*
* Most of the work is done by page migration; but there might
* be a few stable_nodes left over, still pointing to struct
* pages which have been offlined: prune those from the tree.
*/
while ((stable_node = ksm_check_stable_tree(mn->start_pfn,
mn->start_pfn + mn->nr_pages)) != NULL)
remove_node_from_stable_tree(stable_node);
/* fallthrough */
case MEM_CANCEL_OFFLINE:
mutex_unlock(&ksm_thread_mutex);
break;
}
return NOTIFY_OK;
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
#ifdef CONFIG_SYSFS #ifdef CONFIG_SYSFS
/* /*
* This all compiles without CONFIG_SYSFS, but is a waste of space. * This all compiles without CONFIG_SYSFS, but is a waste of space.
...@@ -1946,6 +1997,13 @@ static int __init ksm_init(void) ...@@ -1946,6 +1997,13 @@ static int __init ksm_init(void)
#endif /* CONFIG_SYSFS */ #endif /* CONFIG_SYSFS */
#ifdef CONFIG_MEMORY_HOTREMOVE
/*
* Choose a high priority since the callback takes ksm_thread_mutex:
* later callbacks could only be taking locks which nest within that.
*/
hotplug_memory_notifier(ksm_memory_callback, 100);
#endif
return 0; return 0;
out_free2: out_free2:
......
...@@ -698,7 +698,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) ...@@ -698,7 +698,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
if (list_empty(&source)) if (list_empty(&source))
goto out; goto out;
/* this function returns # of failed pages */ /* this function returns # of failed pages */
ret = migrate_pages(&source, hotremove_migrate_alloc, 0); ret = migrate_pages(&source, hotremove_migrate_alloc, 0, 1);
out: out:
return ret; return ret;
......
...@@ -85,6 +85,7 @@ ...@@ -85,6 +85,7 @@
#include <linux/seq_file.h> #include <linux/seq_file.h>
#include <linux/proc_fs.h> #include <linux/proc_fs.h>
#include <linux/migrate.h> #include <linux/migrate.h>
#include <linux/ksm.h>
#include <linux/rmap.h> #include <linux/rmap.h>
#include <linux/security.h> #include <linux/security.h>
#include <linux/syscalls.h> #include <linux/syscalls.h>
...@@ -413,17 +414,11 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd, ...@@ -413,17 +414,11 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
if (!page) if (!page)
continue; continue;
/* /*
* The check for PageReserved here is important to avoid * vm_normal_page() filters out zero pages, but there might
* handling zero pages and other pages that may have been * still be PageReserved pages to skip, perhaps in a VDSO.
* marked special by the system. * And we cannot move PageKsm pages sensibly or safely yet.
*
* If the PageReserved would not be checked here then f.e.
* the location of the zero page could have an influence
* on MPOL_MF_STRICT, zero pages would be counted for
* the per node stats, and there would be useless attempts
* to put zero pages on the migration list.
*/ */
if (PageReserved(page)) if (PageReserved(page) || PageKsm(page))
continue; continue;
nid = page_to_nid(page); nid = page_to_nid(page);
if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT)) if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
...@@ -839,7 +834,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, ...@@ -839,7 +834,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
flags | MPOL_MF_DISCONTIG_OK, &pagelist); flags | MPOL_MF_DISCONTIG_OK, &pagelist);
if (!list_empty(&pagelist)) if (!list_empty(&pagelist))
err = migrate_pages(&pagelist, new_node_page, dest); err = migrate_pages(&pagelist, new_node_page, dest, 0);
return err; return err;
} }
...@@ -1056,7 +1051,7 @@ static long do_mbind(unsigned long start, unsigned long len, ...@@ -1056,7 +1051,7 @@ static long do_mbind(unsigned long start, unsigned long len,
if (!list_empty(&pagelist)) if (!list_empty(&pagelist))
nr_failed = migrate_pages(&pagelist, new_vma_page, nr_failed = migrate_pages(&pagelist, new_vma_page,
(unsigned long)vma); (unsigned long)vma, 0);
if (!err && nr_failed && (flags & MPOL_MF_STRICT)) if (!err && nr_failed && (flags & MPOL_MF_STRICT))
err = -EIO; err = -EIO;
......
...@@ -543,7 +543,7 @@ static int move_to_new_page(struct page *newpage, struct page *page) ...@@ -543,7 +543,7 @@ static int move_to_new_page(struct page *newpage, struct page *page)
* to the newly allocated page in newpage. * to the newly allocated page in newpage.
*/ */
static int unmap_and_move(new_page_t get_new_page, unsigned long private, static int unmap_and_move(new_page_t get_new_page, unsigned long private,
struct page *page, int force) struct page *page, int force, int offlining)
{ {
int rc = 0; int rc = 0;
int *result = NULL; int *result = NULL;
...@@ -569,6 +569,20 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private, ...@@ -569,6 +569,20 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
lock_page(page); lock_page(page);
} }
/*
* Only memory hotplug's offline_pages() caller has locked out KSM,
* and can safely migrate a KSM page. The other cases have skipped
* PageKsm along with PageReserved - but it is only now when we have
* the page lock that we can be certain it will not go KSM beneath us
* (KSM will not upgrade a page from PageAnon to PageKsm when it sees
* its pagecount raised, but only here do we take the page lock which
* serializes that).
*/
if (PageKsm(page) && !offlining) {
rc = -EBUSY;
goto unlock;
}
/* charge against new page */ /* charge against new page */
charge = mem_cgroup_prepare_migration(page, &mem); charge = mem_cgroup_prepare_migration(page, &mem);
if (charge == -ENOMEM) { if (charge == -ENOMEM) {
...@@ -685,7 +699,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private, ...@@ -685,7 +699,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
* Return: Number of pages not migrated or error code. * Return: Number of pages not migrated or error code.
*/ */
int migrate_pages(struct list_head *from, int migrate_pages(struct list_head *from,
new_page_t get_new_page, unsigned long private) new_page_t get_new_page, unsigned long private, int offlining)
{ {
int retry = 1; int retry = 1;
int nr_failed = 0; int nr_failed = 0;
...@@ -705,7 +719,7 @@ int migrate_pages(struct list_head *from, ...@@ -705,7 +719,7 @@ int migrate_pages(struct list_head *from,
cond_resched(); cond_resched();
rc = unmap_and_move(get_new_page, private, rc = unmap_and_move(get_new_page, private,
page, pass > 2); page, pass > 2, offlining);
switch(rc) { switch(rc) {
case -ENOMEM: case -ENOMEM:
...@@ -801,7 +815,8 @@ static int do_move_page_to_node_array(struct mm_struct *mm, ...@@ -801,7 +815,8 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
if (!page) if (!page)
goto set_status; goto set_status;
if (PageReserved(page)) /* Check for zero page */ /* Use PageReserved to check for zero page */
if (PageReserved(page) || PageKsm(page))
goto put_and_set; goto put_and_set;
pp->page = page; pp->page = page;
...@@ -838,7 +853,7 @@ static int do_move_page_to_node_array(struct mm_struct *mm, ...@@ -838,7 +853,7 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
err = 0; err = 0;
if (!list_empty(&pagelist)) if (!list_empty(&pagelist))
err = migrate_pages(&pagelist, new_page_node, err = migrate_pages(&pagelist, new_page_node,
(unsigned long)pm); (unsigned long)pm, 0);
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
return err; return err;
...@@ -959,7 +974,7 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, ...@@ -959,7 +974,7 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
err = -ENOENT; err = -ENOENT;
/* Use PageReserved to check for zero page */ /* Use PageReserved to check for zero page */
if (!page || PageReserved(page)) if (!page || PageReserved(page) || PageKsm(page))
goto set_status; goto set_status;
err = page_to_nid(page); err = page_to_nid(page);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment