Commit 761ad8d7 authored by Naoya Horiguchi's avatar Naoya Horiguchi Committed by Linus Torvalds

mm: hwpoison: introduce memory_failure_hugetlb()

memory_failure() is a big function and hard to maintain.  Handling
hugetlb- and non-hugetlb- case in a single function is not good, so this
patch separates PageHuge() branch into a new function, which saves many
PageHuge() check.

Link: http://lkml.kernel.org/r/1496305019-5493-7-git-send-email-n-horiguchi@ah.jp.nec.comSigned-off-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent d4a3a60b
...@@ -1009,6 +1009,76 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, ...@@ -1009,6 +1009,76 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
return unmap_success; return unmap_success;
} }
static int memory_failure_hugetlb(unsigned long pfn, int trapno, int flags)
{
struct page_state *ps;
struct page *p = pfn_to_page(pfn);
struct page *head = compound_head(p);
int res;
unsigned long page_flags;
if (TestSetPageHWPoison(head)) {
pr_err("Memory failure: %#lx: already hardware poisoned\n",
pfn);
return 0;
}
num_poisoned_pages_inc();
if (!(flags & MF_COUNT_INCREASED) && !get_hwpoison_page(p)) {
/*
* Check "filter hit" and "race with other subpage."
*/
lock_page(head);
if (PageHWPoison(head)) {
if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
|| (p != head && TestSetPageHWPoison(head))) {
num_poisoned_pages_dec();
unlock_page(head);
return 0;
}
}
unlock_page(head);
dissolve_free_huge_page(p);
action_result(pfn, MF_MSG_FREE_HUGE, MF_DELAYED);
return 0;
}
lock_page(head);
page_flags = head->flags;
if (!PageHWPoison(head)) {
pr_err("Memory failure: %#lx: just unpoisoned\n", pfn);
num_poisoned_pages_dec();
unlock_page(head);
put_hwpoison_page(head);
return 0;
}
if (!hwpoison_user_mappings(p, pfn, trapno, flags, &head)) {
action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);
res = -EBUSY;
goto out;
}
res = -EBUSY;
for (ps = error_states;; ps++)
if ((p->flags & ps->mask) == ps->res)
break;
page_flags |= (p->flags & (1UL << PG_dirty));
if (!ps->mask)
for (ps = error_states;; ps++)
if ((page_flags & ps->mask) == ps->res)
break;
res = page_action(ps, p, pfn);
out:
unlock_page(head);
return res;
}
/** /**
* memory_failure - Handle memory failure of a page. * memory_failure - Handle memory failure of a page.
* @pfn: Page Number of the corrupted page * @pfn: Page Number of the corrupted page
...@@ -1046,33 +1116,22 @@ int memory_failure(unsigned long pfn, int trapno, int flags) ...@@ -1046,33 +1116,22 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
} }
p = pfn_to_page(pfn); p = pfn_to_page(pfn);
orig_head = hpage = compound_head(p); if (PageHuge(p))
return memory_failure_hugetlb(pfn, trapno, flags);
/* tmporary check code, to be updated in later patches */
if (PageHuge(p)) {
if (TestSetPageHWPoison(hpage)) {
pr_err("Memory failure: %#lx: already hardware poisoned\n", pfn);
return 0;
}
goto tmp;
}
if (TestSetPageHWPoison(p)) { if (TestSetPageHWPoison(p)) {
pr_err("Memory failure: %#lx: already hardware poisoned\n", pr_err("Memory failure: %#lx: already hardware poisoned\n",
pfn); pfn);
return 0; return 0;
} }
tmp: orig_head = hpage = compound_head(p);
num_poisoned_pages_inc(); num_poisoned_pages_inc();
/* /*
* We need/can do nothing about count=0 pages. * We need/can do nothing about count=0 pages.
* 1) it's a free page, and therefore in safe hand: * 1) it's a free page, and therefore in safe hand:
* prep_new_page() will be the gate keeper. * prep_new_page() will be the gate keeper.
* 2) it's a free hugepage, which is also safe: * 2) it's part of a non-compound high order page.
* an affected hugepage will be dequeued from hugepage freelist,
* so there's no concern about reusing it ever after.
* 3) it's part of a non-compound high order page.
* Implies some kernel user: cannot stop them from * Implies some kernel user: cannot stop them from
* R/W the page; let's pray that the page has been * R/W the page; let's pray that the page has been
* used and will be freed some time later. * used and will be freed some time later.
...@@ -1083,31 +1142,13 @@ int memory_failure(unsigned long pfn, int trapno, int flags) ...@@ -1083,31 +1142,13 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
if (is_free_buddy_page(p)) { if (is_free_buddy_page(p)) {
action_result(pfn, MF_MSG_BUDDY, MF_DELAYED); action_result(pfn, MF_MSG_BUDDY, MF_DELAYED);
return 0; return 0;
} else if (PageHuge(hpage)) {
/*
* Check "filter hit" and "race with other subpage."
*/
lock_page(hpage);
if (PageHWPoison(hpage)) {
if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
|| (p != hpage && TestSetPageHWPoison(hpage))) {
num_poisoned_pages_dec();
unlock_page(hpage);
return 0;
}
}
res = dequeue_hwpoisoned_huge_page(hpage);
action_result(pfn, MF_MSG_FREE_HUGE,
res ? MF_IGNORED : MF_DELAYED);
unlock_page(hpage);
return res;
} else { } else {
action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);
return -EBUSY; return -EBUSY;
} }
} }
if (!PageHuge(p) && PageTransHuge(hpage)) { if (PageTransHuge(hpage)) {
lock_page(p); lock_page(p);
if (!PageAnon(p) || unlikely(split_huge_page(p))) { if (!PageAnon(p) || unlikely(split_huge_page(p))) {
unlock_page(p); unlock_page(p);
...@@ -1145,7 +1186,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags) ...@@ -1145,7 +1186,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
return 0; return 0;
} }
lock_page(hpage); lock_page(p);
/* /*
* The page could have changed compound pages during the locking. * The page could have changed compound pages during the locking.
...@@ -1175,32 +1216,21 @@ int memory_failure(unsigned long pfn, int trapno, int flags) ...@@ -1175,32 +1216,21 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
if (!PageHWPoison(p)) { if (!PageHWPoison(p)) {
pr_err("Memory failure: %#lx: just unpoisoned\n", pfn); pr_err("Memory failure: %#lx: just unpoisoned\n", pfn);
num_poisoned_pages_dec(); num_poisoned_pages_dec();
unlock_page(hpage); unlock_page(p);
put_hwpoison_page(hpage); put_hwpoison_page(p);
return 0; return 0;
} }
if (hwpoison_filter(p)) { if (hwpoison_filter(p)) {
if (TestClearPageHWPoison(p)) if (TestClearPageHWPoison(p))
num_poisoned_pages_dec(); num_poisoned_pages_dec();
unlock_page(hpage); unlock_page(p);
put_hwpoison_page(hpage); put_hwpoison_page(p);
return 0; return 0;
} }
if (!PageHuge(p) && !PageTransTail(p) && !PageLRU(p)) if (!PageTransTail(p) && !PageLRU(p))
goto identify_page_state; goto identify_page_state;
/*
* For error on the tail page, we should set PG_hwpoison
* on the head page to show that the hugepage is hwpoisoned
*/
if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
action_result(pfn, MF_MSG_POISONED_HUGE, MF_IGNORED);
unlock_page(hpage);
put_hwpoison_page(hpage);
return 0;
}
/* /*
* It's very difficult to mess with pages currently under IO * It's very difficult to mess with pages currently under IO
* and in many cases impossible, so we just avoid it here. * and in many cases impossible, so we just avoid it here.
...@@ -1248,7 +1278,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags) ...@@ -1248,7 +1278,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
break; break;
res = page_action(ps, p, pfn); res = page_action(ps, p, pfn);
out: out:
unlock_page(hpage); unlock_page(p);
return res; return res;
} }
EXPORT_SYMBOL_GPL(memory_failure); EXPORT_SYMBOL_GPL(memory_failure);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment