Commit 76741e77 authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds

mm, page_alloc: don't convert pfn to idx when merging

In __free_one_page() we do the buddy merging arithmetics on "page/buddy
index", which is just the lower MAX_ORDER bits of pfn.  The operations
we do that affect the higher bits are bitwise AND and subtraction (in
that order), where the final result will be the same with the higher
bits left unmasked, as long as these bits are equal for both buddies -
which must be true by the definition of a buddy.

We can therefore use pfn's directly instead of "index" and skip the
zeroing of >MAX_ORDER bits.  This can help a bit by itself, although
compiler might be smart enough already.  It also helps the next patch to
avoid page_to_pfn() for memory hole checks.

Link: http://lkml.kernel.org/r/20161216120009.20064-1-vbabka@suse.czSigned-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarMel Gorman <mgorman@techsingularity.net>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent aa187507
...@@ -133,9 +133,9 @@ struct alloc_context { ...@@ -133,9 +133,9 @@ struct alloc_context {
* Assumption: *_mem_map is contiguous at least up to MAX_ORDER * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
*/ */
static inline unsigned long static inline unsigned long
__find_buddy_index(unsigned long page_idx, unsigned int order) __find_buddy_pfn(unsigned long page_pfn, unsigned int order)
{ {
return page_idx ^ (1 << order); return page_pfn ^ (1 << order);
} }
extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn, extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
......
...@@ -787,9 +787,8 @@ static inline void __free_one_page(struct page *page, ...@@ -787,9 +787,8 @@ static inline void __free_one_page(struct page *page,
struct zone *zone, unsigned int order, struct zone *zone, unsigned int order,
int migratetype) int migratetype)
{ {
unsigned long page_idx; unsigned long combined_pfn;
unsigned long combined_idx; unsigned long uninitialized_var(buddy_pfn);
unsigned long uninitialized_var(buddy_idx);
struct page *buddy; struct page *buddy;
unsigned int max_order; unsigned int max_order;
...@@ -802,15 +801,13 @@ static inline void __free_one_page(struct page *page, ...@@ -802,15 +801,13 @@ static inline void __free_one_page(struct page *page,
if (likely(!is_migrate_isolate(migratetype))) if (likely(!is_migrate_isolate(migratetype)))
__mod_zone_freepage_state(zone, 1 << order, migratetype); __mod_zone_freepage_state(zone, 1 << order, migratetype);
page_idx = pfn & ((1 << MAX_ORDER) - 1); VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
VM_BUG_ON_PAGE(bad_range(zone, page), page); VM_BUG_ON_PAGE(bad_range(zone, page), page);
continue_merging: continue_merging:
while (order < max_order - 1) { while (order < max_order - 1) {
buddy_idx = __find_buddy_index(page_idx, order); buddy_pfn = __find_buddy_pfn(pfn, order);
buddy = page + (buddy_idx - page_idx); buddy = page + (buddy_pfn - pfn);
if (!page_is_buddy(page, buddy, order)) if (!page_is_buddy(page, buddy, order))
goto done_merging; goto done_merging;
/* /*
...@@ -824,9 +821,9 @@ static inline void __free_one_page(struct page *page, ...@@ -824,9 +821,9 @@ static inline void __free_one_page(struct page *page,
zone->free_area[order].nr_free--; zone->free_area[order].nr_free--;
rmv_page_order(buddy); rmv_page_order(buddy);
} }
combined_idx = buddy_idx & page_idx; combined_pfn = buddy_pfn & pfn;
page = page + (combined_idx - page_idx); page = page + (combined_pfn - pfn);
page_idx = combined_idx; pfn = combined_pfn;
order++; order++;
} }
if (max_order < MAX_ORDER) { if (max_order < MAX_ORDER) {
...@@ -841,8 +838,8 @@ static inline void __free_one_page(struct page *page, ...@@ -841,8 +838,8 @@ static inline void __free_one_page(struct page *page,
if (unlikely(has_isolate_pageblock(zone))) { if (unlikely(has_isolate_pageblock(zone))) {
int buddy_mt; int buddy_mt;
buddy_idx = __find_buddy_index(page_idx, order); buddy_pfn = __find_buddy_pfn(pfn, order);
buddy = page + (buddy_idx - page_idx); buddy = page + (buddy_pfn - pfn);
buddy_mt = get_pageblock_migratetype(buddy); buddy_mt = get_pageblock_migratetype(buddy);
if (migratetype != buddy_mt if (migratetype != buddy_mt
...@@ -867,10 +864,10 @@ static inline void __free_one_page(struct page *page, ...@@ -867,10 +864,10 @@ static inline void __free_one_page(struct page *page,
*/ */
if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
struct page *higher_page, *higher_buddy; struct page *higher_page, *higher_buddy;
combined_idx = buddy_idx & page_idx; combined_pfn = buddy_pfn & pfn;
higher_page = page + (combined_idx - page_idx); higher_page = page + (combined_pfn - pfn);
buddy_idx = __find_buddy_index(combined_idx, order + 1); buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
higher_buddy = higher_page + (buddy_idx - combined_idx); higher_buddy = higher_page + (buddy_pfn - combined_pfn);
if (page_is_buddy(higher_page, higher_buddy, order + 1)) { if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
list_add_tail(&page->lru, list_add_tail(&page->lru,
&zone->free_area[order].free_list[migratetype]); &zone->free_area[order].free_list[migratetype]);
......
...@@ -83,7 +83,7 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype) ...@@ -83,7 +83,7 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
unsigned long flags, nr_pages; unsigned long flags, nr_pages;
bool isolated_page = false; bool isolated_page = false;
unsigned int order; unsigned int order;
unsigned long page_idx, buddy_idx; unsigned long pfn, buddy_pfn;
struct page *buddy; struct page *buddy;
zone = page_zone(page); zone = page_zone(page);
...@@ -102,9 +102,9 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype) ...@@ -102,9 +102,9 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
if (PageBuddy(page)) { if (PageBuddy(page)) {
order = page_order(page); order = page_order(page);
if (order >= pageblock_order) { if (order >= pageblock_order) {
page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1); pfn = page_to_pfn(page);
buddy_idx = __find_buddy_index(page_idx, order); buddy_pfn = __find_buddy_pfn(pfn, order);
buddy = page + (buddy_idx - page_idx); buddy = page + (buddy_pfn - pfn);
if (pfn_valid_within(page_to_pfn(buddy)) && if (pfn_valid_within(page_to_pfn(buddy)) &&
!is_migrate_isolate_page(buddy)) { !is_migrate_isolate_page(buddy)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment