Commit 0f3caba2 authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds

bootmem: respect goal more likely

The old node-agnostic code tried allocating on all nodes starting from the
one with the lowest range.  alloc_bootmem_core retried without the goal if
it could not satisfy it and so the goal was only respected at all when it
happened to be on the first (lowest page numbers) node (or theoretically
if allocations failed on all nodes before to the one holding the goal).

Introduce a non-panicking helper that starts allocating from the node
holding the goal and falls back only after all thes tries failed, thus
moving the goal fallback code out of alloc_bootmem_core.

Make all other allocation functions benefit from this new helper.
Signed-off-by: default avatarJohannes Weiner <hannes@saeurebad.de>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Yinghai Lu <yhlu.kernel@gmail.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent e2bf3cae
...@@ -408,6 +408,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -408,6 +408,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
unsigned long size, unsigned long align, unsigned long size, unsigned long align,
unsigned long goal, unsigned long limit) unsigned long goal, unsigned long limit)
{ {
unsigned long fallback = 0;
unsigned long min, max, start, sidx, midx, step; unsigned long min, max, start, sidx, midx, step;
BUG_ON(!size); BUG_ON(!size);
...@@ -443,8 +444,11 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -443,8 +444,11 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
midx = max - PFN_DOWN(bdata->node_boot_start); midx = max - PFN_DOWN(bdata->node_boot_start);
if (bdata->hint_idx > sidx) { if (bdata->hint_idx > sidx) {
/* Make sure we retry on failure */ /*
goal = 1; * Handle the valid case of sidx being zero and still
* catch the fallback below.
*/
fallback = sidx + 1;
sidx = ALIGN(bdata->hint_idx, step); sidx = ALIGN(bdata->hint_idx, step);
} }
...@@ -492,10 +496,39 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -492,10 +496,39 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
return region; return region;
} }
if (fallback) {
sidx = ALIGN(fallback - 1, step);
fallback = 0;
goto find_block;
}
return NULL;
}
static void * __init ___alloc_bootmem_nopanic(unsigned long size,
unsigned long align,
unsigned long goal,
unsigned long limit)
{
bootmem_data_t *bdata;
restart:
list_for_each_entry(bdata, &bdata_list, list) {
void *region;
if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
continue;
if (limit && bdata->node_boot_start >= limit)
break;
region = alloc_bootmem_core(bdata, size, align, goal, limit);
if (region)
return region;
}
if (goal) { if (goal) {
goal = 0; goal = 0;
sidx = 0; goto restart;
goto find_block;
} }
return NULL; return NULL;
...@@ -515,16 +548,23 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata, ...@@ -515,16 +548,23 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
* Returns NULL on failure. * Returns NULL on failure.
*/ */
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
unsigned long goal) unsigned long goal)
{ {
bootmem_data_t *bdata; return ___alloc_bootmem_nopanic(size, align, goal, 0);
void *ptr; }
list_for_each_entry(bdata, &bdata_list, list) { static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
ptr = alloc_bootmem_core(bdata, size, align, goal, 0); unsigned long goal, unsigned long limit)
if (ptr) {
return ptr; void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
}
if (mem)
return mem;
/*
* Whoops, we cannot satisfy the allocation request.
*/
printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
panic("Out of memory");
return NULL; return NULL;
} }
...@@ -544,16 +584,7 @@ void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, ...@@ -544,16 +584,7 @@ void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
void * __init __alloc_bootmem(unsigned long size, unsigned long align, void * __init __alloc_bootmem(unsigned long size, unsigned long align,
unsigned long goal) unsigned long goal)
{ {
void *mem = __alloc_bootmem_nopanic(size,align,goal); return ___alloc_bootmem(size, align, goal, 0);
if (mem)
return mem;
/*
* Whoops, we cannot satisfy the allocation request.
*/
printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
panic("Out of memory");
return NULL;
} }
/** /**
...@@ -653,22 +684,7 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, ...@@ -653,22 +684,7 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
unsigned long goal) unsigned long goal)
{ {
bootmem_data_t *bdata; return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
void *ptr;
list_for_each_entry(bdata, &bdata_list, list) {
ptr = alloc_bootmem_core(bdata, size, align, goal,
ARCH_LOW_ADDRESS_LIMIT);
if (ptr)
return ptr;
}
/*
* Whoops, we cannot satisfy the allocation request.
*/
printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
panic("Out of low memory");
return NULL;
} }
/** /**
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment