Commit ad09315c authored by Yinghai Lu's avatar Yinghai Lu Committed by Ingo Molnar

mm: fix alloc_bootmem_core to use fast searching for all nodes

Make the nodes other than node 0 use bdata->last_success for fast
search too.

We need to use __alloc_bootmem_core() for vmemmap allocation for other
nodes when numa and sparsemem/vmemmap are enabled.

Also, make fail_block path increase i with incr only after ALIGN
to avoid extra increase when size is larger than align.
Signed-off-by: default avatarYinghai Lu <yhlu.kernel@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent e123dd3f
...@@ -238,28 +238,32 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, ...@@ -238,28 +238,32 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
* We try to allocate bootmem pages above 'goal' * We try to allocate bootmem pages above 'goal'
* first, then we try to allocate lower pages. * first, then we try to allocate lower pages.
*/ */
if (goal && goal >= bdata->node_boot_start && PFN_DOWN(goal) < end_pfn) { preferred = 0;
if (goal && PFN_DOWN(goal) < end_pfn) {
if (goal > bdata->node_boot_start)
preferred = goal - bdata->node_boot_start; preferred = goal - bdata->node_boot_start;
if (bdata->last_success >= preferred) if (bdata->last_success >= preferred)
if (!limit || (limit && limit > bdata->last_success)) if (!limit || (limit && limit > bdata->last_success))
preferred = bdata->last_success; preferred = bdata->last_success;
} else }
preferred = 0;
preferred = PFN_DOWN(ALIGN(preferred, align)) + offset; preferred = PFN_DOWN(ALIGN(preferred, align)) + offset;
areasize = (size + PAGE_SIZE-1) / PAGE_SIZE; areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
incr = align >> PAGE_SHIFT ? : 1; incr = align >> PAGE_SHIFT ? : 1;
restart_scan: restart_scan:
for (i = preferred; i < eidx; i += incr) { for (i = preferred; i < eidx;) {
unsigned long j; unsigned long j;
i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i); i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i);
i = ALIGN(i, incr); i = ALIGN(i, incr);
if (i >= eidx) if (i >= eidx)
break; break;
if (test_bit(i, bdata->node_bootmem_map)) if (test_bit(i, bdata->node_bootmem_map)) {
i += incr;
continue; continue;
}
for (j = i + 1; j < i + areasize; ++j) { for (j = i + 1; j < i + areasize; ++j) {
if (j >= eidx) if (j >= eidx)
goto fail_block; goto fail_block;
...@@ -270,6 +274,8 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, ...@@ -270,6 +274,8 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
goto found; goto found;
fail_block: fail_block:
i = ALIGN(j, incr); i = ALIGN(j, incr);
if (i == j)
i += incr;
} }
if (preferred > offset) { if (preferred > offset) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment