Commit afce7191 authored by Andrew Morton's avatar Andrew Morton Committed by Linus Torvalds

[PATCH] percpu: convert global page accounting

Convert global page state accounting to use per-cpu storage

(I think this code remains a little buggy, btw.  Note how I do

	per_cpu(page_states, cpu).member += (delta);

This gets done at interrupt time and hence is assuming that
the "+=" operation on a ulong is atomic wrt interrupts on
all architectures. How do we feel about that assumption?)
parent 999eac41
...@@ -86,4 +86,6 @@ extern void FASTCALL(free_pages(unsigned long addr, unsigned int order)); ...@@ -86,4 +86,6 @@ extern void FASTCALL(free_pages(unsigned long addr, unsigned int order));
#define __free_page(page) __free_pages((page), 0) #define __free_page(page) __free_pages((page), 0)
#define free_page(addr) free_pages((addr),0) #define free_page(addr) free_pages((addr),0)
void page_alloc_init(void);
#endif /* __LINUX_GFP_H */ #endif /* __LINUX_GFP_H */
...@@ -5,6 +5,8 @@ ...@@ -5,6 +5,8 @@
#ifndef PAGE_FLAGS_H #ifndef PAGE_FLAGS_H
#define PAGE_FLAGS_H #define PAGE_FLAGS_H
#include <linux/percpu.h>
/* /*
* Various page->flags bits: * Various page->flags bits:
* *
...@@ -73,7 +75,7 @@ ...@@ -73,7 +75,7 @@
* Global page accounting. One instance per CPU. Only unsigned longs are * Global page accounting. One instance per CPU. Only unsigned longs are
* allowed. * allowed.
*/ */
extern struct page_state { struct page_state {
unsigned long nr_dirty; unsigned long nr_dirty;
unsigned long nr_writeback; unsigned long nr_writeback;
unsigned long nr_pagecache; unsigned long nr_pagecache;
...@@ -103,7 +105,9 @@ extern struct page_state { ...@@ -103,7 +105,9 @@ extern struct page_state {
unsigned long kswapd_steal; unsigned long kswapd_steal;
unsigned long pageoutrun; unsigned long pageoutrun;
unsigned long allocstall; unsigned long allocstall;
} ____cacheline_aligned_in_smp page_states[NR_CPUS]; };
DECLARE_PER_CPU(struct page_state, page_states);
extern void get_page_state(struct page_state *ret); extern void get_page_state(struct page_state *ret);
extern void get_full_page_state(struct page_state *ret); extern void get_full_page_state(struct page_state *ret);
...@@ -111,7 +115,7 @@ extern void get_full_page_state(struct page_state *ret); ...@@ -111,7 +115,7 @@ extern void get_full_page_state(struct page_state *ret);
#define mod_page_state(member, delta) \ #define mod_page_state(member, delta) \
do { \ do { \
int cpu = get_cpu(); \ int cpu = get_cpu(); \
page_states[cpu].member += (delta); \ per_cpu(page_states, cpu).member += (delta); \
put_cpu(); \ put_cpu(); \
} while (0) } while (0)
......
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include <linux/hdreg.h> #include <linux/hdreg.h>
#include <linux/bootmem.h> #include <linux/bootmem.h>
#include <linux/tty.h> #include <linux/tty.h>
#include <linux/gfp.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/kernel_stat.h> #include <linux/kernel_stat.h>
#include <linux/security.h> #include <linux/security.h>
...@@ -388,6 +389,7 @@ asmlinkage void __init start_kernel(void) ...@@ -388,6 +389,7 @@ asmlinkage void __init start_kernel(void)
setup_arch(&command_line); setup_arch(&command_line);
setup_per_cpu_areas(); setup_per_cpu_areas();
build_all_zonelists(); build_all_zonelists();
page_alloc_init();
printk("Kernel command line: %s\n", saved_command_line); printk("Kernel command line: %s\n", saved_command_line);
parse_options(command_line); parse_options(command_line);
trap_init(); trap_init();
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
#include <linux/pagevec.h> #include <linux/pagevec.h>
#include <linux/blkdev.h> #include <linux/blkdev.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/notifier.h>
struct pglist_data *pgdat_list; struct pglist_data *pgdat_list;
unsigned long totalram_pages; unsigned long totalram_pages;
...@@ -573,8 +574,8 @@ unsigned int nr_free_highpages (void) ...@@ -573,8 +574,8 @@ unsigned int nr_free_highpages (void)
* The result is unavoidably approximate - it can change * The result is unavoidably approximate - it can change
* during and after execution of this function. * during and after execution of this function.
*/ */
struct page_state page_states[NR_CPUS] __cacheline_aligned; DEFINE_PER_CPU(struct page_state, page_states) = {0};
EXPORT_SYMBOL(page_states); EXPORT_PER_CPU_SYMBOL(page_states);
void __get_page_state(struct page_state *ret, int nr) void __get_page_state(struct page_state *ret, int nr)
{ {
...@@ -587,7 +588,7 @@ void __get_page_state(struct page_state *ret, int nr) ...@@ -587,7 +588,7 @@ void __get_page_state(struct page_state *ret, int nr)
if (!cpu_online(cpu)) if (!cpu_online(cpu))
continue; continue;
in = (unsigned long *)(page_states + cpu); in = (unsigned long *)&per_cpu(page_states, cpu);
out = (unsigned long *)ret; out = (unsigned long *)ret;
for (off = 0; off < nr; off++) for (off = 0; off < nr; off++)
*out++ += *in++; *out++ += *in++;
...@@ -1197,3 +1198,33 @@ struct seq_operations vmstat_op = { ...@@ -1197,3 +1198,33 @@ struct seq_operations vmstat_op = {
}; };
#endif /* CONFIG_PROC_FS */ #endif /* CONFIG_PROC_FS */
static void __devinit init_page_alloc_cpu(int cpu)
{
struct page_state *ps = &per_cpu(page_states, cpu);
memset(ps, 0, sizeof(*ps));
}
static int __devinit page_alloc_cpu_notify(struct notifier_block *self,
unsigned long action, void *hcpu)
{
int cpu = (unsigned long)hcpu;
switch(action) {
case CPU_UP_PREPARE:
init_page_alloc_cpu(cpu);
break;
default:
break;
}
return NOTIFY_OK;
}
static struct notifier_block __devinitdata page_alloc_nb = {
.notifier_call = page_alloc_cpu_notify,
};
void __init page_alloc_init(void)
{
init_page_alloc_cpu(smp_processor_id());
register_cpu_notifier(&page_alloc_nb);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment