Commit c3d5f5f0 authored by Jiang Liu's avatar Jiang Liu Committed by Linus Torvalds

mm: use a dedicated lock to protect totalram_pages and zone->managed_pages

Currently lock_memory_hotplug()/unlock_memory_hotplug() are used to
protect totalram_pages and zone->managed_pages.  Other than the memory
hotplug driver, totalram_pages and zone->managed_pages may also be
modified at runtime by other drivers, such as Xen balloon,
virtio_balloon etc.  For those cases, memory hotplug lock is a little
too heavy, so introduce a dedicated lock to protect totalram_pages and
zone->managed_pages.

Now we have a simplified locking rules totalram_pages and
zone->managed_pages as:

1) no locking for read accesses because they are unsigned long.
2) no locking for write accesses at boot time in single-threaded context.
3) serialize write accesses at runtime by acquiring the dedicated
   managed_page_count_lock.

Also adjust zone->managed_pages when freeing reserved pages into the
buddy system, to keep totalram_pages and zone->managed_pages in
consistence.

[akpm@linux-foundation.org: don't export adjust_managed_page_count to modules (for now)]
Signed-off-by: default avatarJiang Liu <jiang.liu@huawei.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michel Lespinasse <walken@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: <sworddragon2@aol.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Jianguo Wu <wujianguo@huawei.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Russell King <rmk@arm.linux.org.uk>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 7b4b2a0d
...@@ -1313,6 +1313,7 @@ extern void free_initmem(void); ...@@ -1313,6 +1313,7 @@ extern void free_initmem(void);
*/ */
extern unsigned long free_reserved_area(void *start, void *end, extern unsigned long free_reserved_area(void *start, void *end,
int poison, char *s); int poison, char *s);
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
/* /*
* Free a highmem page into the buddy system, adjusting totalhigh_pages * Free a highmem page into the buddy system, adjusting totalhigh_pages
...@@ -1321,10 +1322,7 @@ extern unsigned long free_reserved_area(void *start, void *end, ...@@ -1321,10 +1322,7 @@ extern unsigned long free_reserved_area(void *start, void *end,
extern void free_highmem_page(struct page *page); extern void free_highmem_page(struct page *page);
#endif #endif
static inline void adjust_managed_page_count(struct page *page, long count) extern void adjust_managed_page_count(struct page *page, long count);
{
totalram_pages += count;
}
/* Free the reserved page into the buddy system, so it gets managed. */ /* Free the reserved page into the buddy system, so it gets managed. */
static inline void __free_reserved_page(struct page *page) static inline void __free_reserved_page(struct page *page)
......
...@@ -474,10 +474,16 @@ struct zone { ...@@ -474,10 +474,16 @@ struct zone {
* frequently read in proximity to zone->lock. It's good to * frequently read in proximity to zone->lock. It's good to
* give them a chance of being in the same cacheline. * give them a chance of being in the same cacheline.
* *
* Write access to present_pages and managed_pages at runtime should * Write access to present_pages at runtime should be protected by
* be protected by lock_memory_hotplug()/unlock_memory_hotplug(). * lock_memory_hotplug()/unlock_memory_hotplug(). Any reader who can't
* Any reader who can't tolerant drift of present_pages and * tolerant drift of present_pages should hold memory hotplug lock to
* managed_pages should hold memory hotplug lock to get a stable value. * get a stable value.
*
* Read access to managed_pages should be safe because it's unsigned
* long. Write access to zone->managed_pages and totalram_pages are
* protected by managed_page_count_lock at runtime. Idealy only
* adjust_managed_page_count() should be used instead of directly
* touching zone->managed_pages and totalram_pages.
*/ */
unsigned long spanned_pages; unsigned long spanned_pages;
unsigned long present_pages; unsigned long present_pages;
......
...@@ -103,6 +103,9 @@ nodemask_t node_states[NR_NODE_STATES] __read_mostly = { ...@@ -103,6 +103,9 @@ nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
}; };
EXPORT_SYMBOL(node_states); EXPORT_SYMBOL(node_states);
/* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock);
unsigned long totalram_pages __read_mostly; unsigned long totalram_pages __read_mostly;
unsigned long totalreserve_pages __read_mostly; unsigned long totalreserve_pages __read_mostly;
/* /*
...@@ -5206,6 +5209,14 @@ early_param("movablecore", cmdline_parse_movablecore); ...@@ -5206,6 +5209,14 @@ early_param("movablecore", cmdline_parse_movablecore);
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
void adjust_managed_page_count(struct page *page, long count)
{
spin_lock(&managed_page_count_lock);
page_zone(page)->managed_pages += count;
totalram_pages += count;
spin_unlock(&managed_page_count_lock);
}
unsigned long free_reserved_area(void *start, void *end, int poison, char *s) unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
{ {
void *pos; void *pos;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment