[PATCH] pagecache accounting speedup

From: Alex Tomas <bzzz@tmi.comex.ru>

This is the second half of the vm_enough_memory() speedup.

When overcommit_memory != 1, vm_enough_memory() calls get_page_state() to
calculate the amount of used pagecache.  It does this on every call to
sys_brk().

get_page_state() is really expensive on SMP.

So the patch arranges for pagecache accounting to be in a global atomic_t,
with per-cpu batching and approximate accounting to amortise the cost of the
global atomic.

The nr_pagecache field of /proc/vmstat is removed.

fs/proc/proc_misc.c

@@ -182,7 +182,7 @@ static int meminfo_read_proc(char *page, char **start, off_t off,
 		K(i.totalram),
 		K(i.freeram),
 		K(i.bufferram),
-		K(ps.nr_pagecache-total_swapcache_pages-i.bufferram),
+		K(get_page_cache_size()-total_swapcache_pages-i.bufferram),
 		K(total_swapcache_pages),
 		K(active),
 		K(inactive),

include/linux/mm.h

@@ -594,7 +594,6 @@ static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * m
 
 extern struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr);
 
-extern unsigned long get_page_cache_size(void);
 extern unsigned int nr_used_zone_pages(void);
 
 #ifdef CONFIG_MMU

include/linux/page-flags.h

@@ -82,7 +82,6 @@
 struct page_state {
 	unsigned long nr_dirty;		/* Dirty writeable pages */
 	unsigned long nr_writeback;	/* Pages under writeback */
-	unsigned long nr_pagecache;	/* Pages in pagecache */
 	unsigned long nr_page_table_pages;/* Pages used for pagetables */
 	unsigned long nr_reverse_maps;	/* includes PageDirect */
 	unsigned long nr_mapped;	/* mapped into pagetables */

include/linux/pagemap.h

@@ -74,6 +74,48 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
 extern void remove_from_page_cache(struct page *page);
 extern void __remove_from_page_cache(struct page *page);
 
+extern atomic_t nr_pagecache;
+
+#ifdef CONFIG_SMP
+
+#define PAGECACHE_ACCT_THRESHOLD        max(16, NR_CPUS * 2)
+DECLARE_PER_CPU(long, nr_pagecache_local);
+
+/*
+ * pagecache_acct implements approximate accounting for pagecache.
+ * vm_enough_memory() do not need high accuracy. Writers will keep
+ * an offset in their per-cpu arena and will spill that into the
+ * global count whenever the absolute value of the local count
+ * exceeds the counter's threshold.
+ *
+ * MUST be protected from preemption.
+ * current protection is mapping->page_lock.
+ */
+static inline void pagecache_acct(int count)
+{
+	long *local;
+
+	local = &__get_cpu_var(nr_pagecache_local);
+	*local += count;
+	if (*local > PAGECACHE_ACCT_THRESHOLD || *local < -PAGECACHE_ACCT_THRESHOLD) {
+		atomic_add(*local, &nr_pagecache);
+		*local = 0;
+	}
+}
+
+#else
+
+static inline void pagecache_acct(int count)
+{
+	atomic_add(count, &nr_pagecache);
+}
+#endif
+
+static inline unsigned long get_page_cache_size(void)
+{
+        return atomic_read(&nr_pagecache);
+}
+
 static inline void ___add_to_page_cache(struct page *page,
 		struct address_space *mapping, unsigned long index)
 {
@@ -82,7 +124,7 @@ static inline void ___add_to_page_cache(struct page *page,
 	page->index = index;
 
 	mapping->nrpages++;
-	inc_page_state(nr_pagecache);
+	pagecache_acct(1);
 }
 
 extern void FASTCALL(__lock_page(struct page *page));

mm/filemap.c

@@ -88,7 +88,7 @@ void __remove_from_page_cache(struct page *page)
 	page->mapping = NULL;
 
 	mapping->nrpages--;
-	dec_page_state(nr_pagecache);
+	pagecache_acct(-1);
 }
 
 void remove_from_page_cache(struct page *page)

mm/page_alloc.c

@@ -804,6 +804,11 @@ static void show_node(struct zone *zone)
 DEFINE_PER_CPU(struct page_state, page_states) = {0};
 EXPORT_PER_CPU_SYMBOL(page_states);
 
+atomic_t nr_pagecache = ATOMIC_INIT(0);
+#ifdef CONFIG_SMP
+DEFINE_PER_CPU(long, nr_pagecache_local) = 0;
+#endif
+
 void __get_page_state(struct page_state *ret, int nr)
 {
 	int cpu = 0;
@@ -857,14 +862,6 @@ void get_zone_counts(unsigned long *active,
 	}
 }
 
-unsigned long get_page_cache_size(void)
-{
-	struct page_state ps;
-
-	get_page_state(&ps);
-	return ps.nr_pagecache;
-}
-
 void si_meminfo(struct sysinfo *val)
 {
 	val->totalram = totalram_pages;
@@ -1434,7 +1431,6 @@ struct seq_operations fragmentation_op = {
 static char *vmstat_text[] = {
 	"nr_dirty",
 	"nr_writeback",
-	"nr_pagecache",
 	"nr_page_table_pages",
 	"nr_reverse_maps",
 	"nr_mapped",