[PATCH] zoned vm counters: conversion of nr_pagecache to per zone counter

Currently a single atomic variable is used to establish the size of the page cache in the whole machine. The zoned VM counters have the same method of implementation as the nr_pagecache code but also allow the determination of the pagecache size per zone. Remove the special implementation for nr_pagecache and make it a zoned counter named NR_FILE_PAGES. Updates of the page cache counters are always performed with interrupts off. We can therefore use the __ variant here. Signed-off-by: Christoph Lameter <clameter@sgi.com> Cc: Trond Myklebust <trond.myklebust@fys.uio.no> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] zoned vm counters: conversion of nr_pagecache to per zone counter
Currently a single atomic variable is used to establish the size of the page cache in the whole machine. The zoned VM counters have the same method of implementation as the nr_pagecache code but also allow the determination of the pagecache size per zone. Remove the special implementation for nr_pagecache and make it a zoned counter named NR_FILE_PAGES. Updates of the page cache counters are always performed with interrupts off. We can therefore use the __ variant here. Signed-off-by: Christoph Lameter <clameter@sgi.com> Cc: Trond Myklebust <trond.myklebust@fys.uio.no> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Christoph Lameter · Linus Torvalds
1 parent 65ba55f500
Showing 13 changed files with 16 additions and 67 deletions Side-by-side Diff
arch/s390/appldata/appldata_mem.c
arch/sparc/kernel/sys_sunos.c
arch/sparc64/kernel/sys_sunos32.c
drivers/base/node.c
fs/proc/proc_misc.c
include/linux/mmzone.h
include/linux/pagemap.h
mm/filemap.c
mm/mmap.c
mm/nommu.c
mm/page_alloc.c
mm/swap_state.c
mm/vmstat.c
@@ -130,7 +130,8 @@
 	mem_data->totalhigh = P2K(val.totalhigh);
 	mem_data->freehigh  = P2K(val.freehigh);
 	mem_data->bufferram = P2K(val.bufferram);
-	mem_data->cached    = P2K(atomic_read(&nr_pagecache) - val.bufferram);
+	mem_data->cached    = P2K(global_page_state(NR_FILE_PAGES)
+				- val.bufferram);
  
 	si_swapinfo(&val);
 	mem_data->totalswap = P2K(val.totalswap);
@@ -196,7 +196,7 @@
 	 * simple, it hopefully works in most obvious cases.. Easy to
 	 * fool it, but this should catch most mistakes.
 	 */
-	freepages = get_page_cache_size();
+	freepages = global_page_state(NR_FILE_PAGES);
 	freepages >>= 1;
 	freepages += nr_free_pages();
 	freepages += nr_swap_pages;
@@ -155,7 +155,7 @@
 	 * simple, it hopefully works in most obvious cases.. Easy to
 	 * fool it, but this should catch most mistakes.
 	 */
-	freepages = get_page_cache_size();
+	freepages = global_page_state(NR_FILE_PAGES);
 	freepages >>= 1;
 	freepages += nr_free_pages();
 	freepages += nr_swap_pages;
@@ -69,6 +69,7 @@
 		       "Node %d LowFree:      %8lu kB\n"
 		       "Node %d Dirty:        %8lu kB\n"
 		       "Node %d Writeback:    %8lu kB\n"
+		       "Node %d FilePages:    %8lu kB\n"
 		       "Node %d Mapped:       %8lu kB\n"
 		       "Node %d Slab:         %8lu kB\n",
 		       nid, K(i.totalram),
@@ -82,6 +83,7 @@
 		       nid, K(i.freeram - i.freehigh),
 		       nid, K(ps.nr_dirty),
 		       nid, K(ps.nr_writeback),
+		       nid, K(node_page_state(nid, NR_FILE_PAGES)),
 		       nid, K(node_page_state(nid, NR_FILE_MAPPED)),
 		       nid, K(ps.nr_slab));
 	n += hugetlb_report_node_meminfo(nid, buf + n);
@@ -142,7 +142,8 @@
 	allowed = ((totalram_pages - hugetlb_total_pages())
 		* sysctl_overcommit_ratio / 100) + total_swap_pages;
  
-	cached = get_page_cache_size() - total_swapcache_pages - i.bufferram;
+	cached = global_page_state(NR_FILE_PAGES) -
+			total_swapcache_pages - i.bufferram;
 	if (cached < 0)
 		cached = 0;
  
@@ -49,7 +49,7 @@
 enum zone_stat_item {
 	NR_FILE_MAPPED,	/* mapped into pagetables.
 			   only modified from process context */
-
+	NR_FILE_PAGES,
 	NR_VM_ZONE_STAT_ITEMS };
  
 struct per_cpu_pages {
@@ -113,51 +113,6 @@
 extern void remove_from_page_cache(struct page *page);
 extern void __remove_from_page_cache(struct page *page);
  
-extern atomic_t nr_pagecache;
-
-#ifdef CONFIG_SMP
-
-#define PAGECACHE_ACCT_THRESHOLD        max(16, NR_CPUS * 2)
-DECLARE_PER_CPU(long, nr_pagecache_local);
-
-/*
- * pagecache_acct implements approximate accounting for pagecache.
- * vm_enough_memory() do not need high accuracy. Writers will keep
- * an offset in their per-cpu arena and will spill that into the
- * global count whenever the absolute value of the local count
- * exceeds the counter's threshold.
- *
- * MUST be protected from preemption.
- * current protection is mapping->page_lock.
- */
-static inline void pagecache_acct(int count)
-{
-	long *local;
-
-	local = &__get_cpu_var(nr_pagecache_local);
-	*local += count;
-	if (*local > PAGECACHE_ACCT_THRESHOLD || *local < -PAGECACHE_ACCT_THRESHOLD) {
-		atomic_add(*local, &nr_pagecache);
-		*local = 0;
-	}
-}
-
-#else
-
-static inline void pagecache_acct(int count)
-{
-	atomic_add(count, &nr_pagecache);
-}
-#endif
-
-static inline unsigned long get_page_cache_size(void)
-{
-	int ret = atomic_read(&nr_pagecache);
-	if (unlikely(ret < 0))
-		ret = 0;
-	return ret;
-}
-
 /*
  * Return byte-offset into filesystem object for page.
  */
@@ -120,7 +120,7 @@
 	radix_tree_delete(&mapping->page_tree, page->index);
 	page->mapping = NULL;
 	mapping->nrpages--;
-	pagecache_acct(-1);
+	__dec_zone_page_state(page, NR_FILE_PAGES);
 }
  
 void remove_from_page_cache(struct page *page)
@@ -449,7 +449,7 @@
 			page->mapping = mapping;
 			page->index = offset;
 			mapping->nrpages++;
-			pagecache_acct(1);
+			__inc_zone_page_state(page, NR_FILE_PAGES);
 		}
 		write_unlock_irq(&mapping->tree_lock);
 		radix_tree_preload_end();
@@ -96,7 +96,7 @@
 	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
 		unsigned long n;
  
-		free = get_page_cache_size();
+		free = global_page_state(NR_FILE_PAGES);
 		free += nr_swap_pages;
  
 		/*
@@ -1122,7 +1122,7 @@
 	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
 		unsigned long n;
  
-		free = get_page_cache_size();
+		free = global_page_state(NR_FILE_PAGES);
 		free += nr_swap_pages;
  
 		/*
@@ -2124,16 +2124,11 @@
 				 unsigned long action, void *hcpu)
 {
 	int cpu = (unsigned long)hcpu;
-	long *count;
 	unsigned long *src, *dest;
  
 	if (action == CPU_DEAD) {
 		int i;
  
-		/* Drain local pagecache count. */
-		count = &per_cpu(nr_pagecache_local, cpu);
-		atomic_add(*count, &nr_pagecache);
-		*count = 0;
 		local_irq_disable();
 		__drain_pages(cpu);
  
@@ -87,7 +87,7 @@
 			SetPageSwapCache(page);
 			set_page_private(page, entry.val);
 			total_swapcache_pages++;
-			pagecache_acct(1);
+			__inc_zone_page_state(page, NR_FILE_PAGES);
 		}
 		write_unlock_irq(&swapper_space.tree_lock);
 		radix_tree_preload_end();
@@ -132,7 +132,7 @@
 	set_page_private(page, 0);
 	ClearPageSwapCache(page);
 	total_swapcache_pages--;
-	pagecache_acct(-1);
+	__dec_zone_page_state(page, NR_FILE_PAGES);
 	INC_CACHE_INFO(del_total);
 }
  
@@ -20,12 +20,6 @@
  */
 DEFINE_PER_CPU(struct page_state, page_states) = {0};
  
-atomic_t nr_pagecache = ATOMIC_INIT(0);
-EXPORT_SYMBOL(nr_pagecache);
-#ifdef CONFIG_SMP
-DEFINE_PER_CPU(long, nr_pagecache_local) = 0;
-#endif
-
 static void __get_page_state(struct page_state *ret, int nr, cpumask_t *cpumask)
 {
 	unsigned cpu;
@@ -402,6 +396,7 @@
 static char *vmstat_text[] = {
 	/* Zoned VM counters */
 	"nr_mapped",
+	"nr_file_pages",
  
 	/* Page state */
 	"nr_dirty",
...	...	@@ -130,7 +130,8 @@
130	130	mem_data->totalhigh = P2K(val.totalhigh);
131	131	mem_data->freehigh = P2K(val.freehigh);
132	132	mem_data->bufferram = P2K(val.bufferram);
133		- mem_data->cached = P2K(atomic_read(&nr_pagecache) - val.bufferram);
	133	+ mem_data->cached = P2K(global_page_state(NR_FILE_PAGES)
	134	+ - val.bufferram);
134	135
135	136	si_swapinfo(&val);
136	137	mem_data->totalswap = P2K(val.totalswap);
...	...	@@ -196,7 +196,7 @@
196	196	* simple, it hopefully works in most obvious cases.. Easy to
197	197	* fool it, but this should catch most mistakes.
198	198	*/
199		- freepages = get_page_cache_size();
	199	+ freepages = global_page_state(NR_FILE_PAGES);
200	200	freepages >>= 1;
201	201	freepages += nr_free_pages();
202	202	freepages += nr_swap_pages;
...	...	@@ -155,7 +155,7 @@
155	155	* simple, it hopefully works in most obvious cases.. Easy to
156	156	* fool it, but this should catch most mistakes.
157	157	*/
158		- freepages = get_page_cache_size();
	158	+ freepages = global_page_state(NR_FILE_PAGES);
159	159	freepages >>= 1;
160	160	freepages += nr_free_pages();
161	161	freepages += nr_swap_pages;
...	...	@@ -69,6 +69,7 @@
69	69	"Node %d LowFree: %8lu kB\n"
70	70	"Node %d Dirty: %8lu kB\n"
71	71	"Node %d Writeback: %8lu kB\n"
	72	+ "Node %d FilePages: %8lu kB\n"
72	73	"Node %d Mapped: %8lu kB\n"
73	74	"Node %d Slab: %8lu kB\n",
74	75	nid, K(i.totalram),
...	...	@@ -82,6 +83,7 @@
82	83	nid, K(i.freeram - i.freehigh),
83	84	nid, K(ps.nr_dirty),
84	85	nid, K(ps.nr_writeback),
	86	+ nid, K(node_page_state(nid, NR_FILE_PAGES)),
85	87	nid, K(node_page_state(nid, NR_FILE_MAPPED)),
86	88	nid, K(ps.nr_slab));
87	89	n += hugetlb_report_node_meminfo(nid, buf + n);
...	...	@@ -142,7 +142,8 @@
142	142	allowed = ((totalram_pages - hugetlb_total_pages())
143	143	* sysctl_overcommit_ratio / 100) + total_swap_pages;
144	144
145		- cached = get_page_cache_size() - total_swapcache_pages - i.bufferram;
	145	+ cached = global_page_state(NR_FILE_PAGES) -
	146	+ total_swapcache_pages - i.bufferram;
146	147	if (cached < 0)
147	148	cached = 0;
148	149
...	...	@@ -49,7 +49,7 @@
49	49	enum zone_stat_item {
50	50	NR_FILE_MAPPED, /* mapped into pagetables.
51	51	only modified from process context */
52		-
	52	+ NR_FILE_PAGES,
53	53	NR_VM_ZONE_STAT_ITEMS };
54	54
55	55	struct per_cpu_pages {
...	...	@@ -113,51 +113,6 @@
113	113	extern void remove_from_page_cache(struct page *page);
114	114	extern void __remove_from_page_cache(struct page *page);
115	115
116		-extern atomic_t nr_pagecache;
117		-
118		-#ifdef CONFIG_SMP
119		-
120		-#define PAGECACHE_ACCT_THRESHOLD max(16, NR_CPUS * 2)
121		-DECLARE_PER_CPU(long, nr_pagecache_local);
122		-
123		-/*
124		- * pagecache_acct implements approximate accounting for pagecache.
125		- * vm_enough_memory() do not need high accuracy. Writers will keep
126		- * an offset in their per-cpu arena and will spill that into the
127		- * global count whenever the absolute value of the local count
128		- * exceeds the counter's threshold.
129		- *
130		- * MUST be protected from preemption.
131		- * current protection is mapping->page_lock.
132		- */
133		-static inline void pagecache_acct(int count)
134		-{
135		- long *local;
136		-
137		- local = &__get_cpu_var(nr_pagecache_local);
138		- *local += count;
139		- if (local > PAGECACHE_ACCT_THRESHOLD \|\| local < -PAGECACHE_ACCT_THRESHOLD) {
140		- atomic_add(*local, &nr_pagecache);
141		- *local = 0;
142		- }
143		-}
144		-
145		-#else
146		-
147		-static inline void pagecache_acct(int count)
148		-{
149		- atomic_add(count, &nr_pagecache);
150		-}
151		-#endif
152		-
153		-static inline unsigned long get_page_cache_size(void)
154		-{
155		- int ret = atomic_read(&nr_pagecache);
156		- if (unlikely(ret < 0))
157		- ret = 0;
158		- return ret;
159		-}
160		-
161	116	/*
162	117	* Return byte-offset into filesystem object for page.
163	118	*/
...	...	@@ -120,7 +120,7 @@
120	120	radix_tree_delete(&mapping->page_tree, page->index);
121	121	page->mapping = NULL;
122	122	mapping->nrpages--;
123		- pagecache_acct(-1);
	123	+ __dec_zone_page_state(page, NR_FILE_PAGES);
124	124	}
125	125
126	126	void remove_from_page_cache(struct page *page)
...	...	@@ -449,7 +449,7 @@
449	449	page->mapping = mapping;
450	450	page->index = offset;
451	451	mapping->nrpages++;
452		- pagecache_acct(1);
	452	+ __inc_zone_page_state(page, NR_FILE_PAGES);
453	453	}
454	454	write_unlock_irq(&mapping->tree_lock);
455	455	radix_tree_preload_end();
...	...	@@ -96,7 +96,7 @@
96	96	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
97	97	unsigned long n;
98	98
99		- free = get_page_cache_size();
	99	+ free = global_page_state(NR_FILE_PAGES);
100	100	free += nr_swap_pages;
101	101
102	102	/*
...	...	@@ -1122,7 +1122,7 @@
1122	1122	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
1123	1123	unsigned long n;
1124	1124
1125		- free = get_page_cache_size();
	1125	+ free = global_page_state(NR_FILE_PAGES);
1126	1126	free += nr_swap_pages;
1127	1127
1128	1128	/*