[PATCH] drain_node_pages: interrupt latency reduction / optimization

1. Only disable interrupts if there is actually something to free 2. Only dirty the pcp cacheline if we actually freed something. 3. Disable interrupts for each single pcp and not for cleaning all the pcps in all zones of a node. drain_node_pages is called every 2 seconds from cache_reap. This fix should avoid most disabling of interrupts. Signed-off-by: Christoph Lameter <clameter@sgi.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] drain_node_pages: interrupt latency reduction / optimization
1. Only disable interrupts if there is actually something to free 2. Only dirty the pcp cacheline if we actually freed something. 3. Disable interrupts for each single pcp and not for cleaning all the pcps in all zones of a node. drain_node_pages is called every 2 seconds from cache_reap. This fix should avoid most disabling of interrupts. Signed-off-by: Christoph Lameter <clameter@sgi.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Christoph Lameter · Linus Torvalds
1 parent b18e7e654d
Showing 1 changed file with 8 additions and 4 deletions Side-by-side Diff
mm/page_alloc.c
@@ -603,13 +603,14 @@
 /*
  * Called from the slab reaper to drain pagesets on a particular node that
  * belong to the currently executing processor.
+ * Note that this function must be called with the thread pinned to
+ * a single processor.
  */
 void drain_node_pages(int nodeid)
 {
 	int i, z;
 	unsigned long flags;
  
-	local_irq_save(flags);
 	for (z = 0; z < MAX_NR_ZONES; z++) {
 		struct zone *zone = NODE_DATA(nodeid)->node_zones + z;
 		struct per_cpu_pageset *pset;
  
@@ -619,11 +620,14 @@
 			struct per_cpu_pages *pcp;
  
 			pcp = &pset->pcp[i];
-			free_pages_bulk(zone, pcp->count, &pcp->list, 0);
-			pcp->count = 0;
+			if (pcp->count) {
+				local_irq_save(flags);
+				free_pages_bulk(zone, pcp->count, &pcp->list, 0);
+				pcp->count = 0;
+				local_irq_restore(flags);
+			}
 		}
 	}
-	local_irq_restore(flags);
 }
 #endif
...	...	@@ -603,13 +603,14 @@
603	603	/*
604	604	* Called from the slab reaper to drain pagesets on a particular node that
605	605	* belong to the currently executing processor.
	606	+ * Note that this function must be called with the thread pinned to
	607	+ * a single processor.
606	608	*/
607	609	void drain_node_pages(int nodeid)
608	610	{
609	611	int i, z;
610	612	unsigned long flags;
611	613
612		- local_irq_save(flags);
613	614	for (z = 0; z < MAX_NR_ZONES; z++) {
614	615	struct zone *zone = NODE_DATA(nodeid)->node_zones + z;
615	616	struct per_cpu_pageset *pset;
616	617
...	...	@@ -619,11 +620,14 @@
619	620	struct per_cpu_pages *pcp;
620	621
621	622	pcp = &pset->pcp[i];
622		- free_pages_bulk(zone, pcp->count, &pcp->list, 0);
623		- pcp->count = 0;
	623	+ if (pcp->count) {
	624	+ local_irq_save(flags);
	625	+ free_pages_bulk(zone, pcp->count, &pcp->list, 0);
	626	+ pcp->count = 0;
	627	+ local_irq_restore(flags);
	628	+ }
624	629	}
625	630	}
626		- local_irq_restore(flags);
627	631	}
628	632	#endif
629	633