page-allocator: maintain rolling count of pages to free from the PCP

When round-robin freeing pages from the PCP lists, empty lists may be encountered. In the event one of the lists has more pages than another, there may be numerous checks for list_empty() which is undesirable. This patch maintains a count of pages to free which is incremented when empty lists are encountered. The intention is that more pages will then be freed from fuller lists than the empty ones reducing the number of empty list checks in the free path. [akpm@linux-foundation.org: coding-style fixes] Signed-off-by: Mel Gorman <mel@csn.ul.ie> Cc: Nick Piggin <npiggin@suse.de> Cc: Christoph Lameter <cl@linux-foundation.org> Reviewed-by: Minchan Kim <minchan.kim@gmail.com> Cc: Pekka Enberg <penberg@cs.helsinki.fi> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

page-allocator: maintain rolling count of pages to free from the PCP
When round-robin freeing pages from the PCP lists, empty lists may be encountered. In the event one of the lists has more pages than another, there may be numerous checks for list_empty() which is undesirable. This patch maintains a count of pages to free which is incremented when empty lists are encountered. The intention is that more pages will then be freed from fuller lists than the empty ones reducing the number of empty list checks in the free path. [akpm@linux-foundation.org: coding-style fixes] Signed-off-by: Mel Gorman <mel@csn.ul.ie> Cc: Nick Piggin <npiggin@suse.de> Cc: Christoph Lameter <cl@linux-foundation.org> Reviewed-by: Minchan Kim <minchan.kim@gmail.com> Cc: Pekka Enberg <penberg@cs.helsinki.fi> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Mel Gorman · Linus Torvalds
1 parent 5f8dcc2121
Showing 1 changed file with 15 additions and 9 deletions Side-by-side Diff
mm/page_alloc.c
@@ -525,32 +525,38 @@
 					struct per_cpu_pages *pcp)
 {
 	int migratetype = 0;
+	int batch_free = 0;
  
 	spin_lock(&zone->lock);
 	zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
 	zone->pages_scanned = 0;
  
 	__mod_zone_page_state(zone, NR_FREE_PAGES, count);
-	while (count--) {
+	while (count) {
 		struct page *page;
 		struct list_head *list;
  
 		/*
-		 * Remove pages from lists in a round-robin fashion. This spinning
-		 * around potentially empty lists is bloody awful, alternatives that
-		 * don't suck are welcome
+		 * Remove pages from lists in a round-robin fashion. A
+		 * batch_free count is maintained that is incremented when an
+		 * empty list is encountered.  This is so more pages are freed
+		 * off fuller lists instead of spinning excessively around empty
+		 * lists
 		 */
 		do {
+			batch_free++;
 			if (++migratetype == MIGRATE_PCPTYPES)
 				migratetype = 0;
 			list = &pcp->lists[migratetype];
 		} while (list_empty(list));
  
-		page = list_entry(list->prev, struct page, lru);
-		/* have to delete it as __free_one_page list manipulates */
-		list_del(&page->lru);
-		trace_mm_page_pcpu_drain(page, 0, migratetype);
-		__free_one_page(page, zone, 0, migratetype);
+		do {
+			page = list_entry(list->prev, struct page, lru);
+			/* must delete as __free_one_page list manipulates */
+			list_del(&page->lru);
+			__free_one_page(page, zone, 0, migratetype);
+			trace_mm_page_pcpu_drain(page, 0, migratetype);
+		} while (--count && --batch_free && !list_empty(list));
 	}
 	spin_unlock(&zone->lock);
 }
...	...	@@ -525,32 +525,38 @@
525	525	struct per_cpu_pages *pcp)
526	526	{
527	527	int migratetype = 0;
	528	+ int batch_free = 0;
528	529
529	530	spin_lock(&zone->lock);
530	531	zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
531	532	zone->pages_scanned = 0;
532	533
533	534	__mod_zone_page_state(zone, NR_FREE_PAGES, count);
534		- while (count--) {
	535	+ while (count) {
535	536	struct page *page;
536	537	struct list_head *list;
537	538
538	539	/*
539		- * Remove pages from lists in a round-robin fashion. This spinning
540		- * around potentially empty lists is bloody awful, alternatives that
541		- * don't suck are welcome
	540	+ * Remove pages from lists in a round-robin fashion. A
	541	+ * batch_free count is maintained that is incremented when an
	542	+ * empty list is encountered. This is so more pages are freed
	543	+ * off fuller lists instead of spinning excessively around empty
	544	+ * lists
542	545	*/
543	546	do {
	547	+ batch_free++;
544	548	if (++migratetype == MIGRATE_PCPTYPES)
545	549	migratetype = 0;
546	550	list = &pcp->lists[migratetype];
547	551	} while (list_empty(list));
548	552
549		- page = list_entry(list->prev, struct page, lru);
550		- /* have to delete it as __free_one_page list manipulates */
551		- list_del(&page->lru);
552		- trace_mm_page_pcpu_drain(page, 0, migratetype);
553		- __free_one_page(page, zone, 0, migratetype);
	553	+ do {
	554	+ page = list_entry(list->prev, struct page, lru);
	555	+ /* must delete as __free_one_page list manipulates */
	556	+ list_del(&page->lru);
	557	+ __free_one_page(page, zone, 0, migratetype);
	558	+ trace_mm_page_pcpu_drain(page, 0, migratetype);
	559	+ } while (--count && --batch_free && !list_empty(list));
554	560	}
555	561	spin_unlock(&zone->lock);
556	562	}