memcg : share event counter rather than duplicate

Memcg has 2 eventcountes which counts "the same" event. Just usages are different from each other. This patch tries to reduce event counter. Now logic uses "only increment, no reset" counter and masks for each checks. Softlimit chesk was done per 1000 evetns. So, the similar check can be done by !(new_counter & 0x3ff). Threshold check was done per 100 events. So, the similar check can be done by (!new_counter & 0x7f) ALL event checks are done right after EVENT percpu counter is updated. Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Kirill A. Shutemov <kirill@shutemov.name> Cc: Balbir Singh <balbir@linux.vnet.ibm.com> Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

memcg : share event counter rather than duplicate
Memcg has 2 eventcountes which counts "the same" event. Just usages are different from each other. This patch tries to reduce event counter. Now logic uses "only increment, no reset" counter and masks for each checks. Softlimit chesk was done per 1000 evetns. So, the similar check can be done by !(new_counter & 0x3ff). Threshold check was done per 100 events. So, the similar check can be done by (!new_counter & 0x7f) ALL event checks are done right after EVENT percpu counter is updated. Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Kirill A. Shutemov <kirill@shutemov.name> Cc: Balbir Singh <balbir@linux.vnet.ibm.com> Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
KAMEZAWA Hiroyuki · Linus Torvalds
1 parent 430e48631e
Showing 1 changed file with 41 additions and 45 deletions Side-by-side Diff
mm/memcontrol.c
@@ -63,8 +63,15 @@
 #define do_swap_account		(0)
 #endif
  
-#define SOFTLIMIT_EVENTS_THRESH (1000)
-#define THRESHOLDS_EVENTS_THRESH (100)
+/*
+ * Per memcg event counter is incremented at every pagein/pageout. This counter
+ * is used for trigger some periodic events. This is straightforward and better
+ * than using jiffies etc. to handle periodic memcg event.
+ *
+ * These values will be used as !((event) & ((1 <<(thresh)) - 1))
+ */
+#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
+#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
  
 /*
  * Statistics for memory cgroup.
@@ -79,10 +86,7 @@
 	MEM_CGROUP_STAT_PGPGIN_COUNT,	/* # of pages paged in */
 	MEM_CGROUP_STAT_PGPGOUT_COUNT,	/* # of pages paged out */
 	MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
-	MEM_CGROUP_STAT_SOFTLIMIT, /* decrements on each page in/out.
-					used by soft limit implementation */
-	MEM_CGROUP_STAT_THRESHOLDS, /* decrements on each page in/out.
-					used by threshold implementation */
+	MEM_CGROUP_EVENTS,	/* incremented at every  pagein/pageout */
  
 	MEM_CGROUP_STAT_NSTATS,
 };
@@ -154,7 +158,6 @@
 	struct mem_cgroup_threshold entries[0];
 };
  
-static bool mem_cgroup_threshold_check(struct mem_cgroup *mem);
 static void mem_cgroup_threshold(struct mem_cgroup *mem);
  
 /*
  
@@ -392,20 +395,7 @@
 	spin_unlock(&mctz->lock);
 }
  
-static bool mem_cgroup_soft_limit_check(struct mem_cgroup *mem)
-{
-	bool ret = false;
-	s64 val;
  
-	val = this_cpu_read(mem->stat->count[MEM_CGROUP_STAT_SOFTLIMIT]);
-	if (unlikely(val < 0)) {
-		this_cpu_write(mem->stat->count[MEM_CGROUP_STAT_SOFTLIMIT],
-				SOFTLIMIT_EVENTS_THRESH);
-		ret = true;
-	}
-	return ret;
-}
-
 static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
 {
 	unsigned long long excess;
@@ -542,8 +532,7 @@
 		__this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
 	else
 		__this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
-	__this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_SOFTLIMIT]);
-	__this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_THRESHOLDS]);
+	__this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
  
 	preempt_enable();
 }
@@ -563,6 +552,29 @@
 	return total;
 }
  
+static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
+{
+	s64 val;
+
+	val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
+
+	return !(val & ((1 << event_mask_shift) - 1));
+}
+
+/*
+ * Check events in order.
+ *
+ */
+static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
+{
+	/* threshold event is triggered in finer grain than soft limit */
+	if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
+		mem_cgroup_threshold(mem);
+		if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
+			mem_cgroup_update_tree(mem, page);
+	}
+}
+
 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 {
 	return container_of(cgroup_subsys_state(cont,
@@ -1686,11 +1698,7 @@
 	 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
 	 * if they exceeds softlimit.
 	 */
-	if (mem_cgroup_soft_limit_check(mem))
-		mem_cgroup_update_tree(mem, pc->page);
-	if (mem_cgroup_threshold_check(mem))
-		mem_cgroup_threshold(mem);
-
+	memcg_check_events(mem, pc->page);
 }
  
 /**
@@ -1760,6 +1768,11 @@
 		ret = 0;
 	}
 	unlock_page_cgroup(pc);
+	/*
+	 * check events
+	 */
+	memcg_check_events(to, pc->page);
+	memcg_check_events(from, pc->page);
 	return ret;
 }
  
@@ -2128,10 +2141,7 @@
 	mz = page_cgroup_zoneinfo(pc);
 	unlock_page_cgroup(pc);
  
-	if (mem_cgroup_soft_limit_check(mem))
-		mem_cgroup_update_tree(mem, page);
-	if (mem_cgroup_threshold_check(mem))
-		mem_cgroup_threshold(mem);
+	memcg_check_events(mem, page);
 	/* at swapout, this memcg will be accessed to record to swap */
 	if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
 		css_put(&mem->css);
@@ -3213,20 +3223,6 @@
 	cgroup_unlock();
  
 	return 0;
-}
-
-static bool mem_cgroup_threshold_check(struct mem_cgroup *mem)
-{
-	bool ret = false;
-	s64 val;
-
-	val = this_cpu_read(mem->stat->count[MEM_CGROUP_STAT_THRESHOLDS]);
-	if (unlikely(val < 0)) {
-		this_cpu_write(mem->stat->count[MEM_CGROUP_STAT_THRESHOLDS],
-				THRESHOLDS_EVENTS_THRESH);
-		ret = true;
-	}
-	return ret;
 }
  
 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
...	...	@@ -63,8 +63,15 @@
63	63	#define do_swap_account (0)
64	64	#endif
65	65
66		-#define SOFTLIMIT_EVENTS_THRESH (1000)
67		-#define THRESHOLDS_EVENTS_THRESH (100)
	66	+/*
	67	+ * Per memcg event counter is incremented at every pagein/pageout. This counter
	68	+ * is used for trigger some periodic events. This is straightforward and better
	69	+ * than using jiffies etc. to handle periodic memcg event.
	70	+ *
	71	+ * These values will be used as !((event) & ((1 <<(thresh)) - 1))
	72	+ */
	73	+#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
	74	+#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
68	75
69	76	/*
70	77	* Statistics for memory cgroup.
...	...	@@ -79,10 +86,7 @@
79	86	MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */
80	87	MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */
81	88	MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
82		- MEM_CGROUP_STAT_SOFTLIMIT, /* decrements on each page in/out.
83		- used by soft limit implementation */
84		- MEM_CGROUP_STAT_THRESHOLDS, /* decrements on each page in/out.
85		- used by threshold implementation */
	89	+ MEM_CGROUP_EVENTS, /* incremented at every pagein/pageout */
86	90
87	91	MEM_CGROUP_STAT_NSTATS,
88	92	};
...	...	@@ -154,7 +158,6 @@
154	158	struct mem_cgroup_threshold entries[0];
155	159	};
156	160
157		-static bool mem_cgroup_threshold_check(struct mem_cgroup *mem);
158	161	static void mem_cgroup_threshold(struct mem_cgroup *mem);
159	162
160	163	/*
161	164
...	...	@@ -392,20 +395,7 @@
392	395	spin_unlock(&mctz->lock);
393	396	}
394	397
395		-static bool mem_cgroup_soft_limit_check(struct mem_cgroup *mem)
396		-{
397		- bool ret = false;
398		- s64 val;
399	398
400		- val = this_cpu_read(mem->stat->count[MEM_CGROUP_STAT_SOFTLIMIT]);
401		- if (unlikely(val < 0)) {
402		- this_cpu_write(mem->stat->count[MEM_CGROUP_STAT_SOFTLIMIT],
403		- SOFTLIMIT_EVENTS_THRESH);
404		- ret = true;
405		- }
406		- return ret;
407		-}
408		-
409	399	static void mem_cgroup_update_tree(struct mem_cgroup mem, struct page page)
410	400	{
411	401	unsigned long long excess;
...	...	@@ -542,8 +532,7 @@
542	532	__this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
543	533	else
544	534	__this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
545		- __this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_SOFTLIMIT]);
546		- __this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_THRESHOLDS]);
	535	+ __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
547	536
548	537	preempt_enable();
549	538	}
...	...	@@ -563,6 +552,29 @@
563	552	return total;
564	553	}
565	554
	555	+static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
	556	+{
	557	+ s64 val;
	558	+
	559	+ val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
	560	+
	561	+ return !(val & ((1 << event_mask_shift) - 1));
	562	+}
	563	+
	564	+/*
	565	+ * Check events in order.
	566	+ *
	567	+ */
	568	+static void memcg_check_events(struct mem_cgroup mem, struct page page)
	569	+{
	570	+ /* threshold event is triggered in finer grain than soft limit */
	571	+ if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
	572	+ mem_cgroup_threshold(mem);
	573	+ if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
	574	+ mem_cgroup_update_tree(mem, page);
	575	+ }
	576	+}
	577	+
566	578	static struct mem_cgroup mem_cgroup_from_cont(struct cgroup cont)
567	579	{
568	580	return container_of(cgroup_subsys_state(cont,
...	...	@@ -1686,11 +1698,7 @@
1686	1698	* Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
1687	1699	* if they exceeds softlimit.
1688	1700	*/
1689		- if (mem_cgroup_soft_limit_check(mem))
1690		- mem_cgroup_update_tree(mem, pc->page);
1691		- if (mem_cgroup_threshold_check(mem))
1692		- mem_cgroup_threshold(mem);
1693		-
	1701	+ memcg_check_events(mem, pc->page);
1694	1702	}
1695	1703
1696	1704	/**
...	...	@@ -1760,6 +1768,11 @@
1760	1768	ret = 0;
1761	1769	}
1762	1770	unlock_page_cgroup(pc);
	1771	+ /*
	1772	+ * check events
	1773	+ */
	1774	+ memcg_check_events(to, pc->page);
	1775	+ memcg_check_events(from, pc->page);
1763	1776	return ret;
1764	1777	}
1765	1778
...	...	@@ -2128,10 +2141,7 @@
2128	2141	mz = page_cgroup_zoneinfo(pc);
2129	2142	unlock_page_cgroup(pc);
2130	2143
2131		- if (mem_cgroup_soft_limit_check(mem))
2132		- mem_cgroup_update_tree(mem, page);
2133		- if (mem_cgroup_threshold_check(mem))
2134		- mem_cgroup_threshold(mem);
	2144	+ memcg_check_events(mem, page);
2135	2145	/* at swapout, this memcg will be accessed to record to swap */
2136	2146	if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2137	2147	css_put(&mem->css);
...	...	@@ -3213,20 +3223,6 @@
3213	3223	cgroup_unlock();
3214	3224
3215	3225	return 0;
3216		-}
3217		-
3218		-static bool mem_cgroup_threshold_check(struct mem_cgroup *mem)
3219		-{
3220		- bool ret = false;
3221		- s64 val;
3222		-
3223		- val = this_cpu_read(mem->stat->count[MEM_CGROUP_STAT_THRESHOLDS]);
3224		- if (unlikely(val < 0)) {
3225		- this_cpu_write(mem->stat->count[MEM_CGROUP_STAT_THRESHOLDS],
3226		- THRESHOLDS_EVENTS_THRESH);
3227		- ret = true;
3228		- }
3229		- return ret;
3230	3226	}
3231	3227
3232	3228	static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)