rcu: Make RCU_FAST_NO_HZ account for pauses out of idle

Both Steven Rostedt's new idle-capable trace macros and the RCU_NONIDLE() macro can cause RCU to momentarily pause out of idle without the rest of the system being involved. This can cause rcu_prepare_for_idle() to run through its state machine too quickly, which can in turn result in needless scheduling-clock interrupts. This commit therefore adds code to enable rcu_prepare_for_idle() to distinguish between an initial entry to idle on the one hand (which needs to advance the rcu_prepare_for_idle() state machine) and an idle reentry due to idle-capable trace macros and RCU_NONIDLE() on the other hand (which should avoid advancing the rcu_prepare_for_idle() state machine). Additional state is maintained to allow the timer to be correctly reposted when returning after a momentary pause out of idle, and even more state is maintained to detect when new non-lazy callbacks have been enqueued (which may require re-evaluation of the approach to idleness). Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

rcu: Make RCU_FAST_NO_HZ account for pauses out of idle
Both Steven Rostedt's new idle-capable trace macros and the RCU_NONIDLE() macro can cause RCU to momentarily pause out of idle without the rest of the system being involved. This can cause rcu_prepare_for_idle() to run through its state machine too quickly, which can in turn result in needless scheduling-clock interrupts. This commit therefore adds code to enable rcu_prepare_for_idle() to distinguish between an initial entry to idle on the one hand (which needs to advance the rcu_prepare_for_idle() state machine) and an idle reentry due to idle-capable trace macros and RCU_NONIDLE() on the other hand (which should avoid advancing the rcu_prepare_for_idle() state machine). Additional state is maintained to allow the timer to be correctly reposted when returning after a momentary pause out of idle, and even more state is maintained to detect when new non-lazy callbacks have been enqueued (which may require re-evaluation of the approach to idleness). Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Paul E. McKenney · Paul E. McKenney
1 parent 2ee3dc8066
Showing 3 changed files with 56 additions and 4 deletions Side-by-side Diff
kernel/rcutree.c
kernel/rcutree.h
kernel/rcutree_plugin.h
@@ -1829,6 +1829,8 @@
 	rdp->qlen++;
 	if (lazy)
 		rdp->qlen_lazy++;
+	else
+		rcu_idle_count_callbacks_posted();
  
 	if (__is_kfree_rcu_offset((unsigned long)func))
 		trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
@@ -471,6 +471,7 @@
 static void rcu_prepare_for_idle_init(int cpu);
 static void rcu_cleanup_after_idle(int cpu);
 static void rcu_prepare_for_idle(int cpu);
+static void rcu_idle_count_callbacks_posted(void);
 static void print_cpu_stall_info_begin(void);
 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu);
 static void print_cpu_stall_info_end(void);
@@ -1938,6 +1938,14 @@
 {
 }
  
+/*
+ * Don't bother keeping a running count of the number of RCU callbacks
+ * posted because CONFIG_RCU_FAST_NO_HZ=n.
+ */
+static void rcu_idle_count_callbacks_posted(void)
+{
+}
+
 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
  
 /*
@@ -1981,6 +1989,10 @@
 static DEFINE_PER_CPU(int, rcu_dyntick_drain);
 static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
 static DEFINE_PER_CPU(struct timer_list, rcu_idle_gp_timer);
+static DEFINE_PER_CPU(unsigned long, rcu_idle_gp_timer_expires);
+static DEFINE_PER_CPU(bool, rcu_idle_first_pass);
+static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted);
+static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap);
  
 /*
  * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
@@ -1993,6 +2005,8 @@
  */
 int rcu_needs_cpu(int cpu)
 {
+	/* Flag a new idle sojourn to the idle-entry state machine. */
+	per_cpu(rcu_idle_first_pass, cpu) = 1;
 	/* If no callbacks, RCU doesn't need the CPU. */
 	if (!rcu_cpu_has_callbacks(cpu))
 		return 0;
@@ -2096,6 +2110,26 @@
 static void rcu_prepare_for_idle(int cpu)
 {
 	/*
+	 * If this is an idle re-entry, for example, due to use of
+	 * RCU_NONIDLE() or the new idle-loop tracing API within the idle
+	 * loop, then don't take any state-machine actions, unless the
+	 * momentary exit from idle queued additional non-lazy callbacks.
+	 * Instead, repost the rcu_idle_gp_timer if this CPU has callbacks
+	 * pending.
+	 */
+	if (!per_cpu(rcu_idle_first_pass, cpu) &&
+	    (per_cpu(rcu_nonlazy_posted, cpu) ==
+	     per_cpu(rcu_nonlazy_posted_snap, cpu))) {
+		if (rcu_cpu_has_callbacks(cpu))
+			mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
+				  per_cpu(rcu_idle_gp_timer_expires, cpu));
+		return;
+	}
+	per_cpu(rcu_idle_first_pass, cpu) = 0;
+	per_cpu(rcu_nonlazy_posted_snap, cpu) =
+		per_cpu(rcu_nonlazy_posted, cpu) - 1;
+
+	/*
 	 * If there are no callbacks on this CPU, enter dyntick-idle mode.
 	 * Also reset state to avoid prejudicing later attempts.
 	 */
  
@@ -2127,11 +2161,15 @@
 		per_cpu(rcu_dyntick_drain, cpu) = 0;
 		per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
 		if (rcu_cpu_has_nonlazy_callbacks(cpu))
-			mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
-					   jiffies + RCU_IDLE_GP_DELAY);
+			per_cpu(rcu_idle_gp_timer_expires, cpu) =
+					   jiffies + RCU_IDLE_GP_DELAY;
 		else
-			mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
-					   jiffies + RCU_IDLE_LAZY_GP_DELAY);
+			per_cpu(rcu_idle_gp_timer_expires, cpu) =
+					   jiffies + RCU_IDLE_LAZY_GP_DELAY;
+		mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
+			  per_cpu(rcu_idle_gp_timer_expires, cpu));
+		per_cpu(rcu_nonlazy_posted_snap, cpu) =
+			per_cpu(rcu_nonlazy_posted, cpu);
 		return; /* Nothing more to do immediately. */
 	} else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
 		/* We have hit the limit, so time to give up. */
@@ -2169,6 +2207,17 @@
 		invoke_rcu_core();
 	} else
 		trace_rcu_prep_idle("Callbacks drained");
+}
+
+/*
+ * Keep a running count of callbacks posted so that rcu_prepare_for_idle()
+ * can detect when something out of the idle loop posts a callback.
+ * Of course, it had better do so either from a trace event designed to
+ * be called from idle or from within RCU_NONIDLE().
+ */
+static void rcu_idle_count_callbacks_posted(void)
+{
+	__this_cpu_add(rcu_nonlazy_posted, 1);
 }
  
 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
...	...	@@ -1829,6 +1829,8 @@
1829	1829	rdp->qlen++;
1830	1830	if (lazy)
1831	1831	rdp->qlen_lazy++;
	1832	+ else
	1833	+ rcu_idle_count_callbacks_posted();
1832	1834
1833	1835	if (__is_kfree_rcu_offset((unsigned long)func))
1834	1836	trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
...	...	@@ -471,6 +471,7 @@
471	471	static void rcu_prepare_for_idle_init(int cpu);
472	472	static void rcu_cleanup_after_idle(int cpu);
473	473	static void rcu_prepare_for_idle(int cpu);
	474	+static void rcu_idle_count_callbacks_posted(void);
474	475	static void print_cpu_stall_info_begin(void);
475	476	static void print_cpu_stall_info(struct rcu_state *rsp, int cpu);
476	477	static void print_cpu_stall_info_end(void);
...	...	@@ -1938,6 +1938,14 @@
1938	1938	{
1939	1939	}
1940	1940
	1941	+/*
	1942	+ * Don't bother keeping a running count of the number of RCU callbacks
	1943	+ * posted because CONFIG_RCU_FAST_NO_HZ=n.
	1944	+ */
	1945	+static void rcu_idle_count_callbacks_posted(void)
	1946	+{
	1947	+}
	1948	+
1941	1949	#else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1942	1950
1943	1951	/*
...	...	@@ -1981,6 +1989,10 @@
1981	1989	static DEFINE_PER_CPU(int, rcu_dyntick_drain);
1982	1990	static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
1983	1991	static DEFINE_PER_CPU(struct timer_list, rcu_idle_gp_timer);
	1992	+static DEFINE_PER_CPU(unsigned long, rcu_idle_gp_timer_expires);
	1993	+static DEFINE_PER_CPU(bool, rcu_idle_first_pass);
	1994	+static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted);
	1995	+static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap);
1984	1996
1985	1997	/*
1986	1998	* Allow the CPU to enter dyntick-idle mode if either: (1) There are no
...	...	@@ -1993,6 +2005,8 @@
1993	2005	*/
1994	2006	int rcu_needs_cpu(int cpu)
1995	2007	{
	2008	+ /* Flag a new idle sojourn to the idle-entry state machine. */
	2009	+ per_cpu(rcu_idle_first_pass, cpu) = 1;
1996	2010	/* If no callbacks, RCU doesn't need the CPU. */
1997	2011	if (!rcu_cpu_has_callbacks(cpu))
1998	2012	return 0;
...	...	@@ -2096,6 +2110,26 @@
2096	2110	static void rcu_prepare_for_idle(int cpu)
2097	2111	{
2098	2112	/*
	2113	+ * If this is an idle re-entry, for example, due to use of
	2114	+ * RCU_NONIDLE() or the new idle-loop tracing API within the idle
	2115	+ * loop, then don't take any state-machine actions, unless the
	2116	+ * momentary exit from idle queued additional non-lazy callbacks.
	2117	+ * Instead, repost the rcu_idle_gp_timer if this CPU has callbacks
	2118	+ * pending.
	2119	+ */
	2120	+ if (!per_cpu(rcu_idle_first_pass, cpu) &&
	2121	+ (per_cpu(rcu_nonlazy_posted, cpu) ==
	2122	+ per_cpu(rcu_nonlazy_posted_snap, cpu))) {
	2123	+ if (rcu_cpu_has_callbacks(cpu))
	2124	+ mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
	2125	+ per_cpu(rcu_idle_gp_timer_expires, cpu));
	2126	+ return;
	2127	+ }
	2128	+ per_cpu(rcu_idle_first_pass, cpu) = 0;
	2129	+ per_cpu(rcu_nonlazy_posted_snap, cpu) =
	2130	+ per_cpu(rcu_nonlazy_posted, cpu) - 1;
	2131	+
	2132	+ /*
2099	2133	* If there are no callbacks on this CPU, enter dyntick-idle mode.
2100	2134	* Also reset state to avoid prejudicing later attempts.
2101	2135	*/
2102	2136
...	...	@@ -2127,11 +2161,15 @@
2127	2161	per_cpu(rcu_dyntick_drain, cpu) = 0;
2128	2162	per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
2129	2163	if (rcu_cpu_has_nonlazy_callbacks(cpu))
2130		- mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
2131		- jiffies + RCU_IDLE_GP_DELAY);
	2164	+ per_cpu(rcu_idle_gp_timer_expires, cpu) =
	2165	+ jiffies + RCU_IDLE_GP_DELAY;
2132	2166	else
2133		- mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
2134		- jiffies + RCU_IDLE_LAZY_GP_DELAY);
	2167	+ per_cpu(rcu_idle_gp_timer_expires, cpu) =
	2168	+ jiffies + RCU_IDLE_LAZY_GP_DELAY;
	2169	+ mod_timer(&per_cpu(rcu_idle_gp_timer, cpu),
	2170	+ per_cpu(rcu_idle_gp_timer_expires, cpu));
	2171	+ per_cpu(rcu_nonlazy_posted_snap, cpu) =
	2172	+ per_cpu(rcu_nonlazy_posted, cpu);
2135	2173	return; /* Nothing more to do immediately. */
2136	2174	} else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
2137	2175	/* We have hit the limit, so time to give up. */
...	...	@@ -2169,6 +2207,17 @@
2169	2207	invoke_rcu_core();
2170	2208	} else
2171	2209	trace_rcu_prep_idle("Callbacks drained");
	2210	+}
	2211	+
	2212	+/*
	2213	+ * Keep a running count of callbacks posted so that rcu_prepare_for_idle()
	2214	+ * can detect when something out of the idle loop posts a callback.
	2215	+ * Of course, it had better do so either from a trace event designed to
	2216	+ * be called from idle or from within RCU_NONIDLE().
	2217	+ */
	2218	+static void rcu_idle_count_callbacks_posted(void)
	2219	+{
	2220	+ __this_cpu_add(rcu_nonlazy_posted, 1);
2172	2221	}
2173	2222
2174	2223	#endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */