rcu: Permit dyntick-idle with callbacks pending

The current implementation of RCU_FAST_NO_HZ prevents CPUs from entering dyntick-idle state if they have RCU callbacks pending. Unfortunately, this has the side-effect of often preventing them from entering this state, especially if at least one other CPU is not in dyntick-idle state. However, the resulting per-tick wakeup is wasteful in many cases: if the CPU has already fully responded to the current RCU grace period, there will be nothing for it to do until this grace period ends, which will frequently take several jiffies. This commit therefore permits a CPU that has done everything that the current grace period has asked of it (rcu_pending() == 0) even if it still as RCU callbacks pending. However, such a CPU posts a timer to wake it up several jiffies later (6 jiffies, based on experience with grace-period lengths). This wakeup is required to handle situations that can result in all CPUs being in dyntick-idle mode, thus failing to ever complete the current grace period. If a CPU wakes up before the timer goes off, then it cancels that timer, thus avoiding spurious wakeups. Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

rcu: Permit dyntick-idle with callbacks pending
The current implementation of RCU_FAST_NO_HZ prevents CPUs from entering dyntick-idle state if they have RCU callbacks pending. Unfortunately, this has the side-effect of often preventing them from entering this state, especially if at least one other CPU is not in dyntick-idle state. However, the resulting per-tick wakeup is wasteful in many cases: if the CPU has already fully responded to the current RCU grace period, there will be nothing for it to do until this grace period ends, which will frequently take several jiffies. This commit therefore permits a CPU that has done everything that the current grace period has asked of it (rcu_pending() == 0) even if it still as RCU callbacks pending. However, such a CPU posts a timer to wake it up several jiffies later (6 jiffies, based on experience with grace-period lengths). This wakeup is required to handle situations that can result in all CPUs being in dyntick-idle mode, thus failing to ever complete the current grace period. If a CPU wakes up before the timer goes off, then it cancels that timer, thus avoiding spurious wakeups. Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Paul E. McKenney · Paul E. McKenney
1 parent 3842a0832a
Showing 4 changed files with 78 additions and 5 deletions Side-by-side Diff
include/trace/events/rcu.h
kernel/rcutree.c
kernel/rcutree.h
kernel/rcutree_plugin.h
@@ -288,9 +288,10 @@
  *	"No callbacks": Nothing to do, no callbacks on this CPU.
  *	"In holdoff": Nothing to do, holding off after unsuccessful attempt.
  *	"Begin holdoff": Attempt failed, don't retry until next jiffy.
+ *	"Dyntick with callbacks": Entering dyntick-idle despite callbacks.
  *	"More callbacks": Still more callbacks, try again to clear them out.
  *	"Callbacks drained": All callbacks processed, off to dyntick idle!
- *	"CPU awakened at GP end":
+ *	"Timer": Timer fired to cause CPU to continue processing callbacks.
  */
 TRACE_EVENT(rcu_prep_idle,
  
@@ -448,6 +448,7 @@
 	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
 	smp_mb__after_atomic_inc();  /* See above. */
 	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+	rcu_cleanup_after_idle(smp_processor_id());
 	trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting);
 	if (!is_idle_task(current)) {
 		struct task_struct *idle = idle_task(smp_processor_id());
@@ -2057,6 +2058,7 @@
 	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_NESTING;
 	atomic_set(&rdp->dynticks->dynticks,
 		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+	rcu_prepare_for_idle_init(cpu);
 	raw_spin_unlock(&rnp->lock);		/* irqs remain disabled. */
  
 	/*
@@ -2138,6 +2140,7 @@
 		rcu_send_cbs_to_online(&rcu_bh_state);
 		rcu_send_cbs_to_online(&rcu_sched_state);
 		rcu_preempt_send_cbs_to_online();
+		rcu_cleanup_after_idle(cpu);
 		break;
 	case CPU_DEAD:
 	case CPU_DEAD_FROZEN:
@@ -467,6 +467,8 @@
 #endif /* #ifdef CONFIG_RCU_BOOST */
 static void rcu_cpu_kthread_setrt(int cpu, int to_rt);
 static void __cpuinit rcu_prepare_kthreads(int cpu);
+static void rcu_prepare_for_idle_init(int cpu);
+static void rcu_cleanup_after_idle(int cpu);
 static void rcu_prepare_for_idle(int cpu);
  
 #endif /* #ifndef RCU_TREE_NONCORE */
@@ -1947,9 +1947,8 @@
  * 1 if so.  This function is part of the RCU implementation; it is -not-
  * an exported member of the RCU API.
  *
- * Because we have preemptible RCU, just check whether this CPU needs
- * any flavor of RCU.  Do not chew up lots of CPU cycles with preemption
- * disabled in a most-likely vain attempt to cause RCU not to need this CPU.
+ * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
+ * any flavor of RCU.
  */
 int rcu_needs_cpu(int cpu)
 {
@@ -1957,6 +1956,21 @@
 }
  
 /*
+ * Because we do not have RCU_FAST_NO_HZ, don't bother initializing for it.
+ */
+static void rcu_prepare_for_idle_init(int cpu)
+{
+}
+
+/*
+ * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
+ * after it.
+ */
+static void rcu_cleanup_after_idle(int cpu)
+{
+}
+
+/*
  * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=y,
  * is nothing.
  */
  
@@ -1966,9 +1980,12 @@
  
 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
  
-#define RCU_NEEDS_CPU_FLUSHES 5
+#define RCU_NEEDS_CPU_FLUSHES 5		/* Allow for callback self-repost. */
+#define RCU_IDLE_GP_DELAY 6		/* Roughly one grace period. */
 static DEFINE_PER_CPU(int, rcu_dyntick_drain);
 static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
+static DEFINE_PER_CPU(struct hrtimer, rcu_idle_gp_timer);
+static ktime_t rcu_idle_gp_wait;
  
 /*
  * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
@@ -1989,6 +2006,47 @@
 }
  
 /*
+ * Timer handler used to force CPU to start pushing its remaining RCU
+ * callbacks in the case where it entered dyntick-idle mode with callbacks
+ * pending.  The hander doesn't really need to do anything because the
+ * real work is done upon re-entry to idle, or by the next scheduling-clock
+ * interrupt should idle not be re-entered.
+ */
+static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp)
+{
+	trace_rcu_prep_idle("Timer");
+	return HRTIMER_NORESTART;
+}
+
+/*
+ * Initialize the timer used to pull CPUs out of dyntick-idle mode.
+ */
+static void rcu_prepare_for_idle_init(int cpu)
+{
+	static int firsttime = 1;
+	struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu);
+
+	hrtimer_init(hrtp, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	hrtp->function = rcu_idle_gp_timer_func;
+	if (firsttime) {
+		unsigned int upj = jiffies_to_usecs(RCU_IDLE_GP_DELAY);
+
+		rcu_idle_gp_wait = ns_to_ktime(upj * (u64)1000);
+		firsttime = 0;
+	}
+}
+
+/*
+ * Clean up for exit from idle.  Because we are exiting from idle, there
+ * is no longer any point to rcu_idle_gp_timer, so cancel it.  This will
+ * do nothing if this timer is not active, so just cancel it unconditionally.
+ */
+static void rcu_cleanup_after_idle(int cpu)
+{
+	hrtimer_cancel(&per_cpu(rcu_idle_gp_timer, cpu));
+}
+
+/*
  * Check to see if any RCU-related work can be done by the current CPU,
  * and if so, schedule a softirq to get it done.  This function is part
  * of the RCU implementation; it is -not- an exported member of the RCU API.
@@ -2040,6 +2098,15 @@
 		/* First time through, initialize the counter. */
 		per_cpu(rcu_dyntick_drain, cpu) = RCU_NEEDS_CPU_FLUSHES;
 	} else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
+		/* Can we go dyntick-idle despite still having callbacks? */
+		if (!rcu_pending(cpu)) {
+			trace_rcu_prep_idle("Dyntick with callbacks");
+			per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
+			hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
+				      rcu_idle_gp_wait, HRTIMER_MODE_REL);
+			return; /* Nothing more to do immediately. */
+		}
+
 		/* We have hit the limit, so time to give up. */
 		per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
 		local_irq_restore(flags);
...	...	@@ -288,9 +288,10 @@
288	288	* "No callbacks": Nothing to do, no callbacks on this CPU.
289	289	* "In holdoff": Nothing to do, holding off after unsuccessful attempt.
290	290	* "Begin holdoff": Attempt failed, don't retry until next jiffy.
	291	+ * "Dyntick with callbacks": Entering dyntick-idle despite callbacks.
291	292	* "More callbacks": Still more callbacks, try again to clear them out.
292	293	* "Callbacks drained": All callbacks processed, off to dyntick idle!
293		- * "CPU awakened at GP end":
	294	+ * "Timer": Timer fired to cause CPU to continue processing callbacks.
294	295	*/
295	296	TRACE_EVENT(rcu_prep_idle,
296	297
...	...	@@ -448,6 +448,7 @@
448	448	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
449	449	smp_mb__after_atomic_inc(); /* See above. */
450	450	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
	451	+ rcu_cleanup_after_idle(smp_processor_id());
451	452	trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting);
452	453	if (!is_idle_task(current)) {
453	454	struct task_struct *idle = idle_task(smp_processor_id());
...	...	@@ -2057,6 +2058,7 @@
2057	2058	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_NESTING;
2058	2059	atomic_set(&rdp->dynticks->dynticks,
2059	2060	(atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
	2061	+ rcu_prepare_for_idle_init(cpu);
2060	2062	raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
2061	2063
2062	2064	/*
...	...	@@ -2138,6 +2140,7 @@
2138	2140	rcu_send_cbs_to_online(&rcu_bh_state);
2139	2141	rcu_send_cbs_to_online(&rcu_sched_state);
2140	2142	rcu_preempt_send_cbs_to_online();
	2143	+ rcu_cleanup_after_idle(cpu);
2141	2144	break;
2142	2145	case CPU_DEAD:
2143	2146	case CPU_DEAD_FROZEN:
...	...	@@ -467,6 +467,8 @@
467	467	#endif /* #ifdef CONFIG_RCU_BOOST */
468	468	static void rcu_cpu_kthread_setrt(int cpu, int to_rt);
469	469	static void __cpuinit rcu_prepare_kthreads(int cpu);
	470	+static void rcu_prepare_for_idle_init(int cpu);
	471	+static void rcu_cleanup_after_idle(int cpu);
470	472	static void rcu_prepare_for_idle(int cpu);
471	473
472	474	#endif /* #ifndef RCU_TREE_NONCORE */
...	...	@@ -1947,9 +1947,8 @@
1947	1947	* 1 if so. This function is part of the RCU implementation; it is -not-
1948	1948	* an exported member of the RCU API.
1949	1949	*
1950		- * Because we have preemptible RCU, just check whether this CPU needs
1951		- * any flavor of RCU. Do not chew up lots of CPU cycles with preemption
1952		- * disabled in a most-likely vain attempt to cause RCU not to need this CPU.
	1950	+ * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
	1951	+ * any flavor of RCU.
1953	1952	*/
1954	1953	int rcu_needs_cpu(int cpu)
1955	1954	{
...	...	@@ -1957,6 +1956,21 @@
1957	1956	}
1958	1957
1959	1958	/*
	1959	+ * Because we do not have RCU_FAST_NO_HZ, don't bother initializing for it.
	1960	+ */
	1961	+static void rcu_prepare_for_idle_init(int cpu)
	1962	+{
	1963	+}
	1964	+
	1965	+/*
	1966	+ * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
	1967	+ * after it.
	1968	+ */
	1969	+static void rcu_cleanup_after_idle(int cpu)
	1970	+{
	1971	+}
	1972	+
	1973	+/*
1960	1974	* Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=y,
1961	1975	* is nothing.
1962	1976	*/
1963	1977
...	...	@@ -1966,9 +1980,12 @@
1966	1980
1967	1981	#else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1968	1982
1969		-#define RCU_NEEDS_CPU_FLUSHES 5
	1983	+#define RCU_NEEDS_CPU_FLUSHES 5 /* Allow for callback self-repost. */
	1984	+#define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */
1970	1985	static DEFINE_PER_CPU(int, rcu_dyntick_drain);
1971	1986	static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
	1987	+static DEFINE_PER_CPU(struct hrtimer, rcu_idle_gp_timer);
	1988	+static ktime_t rcu_idle_gp_wait;
1972	1989
1973	1990	/*
1974	1991	* Allow the CPU to enter dyntick-idle mode if either: (1) There are no
...	...	@@ -1989,6 +2006,47 @@
1989	2006	}
1990	2007
1991	2008	/*
	2009	+ * Timer handler used to force CPU to start pushing its remaining RCU
	2010	+ * callbacks in the case where it entered dyntick-idle mode with callbacks
	2011	+ * pending. The hander doesn't really need to do anything because the
	2012	+ * real work is done upon re-entry to idle, or by the next scheduling-clock
	2013	+ * interrupt should idle not be re-entered.
	2014	+ */
	2015	+static enum hrtimer_restart rcu_idle_gp_timer_func(struct hrtimer *hrtp)
	2016	+{
	2017	+ trace_rcu_prep_idle("Timer");
	2018	+ return HRTIMER_NORESTART;
	2019	+}
	2020	+
	2021	+/*
	2022	+ * Initialize the timer used to pull CPUs out of dyntick-idle mode.
	2023	+ */
	2024	+static void rcu_prepare_for_idle_init(int cpu)
	2025	+{
	2026	+ static int firsttime = 1;
	2027	+ struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu);
	2028	+
	2029	+ hrtimer_init(hrtp, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	2030	+ hrtp->function = rcu_idle_gp_timer_func;
	2031	+ if (firsttime) {
	2032	+ unsigned int upj = jiffies_to_usecs(RCU_IDLE_GP_DELAY);
	2033	+
	2034	+ rcu_idle_gp_wait = ns_to_ktime(upj * (u64)1000);
	2035	+ firsttime = 0;
	2036	+ }
	2037	+}
	2038	+
	2039	+/*
	2040	+ * Clean up for exit from idle. Because we are exiting from idle, there
	2041	+ * is no longer any point to rcu_idle_gp_timer, so cancel it. This will
	2042	+ * do nothing if this timer is not active, so just cancel it unconditionally.
	2043	+ */
	2044	+static void rcu_cleanup_after_idle(int cpu)
	2045	+{
	2046	+ hrtimer_cancel(&per_cpu(rcu_idle_gp_timer, cpu));
	2047	+}
	2048	+
	2049	+/*
1992	2050	* Check to see if any RCU-related work can be done by the current CPU,
1993	2051	* and if so, schedule a softirq to get it done. This function is part
1994	2052	* of the RCU implementation; it is -not- an exported member of the RCU API.
...	...	@@ -2040,6 +2098,15 @@
2040	2098	/* First time through, initialize the counter. */
2041	2099	per_cpu(rcu_dyntick_drain, cpu) = RCU_NEEDS_CPU_FLUSHES;
2042	2100	} else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
	2101	+ /* Can we go dyntick-idle despite still having callbacks? */
	2102	+ if (!rcu_pending(cpu)) {
	2103	+ trace_rcu_prep_idle("Dyntick with callbacks");
	2104	+ per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
	2105	+ hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
	2106	+ rcu_idle_gp_wait, HRTIMER_MODE_REL);
	2107	+ return; /* Nothing more to do immediately. */
	2108	+ }
	2109	+
2043	2110	/* We have hit the limit, so time to give up. */
2044	2111	per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
2045	2112	local_irq_restore(flags);