highres/dyntick: prevent xtime lock contention

While the !highres/!dyntick code assigns the duty of the do_timer() call to one specific CPU, this was dropped in the highres/dyntick part during development. Steven Rostedt discovered the xtime lock contention on highres/dyntick due to several CPUs trying to update jiffies. Add the single CPU assignement back. In the dyntick case this needs to be handled carefully, as the CPU which has the do_timer() duty must drop the assignement and let it be grabbed by another CPU, which is active. Otherwise the do_timer() calls would not happen during the long sleep. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Ingo Molnar <mingo@elte.hu> Cc: Steven Rostedt <rostedt@goodmis.org> Acked-by: Mark Lord <mlord@pobox.com> Cc: <stable@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

highres/dyntick: prevent xtime lock contention
While the !highres/!dyntick code assigns the duty of the do_timer() call to one specific CPU, this was dropped in the highres/dyntick part during development. Steven Rostedt discovered the xtime lock contention on highres/dyntick due to several CPUs trying to update jiffies. Add the single CPU assignement back. In the dyntick case this needs to be handled carefully, as the CPU which has the do_timer() duty must drop the assignement and let it be grabbed by another CPU, which is active. Otherwise the do_timer() calls would not happen during the long sleep. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Ingo Molnar <mingo@elte.hu> Cc: Steven Rostedt <rostedt@goodmis.org> Acked-by: Mark Lord <mlord@pobox.com> Cc: <stable@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Thomas Gleixner · Linus Torvalds
1 parent d5d3b736e3
Showing 3 changed files with 48 additions and 3 deletions Side-by-side Diff
kernel/time/tick-common.c
kernel/time/tick-internal.h
kernel/time/tick-sched.c
@@ -31,7 +31,7 @@
  */
 ktime_t tick_next_period;
 ktime_t tick_period;
-static int tick_do_timer_cpu = -1;
+int tick_do_timer_cpu __read_mostly = -1;
 DEFINE_SPINLOCK(tick_device_lock);
  
 /*
@@ -294,6 +294,12 @@
 		dev->mode = CLOCK_EVT_MODE_UNUSED;
 		clockevents_exchange_device(dev, NULL);
 		td->evtdev = NULL;
+	}
+	/* Transfer the do_timer job away from this cpu */
+	if (*cpup == tick_do_timer_cpu) {
+		int cpu = first_cpu(cpu_online_map);
+
+		tick_do_timer_cpu = (cpu != NR_CPUS) ? cpu : -1;
 	}
 	spin_unlock_irqrestore(&tick_device_lock, flags);
 }
@@ -5,6 +5,7 @@
 extern spinlock_t tick_device_lock;
 extern ktime_t tick_next_period;
 extern ktime_t tick_period;
+extern int tick_do_timer_cpu __read_mostly;
  
 extern void tick_setup_periodic(struct clock_event_device *dev, int broadcast);
 extern void tick_handle_periodic(struct clock_event_device *dev);
@@ -221,7 +221,19 @@
 			ts->tick_stopped = 1;
 			ts->idle_jiffies = last_jiffies;
 		}
+
 		/*
+		 * If this cpu is the one which updates jiffies, then
+		 * give up the assignment and let it be taken by the
+		 * cpu which runs the tick timer next, which might be
+		 * this cpu as well. If we don't drop this here the
+		 * jiffies might be stale and do_timer() never
+		 * invoked.
+		 */
+		if (cpu == tick_do_timer_cpu)
+			tick_do_timer_cpu = -1;
+
+		/*
 		 * calculate the expiry time for the next timer wheel
 		 * timer
 		 */
  
  
@@ -338,12 +350,24 @@
 {
 	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 	struct pt_regs *regs = get_irq_regs();
+	int cpu = smp_processor_id();
 	ktime_t now = ktime_get();
  
 	dev->next_event.tv64 = KTIME_MAX;
  
+	/*
+	 * Check if the do_timer duty was dropped. We don't care about
+	 * concurrency: This happens only when the cpu in charge went
+	 * into a long sleep. If two cpus happen to assign themself to
+	 * this duty, then the jiffies update is still serialized by
+	 * xtime_lock.
+	 */
+	if (unlikely(tick_do_timer_cpu == -1))
+		tick_do_timer_cpu = cpu;
+
 	/* Check, if the jiffies need an update */
-	tick_do_update_jiffies64(now);
+	if (tick_do_timer_cpu == cpu)
+		tick_do_update_jiffies64(now);
  
 	/*
 	 * When we are idle and the tick is stopped, we have to touch
  
  
@@ -431,9 +455,23 @@
 	struct hrtimer_cpu_base *base = timer->base->cpu_base;
 	struct pt_regs *regs = get_irq_regs();
 	ktime_t now = ktime_get();
+	int cpu = smp_processor_id();
  
+#ifdef CONFIG_NO_HZ
+	/*
+	 * Check if the do_timer duty was dropped. We don't care about
+	 * concurrency: This happens only when the cpu in charge went
+	 * into a long sleep. If two cpus happen to assign themself to
+	 * this duty, then the jiffies update is still serialized by
+	 * xtime_lock.
+	 */
+	if (unlikely(tick_do_timer_cpu == -1))
+		tick_do_timer_cpu = cpu;
+#endif
+
 	/* Check, if the jiffies need an update */
-	tick_do_update_jiffies64(now);
+	if (tick_do_timer_cpu == cpu)
+		tick_do_update_jiffies64(now);
  
 	/*
 	 * Do not call, when we are not in irq context and have
...	...	@@ -31,7 +31,7 @@
31	31	*/
32	32	ktime_t tick_next_period;
33	33	ktime_t tick_period;
34		-static int tick_do_timer_cpu = -1;
	34	+int tick_do_timer_cpu __read_mostly = -1;
35	35	DEFINE_SPINLOCK(tick_device_lock);
36	36
37	37	/*
...	...	@@ -294,6 +294,12 @@
294	294	dev->mode = CLOCK_EVT_MODE_UNUSED;
295	295	clockevents_exchange_device(dev, NULL);
296	296	td->evtdev = NULL;
	297	+ }
	298	+ /* Transfer the do_timer job away from this cpu */
	299	+ if (*cpup == tick_do_timer_cpu) {
	300	+ int cpu = first_cpu(cpu_online_map);
	301	+
	302	+ tick_do_timer_cpu = (cpu != NR_CPUS) ? cpu : -1;
297	303	}
298	304	spin_unlock_irqrestore(&tick_device_lock, flags);
299	305	}
...	...	@@ -5,6 +5,7 @@
5	5	extern spinlock_t tick_device_lock;
6	6	extern ktime_t tick_next_period;
7	7	extern ktime_t tick_period;
	8	+extern int tick_do_timer_cpu __read_mostly;
8	9
9	10	extern void tick_setup_periodic(struct clock_event_device *dev, int broadcast);
10	11	extern void tick_handle_periodic(struct clock_event_device *dev);
...	...	@@ -221,7 +221,19 @@
221	221	ts->tick_stopped = 1;
222	222	ts->idle_jiffies = last_jiffies;
223	223	}
	224	+
224	225	/*
	226	+ * If this cpu is the one which updates jiffies, then
	227	+ * give up the assignment and let it be taken by the
	228	+ * cpu which runs the tick timer next, which might be
	229	+ * this cpu as well. If we don't drop this here the
	230	+ * jiffies might be stale and do_timer() never
	231	+ * invoked.
	232	+ */
	233	+ if (cpu == tick_do_timer_cpu)
	234	+ tick_do_timer_cpu = -1;
	235	+
	236	+ /*
225	237	* calculate the expiry time for the next timer wheel
226	238	* timer
227	239	*/
228	240
229	241
...	...	@@ -338,12 +350,24 @@
338	350	{
339	351	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
340	352	struct pt_regs *regs = get_irq_regs();
	353	+ int cpu = smp_processor_id();
341	354	ktime_t now = ktime_get();
342	355
343	356	dev->next_event.tv64 = KTIME_MAX;
344	357
	358	+ /*
	359	+ * Check if the do_timer duty was dropped. We don't care about
	360	+ * concurrency: This happens only when the cpu in charge went
	361	+ * into a long sleep. If two cpus happen to assign themself to
	362	+ * this duty, then the jiffies update is still serialized by
	363	+ * xtime_lock.
	364	+ */
	365	+ if (unlikely(tick_do_timer_cpu == -1))
	366	+ tick_do_timer_cpu = cpu;
	367	+
345	368	/* Check, if the jiffies need an update */
346		- tick_do_update_jiffies64(now);
	369	+ if (tick_do_timer_cpu == cpu)
	370	+ tick_do_update_jiffies64(now);
347	371
348	372	/*
349	373	* When we are idle and the tick is stopped, we have to touch
350	374
351	375
...	...	@@ -431,9 +455,23 @@
431	455	struct hrtimer_cpu_base *base = timer->base->cpu_base;
432	456	struct pt_regs *regs = get_irq_regs();
433	457	ktime_t now = ktime_get();
	458	+ int cpu = smp_processor_id();
434	459
	460	+#ifdef CONFIG_NO_HZ
	461	+ /*
	462	+ * Check if the do_timer duty was dropped. We don't care about
	463	+ * concurrency: This happens only when the cpu in charge went
	464	+ * into a long sleep. If two cpus happen to assign themself to
	465	+ * this duty, then the jiffies update is still serialized by
	466	+ * xtime_lock.
	467	+ */
	468	+ if (unlikely(tick_do_timer_cpu == -1))
	469	+ tick_do_timer_cpu = cpu;
	470	+#endif
	471	+
435	472	/* Check, if the jiffies need an update */
436		- tick_do_update_jiffies64(now);
	473	+ if (tick_do_timer_cpu == cpu)
	474	+ tick_do_update_jiffies64(now);
437	475
438	476	/*
439	477	* Do not call, when we are not in irq context and have