[PATCH] hrtimers: optimize softirq runqueues

The hrtimer softirq is called from the timer softirq every tick. Retrieve the current time from xtime and wall_to_monotonic instead of calling base->get_time() for each timer base. Store the time in the base structure and provide a hook once clock source abstractions are in place and to keep the code open for new base clocks. Based on a patch from: Roman Zippel <zippel@linux-m68k.org> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] hrtimers: optimize softirq runqueues
The hrtimer softirq is called from the timer softirq every tick. Retrieve the current time from xtime and wall_to_monotonic instead of calling base->get_time() for each timer base. Store the time in the base structure and provide a hook once clock source abstractions are in place and to keep the code open for new base clocks. Based on a patch from: Roman Zippel <zippel@linux-m68k.org> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Thomas Gleixner · Linus Torvalds
1 parent a0e9285233
Showing 2 changed files with 38 additions and 10 deletions Side-by-side Diff
include/linux/hrtimer.h
kernel/hrtimer.c
@@ -72,14 +72,16 @@
 /**
  * struct hrtimer_base - the timer base for a specific clock
  *
- * @index:	clock type index for per_cpu support when moving a timer
- *		to a base on another cpu.
- * @lock:	lock protecting the base and associated timers
- * @active:	red black tree root node for the active timers
- * @first:	pointer to the timer node which expires first
- * @resolution:	the resolution of the clock, in nanoseconds
- * @get_time:	function to retrieve the current time of the clock
- * @curr_timer:	the timer which is executing a callback right now
+ * @index:		clock type index for per_cpu support when moving a timer
+ *			to a base on another cpu.
+ * @lock:		lock protecting the base and associated timers
+ * @active:		red black tree root node for the active timers
+ * @first:		pointer to the timer node which expires first
+ * @resolution:		the resolution of the clock, in nanoseconds
+ * @get_time:		function to retrieve the current time of the clock
+ * @get_sofirq_time:	function to retrieve the current time from the softirq
+ * @curr_timer:		the timer which is executing a callback right now
+ * @softirq_time:	the time when running the hrtimer queue in the softirq
  */
 struct hrtimer_base {
 	clockid_t		index;
  
@@ -88,7 +90,9 @@
 	struct rb_node		*first;
 	ktime_t			resolution;
 	ktime_t			(*get_time)(void);
+	ktime_t			(*get_softirq_time)(void);
 	struct hrtimer		*curr_timer;
+	ktime_t			softirq_time;
 };
  
 /*
@@ -123,6 +123,26 @@
 EXPORT_SYMBOL_GPL(ktime_get_ts);
  
 /*
+ * Get the coarse grained time at the softirq based on xtime and
+ * wall_to_monotonic.
+ */
+static void hrtimer_get_softirq_time(struct hrtimer_base *base)
+{
+	ktime_t xtim, tomono;
+	unsigned long seq;
+
+	do {
+		seq = read_seqbegin(&xtime_lock);
+		xtim = timespec_to_ktime(xtime);
+		tomono = timespec_to_ktime(wall_to_monotonic);
+
+	} while (read_seqretry(&xtime_lock, seq));
+
+	base[CLOCK_REALTIME].softirq_time = xtim;
+	base[CLOCK_MONOTONIC].softirq_time = ktime_add(xtim, tomono);
+}
+
+/*
  * Functions and macros which are different for UP/SMP systems are kept in a
  * single place
  */
  
@@ -586,9 +606,11 @@
  */
 static inline void run_hrtimer_queue(struct hrtimer_base *base)
 {
-	ktime_t now = base->get_time();
 	struct rb_node *node;
  
+	if (base->get_softirq_time)
+		base->softirq_time = base->get_softirq_time();
+
 	spin_lock_irq(&base->lock);
  
 	while ((node = base->first)) {
@@ -598,7 +620,7 @@
 		void *data;
  
 		timer = rb_entry(node, struct hrtimer, node);
-		if (now.tv64 <= timer->expires.tv64)
+		if (base->softirq_time.tv64 <= timer->expires.tv64)
 			break;
  
 		fn = timer->function;
@@ -640,6 +662,8 @@
 {
 	struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
 	int i;
+
+	hrtimer_get_softirq_time(base);
  
 	for (i = 0; i < MAX_HRTIMER_BASES; i++)
 		run_hrtimer_queue(&base[i]);
...	...	@@ -72,14 +72,16 @@
72	72	/**
73	73	* struct hrtimer_base - the timer base for a specific clock
74	74	*
75		- * @index: clock type index for per_cpu support when moving a timer
76		- * to a base on another cpu.
77		- * @lock: lock protecting the base and associated timers
78		- * @active: red black tree root node for the active timers
79		- * @first: pointer to the timer node which expires first
80		- * @resolution: the resolution of the clock, in nanoseconds
81		- * @get_time: function to retrieve the current time of the clock
82		- * @curr_timer: the timer which is executing a callback right now
	75	+ * @index: clock type index for per_cpu support when moving a timer
	76	+ * to a base on another cpu.
	77	+ * @lock: lock protecting the base and associated timers
	78	+ * @active: red black tree root node for the active timers
	79	+ * @first: pointer to the timer node which expires first
	80	+ * @resolution: the resolution of the clock, in nanoseconds
	81	+ * @get_time: function to retrieve the current time of the clock
	82	+ * @get_sofirq_time: function to retrieve the current time from the softirq
	83	+ * @curr_timer: the timer which is executing a callback right now
	84	+ * @softirq_time: the time when running the hrtimer queue in the softirq
83	85	*/
84	86	struct hrtimer_base {
85	87	clockid_t index;
86	88
...	...	@@ -88,7 +90,9 @@
88	90	struct rb_node *first;
89	91	ktime_t resolution;
90	92	ktime_t (*get_time)(void);
	93	+ ktime_t (*get_softirq_time)(void);
91	94	struct hrtimer *curr_timer;
	95	+ ktime_t softirq_time;
92	96	};
93	97
94	98	/*
...	...	@@ -123,6 +123,26 @@
123	123	EXPORT_SYMBOL_GPL(ktime_get_ts);
124	124
125	125	/*
	126	+ * Get the coarse grained time at the softirq based on xtime and
	127	+ * wall_to_monotonic.
	128	+ */
	129	+static void hrtimer_get_softirq_time(struct hrtimer_base *base)
	130	+{
	131	+ ktime_t xtim, tomono;
	132	+ unsigned long seq;
	133	+
	134	+ do {
	135	+ seq = read_seqbegin(&xtime_lock);
	136	+ xtim = timespec_to_ktime(xtime);
	137	+ tomono = timespec_to_ktime(wall_to_monotonic);
	138	+
	139	+ } while (read_seqretry(&xtime_lock, seq));
	140	+
	141	+ base[CLOCK_REALTIME].softirq_time = xtim;
	142	+ base[CLOCK_MONOTONIC].softirq_time = ktime_add(xtim, tomono);
	143	+}
	144	+
	145	+/*
126	146	* Functions and macros which are different for UP/SMP systems are kept in a
127	147	* single place
128	148	*/
129	149
...	...	@@ -586,9 +606,11 @@
586	606	*/
587	607	static inline void run_hrtimer_queue(struct hrtimer_base *base)
588	608	{
589		- ktime_t now = base->get_time();
590	609	struct rb_node *node;
591	610
	611	+ if (base->get_softirq_time)
	612	+ base->softirq_time = base->get_softirq_time();
	613	+
592	614	spin_lock_irq(&base->lock);
593	615
594	616	while ((node = base->first)) {
...	...	@@ -598,7 +620,7 @@
598	620	void *data;
599	621
600	622	timer = rb_entry(node, struct hrtimer, node);
601		- if (now.tv64 <= timer->expires.tv64)
	623	+ if (base->softirq_time.tv64 <= timer->expires.tv64)
602	624	break;
603	625
604	626	fn = timer->function;
...	...	@@ -640,6 +662,8 @@
640	662	{
641	663	struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
642	664	int i;
	665	+
	666	+ hrtimer_get_softirq_time(base);
643	667
644	668	for (i = 0; i < MAX_HRTIMER_BASES; i++)
645	669	run_hrtimer_queue(&base[i]);