cputime: Optimize jiffies_to_cputime(1)

For powerpc with CONFIG_VIRT_CPU_ACCOUNTING jiffies_to_cputime(1) is not compile time constant and run time calculations are quite expensive. To optimize we use precomputed value. For all other architectures is is preprocessor definition. Signed-off-by: Stanislaw Gruszka <sgruszka@redhat.com> Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Acked-by: Thomas Gleixner <tglx@linutronix.de> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Paul Mackerras <paulus@samba.org> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> LKML-Reference: <1248862529-6063-5-git-send-email-sgruszka@redhat.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>

cputime: Optimize jiffies_to_cputime(1)
For powerpc with CONFIG_VIRT_CPU_ACCOUNTING jiffies_to_cputime(1) is not compile time constant and run time calculations are quite expensive. To optimize we use precomputed value. For all other architectures is is preprocessor definition. Signed-off-by: Stanislaw Gruszka <sgruszka@redhat.com> Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Acked-by: Thomas Gleixner <tglx@linutronix.de> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Paul Mackerras <paulus@samba.org> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> LKML-Reference: <1248862529-6063-5-git-send-email-sgruszka@redhat.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Stanislaw Gruszka · Ingo Molnar
1 parent d1e3b6d195
Showing 8 changed files with 29 additions and 10 deletions Side-by-side Diff
arch/ia64/include/asm/cputime.h
arch/powerpc/include/asm/cputime.h
arch/powerpc/kernel/time.c
arch/s390/include/asm/cputime.h
include/asm-generic/cputime.h
kernel/itimer.c
kernel/posix-cpu-timers.c
kernel/sched.c
@@ -30,6 +30,7 @@
 typedef u64 cputime64_t;
  
 #define cputime_zero			((cputime_t)0)
+#define cputime_one_jiffy		jiffies_to_cputime(1)
 #define cputime_max			((~((cputime_t)0) >> 1) - 1)
 #define cputime_add(__a, __b)		((__a) +  (__b))
 #define cputime_sub(__a, __b)		((__a) -  (__b))
@@ -18,6 +18,9 @@
  
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
 #include <asm-generic/cputime.h>
+#ifdef __KERNEL__
+static inline void setup_cputime_one_jiffy(void) { }
+#endif
 #else
  
 #include <linux/types.h>
@@ -49,6 +52,11 @@
 #ifdef __KERNEL__
  
 /*
+ * One jiffy in timebase units computed during initialization
+ */
+extern cputime_t cputime_one_jiffy;
+
+/*
  * Convert cputime <-> jiffies
  */
 extern u64 __cputime_jiffies_factor;
@@ -87,6 +95,11 @@
 	if (sec)
 		ct += (cputime_t) sec * tb_ticks_per_sec;
 	return ct;
+}
+
+static inline void setup_cputime_one_jiffy(void)
+{
+	cputime_one_jiffy = jiffies_to_cputime(1);
 }
  
 static inline cputime64_t jiffies64_to_cputime64(const u64 jif)
@@ -193,6 +193,8 @@
 DEFINE_PER_CPU(unsigned long, cputime_last_delta);
 DEFINE_PER_CPU(unsigned long, cputime_scaled_last_delta);
  
+cputime_t cputime_one_jiffy;
+
 static void calc_cputime_factors(void)
 {
 	struct div_result res;
@@ -500,6 +502,7 @@
 				tb_to_xs = divres.result_low;
 				vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
 				vdso_data->tb_to_xs = tb_to_xs;
+				setup_cputime_one_jiffy();
 			}
 			else {
 				printk( "Titan recalibrate: FAILED (difference > 4 percent)\n"
@@ -945,6 +948,7 @@
 	tb_ticks_per_usec = ppc_tb_freq / 1000000;
 	tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000);
 	calc_cputime_factors();
+	setup_cputime_one_jiffy();
  
 	/*
 	 * Calculate the length of each tick in ns.  It will not be
@@ -42,6 +42,7 @@
 #endif /* __s390x__ */
  
 #define cputime_zero			(0ULL)
+#define cputime_one_jiffy		jiffies_to_cputime(1)
 #define cputime_max			((~0UL >> 1) - 1)
 #define cputime_add(__a, __b)		((__a) +  (__b))
 #define cputime_sub(__a, __b)		((__a) -  (__b))
@@ -7,6 +7,7 @@
 typedef unsigned long cputime_t;
  
 #define cputime_zero			(0UL)
+#define cputime_one_jiffy		jiffies_to_cputime(1)
 #define cputime_max			((~0UL >> 1) - 1)
 #define cputime_add(__a, __b)		((__a) +  (__b))
 #define cputime_sub(__a, __b)		((__a) -  (__b))
@@ -64,7 +64,7 @@
  
 		if (cputime_le(cval, t))
 			/* about to fire */
-			cval = jiffies_to_cputime(1);
+			cval = cputime_one_jiffy;
 		else
 			cval = cputime_sub(cval, t);
 	}
@@ -161,7 +161,7 @@
 	if (!cputime_eq(cval, cputime_zero) ||
 	    !cputime_eq(nval, cputime_zero)) {
 		if (cputime_gt(nval, cputime_zero))
-			nval = cputime_add(nval, jiffies_to_cputime(1));
+			nval = cputime_add(nval, cputime_one_jiffy);
 		set_process_cpu_timer(tsk, clock_id, &nval, &cval);
 	}
 	it->expires = nval;
@@ -1086,7 +1086,7 @@
 			it->error += it->incr_error;
 			if (it->error >= onecputick) {
 				it->expires = cputime_sub(it->expires,
-							jiffies_to_cputime(1));
+							  cputime_one_jiffy);
 				it->error -= onecputick;
 			}
 		} else
@@ -1461,7 +1461,7 @@
 		if (!cputime_eq(*oldval, cputime_zero)) {
 			if (cputime_le(*oldval, now.cpu)) {
 				/* Just about to fire. */
-				*oldval = jiffies_to_cputime(1);
+				*oldval = cputime_one_jiffy;
 			} else {
 				*oldval = cputime_sub(*oldval, now.cpu);
 			}
@@ -1712,7 +1712,7 @@
 	register_posix_clock(CLOCK_PROCESS_CPUTIME_ID, &process);
 	register_posix_clock(CLOCK_THREAD_CPUTIME_ID, &thread);
  
-	cputime_to_timespec(jiffies_to_cputime(1), &ts);
+	cputime_to_timespec(cputime_one_jiffy, &ts);
 	onecputick = ts.tv_nsec;
 	WARN_ON(ts.tv_sec != 0);
  
@@ -5031,17 +5031,16 @@
  */
 void account_process_tick(struct task_struct *p, int user_tick)
 {
-	cputime_t one_jiffy = jiffies_to_cputime(1);
-	cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
+	cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
 	struct rq *rq = this_rq();
  
 	if (user_tick)
-		account_user_time(p, one_jiffy, one_jiffy_scaled);
+		account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
 	else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
-		account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
+		account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
 				    one_jiffy_scaled);
 	else
-		account_idle_time(one_jiffy);
+		account_idle_time(cputime_one_jiffy);
 }
  
 /*
...	...	@@ -30,6 +30,7 @@
30	30	typedef u64 cputime64_t;
31	31
32	32	#define cputime_zero ((cputime_t)0)
	33	+#define cputime_one_jiffy jiffies_to_cputime(1)
33	34	#define cputime_max ((~((cputime_t)0) >> 1) - 1)
34	35	#define cputime_add(__a, __b) ((__a) + (__b))
35	36	#define cputime_sub(__a, __b) ((__a) - (__b))
...	...	@@ -18,6 +18,9 @@
18	18
19	19	#ifndef CONFIG_VIRT_CPU_ACCOUNTING
20	20	#include <asm-generic/cputime.h>
	21	+#ifdef __KERNEL__
	22	+static inline void setup_cputime_one_jiffy(void) { }
	23	+#endif
21	24	#else
22	25
23	26	#include <linux/types.h>
...	...	@@ -49,6 +52,11 @@
49	52	#ifdef __KERNEL__
50	53
51	54	/*
	55	+ * One jiffy in timebase units computed during initialization
	56	+ */
	57	+extern cputime_t cputime_one_jiffy;
	58	+
	59	+/*
52	60	* Convert cputime <-> jiffies
53	61	*/
54	62	extern u64 __cputime_jiffies_factor;
...	...	@@ -87,6 +95,11 @@
87	95	if (sec)
88	96	ct += (cputime_t) sec * tb_ticks_per_sec;
89	97	return ct;
	98	+}
	99	+
	100	+static inline void setup_cputime_one_jiffy(void)
	101	+{
	102	+ cputime_one_jiffy = jiffies_to_cputime(1);
90	103	}
91	104
92	105	static inline cputime64_t jiffies64_to_cputime64(const u64 jif)
...	...	@@ -193,6 +193,8 @@
193	193	DEFINE_PER_CPU(unsigned long, cputime_last_delta);
194	194	DEFINE_PER_CPU(unsigned long, cputime_scaled_last_delta);
195	195
	196	+cputime_t cputime_one_jiffy;
	197	+
196	198	static void calc_cputime_factors(void)
197	199	{
198	200	struct div_result res;
...	...	@@ -500,6 +502,7 @@
500	502	tb_to_xs = divres.result_low;
501	503	vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
502	504	vdso_data->tb_to_xs = tb_to_xs;
	505	+ setup_cputime_one_jiffy();
503	506	}
504	507	else {
505	508	printk( "Titan recalibrate: FAILED (difference > 4 percent)\n"
...	...	@@ -945,6 +948,7 @@
945	948	tb_ticks_per_usec = ppc_tb_freq / 1000000;
946	949	tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000);
947	950	calc_cputime_factors();
	951	+ setup_cputime_one_jiffy();
948	952
949	953	/*
950	954	* Calculate the length of each tick in ns. It will not be
...	...	@@ -42,6 +42,7 @@
42	42	#endif /* __s390x__ */
43	43
44	44	#define cputime_zero (0ULL)
	45	+#define cputime_one_jiffy jiffies_to_cputime(1)
45	46	#define cputime_max ((~0UL >> 1) - 1)
46	47	#define cputime_add(__a, __b) ((__a) + (__b))
47	48	#define cputime_sub(__a, __b) ((__a) - (__b))
...	...	@@ -7,6 +7,7 @@
7	7	typedef unsigned long cputime_t;
8	8
9	9	#define cputime_zero (0UL)
	10	+#define cputime_one_jiffy jiffies_to_cputime(1)
10	11	#define cputime_max ((~0UL >> 1) - 1)
11	12	#define cputime_add(__a, __b) ((__a) + (__b))
12	13	#define cputime_sub(__a, __b) ((__a) - (__b))
...	...	@@ -64,7 +64,7 @@
64	64
65	65	if (cputime_le(cval, t))
66	66	/* about to fire */
67		- cval = jiffies_to_cputime(1);
	67	+ cval = cputime_one_jiffy;
68	68	else
69	69	cval = cputime_sub(cval, t);
70	70	}
...	...	@@ -161,7 +161,7 @@
161	161	if (!cputime_eq(cval, cputime_zero) \|\|
162	162	!cputime_eq(nval, cputime_zero)) {
163	163	if (cputime_gt(nval, cputime_zero))
164		- nval = cputime_add(nval, jiffies_to_cputime(1));
	164	+ nval = cputime_add(nval, cputime_one_jiffy);
165	165	set_process_cpu_timer(tsk, clock_id, &nval, &cval);
166	166	}
167	167	it->expires = nval;
...	...	@@ -1086,7 +1086,7 @@
1086	1086	it->error += it->incr_error;
1087	1087	if (it->error >= onecputick) {
1088	1088	it->expires = cputime_sub(it->expires,
1089		- jiffies_to_cputime(1));
	1089	+ cputime_one_jiffy);
1090	1090	it->error -= onecputick;
1091	1091	}
1092	1092	} else
...	...	@@ -1461,7 +1461,7 @@
1461	1461	if (!cputime_eq(*oldval, cputime_zero)) {
1462	1462	if (cputime_le(*oldval, now.cpu)) {
1463	1463	/* Just about to fire. */
1464		- *oldval = jiffies_to_cputime(1);
	1464	+ *oldval = cputime_one_jiffy;
1465	1465	} else {
1466	1466	oldval = cputime_sub(oldval, now.cpu);
1467	1467	}
...	...	@@ -1712,7 +1712,7 @@
1712	1712	register_posix_clock(CLOCK_PROCESS_CPUTIME_ID, &process);
1713	1713	register_posix_clock(CLOCK_THREAD_CPUTIME_ID, &thread);
1714	1714
1715		- cputime_to_timespec(jiffies_to_cputime(1), &ts);
	1715	+ cputime_to_timespec(cputime_one_jiffy, &ts);
1716	1716	onecputick = ts.tv_nsec;
1717	1717	WARN_ON(ts.tv_sec != 0);
1718	1718
...	...	@@ -5031,17 +5031,16 @@
5031	5031	*/
5032	5032	void account_process_tick(struct task_struct *p, int user_tick)
5033	5033	{
5034		- cputime_t one_jiffy = jiffies_to_cputime(1);
5035		- cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
	5034	+ cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
5036	5035	struct rq *rq = this_rq();
5037	5036
5038	5037	if (user_tick)
5039		- account_user_time(p, one_jiffy, one_jiffy_scaled);
	5038	+ account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
5040	5039	else if ((p != rq->idle) \|\| (irq_count() != HARDIRQ_OFFSET))
5041		- account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
	5040	+ account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
5042	5041	one_jiffy_scaled);
5043	5042	else
5044		- account_idle_time(one_jiffy);
	5043	+ account_idle_time(cputime_one_jiffy);
5045	5044	}
5046	5045
5047	5046	/*