sched: Introduce task_times() to replace task_{u,s}time() pair

Functions task_{u,s}time() are called in pair in almost all cases. However task_stime() is implemented to call task_utime() from its inside, so such paired calls run task_utime() twice. It means we do heavy divisions (div_u64 + do_div) twice to get utime and stime which can be obtained at same time by one set of divisions. This patch introduces a function task_times(*tsk, *utime, *stime) to retrieve utime and stime at once in better, optimized way. Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com> Acked-by: Peter Zijlstra <peterz@infradead.org> Cc: Stanislaw Gruszka <sgruszka@redhat.com> Cc: Spencer Candland <spencer@bluehost.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Balbir Singh <balbir@in.ibm.com> Cc: Americo Wang <xiyou.wangcong@gmail.com> LKML-Reference: <4B0E16AE.906@jp.fujitsu.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>

sched: Introduce task_times() to replace task_{u,s}time() pair
Functions task_{u,s}time() are called in pair in almost all cases. However task_stime() is implemented to call task_utime() from its inside, so such paired calls run task_utime() twice. It means we do heavy divisions (div_u64 + do_div) twice to get utime and stime which can be obtained at same time by one set of divisions. This patch introduces a function task_times(*tsk, *utime, *stime) to retrieve utime and stime at once in better, optimized way. Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com> Acked-by: Peter Zijlstra <peterz@infradead.org> Cc: Stanislaw Gruszka <sgruszka@redhat.com> Cc: Spencer Candland <spencer@bluehost.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Balbir Singh <balbir@in.ibm.com> Cc: Americo Wang <xiyou.wangcong@gmail.com> LKML-Reference: <4B0E16AE.906@jp.fujitsu.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Hidetoshi Seto · Ingo Molnar
1 parent 16bc67edeb
Showing 5 changed files with 43 additions and 26 deletions Side-by-side Diff
fs/proc/array.c
include/linux/sched.h
kernel/exit.c
kernel/sched.c
kernel/sys.c
@@ -535,8 +535,7 @@
 	if (!whole) {
 		min_flt = task->min_flt;
 		maj_flt = task->maj_flt;
-		utime = task_utime(task);
-		stime = task_stime(task);
+		task_times(task, &utime, &stime);
 		gtime = task_gtime(task);
 	}
  
@@ -1723,6 +1723,7 @@
 extern cputime_t task_utime(struct task_struct *p);
 extern cputime_t task_stime(struct task_struct *p);
 extern cputime_t task_gtime(struct task_struct *p);
+extern void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st);
  
 /*
  * Per process flags
@@ -91,6 +91,8 @@
 	if (atomic_dec_and_test(&sig->count))
 		posix_cpu_timers_exit_group(tsk);
 	else {
+		cputime_t utime, stime;
+
 		/*
 		 * If there is any task waiting for the group exit
 		 * then notify it:
@@ -110,8 +112,9 @@
 		 * We won't ever get here for the group leader, since it
 		 * will have been the last reference on the signal_struct.
 		 */
-		sig->utime = cputime_add(sig->utime, task_utime(tsk));
-		sig->stime = cputime_add(sig->stime, task_stime(tsk));
+		task_times(tsk, &utime, &stime);
+		sig->utime = cputime_add(sig->utime, utime);
+		sig->stime = cputime_add(sig->stime, stime);
 		sig->gtime = cputime_add(sig->gtime, task_gtime(tsk));
 		sig->min_flt += tsk->min_flt;
 		sig->maj_flt += tsk->maj_flt;
@@ -5191,6 +5191,14 @@
 {
 	return p->stime;
 }
+
+void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
+{
+	if (ut)
+		*ut = task_utime(p);
+	if (st)
+		*st = task_stime(p);
+}
 #else
  
 #ifndef nsecs_to_cputime
  
  
  
  
  
  
  
  
@@ -5198,41 +5206,48 @@
 	msecs_to_cputime(div_u64((__nsecs), NSEC_PER_MSEC))
 #endif
  
-cputime_t task_utime(struct task_struct *p)
+void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
-	cputime_t utime = p->utime, total = utime + p->stime;
-	u64 temp;
+	cputime_t rtime, utime = p->utime, total = utime + p->stime;
  
 	/*
 	 * Use CFS's precise accounting:
 	 */
-	temp = (u64)nsecs_to_cputime(p->se.sum_exec_runtime);
+	rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
  
 	if (total) {
-		temp *= utime;
+		u64 temp;
+
+		temp = (u64)(rtime * utime);
 		do_div(temp, total);
-	}
-	utime = (cputime_t)temp;
+		utime = (cputime_t)temp;
+	} else
+		utime = rtime;
  
+	/*
+	 * Compare with previous values, to keep monotonicity:
+	 */
 	p->prev_utime = max(p->prev_utime, utime);
-	return p->prev_utime;
+	p->prev_stime = max(p->prev_stime, rtime - p->prev_utime);
+
+	if (ut)
+		*ut = p->prev_utime;
+	if (st)
+		*st = p->prev_stime;
 }
  
+cputime_t task_utime(struct task_struct *p)
+{
+	cputime_t utime;
+	task_times(p, &utime, NULL);
+	return utime;
+}
+
 cputime_t task_stime(struct task_struct *p)
 {
 	cputime_t stime;
-
-	/*
-	 * Use CFS's precise accounting. (we subtract utime from
-	 * the total, to make sure the total observed by userspace
-	 * grows monotonically - apps rely on that):
-	 */
-	stime = nsecs_to_cputime(p->se.sum_exec_runtime) - task_utime(p);
-
-	if (stime >= 0)
-		p->prev_stime = max(p->prev_stime, stime);
-
-	return p->prev_stime;
+	task_times(p, NULL, &stime);
+	return stime;
 }
 #endif
  
@@ -1346,8 +1346,7 @@
 	utime = stime = cputime_zero;
  
 	if (who == RUSAGE_THREAD) {
-		utime = task_utime(current);
-		stime = task_stime(current);
+		task_times(current, &utime, &stime);
 		accumulate_thread_rusage(p, r);
 		maxrss = p->signal->maxrss;
 		goto out;
...	...	@@ -535,8 +535,7 @@
535	535	if (!whole) {
536	536	min_flt = task->min_flt;
537	537	maj_flt = task->maj_flt;
538		- utime = task_utime(task);
539		- stime = task_stime(task);
	538	+ task_times(task, &utime, &stime);
540	539	gtime = task_gtime(task);
541	540	}
542	541
...	...	@@ -1723,6 +1723,7 @@
1723	1723	extern cputime_t task_utime(struct task_struct *p);
1724	1724	extern cputime_t task_stime(struct task_struct *p);
1725	1725	extern cputime_t task_gtime(struct task_struct *p);
	1726	+extern void task_times(struct task_struct p, cputime_t ut, cputime_t *st);
1726	1727
1727	1728	/*
1728	1729	* Per process flags
...	...	@@ -91,6 +91,8 @@
91	91	if (atomic_dec_and_test(&sig->count))
92	92	posix_cpu_timers_exit_group(tsk);
93	93	else {
	94	+ cputime_t utime, stime;
	95	+
94	96	/*
95	97	* If there is any task waiting for the group exit
96	98	* then notify it:
...	...	@@ -110,8 +112,9 @@
110	112	* We won't ever get here for the group leader, since it
111	113	* will have been the last reference on the signal_struct.
112	114	*/
113		- sig->utime = cputime_add(sig->utime, task_utime(tsk));
114		- sig->stime = cputime_add(sig->stime, task_stime(tsk));
	115	+ task_times(tsk, &utime, &stime);
	116	+ sig->utime = cputime_add(sig->utime, utime);
	117	+ sig->stime = cputime_add(sig->stime, stime);
115	118	sig->gtime = cputime_add(sig->gtime, task_gtime(tsk));
116	119	sig->min_flt += tsk->min_flt;
117	120	sig->maj_flt += tsk->maj_flt;
...	...	@@ -5191,6 +5191,14 @@
5191	5191	{
5192	5192	return p->stime;
5193	5193	}
	5194	+
	5195	+void task_times(struct task_struct p, cputime_t ut, cputime_t *st)
	5196	+{
	5197	+ if (ut)
	5198	+ *ut = task_utime(p);
	5199	+ if (st)
	5200	+ *st = task_stime(p);
	5201	+}
5194	5202	#else
5195	5203
5196	5204	#ifndef nsecs_to_cputime
5197	5205
5198	5206
5199	5207
5200	5208
5201	5209
5202	5210
5203	5211
5204	5212
...	...	@@ -5198,41 +5206,48 @@
5198	5206	msecs_to_cputime(div_u64((__nsecs), NSEC_PER_MSEC))
5199	5207	#endif
5200	5208
5201		-cputime_t task_utime(struct task_struct *p)
	5209	+void task_times(struct task_struct p, cputime_t ut, cputime_t *st)
5202	5210	{
5203		- cputime_t utime = p->utime, total = utime + p->stime;
5204		- u64 temp;
	5211	+ cputime_t rtime, utime = p->utime, total = utime + p->stime;
5205	5212
5206	5213	/*
5207	5214	* Use CFS's precise accounting:
5208	5215	*/
5209		- temp = (u64)nsecs_to_cputime(p->se.sum_exec_runtime);
	5216	+ rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
5210	5217
5211	5218	if (total) {
5212		- temp *= utime;
	5219	+ u64 temp;
	5220	+
	5221	+ temp = (u64)(rtime * utime);
5213	5222	do_div(temp, total);
5214		- }
5215		- utime = (cputime_t)temp;
	5223	+ utime = (cputime_t)temp;
	5224	+ } else
	5225	+ utime = rtime;
5216	5226
	5227	+ /*
	5228	+ * Compare with previous values, to keep monotonicity:
	5229	+ */
5217	5230	p->prev_utime = max(p->prev_utime, utime);
5218		- return p->prev_utime;
	5231	+ p->prev_stime = max(p->prev_stime, rtime - p->prev_utime);
	5232	+
	5233	+ if (ut)
	5234	+ *ut = p->prev_utime;
	5235	+ if (st)
	5236	+ *st = p->prev_stime;
5219	5237	}
5220	5238
	5239	+cputime_t task_utime(struct task_struct *p)
	5240	+{
	5241	+ cputime_t utime;
	5242	+ task_times(p, &utime, NULL);
	5243	+ return utime;
	5244	+}
	5245	+
5221	5246	cputime_t task_stime(struct task_struct *p)
5222	5247	{
5223	5248	cputime_t stime;
5224		-
5225		- /*
5226		- * Use CFS's precise accounting. (we subtract utime from
5227		- * the total, to make sure the total observed by userspace
5228		- * grows monotonically - apps rely on that):
5229		- */
5230		- stime = nsecs_to_cputime(p->se.sum_exec_runtime) - task_utime(p);
5231		-
5232		- if (stime >= 0)
5233		- p->prev_stime = max(p->prev_stime, stime);
5234		-
5235		- return p->prev_stime;
	5249	+ task_times(p, NULL, &stime);
	5250	+ return stime;
5236	5251	}
5237	5252	#endif
5238	5253
...	...	@@ -1346,8 +1346,7 @@
1346	1346	utime = stime = cputime_zero;
1347	1347
1348	1348	if (who == RUSAGE_THREAD) {
1349		- utime = task_utime(current);
1350		- stime = task_stime(current);
	1349	+ task_times(current, &utime, &stime);
1351	1350	accumulate_thread_rusage(p, r);
1352	1351	maxrss = p->signal->maxrss;
1353	1352	goto out;