Blame view
kernel/posix-cpu-timers.c
39.5 KB
1da177e4c
|
1 2 3 4 5 6 |
/* * Implement CPU time clocks for the POSIX clock interface. */ #include <linux/sched.h> #include <linux/posix-timers.h> |
1da177e4c
|
7 |
#include <linux/errno.h> |
f8bd2258e
|
8 9 |
#include <linux/math64.h> #include <asm/uaccess.h> |
bb34d92f6
|
10 |
#include <linux/kernel_stat.h> |
3f0a525eb
|
11 |
#include <trace/events/timer.h> |
613370549
|
12 |
#include <linux/random.h> |
a85721601
|
13 14 |
#include <linux/tick.h> #include <linux/workqueue.h> |
1da177e4c
|
15 |
|
f06febc96
|
16 |
/* |
f55db6090
|
17 18 19 20 |
* Called after updating RLIMIT_CPU to run cpu timer and update * tsk->signal->cputime_expires expiration cache if necessary. Needs * siglock protection since other code may update expiration cache as * well. |
f06febc96
|
21 |
*/ |
5ab46b345
|
22 |
void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) |
f06febc96
|
23 |
{ |
42c4ab41a
|
24 |
cputime_t cputime = secs_to_cputime(rlim_new); |
f06febc96
|
25 |
|
5ab46b345
|
26 27 28 |
spin_lock_irq(&task->sighand->siglock); set_process_cpu_timer(task, CPUCLOCK_PROF, &cputime, NULL); spin_unlock_irq(&task->sighand->siglock); |
f06febc96
|
29 |
} |
a924b04dd
|
30 |
static int check_clock(const clockid_t which_clock) |
1da177e4c
|
31 32 33 34 35 36 37 38 39 40 |
{ int error = 0; struct task_struct *p; const pid_t pid = CPUCLOCK_PID(which_clock); if (CPUCLOCK_WHICH(which_clock) >= CPUCLOCK_MAX) return -EINVAL; if (pid == 0) return 0; |
c0deae8c9
|
41 |
rcu_read_lock(); |
8dc86af00
|
42 |
p = find_task_by_vpid(pid); |
bac0abd61
|
43 |
if (!p || !(CPUCLOCK_PERTHREAD(which_clock) ? |
c0deae8c9
|
44 |
same_thread_group(p, current) : has_group_leader_pid(p))) { |
1da177e4c
|
45 46 |
error = -EINVAL; } |
c0deae8c9
|
47 |
rcu_read_unlock(); |
1da177e4c
|
48 49 50 |
return error; } |
55ccb616a
|
51 |
static inline unsigned long long |
a924b04dd
|
52 |
timespec_to_sample(const clockid_t which_clock, const struct timespec *tp) |
1da177e4c
|
53 |
{ |
55ccb616a
|
54 55 56 |
unsigned long long ret; ret = 0; /* high half always zero when .cpu used */ |
1da177e4c
|
57 |
if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) { |
55ccb616a
|
58 |
ret = (unsigned long long)tp->tv_sec * NSEC_PER_SEC + tp->tv_nsec; |
1da177e4c
|
59 |
} else { |
55ccb616a
|
60 |
ret = cputime_to_expires(timespec_to_cputime(tp)); |
1da177e4c
|
61 62 63 |
} return ret; } |
a924b04dd
|
64 |
static void sample_to_timespec(const clockid_t which_clock, |
55ccb616a
|
65 |
unsigned long long expires, |
1da177e4c
|
66 67 |
struct timespec *tp) { |
f8bd2258e
|
68 |
if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) |
55ccb616a
|
69 |
*tp = ns_to_timespec(expires); |
f8bd2258e
|
70 |
else |
55ccb616a
|
71 |
cputime_to_timespec((__force cputime_t)expires, tp); |
1da177e4c
|
72 73 74 75 76 77 |
} /* * Update expiry time from increment, and increase overrun count, * given the current clock sample. */ |
7a4ed937a
|
78 |
static void bump_cpu_timer(struct k_itimer *timer, |
55ccb616a
|
79 |
unsigned long long now) |
1da177e4c
|
80 81 |
{ int i; |
55ccb616a
|
82 |
unsigned long long delta, incr; |
1da177e4c
|
83 |
|
55ccb616a
|
84 |
if (timer->it.cpu.incr == 0) |
1da177e4c
|
85 |
return; |
55ccb616a
|
86 87 |
if (now < timer->it.cpu.expires) return; |
1da177e4c
|
88 |
|
55ccb616a
|
89 90 |
incr = timer->it.cpu.incr; delta = now + incr - timer->it.cpu.expires; |
1da177e4c
|
91 |
|
55ccb616a
|
92 93 94 95 96 97 98 99 100 101 102 |
/* Don't use (incr*2 < delta), incr*2 might overflow. */ for (i = 0; incr < delta - incr; i++) incr = incr << 1; for (; i >= 0; incr >>= 1, i--) { if (delta < incr) continue; timer->it.cpu.expires += incr; timer->it_overrun += 1 << i; delta -= incr; |
1da177e4c
|
103 104 |
} } |
555347f6c
|
105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
/** * task_cputime_zero - Check a task_cputime struct for all zero fields. * * @cputime: The struct to compare. * * Checks @cputime to see if all fields are zero. Returns true if all fields * are zero, false if any field is nonzero. */ static inline int task_cputime_zero(const struct task_cputime *cputime) { if (!cputime->utime && !cputime->stime && !cputime->sum_exec_runtime) return 1; return 0; } |
55ccb616a
|
119 |
static inline unsigned long long prof_ticks(struct task_struct *p) |
1da177e4c
|
120 |
{ |
6fac4829c
|
121 122 123 |
cputime_t utime, stime; task_cputime(p, &utime, &stime); |
55ccb616a
|
124 |
return cputime_to_expires(utime + stime); |
1da177e4c
|
125 |
} |
55ccb616a
|
126 |
static inline unsigned long long virt_ticks(struct task_struct *p) |
1da177e4c
|
127 |
{ |
6fac4829c
|
128 129 130 |
cputime_t utime; task_cputime(p, &utime, NULL); |
55ccb616a
|
131 |
return cputime_to_expires(utime); |
1da177e4c
|
132 |
} |
1da177e4c
|
133 |
|
bc2c8ea48
|
134 135 |
static int posix_cpu_clock_getres(const clockid_t which_clock, struct timespec *tp) |
1da177e4c
|
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
{ int error = check_clock(which_clock); if (!error) { tp->tv_sec = 0; tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ); if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) { /* * If sched_clock is using a cycle counter, we * don't have any idea of its true resolution * exported, but it is much more than 1s/HZ. */ tp->tv_nsec = 1; } } return error; } |
bc2c8ea48
|
152 153 |
static int posix_cpu_clock_set(const clockid_t which_clock, const struct timespec *tp) |
1da177e4c
|
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
{ /* * You can never reset a CPU clock, but we check for other errors * in the call before failing with EPERM. */ int error = check_clock(which_clock); if (error == 0) { error = -EPERM; } return error; } /* * Sample a per-thread clock for the given task. */ |
a924b04dd
|
170 |
static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p, |
55ccb616a
|
171 |
unsigned long long *sample) |
1da177e4c
|
172 173 174 175 176 |
{ switch (CPUCLOCK_WHICH(which_clock)) { default: return -EINVAL; case CPUCLOCK_PROF: |
55ccb616a
|
177 |
*sample = prof_ticks(p); |
1da177e4c
|
178 179 |
break; case CPUCLOCK_VIRT: |
55ccb616a
|
180 |
*sample = virt_ticks(p); |
1da177e4c
|
181 182 |
break; case CPUCLOCK_SCHED: |
55ccb616a
|
183 |
*sample = task_sched_runtime(p); |
1da177e4c
|
184 185 186 187 |
break; } return 0; } |
4da94d49b
|
188 189 |
static void update_gt_cputime(struct task_cputime *a, struct task_cputime *b) { |
648616343
|
190 |
if (b->utime > a->utime) |
4da94d49b
|
191 |
a->utime = b->utime; |
648616343
|
192 |
if (b->stime > a->stime) |
4da94d49b
|
193 194 195 196 197 198 199 200 201 202 203 |
a->stime = b->stime; if (b->sum_exec_runtime > a->sum_exec_runtime) a->sum_exec_runtime = b->sum_exec_runtime; } void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) { struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; struct task_cputime sum; unsigned long flags; |
4da94d49b
|
204 |
if (!cputimer->running) { |
4da94d49b
|
205 206 207 208 209 210 211 |
/* * The POSIX timer interface allows for absolute time expiry * values through the TIMER_ABSTIME flag, therefore we have * to synchronize the timer to the clock every time we start * it. */ thread_group_cputime(tsk, &sum); |
3cfef9524
|
212 |
raw_spin_lock_irqsave(&cputimer->lock, flags); |
bcd5cff72
|
213 |
cputimer->running = 1; |
4da94d49b
|
214 |
update_gt_cputime(&cputimer->cputime, &sum); |
bcd5cff72
|
215 |
} else |
3cfef9524
|
216 |
raw_spin_lock_irqsave(&cputimer->lock, flags); |
4da94d49b
|
217 |
*times = cputimer->cputime; |
ee30a7b2f
|
218 |
raw_spin_unlock_irqrestore(&cputimer->lock, flags); |
4da94d49b
|
219 |
} |
1da177e4c
|
220 221 222 |
/* * Sample a process (thread group) clock for the given group_leader task. * Must be called with tasklist_lock held for reading. |
1da177e4c
|
223 |
*/ |
bb34d92f6
|
224 225 |
static int cpu_clock_sample_group(const clockid_t which_clock, struct task_struct *p, |
55ccb616a
|
226 |
unsigned long long *sample) |
1da177e4c
|
227 |
{ |
f06febc96
|
228 |
struct task_cputime cputime; |
eccdaeafa
|
229 |
switch (CPUCLOCK_WHICH(which_clock)) { |
1da177e4c
|
230 231 232 |
default: return -EINVAL; case CPUCLOCK_PROF: |
c5f8d9958
|
233 |
thread_group_cputime(p, &cputime); |
55ccb616a
|
234 |
*sample = cputime_to_expires(cputime.utime + cputime.stime); |
1da177e4c
|
235 236 |
break; case CPUCLOCK_VIRT: |
c5f8d9958
|
237 |
thread_group_cputime(p, &cputime); |
55ccb616a
|
238 |
*sample = cputime_to_expires(cputime.utime); |
1da177e4c
|
239 240 |
break; case CPUCLOCK_SCHED: |
d670ec131
|
241 |
thread_group_cputime(p, &cputime); |
55ccb616a
|
242 |
*sample = cputime.sum_exec_runtime; |
1da177e4c
|
243 244 245 246 |
break; } return 0; } |
1da177e4c
|
247 |
|
bc2c8ea48
|
248 |
static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp) |
1da177e4c
|
249 250 251 |
{ const pid_t pid = CPUCLOCK_PID(which_clock); int error = -EINVAL; |
55ccb616a
|
252 |
unsigned long long rtn; |
1da177e4c
|
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 |
if (pid == 0) { /* * Special case constant value for our own clocks. * We don't have to do any lookup to find ourselves. */ if (CPUCLOCK_PERTHREAD(which_clock)) { /* * Sampling just ourselves we can do with no locking. */ error = cpu_clock_sample(which_clock, current, &rtn); } else { read_lock(&tasklist_lock); error = cpu_clock_sample_group(which_clock, current, &rtn); read_unlock(&tasklist_lock); } } else { /* * Find the given PID, and validate that the caller * should be able to see it. */ struct task_struct *p; |
1f2ea0837
|
277 |
rcu_read_lock(); |
8dc86af00
|
278 |
p = find_task_by_vpid(pid); |
1da177e4c
|
279 280 |
if (p) { if (CPUCLOCK_PERTHREAD(which_clock)) { |
bac0abd61
|
281 |
if (same_thread_group(p, current)) { |
1da177e4c
|
282 283 284 |
error = cpu_clock_sample(which_clock, p, &rtn); } |
1f2ea0837
|
285 286 |
} else { read_lock(&tasklist_lock); |
d30fda355
|
287 |
if (thread_group_leader(p) && p->sighand) { |
1f2ea0837
|
288 289 290 291 292 |
error = cpu_clock_sample_group(which_clock, p, &rtn); } read_unlock(&tasklist_lock); |
1da177e4c
|
293 294 |
} } |
1f2ea0837
|
295 |
rcu_read_unlock(); |
1da177e4c
|
296 297 298 299 300 301 302 303 304 305 306 |
} if (error) return error; sample_to_timespec(which_clock, rtn, tp); return 0; } /* * Validate the clockid_t for a new CPU-clock timer, and initialize the timer. |
ba5ea951d
|
307 308 |
* This is called from sys_timer_create() and do_cpu_nanosleep() with the * new timer already all-zeros initialized. |
1da177e4c
|
309 |
*/ |
bc2c8ea48
|
310 |
static int posix_cpu_timer_create(struct k_itimer *new_timer) |
1da177e4c
|
311 312 313 314 315 316 317 318 319 |
{ int ret = 0; const pid_t pid = CPUCLOCK_PID(new_timer->it_clock); struct task_struct *p; if (CPUCLOCK_WHICH(new_timer->it_clock) >= CPUCLOCK_MAX) return -EINVAL; INIT_LIST_HEAD(&new_timer->it.cpu.entry); |
1da177e4c
|
320 |
|
c0deae8c9
|
321 |
rcu_read_lock(); |
1da177e4c
|
322 323 324 325 |
if (CPUCLOCK_PERTHREAD(new_timer->it_clock)) { if (pid == 0) { p = current; } else { |
8dc86af00
|
326 |
p = find_task_by_vpid(pid); |
bac0abd61
|
327 |
if (p && !same_thread_group(p, current)) |
1da177e4c
|
328 329 330 331 332 333 |
p = NULL; } } else { if (pid == 0) { p = current->group_leader; } else { |
8dc86af00
|
334 |
p = find_task_by_vpid(pid); |
c0deae8c9
|
335 |
if (p && !has_group_leader_pid(p)) |
1da177e4c
|
336 337 338 339 340 341 342 343 344 |
p = NULL; } } new_timer->it.cpu.task = p; if (p) { get_task_struct(p); } else { ret = -EINVAL; } |
c0deae8c9
|
345 |
rcu_read_unlock(); |
1da177e4c
|
346 347 348 349 350 351 352 353 354 355 |
return ret; } /* * Clean up a CPU-clock timer that is about to be destroyed. * This is called from timer deletion with the timer already locked. * If we return TIMER_RETRY, it's necessary to release the timer's lock * and try again. (This happens when the timer is in the middle of firing.) */ |
bc2c8ea48
|
356 |
static int posix_cpu_timer_del(struct k_itimer *timer) |
1da177e4c
|
357 358 |
{ struct task_struct *p = timer->it.cpu.task; |
108150ea7
|
359 |
int ret = 0; |
1da177e4c
|
360 |
|
108150ea7
|
361 |
if (likely(p != NULL)) { |
9465bee86
|
362 |
read_lock(&tasklist_lock); |
d30fda355
|
363 |
if (unlikely(p->sighand == NULL)) { |
9465bee86
|
364 365 366 367 368 369 |
/* * We raced with the reaping of the task. * The deletion should have cleared us off the list. */ BUG_ON(!list_empty(&timer->it.cpu.entry)); } else { |
9465bee86
|
370 |
spin_lock(&p->sighand->siglock); |
108150ea7
|
371 372 373 374 |
if (timer->it.cpu.firing) ret = TIMER_RETRY; else list_del(&timer->it.cpu.entry); |
9465bee86
|
375 376 377 |
spin_unlock(&p->sighand->siglock); } read_unlock(&tasklist_lock); |
108150ea7
|
378 379 380 |
if (!ret) put_task_struct(p); |
1da177e4c
|
381 |
} |
1da177e4c
|
382 |
|
108150ea7
|
383 |
return ret; |
1da177e4c
|
384 |
} |
1a7fa510b
|
385 386 387 388 |
static void cleanup_timers_list(struct list_head *head, unsigned long long curr) { struct cpu_timer_list *timer, *next; |
a0b2062b0
|
389 |
list_for_each_entry_safe(timer, next, head, entry) |
1a7fa510b
|
390 |
list_del_init(&timer->entry); |
1a7fa510b
|
391 |
} |
1da177e4c
|
392 393 394 395 396 397 398 399 |
/* * Clean out CPU timers still ticking when a thread exited. The task * pointer is cleared, and the expiry time is replaced with the residual * time for later timer_gettime calls to return. * This must be called with the siglock held. */ static void cleanup_timers(struct list_head *head, cputime_t utime, cputime_t stime, |
41b86e9c5
|
400 |
unsigned long long sum_exec_runtime) |
1da177e4c
|
401 |
{ |
1da177e4c
|
402 |
|
1a7fa510b
|
403 |
cputime_t ptime = utime + stime; |
1da177e4c
|
404 |
|
1a7fa510b
|
405 406 407 |
cleanup_timers_list(head, cputime_to_expires(ptime)); cleanup_timers_list(++head, cputime_to_expires(utime)); cleanup_timers_list(++head, sum_exec_runtime); |
1da177e4c
|
408 409 410 411 412 413 414 415 416 |
} /* * These are both called with the siglock held, when the current thread * is being reaped. When the final (leader) thread in the group is reaped, * posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit. */ void posix_cpu_timers_exit(struct task_struct *tsk) { |
6fac4829c
|
417 |
cputime_t utime, stime; |
613370549
|
418 419 |
add_device_randomness((const void*) &tsk->se.sum_exec_runtime, sizeof(unsigned long long)); |
6fac4829c
|
420 |
task_cputime(tsk, &utime, &stime); |
1da177e4c
|
421 |
cleanup_timers(tsk->cpu_timers, |
6fac4829c
|
422 |
utime, stime, tsk->se.sum_exec_runtime); |
1da177e4c
|
423 424 425 426 |
} void posix_cpu_timers_exit_group(struct task_struct *tsk) { |
17d42c1c4
|
427 |
struct signal_struct *const sig = tsk->signal; |
6fac4829c
|
428 |
cputime_t utime, stime; |
ca531a0a5
|
429 |
|
6fac4829c
|
430 |
task_cputime(tsk, &utime, &stime); |
f06febc96
|
431 |
cleanup_timers(tsk->signal->cpu_timers, |
6fac4829c
|
432 |
utime + sig->utime, stime + sig->stime, |
17d42c1c4
|
433 |
tsk->se.sum_exec_runtime + sig->sum_sched_runtime); |
1da177e4c
|
434 |
} |
a0b2062b0
|
435 |
static void clear_dead_task(struct k_itimer *itimer, unsigned long long now) |
1da177e4c
|
436 |
{ |
a0b2062b0
|
437 |
struct cpu_timer_list *timer = &itimer->it.cpu; |
1da177e4c
|
438 439 440 441 |
/* * That's all for this thread or process. * We leave our residual in expires to be reported. */ |
a0b2062b0
|
442 443 444 445 446 447 448 |
put_task_struct(timer->task); timer->task = NULL; if (timer->expires < now) { timer->expires = 0; } else { timer->expires -= now; } |
1da177e4c
|
449 |
} |
d1e3b6d19
|
450 451 |
static inline int expires_gt(cputime_t expires, cputime_t new_exp) { |
648616343
|
452 |
return expires == 0 || expires > new_exp; |
d1e3b6d19
|
453 |
} |
1da177e4c
|
454 455 456 |
/* * Insert the timer on the appropriate list before any timers that * expire later. This must be called with the tasklist_lock held |
c28739375
|
457 |
* for reading, interrupts disabled and p->sighand->siglock taken. |
1da177e4c
|
458 |
*/ |
5eb9aa641
|
459 |
static void arm_timer(struct k_itimer *timer) |
1da177e4c
|
460 461 462 |
{ struct task_struct *p = timer->it.cpu.task; struct list_head *head, *listpos; |
5eb9aa641
|
463 |
struct task_cputime *cputime_expires; |
1da177e4c
|
464 465 |
struct cpu_timer_list *const nt = &timer->it.cpu; struct cpu_timer_list *next; |
1da177e4c
|
466 |
|
5eb9aa641
|
467 468 469 470 471 472 473 |
if (CPUCLOCK_PERTHREAD(timer->it_clock)) { head = p->cpu_timers; cputime_expires = &p->cputime_expires; } else { head = p->signal->cpu_timers; cputime_expires = &p->signal->cputime_expires; } |
1da177e4c
|
474 |
head += CPUCLOCK_WHICH(timer->it_clock); |
1da177e4c
|
475 |
listpos = head; |
5eb9aa641
|
476 |
list_for_each_entry(next, head, entry) { |
55ccb616a
|
477 |
if (nt->expires < next->expires) |
5eb9aa641
|
478 479 |
break; listpos = &next->entry; |
1da177e4c
|
480 481 482 483 |
} list_add(&nt->entry, listpos); if (listpos == head) { |
55ccb616a
|
484 |
unsigned long long exp = nt->expires; |
5eb9aa641
|
485 |
|
1da177e4c
|
486 |
/* |
5eb9aa641
|
487 488 489 490 |
* We are the new earliest-expiring POSIX 1.b timer, hence * need to update expiration cache. Take into account that * for process timers we share expiration cache with itimers * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME. |
1da177e4c
|
491 |
*/ |
5eb9aa641
|
492 493 |
switch (CPUCLOCK_WHICH(timer->it_clock)) { case CPUCLOCK_PROF: |
55ccb616a
|
494 495 |
if (expires_gt(cputime_expires->prof_exp, expires_to_cputime(exp))) cputime_expires->prof_exp = expires_to_cputime(exp); |
5eb9aa641
|
496 497 |
break; case CPUCLOCK_VIRT: |
55ccb616a
|
498 499 |
if (expires_gt(cputime_expires->virt_exp, expires_to_cputime(exp))) cputime_expires->virt_exp = expires_to_cputime(exp); |
5eb9aa641
|
500 501 502 |
break; case CPUCLOCK_SCHED: if (cputime_expires->sched_exp == 0 || |
55ccb616a
|
503 504 |
cputime_expires->sched_exp > exp) cputime_expires->sched_exp = exp; |
5eb9aa641
|
505 |
break; |
1da177e4c
|
506 507 |
} } |
1da177e4c
|
508 509 510 511 512 513 514 |
} /* * The timer is locked, fire it and arrange for its reload. */ static void cpu_timer_fire(struct k_itimer *timer) { |
1f169f84d
|
515 516 517 518 |
if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) { /* * User don't want any signal. */ |
55ccb616a
|
519 |
timer->it.cpu.expires = 0; |
1f169f84d
|
520 |
} else if (unlikely(timer->sigq == NULL)) { |
1da177e4c
|
521 522 523 524 525 |
/* * This a special case for clock_nanosleep, * not a normal timer from sys_timer_create. */ wake_up_process(timer->it_process); |
55ccb616a
|
526 527 |
timer->it.cpu.expires = 0; } else if (timer->it.cpu.incr == 0) { |
1da177e4c
|
528 529 530 531 |
/* * One-shot timer. Clear it as soon as it's fired. */ posix_timer_event(timer, 0); |
55ccb616a
|
532 |
timer->it.cpu.expires = 0; |
1da177e4c
|
533 534 535 536 537 538 539 540 541 542 543 544 |
} else if (posix_timer_event(timer, ++timer->it_requeue_pending)) { /* * The signal did not get queued because the signal * was ignored, so we won't get any callback to * reload the timer. But we need to keep it * ticking in case the signal is deliverable next time. */ posix_cpu_timer_schedule(timer); } } /* |
3997ad317
|
545 546 547 548 549 |
* Sample a process (thread group) timer for the given group_leader task. * Must be called with tasklist_lock held for reading. */ static int cpu_timer_sample_group(const clockid_t which_clock, struct task_struct *p, |
55ccb616a
|
550 |
unsigned long long *sample) |
3997ad317
|
551 552 553 554 555 556 557 558 |
{ struct task_cputime cputime; thread_group_cputimer(p, &cputime); switch (CPUCLOCK_WHICH(which_clock)) { default: return -EINVAL; case CPUCLOCK_PROF: |
55ccb616a
|
559 |
*sample = cputime_to_expires(cputime.utime + cputime.stime); |
3997ad317
|
560 561 |
break; case CPUCLOCK_VIRT: |
55ccb616a
|
562 |
*sample = cputime_to_expires(cputime.utime); |
3997ad317
|
563 564 |
break; case CPUCLOCK_SCHED: |
55ccb616a
|
565 |
*sample = cputime.sum_exec_runtime + task_delta_exec(p); |
3997ad317
|
566 567 568 569 |
break; } return 0; } |
a85721601
|
570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 |
#ifdef CONFIG_NO_HZ_FULL static void nohz_kick_work_fn(struct work_struct *work) { tick_nohz_full_kick_all(); } static DECLARE_WORK(nohz_kick_work, nohz_kick_work_fn); /* * We need the IPIs to be sent from sane process context. * The posix cpu timers are always set with irqs disabled. */ static void posix_cpu_timer_kick_nohz(void) { schedule_work(&nohz_kick_work); } |
555347f6c
|
586 587 588 589 |
bool posix_cpu_timers_can_stop_tick(struct task_struct *tsk) { if (!task_cputime_zero(&tsk->cputime_expires)) |
6ac29178b
|
590 |
return false; |
555347f6c
|
591 592 |
if (tsk->signal->cputimer.running) |
6ac29178b
|
593 |
return false; |
555347f6c
|
594 |
|
6ac29178b
|
595 |
return true; |
555347f6c
|
596 |
} |
a85721601
|
597 598 599 |
#else static inline void posix_cpu_timer_kick_nohz(void) { } #endif |
3997ad317
|
600 |
/* |
1da177e4c
|
601 602 603 604 605 |
* Guts of sys_timer_settime for CPU timers. * This is called with the timer locked and interrupts disabled. * If we return TIMER_RETRY, it's necessary to release the timer's lock * and try again. (This happens when the timer is in the middle of firing.) */ |
bc2c8ea48
|
606 607 |
static int posix_cpu_timer_set(struct k_itimer *timer, int flags, struct itimerspec *new, struct itimerspec *old) |
1da177e4c
|
608 609 |
{ struct task_struct *p = timer->it.cpu.task; |
55ccb616a
|
610 |
unsigned long long old_expires, new_expires, old_incr, val; |
1da177e4c
|
611 612 613 614 615 616 617 618 619 620 621 622 623 624 |
int ret; if (unlikely(p == NULL)) { /* * Timer refers to a dead task's clock. */ return -ESRCH; } new_expires = timespec_to_sample(timer->it_clock, &new->it_value); read_lock(&tasklist_lock); /* * We need the tasklist_lock to protect against reaping that |
d30fda355
|
625 |
* clears p->sighand. If p has just been reaped, we can no |
1da177e4c
|
626 627 |
* longer get any information about it at all. */ |
d30fda355
|
628 |
if (unlikely(p->sighand == NULL)) { |
1da177e4c
|
629 630 631 632 633 634 635 636 637 638 |
read_unlock(&tasklist_lock); put_task_struct(p); timer->it.cpu.task = NULL; return -ESRCH; } /* * Disarm any old timer after extracting its expiry time. */ BUG_ON(!irqs_disabled()); |
a69ac4a78
|
639 640 |
ret = 0; |
ae1a78eec
|
641 |
old_incr = timer->it.cpu.incr; |
1da177e4c
|
642 643 |
spin_lock(&p->sighand->siglock); old_expires = timer->it.cpu.expires; |
a69ac4a78
|
644 645 646 647 648 |
if (unlikely(timer->it.cpu.firing)) { timer->it.cpu.firing = -1; ret = TIMER_RETRY; } else list_del_init(&timer->it.cpu.entry); |
1da177e4c
|
649 650 651 652 653 654 655 656 657 658 659 660 |
/* * We need to sample the current value to convert the new * value from to relative and absolute, and to convert the * old value from absolute to relative. To set a process * timer, we need a sample to balance the thread expiry * times (in arm_timer). With an absolute time, we must * check if it's already passed. In short, we need a sample. */ if (CPUCLOCK_PERTHREAD(timer->it_clock)) { cpu_clock_sample(timer->it_clock, p, &val); } else { |
3997ad317
|
661 |
cpu_timer_sample_group(timer->it_clock, p, &val); |
1da177e4c
|
662 663 664 |
} if (old) { |
55ccb616a
|
665 |
if (old_expires == 0) { |
1da177e4c
|
666 667 668 669 670 671 672 673 674 675 676 677 678 679 |
old->it_value.tv_sec = 0; old->it_value.tv_nsec = 0; } else { /* * Update the timer in case it has * overrun already. If it has, * we'll report it as having overrun * and with the next reloaded timer * already ticking, though we are * swallowing that pending * notification here to install the * new setting. */ bump_cpu_timer(timer, val); |
55ccb616a
|
680 681 |
if (val < timer->it.cpu.expires) { old_expires = timer->it.cpu.expires - val; |
1da177e4c
|
682 683 684 685 686 687 688 689 690 |
sample_to_timespec(timer->it_clock, old_expires, &old->it_value); } else { old->it_value.tv_nsec = 1; old->it_value.tv_sec = 0; } } } |
a69ac4a78
|
691 |
if (unlikely(ret)) { |
1da177e4c
|
692 693 694 695 696 697 |
/* * We are colliding with the timer actually firing. * Punt after filling in the timer's old value, and * disable this firing since we are already reporting * it as an overrun (thanks to bump_cpu_timer above). */ |
c28739375
|
698 |
spin_unlock(&p->sighand->siglock); |
1da177e4c
|
699 |
read_unlock(&tasklist_lock); |
1da177e4c
|
700 701 |
goto out; } |
55ccb616a
|
702 703 |
if (new_expires != 0 && !(flags & TIMER_ABSTIME)) { new_expires += val; |
1da177e4c
|
704 705 706 707 708 709 710 711 |
} /* * Install the new expiry time (or zero). * For a timer with no notification action, we don't actually * arm the timer (we'll just fake it for timer_gettime). */ timer->it.cpu.expires = new_expires; |
55ccb616a
|
712 |
if (new_expires != 0 && val < new_expires) { |
5eb9aa641
|
713 |
arm_timer(timer); |
1da177e4c
|
714 |
} |
c28739375
|
715 |
spin_unlock(&p->sighand->siglock); |
1da177e4c
|
716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 |
read_unlock(&tasklist_lock); /* * Install the new reload setting, and * set up the signal and overrun bookkeeping. */ timer->it.cpu.incr = timespec_to_sample(timer->it_clock, &new->it_interval); /* * This acts as a modification timestamp for the timer, * so any automatic reload attempt will punt on seeing * that we have reset the timer manually. */ timer->it_requeue_pending = (timer->it_requeue_pending + 2) & ~REQUEUE_PENDING; timer->it_overrun_last = 0; timer->it_overrun = -1; |
55ccb616a
|
734 |
if (new_expires != 0 && !(val < new_expires)) { |
1da177e4c
|
735 736 737 738 739 740 741 742 743 744 745 746 |
/* * The designated time already passed, so we notify * immediately, even if the thread never runs to * accumulate more time on this clock. */ cpu_timer_fire(timer); } ret = 0; out: if (old) { sample_to_timespec(timer->it_clock, |
ae1a78eec
|
747 |
old_incr, &old->it_interval); |
1da177e4c
|
748 |
} |
a85721601
|
749 750 |
if (!ret) posix_cpu_timer_kick_nohz(); |
1da177e4c
|
751 752 |
return ret; } |
bc2c8ea48
|
753 |
static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) |
1da177e4c
|
754 |
{ |
55ccb616a
|
755 |
unsigned long long now; |
1da177e4c
|
756 757 758 759 760 761 762 763 |
struct task_struct *p = timer->it.cpu.task; int clear_dead; /* * Easy part: convert the reload time. */ sample_to_timespec(timer->it_clock, timer->it.cpu.incr, &itp->it_interval); |
55ccb616a
|
764 |
if (timer->it.cpu.expires == 0) { /* Timer not armed at all. */ |
1da177e4c
|
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 |
itp->it_value.tv_sec = itp->it_value.tv_nsec = 0; return; } if (unlikely(p == NULL)) { /* * This task already died and the timer will never fire. * In this case, expires is actually the dead value. */ dead: sample_to_timespec(timer->it_clock, timer->it.cpu.expires, &itp->it_value); return; } /* * Sample the clock to take the difference with the expiry time. */ if (CPUCLOCK_PERTHREAD(timer->it_clock)) { cpu_clock_sample(timer->it_clock, p, &now); clear_dead = p->exit_state; } else { read_lock(&tasklist_lock); |
d30fda355
|
788 |
if (unlikely(p->sighand == NULL)) { |
1da177e4c
|
789 790 791 792 793 794 795 |
/* * The process has been reaped. * We can't even collect a sample any more. * Call the timer disarmed, nothing else to do. */ put_task_struct(p); timer->it.cpu.task = NULL; |
55ccb616a
|
796 |
timer->it.cpu.expires = 0; |
1da177e4c
|
797 798 799 |
read_unlock(&tasklist_lock); goto dead; } else { |
3997ad317
|
800 |
cpu_timer_sample_group(timer->it_clock, p, &now); |
1da177e4c
|
801 802 803 804 805 |
clear_dead = (unlikely(p->exit_state) && thread_group_empty(p)); } read_unlock(&tasklist_lock); } |
1da177e4c
|
806 807 808 809 810 811 812 813 814 |
if (unlikely(clear_dead)) { /* * We've noticed that the thread is dead, but * not yet reaped. Take this opportunity to * drop our task ref. */ clear_dead_task(timer, now); goto dead; } |
55ccb616a
|
815 |
if (now < timer->it.cpu.expires) { |
1da177e4c
|
816 |
sample_to_timespec(timer->it_clock, |
55ccb616a
|
817 |
timer->it.cpu.expires - now, |
1da177e4c
|
818 819 820 821 822 823 824 825 826 827 |
&itp->it_value); } else { /* * The timer should have expired already, but the firing * hasn't taken place yet. Say it's just about to expire. */ itp->it_value.tv_nsec = 1; itp->it_value.tv_sec = 0; } } |
2473f3e7a
|
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 |
static unsigned long long check_timers_list(struct list_head *timers, struct list_head *firing, unsigned long long curr) { int maxfire = 20; while (!list_empty(timers)) { struct cpu_timer_list *t; t = list_first_entry(timers, struct cpu_timer_list, entry); if (!--maxfire || curr < t->expires) return t->expires; t->firing = 1; list_move_tail(&t->entry, firing); } return 0; } |
1da177e4c
|
849 850 851 852 853 854 855 856 857 |
/* * Check for any per-thread CPU timers that have fired and move them off * the tsk->cpu_timers[N] list onto the firing list. Here we update the * tsk->it_*_expires values to reflect the remaining thread CPU timers. */ static void check_thread_timers(struct task_struct *tsk, struct list_head *firing) { struct list_head *timers = tsk->cpu_timers; |
78f2c7db6
|
858 |
struct signal_struct *const sig = tsk->signal; |
2473f3e7a
|
859 860 |
struct task_cputime *tsk_expires = &tsk->cputime_expires; unsigned long long expires; |
d4bb52743
|
861 |
unsigned long soft; |
1da177e4c
|
862 |
|
2473f3e7a
|
863 864 |
expires = check_timers_list(timers, firing, prof_ticks(tsk)); tsk_expires->prof_exp = expires_to_cputime(expires); |
1da177e4c
|
865 |
|
2473f3e7a
|
866 867 |
expires = check_timers_list(++timers, firing, virt_ticks(tsk)); tsk_expires->virt_exp = expires_to_cputime(expires); |
1da177e4c
|
868 |
|
2473f3e7a
|
869 870 |
tsk_expires->sched_exp = check_timers_list(++timers, firing, tsk->se.sum_exec_runtime); |
78f2c7db6
|
871 872 873 874 |
/* * Check for the special case thread timers. */ |
78d7d407b
|
875 |
soft = ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_cur); |
d4bb52743
|
876 |
if (soft != RLIM_INFINITY) { |
78d7d407b
|
877 878 |
unsigned long hard = ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_max); |
78f2c7db6
|
879 |
|
5a52dd500
|
880 881 |
if (hard != RLIM_INFINITY && tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) { |
78f2c7db6
|
882 883 884 885 886 887 888 |
/* * At the hard limit, we just die. * No need to calculate anything else now. */ __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk); return; } |
d4bb52743
|
889 |
if (tsk->rt.timeout > DIV_ROUND_UP(soft, USEC_PER_SEC/HZ)) { |
78f2c7db6
|
890 891 892 |
/* * At the soft limit, send a SIGXCPU every second. */ |
d4bb52743
|
893 894 895 |
if (soft < hard) { soft += USEC_PER_SEC; sig->rlim[RLIMIT_RTTIME].rlim_cur = soft; |
78f2c7db6
|
896 |
} |
81d50bb25
|
897 898 899 900 |
printk(KERN_INFO "RT Watchdog Timeout: %s[%d] ", tsk->comm, task_pid_nr(tsk)); |
78f2c7db6
|
901 902 903 |
__group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk); } } |
1da177e4c
|
904 |
} |
15365c108
|
905 |
static void stop_process_timers(struct signal_struct *sig) |
3fccfd67d
|
906 |
{ |
15365c108
|
907 |
struct thread_group_cputimer *cputimer = &sig->cputimer; |
3fccfd67d
|
908 |
unsigned long flags; |
ee30a7b2f
|
909 |
raw_spin_lock_irqsave(&cputimer->lock, flags); |
3fccfd67d
|
910 |
cputimer->running = 0; |
ee30a7b2f
|
911 |
raw_spin_unlock_irqrestore(&cputimer->lock, flags); |
3fccfd67d
|
912 |
} |
8356b5f9c
|
913 |
static u32 onecputick; |
42c4ab41a
|
914 |
static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, |
55ccb616a
|
915 916 |
unsigned long long *expires, unsigned long long cur_time, int signo) |
42c4ab41a
|
917 |
{ |
648616343
|
918 |
if (!it->expires) |
42c4ab41a
|
919 |
return; |
648616343
|
920 921 922 |
if (cur_time >= it->expires) { if (it->incr) { it->expires += it->incr; |
8356b5f9c
|
923 924 |
it->error += it->incr_error; if (it->error >= onecputick) { |
648616343
|
925 |
it->expires -= cputime_one_jiffy; |
8356b5f9c
|
926 927 |
it->error -= onecputick; } |
3f0a525eb
|
928 |
} else { |
648616343
|
929 |
it->expires = 0; |
3f0a525eb
|
930 |
} |
42c4ab41a
|
931 |
|
3f0a525eb
|
932 933 934 |
trace_itimer_expire(signo == SIGPROF ? ITIMER_PROF : ITIMER_VIRTUAL, tsk->signal->leader_pid, cur_time); |
42c4ab41a
|
935 936 |
__group_send_sig_info(signo, SEND_SIG_PRIV, tsk); } |
648616343
|
937 |
if (it->expires && (!*expires || it->expires < *expires)) { |
42c4ab41a
|
938 939 940 |
*expires = it->expires; } } |
1da177e4c
|
941 942 943 944 945 946 947 948 949 |
/* * Check for any per-thread CPU timers that have fired and move them * off the tsk->*_timers list onto the firing list. Per-thread timers * have already been taken off. */ static void check_process_timers(struct task_struct *tsk, struct list_head *firing) { struct signal_struct *const sig = tsk->signal; |
55ccb616a
|
950 |
unsigned long long utime, ptime, virt_expires, prof_expires; |
41b86e9c5
|
951 |
unsigned long long sum_sched_runtime, sched_expires; |
1da177e4c
|
952 |
struct list_head *timers = sig->cpu_timers; |
f06febc96
|
953 |
struct task_cputime cputime; |
d4bb52743
|
954 |
unsigned long soft; |
1da177e4c
|
955 956 |
/* |
1da177e4c
|
957 958 |
* Collect the current process totals. */ |
4cd4c1b40
|
959 |
thread_group_cputimer(tsk, &cputime); |
55ccb616a
|
960 961 |
utime = cputime_to_expires(cputime.utime); ptime = utime + cputime_to_expires(cputime.stime); |
f06febc96
|
962 |
sum_sched_runtime = cputime.sum_exec_runtime; |
1da177e4c
|
963 |
|
2473f3e7a
|
964 965 966 |
prof_expires = check_timers_list(timers, firing, ptime); virt_expires = check_timers_list(++timers, firing, utime); sched_expires = check_timers_list(++timers, firing, sum_sched_runtime); |
1da177e4c
|
967 968 969 970 |
/* * Check for the special case process timers. */ |
42c4ab41a
|
971 972 973 974 |
check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], &prof_expires, ptime, SIGPROF); check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_expires, utime, SIGVTALRM); |
78d7d407b
|
975 |
soft = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); |
d4bb52743
|
976 |
if (soft != RLIM_INFINITY) { |
1da177e4c
|
977 |
unsigned long psecs = cputime_to_secs(ptime); |
78d7d407b
|
978 979 |
unsigned long hard = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_max); |
1da177e4c
|
980 |
cputime_t x; |
d4bb52743
|
981 |
if (psecs >= hard) { |
1da177e4c
|
982 983 984 985 986 987 988 |
/* * At the hard limit, we just die. * No need to calculate anything else now. */ __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk); return; } |
d4bb52743
|
989 |
if (psecs >= soft) { |
1da177e4c
|
990 991 992 993 |
/* * At the soft limit, send a SIGXCPU every second. */ __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk); |
d4bb52743
|
994 995 996 |
if (soft < hard) { soft++; sig->rlim[RLIMIT_CPU].rlim_cur = soft; |
1da177e4c
|
997 998 |
} } |
d4bb52743
|
999 |
x = secs_to_cputime(soft); |
648616343
|
1000 |
if (!prof_expires || x < prof_expires) { |
1da177e4c
|
1001 1002 1003 |
prof_expires = x; } } |
55ccb616a
|
1004 1005 |
sig->cputime_expires.prof_exp = expires_to_cputime(prof_expires); sig->cputime_expires.virt_exp = expires_to_cputime(virt_expires); |
29f87b793
|
1006 1007 1008 |
sig->cputime_expires.sched_exp = sched_expires; if (task_cputime_zero(&sig->cputime_expires)) stop_process_timers(sig); |
1da177e4c
|
1009 1010 1011 1012 1013 1014 1015 1016 1017 |
} /* * This is called from the signal code (via do_schedule_next_timer) * when the last timer signal was delivered and we have to reload the timer. */ void posix_cpu_timer_schedule(struct k_itimer *timer) { struct task_struct *p = timer->it.cpu.task; |
55ccb616a
|
1018 |
unsigned long long now; |
1da177e4c
|
1019 1020 1021 1022 1023 |
if (unlikely(p == NULL)) /* * The task was cleaned up already, no future firings. */ |
708f430dc
|
1024 |
goto out; |
1da177e4c
|
1025 1026 1027 1028 1029 1030 1031 1032 1033 |
/* * Fetch the current sample and update the timer's expiry time. */ if (CPUCLOCK_PERTHREAD(timer->it_clock)) { cpu_clock_sample(timer->it_clock, p, &now); bump_cpu_timer(timer, now); if (unlikely(p->exit_state)) { clear_dead_task(timer, now); |
708f430dc
|
1034 |
goto out; |
1da177e4c
|
1035 1036 |
} read_lock(&tasklist_lock); /* arm_timer needs it. */ |
c28739375
|
1037 |
spin_lock(&p->sighand->siglock); |
1da177e4c
|
1038 1039 |
} else { read_lock(&tasklist_lock); |
d30fda355
|
1040 |
if (unlikely(p->sighand == NULL)) { |
1da177e4c
|
1041 1042 1043 1044 1045 1046 |
/* * The process has been reaped. * We can't even collect a sample any more. */ put_task_struct(p); timer->it.cpu.task = p = NULL; |
55ccb616a
|
1047 |
timer->it.cpu.expires = 0; |
708f430dc
|
1048 |
goto out_unlock; |
1da177e4c
|
1049 1050 1051 1052 1053 1054 |
} else if (unlikely(p->exit_state) && thread_group_empty(p)) { /* * We've noticed that the thread is dead, but * not yet reaped. Take this opportunity to * drop our task ref. */ |
76cdcdd97
|
1055 |
cpu_timer_sample_group(timer->it_clock, p, &now); |
1da177e4c
|
1056 |
clear_dead_task(timer, now); |
708f430dc
|
1057 |
goto out_unlock; |
1da177e4c
|
1058 |
} |
c28739375
|
1059 |
spin_lock(&p->sighand->siglock); |
3997ad317
|
1060 |
cpu_timer_sample_group(timer->it_clock, p, &now); |
1da177e4c
|
1061 1062 1063 1064 1065 1066 1067 |
bump_cpu_timer(timer, now); /* Leave the tasklist_lock locked for the call below. */ } /* * Now re-arm for the new expiry time. */ |
c28739375
|
1068 |
BUG_ON(!irqs_disabled()); |
5eb9aa641
|
1069 |
arm_timer(timer); |
c28739375
|
1070 |
spin_unlock(&p->sighand->siglock); |
1da177e4c
|
1071 |
|
708f430dc
|
1072 |
out_unlock: |
1da177e4c
|
1073 |
read_unlock(&tasklist_lock); |
708f430dc
|
1074 1075 1076 1077 1078 |
out: timer->it_overrun_last = timer->it_overrun; timer->it_overrun = -1; ++timer->it_requeue_pending; |
1da177e4c
|
1079 |
} |
f06febc96
|
1080 |
/** |
f06febc96
|
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 |
* task_cputime_expired - Compare two task_cputime entities. * * @sample: The task_cputime structure to be checked for expiration. * @expires: Expiration times, against which @sample will be checked. * * Checks @sample against @expires to see if any field of @sample has expired. * Returns true if any field of the former is greater than the corresponding * field of the latter if the latter field is set. Otherwise returns false. */ static inline int task_cputime_expired(const struct task_cputime *sample, const struct task_cputime *expires) { |
648616343
|
1093 |
if (expires->utime && sample->utime >= expires->utime) |
f06febc96
|
1094 |
return 1; |
648616343
|
1095 |
if (expires->stime && sample->utime + sample->stime >= expires->stime) |
f06febc96
|
1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 |
return 1; if (expires->sum_exec_runtime != 0 && sample->sum_exec_runtime >= expires->sum_exec_runtime) return 1; return 0; } /** * fastpath_timer_check - POSIX CPU timers fast path. * * @tsk: The task (thread) being checked. |
f06febc96
|
1107 |
* |
bb34d92f6
|
1108 1109 1110 1111 |
* Check the task and thread group timers. If both are zero (there are no * timers set) return false. Otherwise snapshot the task and thread group * timers and compare them with the corresponding expiration times. Return * true if a timer has expired, else return false. |
f06febc96
|
1112 |
*/ |
bb34d92f6
|
1113 |
static inline int fastpath_timer_check(struct task_struct *tsk) |
f06febc96
|
1114 |
{ |
ad133ba3d
|
1115 |
struct signal_struct *sig; |
6fac4829c
|
1116 1117 1118 |
cputime_t utime, stime; task_cputime(tsk, &utime, &stime); |
bb34d92f6
|
1119 |
|
bb34d92f6
|
1120 1121 |
if (!task_cputime_zero(&tsk->cputime_expires)) { struct task_cputime task_sample = { |
6fac4829c
|
1122 1123 |
.utime = utime, .stime = stime, |
bb34d92f6
|
1124 1125 1126 1127 1128 1129 |
.sum_exec_runtime = tsk->se.sum_exec_runtime }; if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) return 1; } |
ad133ba3d
|
1130 1131 |
sig = tsk->signal; |
29f87b793
|
1132 |
if (sig->cputimer.running) { |
bb34d92f6
|
1133 |
struct task_cputime group_sample; |
ee30a7b2f
|
1134 |
raw_spin_lock(&sig->cputimer.lock); |
8d1f431cb
|
1135 |
group_sample = sig->cputimer.cputime; |
ee30a7b2f
|
1136 |
raw_spin_unlock(&sig->cputimer.lock); |
8d1f431cb
|
1137 |
|
bb34d92f6
|
1138 1139 1140 |
if (task_cputime_expired(&group_sample, &sig->cputime_expires)) return 1; } |
37bebc70d
|
1141 |
|
f55db6090
|
1142 |
return 0; |
f06febc96
|
1143 |
} |
1da177e4c
|
1144 1145 1146 1147 1148 1149 1150 1151 1152 |
/* * This is called from the timer interrupt handler. The irq handler has * already updated our counts. We need to check if any timers fire now. * Interrupts are disabled. */ void run_posix_cpu_timers(struct task_struct *tsk) { LIST_HEAD(firing); struct k_itimer *timer, *next; |
0bdd2ed41
|
1153 |
unsigned long flags; |
1da177e4c
|
1154 1155 |
BUG_ON(!irqs_disabled()); |
1da177e4c
|
1156 |
/* |
f06febc96
|
1157 |
* The fast path checks that there are no expired thread or thread |
bb34d92f6
|
1158 |
* group timers. If that's so, just return. |
1da177e4c
|
1159 |
*/ |
bb34d92f6
|
1160 |
if (!fastpath_timer_check(tsk)) |
f06febc96
|
1161 |
return; |
5ce73a4a5
|
1162 |
|
0bdd2ed41
|
1163 1164 |
if (!lock_task_sighand(tsk, &flags)) return; |
bb34d92f6
|
1165 1166 1167 1168 1169 1170 |
/* * Here we take off tsk->signal->cpu_timers[N] and * tsk->cpu_timers[N] all the timers that are firing, and * put them on the firing list. */ check_thread_timers(tsk, &firing); |
29f87b793
|
1171 1172 1173 1174 1175 1176 |
/* * If there are any active process wide timers (POSIX 1.b, itimers, * RLIMIT_CPU) cputimer must be running. */ if (tsk->signal->cputimer.running) check_process_timers(tsk, &firing); |
1da177e4c
|
1177 |
|
bb34d92f6
|
1178 1179 1180 1181 1182 1183 1184 1185 |
/* * We must release these locks before taking any timer's lock. * There is a potential race with timer deletion here, as the * siglock now protects our private firing list. We have set * the firing flag in each timer, so that a deletion attempt * that gets the timer lock before we do will give it up and * spin until we've taken care of that timer below. */ |
0bdd2ed41
|
1186 |
unlock_task_sighand(tsk, &flags); |
1da177e4c
|
1187 1188 1189 |
/* * Now that all the timers on our list have the firing flag, |
25985edce
|
1190 |
* no one will touch their list entries but us. We'll take |
1da177e4c
|
1191 1192 1193 1194 |
* each timer's lock before clearing its firing flag, so no * timer call will interfere. */ list_for_each_entry_safe(timer, next, &firing, it.cpu.entry) { |
6e85c5ba7
|
1195 |
int cpu_firing; |
1da177e4c
|
1196 1197 |
spin_lock(&timer->it_lock); list_del_init(&timer->it.cpu.entry); |
6e85c5ba7
|
1198 |
cpu_firing = timer->it.cpu.firing; |
1da177e4c
|
1199 1200 1201 1202 1203 1204 |
timer->it.cpu.firing = 0; /* * The firing flag is -1 if we collided with a reset * of the timer, which already reported this * almost-firing as an overrun. So don't generate an event. */ |
6e85c5ba7
|
1205 |
if (likely(cpu_firing >= 0)) |
1da177e4c
|
1206 |
cpu_timer_fire(timer); |
1da177e4c
|
1207 1208 |
spin_unlock(&timer->it_lock); } |
a85721601
|
1209 1210 1211 1212 1213 1214 1215 |
/* * In case some timers were rescheduled after the queue got emptied, * wake up full dynticks CPUs. */ if (tsk->signal->cputimer.running) posix_cpu_timer_kick_nohz(); |
1da177e4c
|
1216 1217 1218 |
} /* |
f55db6090
|
1219 |
* Set one of the process-wide special case CPU timers or RLIMIT_CPU. |
f06febc96
|
1220 |
* The tsk->sighand->siglock must be held by the caller. |
1da177e4c
|
1221 1222 1223 1224 |
*/ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, cputime_t *newval, cputime_t *oldval) { |
55ccb616a
|
1225 |
unsigned long long now; |
1da177e4c
|
1226 1227 |
BUG_ON(clock_idx == CPUCLOCK_SCHED); |
4cd4c1b40
|
1228 |
cpu_timer_sample_group(clock_idx, tsk, &now); |
1da177e4c
|
1229 1230 |
if (oldval) { |
f55db6090
|
1231 1232 1233 1234 1235 |
/* * We are setting itimer. The *oldval is absolute and we update * it to be relative, *newval argument is relative and we update * it to be absolute. */ |
648616343
|
1236 |
if (*oldval) { |
55ccb616a
|
1237 |
if (*oldval <= now) { |
1da177e4c
|
1238 |
/* Just about to fire. */ |
a42548a18
|
1239 |
*oldval = cputime_one_jiffy; |
1da177e4c
|
1240 |
} else { |
55ccb616a
|
1241 |
*oldval -= now; |
1da177e4c
|
1242 1243 |
} } |
648616343
|
1244 |
if (!*newval) |
a85721601
|
1245 |
goto out; |
55ccb616a
|
1246 |
*newval += now; |
1da177e4c
|
1247 1248 1249 |
} /* |
f55db6090
|
1250 1251 |
* Update expiration cache if we are the earliest timer, or eventually * RLIMIT_CPU limit is earlier than prof_exp cpu timer expire. |
1da177e4c
|
1252 |
*/ |
f55db6090
|
1253 1254 1255 |
switch (clock_idx) { case CPUCLOCK_PROF: if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval)) |
f06febc96
|
1256 |
tsk->signal->cputime_expires.prof_exp = *newval; |
f55db6090
|
1257 1258 1259 |
break; case CPUCLOCK_VIRT: if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval)) |
f06febc96
|
1260 |
tsk->signal->cputime_expires.virt_exp = *newval; |
f55db6090
|
1261 |
break; |
1da177e4c
|
1262 |
} |
a85721601
|
1263 1264 |
out: posix_cpu_timer_kick_nohz(); |
1da177e4c
|
1265 |
} |
e4b765551
|
1266 1267 |
static int do_cpu_nanosleep(const clockid_t which_clock, int flags, struct timespec *rqtp, struct itimerspec *it) |
1da177e4c
|
1268 |
{ |
1da177e4c
|
1269 1270 1271 1272 |
struct k_itimer timer; int error; /* |
1da177e4c
|
1273 1274 1275 1276 1277 1278 1279 1280 1281 |
* Set up a temporary timer and then wait for it to go off. */ memset(&timer, 0, sizeof timer); spin_lock_init(&timer.it_lock); timer.it_clock = which_clock; timer.it_overrun = -1; error = posix_cpu_timer_create(&timer); timer.it_process = current; if (!error) { |
1da177e4c
|
1282 |
static struct itimerspec zero_it; |
e4b765551
|
1283 1284 1285 |
memset(it, 0, sizeof *it); it->it_value = *rqtp; |
1da177e4c
|
1286 1287 |
spin_lock_irq(&timer.it_lock); |
e4b765551
|
1288 |
error = posix_cpu_timer_set(&timer, flags, it, NULL); |
1da177e4c
|
1289 1290 1291 1292 1293 1294 |
if (error) { spin_unlock_irq(&timer.it_lock); return error; } while (!signal_pending(current)) { |
55ccb616a
|
1295 |
if (timer.it.cpu.expires == 0) { |
1da177e4c
|
1296 |
/* |
e6c42c295
|
1297 1298 |
* Our timer fired and was reset, below * deletion can not fail. |
1da177e4c
|
1299 |
*/ |
e6c42c295
|
1300 |
posix_cpu_timer_del(&timer); |
1da177e4c
|
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 |
spin_unlock_irq(&timer.it_lock); return 0; } /* * Block until cpu_timer_fire (or a signal) wakes us. */ __set_current_state(TASK_INTERRUPTIBLE); spin_unlock_irq(&timer.it_lock); schedule(); spin_lock_irq(&timer.it_lock); } /* * We were interrupted by a signal. */ sample_to_timespec(which_clock, timer.it.cpu.expires, rqtp); |
e6c42c295
|
1318 1319 1320 1321 1322 1323 1324 |
error = posix_cpu_timer_set(&timer, 0, &zero_it, it); if (!error) { /* * Timer is now unarmed, deletion can not fail. */ posix_cpu_timer_del(&timer); } |
1da177e4c
|
1325 |
spin_unlock_irq(&timer.it_lock); |
e6c42c295
|
1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 |
while (error == TIMER_RETRY) { /* * We need to handle case when timer was or is in the * middle of firing. In other cases we already freed * resources. */ spin_lock_irq(&timer.it_lock); error = posix_cpu_timer_del(&timer); spin_unlock_irq(&timer.it_lock); } |
e4b765551
|
1336 |
if ((it->it_value.tv_sec | it->it_value.tv_nsec) == 0) { |
1da177e4c
|
1337 1338 1339 1340 1341 |
/* * It actually did fire already. */ return 0; } |
e4b765551
|
1342 1343 1344 1345 1346 |
error = -ERESTART_RESTARTBLOCK; } return error; } |
bc2c8ea48
|
1347 1348 1349 1350 |
static long posix_cpu_nsleep_restart(struct restart_block *restart_block); static int posix_cpu_nsleep(const clockid_t which_clock, int flags, struct timespec *rqtp, struct timespec __user *rmtp) |
e4b765551
|
1351 1352 |
{ struct restart_block *restart_block = |
3751f9f29
|
1353 |
¤t_thread_info()->restart_block; |
e4b765551
|
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 |
struct itimerspec it; int error; /* * Diagnose required errors first. */ if (CPUCLOCK_PERTHREAD(which_clock) && (CPUCLOCK_PID(which_clock) == 0 || CPUCLOCK_PID(which_clock) == current->pid)) return -EINVAL; error = do_cpu_nanosleep(which_clock, flags, rqtp, &it); if (error == -ERESTART_RESTARTBLOCK) { |
3751f9f29
|
1368 |
if (flags & TIMER_ABSTIME) |
e4b765551
|
1369 |
return -ERESTARTNOHAND; |
1da177e4c
|
1370 |
/* |
3751f9f29
|
1371 1372 1373 |
* Report back to the user the time still remaining. */ if (rmtp && copy_to_user(rmtp, &it.it_value, sizeof *rmtp)) |
1da177e4c
|
1374 |
return -EFAULT; |
1711ef386
|
1375 |
restart_block->fn = posix_cpu_nsleep_restart; |
ab8177bc5
|
1376 |
restart_block->nanosleep.clockid = which_clock; |
3751f9f29
|
1377 1378 |
restart_block->nanosleep.rmtp = rmtp; restart_block->nanosleep.expires = timespec_to_ns(rqtp); |
1da177e4c
|
1379 |
} |
1da177e4c
|
1380 1381 |
return error; } |
bc2c8ea48
|
1382 |
static long posix_cpu_nsleep_restart(struct restart_block *restart_block) |
1da177e4c
|
1383 |
{ |
ab8177bc5
|
1384 |
clockid_t which_clock = restart_block->nanosleep.clockid; |
97735f25d
|
1385 |
struct timespec t; |
e4b765551
|
1386 1387 |
struct itimerspec it; int error; |
97735f25d
|
1388 |
|
3751f9f29
|
1389 |
t = ns_to_timespec(restart_block->nanosleep.expires); |
97735f25d
|
1390 |
|
e4b765551
|
1391 1392 1393 |
error = do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t, &it); if (error == -ERESTART_RESTARTBLOCK) { |
3751f9f29
|
1394 |
struct timespec __user *rmtp = restart_block->nanosleep.rmtp; |
e4b765551
|
1395 |
/* |
3751f9f29
|
1396 1397 1398 |
* Report back to the user the time still remaining. */ if (rmtp && copy_to_user(rmtp, &it.it_value, sizeof *rmtp)) |
e4b765551
|
1399 |
return -EFAULT; |
3751f9f29
|
1400 |
restart_block->nanosleep.expires = timespec_to_ns(&t); |
e4b765551
|
1401 1402 |
} return error; |
1da177e4c
|
1403 |
} |
1da177e4c
|
1404 1405 |
#define PROCESS_CLOCK MAKE_PROCESS_CPUCLOCK(0, CPUCLOCK_SCHED) #define THREAD_CLOCK MAKE_THREAD_CPUCLOCK(0, CPUCLOCK_SCHED) |
a924b04dd
|
1406 1407 |
static int process_cpu_clock_getres(const clockid_t which_clock, struct timespec *tp) |
1da177e4c
|
1408 1409 1410 |
{ return posix_cpu_clock_getres(PROCESS_CLOCK, tp); } |
a924b04dd
|
1411 1412 |
static int process_cpu_clock_get(const clockid_t which_clock, struct timespec *tp) |
1da177e4c
|
1413 1414 1415 1416 1417 1418 1419 1420 |
{ return posix_cpu_clock_get(PROCESS_CLOCK, tp); } static int process_cpu_timer_create(struct k_itimer *timer) { timer->it_clock = PROCESS_CLOCK; return posix_cpu_timer_create(timer); } |
a924b04dd
|
1421 |
static int process_cpu_nsleep(const clockid_t which_clock, int flags, |
97735f25d
|
1422 1423 |
struct timespec *rqtp, struct timespec __user *rmtp) |
1da177e4c
|
1424 |
{ |
97735f25d
|
1425 |
return posix_cpu_nsleep(PROCESS_CLOCK, flags, rqtp, rmtp); |
1da177e4c
|
1426 |
} |
1711ef386
|
1427 1428 1429 1430 |
static long process_cpu_nsleep_restart(struct restart_block *restart_block) { return -EINVAL; } |
a924b04dd
|
1431 1432 |
static int thread_cpu_clock_getres(const clockid_t which_clock, struct timespec *tp) |
1da177e4c
|
1433 1434 1435 |
{ return posix_cpu_clock_getres(THREAD_CLOCK, tp); } |
a924b04dd
|
1436 1437 |
static int thread_cpu_clock_get(const clockid_t which_clock, struct timespec *tp) |
1da177e4c
|
1438 1439 1440 1441 1442 1443 1444 1445 |
{ return posix_cpu_clock_get(THREAD_CLOCK, tp); } static int thread_cpu_timer_create(struct k_itimer *timer) { timer->it_clock = THREAD_CLOCK; return posix_cpu_timer_create(timer); } |
1da177e4c
|
1446 |
|
1976945ee
|
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 |
struct k_clock clock_posix_cpu = { .clock_getres = posix_cpu_clock_getres, .clock_set = posix_cpu_clock_set, .clock_get = posix_cpu_clock_get, .timer_create = posix_cpu_timer_create, .nsleep = posix_cpu_nsleep, .nsleep_restart = posix_cpu_nsleep_restart, .timer_set = posix_cpu_timer_set, .timer_del = posix_cpu_timer_del, .timer_get = posix_cpu_timer_get, }; |
1da177e4c
|
1458 1459 1460 |
static __init int init_posix_cpu_timers(void) { struct k_clock process = { |
2fd1f0408
|
1461 1462 |
.clock_getres = process_cpu_clock_getres, .clock_get = process_cpu_clock_get, |
2fd1f0408
|
1463 1464 1465 |
.timer_create = process_cpu_timer_create, .nsleep = process_cpu_nsleep, .nsleep_restart = process_cpu_nsleep_restart, |
1da177e4c
|
1466 1467 |
}; struct k_clock thread = { |
2fd1f0408
|
1468 1469 |
.clock_getres = thread_cpu_clock_getres, .clock_get = thread_cpu_clock_get, |
2fd1f0408
|
1470 |
.timer_create = thread_cpu_timer_create, |
1da177e4c
|
1471 |
}; |
8356b5f9c
|
1472 |
struct timespec ts; |
1da177e4c
|
1473 |
|
527087374
|
1474 1475 |
posix_timers_register_clock(CLOCK_PROCESS_CPUTIME_ID, &process); posix_timers_register_clock(CLOCK_THREAD_CPUTIME_ID, &thread); |
1da177e4c
|
1476 |
|
a42548a18
|
1477 |
cputime_to_timespec(cputime_one_jiffy, &ts); |
8356b5f9c
|
1478 1479 |
onecputick = ts.tv_nsec; WARN_ON(ts.tv_sec != 0); |
1da177e4c
|
1480 1481 1482 |
return 0; } __initcall(init_posix_cpu_timers); |