Blame view
kernel/hrtimer.c
35.2 KB
c0a313296 [PATCH] hrtimer: ... |
1 2 3 |
/* * linux/kernel/hrtimer.c * |
3c8aa39d7 [PATCH] hrtimers:... |
4 |
* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> |
79bf2bb33 [PATCH] tick-mana... |
5 |
* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar |
54cdfdb47 [PATCH] hrtimers:... |
6 |
* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner |
c0a313296 [PATCH] hrtimer: ... |
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 |
* * High-resolution kernel timers * * In contrast to the low-resolution timeout API implemented in * kernel/timer.c, hrtimers provide finer resolution and accuracy * depending on system configuration and capabilities. * * These timers are currently used for: * - itimers * - POSIX timers * - nanosleep * - precise in-kernel timing * * Started by: Thomas Gleixner and Ingo Molnar * * Credits: * based on kernel/timer.c * |
66188fae3 [PATCH] hrtimers:... |
25 26 27 28 29 30 |
* Help, testing, suggestions, bugfixes, improvements were * provided by: * * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel * et. al. * |
c0a313296 [PATCH] hrtimer: ... |
31 32 33 34 |
* For licencing details see kernel-base/COPYING */ #include <linux/cpu.h> |
54cdfdb47 [PATCH] hrtimers:... |
35 |
#include <linux/irq.h> |
c0a313296 [PATCH] hrtimer: ... |
36 37 38 39 40 |
#include <linux/module.h> #include <linux/percpu.h> #include <linux/hrtimer.h> #include <linux/notifier.h> #include <linux/syscalls.h> |
54cdfdb47 [PATCH] hrtimers:... |
41 |
#include <linux/kallsyms.h> |
c0a313296 [PATCH] hrtimer: ... |
42 |
#include <linux/interrupt.h> |
79bf2bb33 [PATCH] tick-mana... |
43 |
#include <linux/tick.h> |
54cdfdb47 [PATCH] hrtimers:... |
44 45 |
#include <linux/seq_file.h> #include <linux/err.h> |
c0a313296 [PATCH] hrtimer: ... |
46 47 48 49 50 51 52 53 |
#include <asm/uaccess.h> /** * ktime_get - get the monotonic time in ktime_t format * * returns the time in ktime_t format */ |
d316c57ff [PATCH] clockeven... |
54 |
ktime_t ktime_get(void) |
c0a313296 [PATCH] hrtimer: ... |
55 56 57 58 59 60 61 |
{ struct timespec now; ktime_get_ts(&now); return timespec_to_ktime(now); } |
641b9e0e8 [NET_SCHED]: Use ... |
62 |
EXPORT_SYMBOL_GPL(ktime_get); |
c0a313296 [PATCH] hrtimer: ... |
63 64 65 66 67 68 |
/** * ktime_get_real - get the real (wall-) time in ktime_t format * * returns the time in ktime_t format */ |
d316c57ff [PATCH] clockeven... |
69 |
ktime_t ktime_get_real(void) |
c0a313296 [PATCH] hrtimer: ... |
70 71 72 73 74 75 76 77 78 79 80 81 |
{ struct timespec now; getnstimeofday(&now); return timespec_to_ktime(now); } EXPORT_SYMBOL_GPL(ktime_get_real); /* * The timer bases: |
7978672c4 [PATCH] hrtimers:... |
82 83 84 85 86 87 |
* * Note: If we want to add new timer bases, we have to skip the two * clock ids captured by the cpu-timers. We do this by holding empty * entries rather than doing math adjustment of the clock ids. * This ensures that we capture erroneous accesses to these clock ids * rather than moving them into the range of valid clock id's. |
c0a313296 [PATCH] hrtimer: ... |
88 |
*/ |
54cdfdb47 [PATCH] hrtimers:... |
89 |
DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = |
c0a313296 [PATCH] hrtimer: ... |
90 |
{ |
3c8aa39d7 [PATCH] hrtimers:... |
91 92 |
.clock_base = |
c0a313296 [PATCH] hrtimer: ... |
93 |
{ |
3c8aa39d7 [PATCH] hrtimers:... |
94 95 96 |
{ .index = CLOCK_REALTIME, .get_time = &ktime_get_real, |
54cdfdb47 [PATCH] hrtimers:... |
97 |
.resolution = KTIME_LOW_RES, |
3c8aa39d7 [PATCH] hrtimers:... |
98 99 100 101 |
}, { .index = CLOCK_MONOTONIC, .get_time = &ktime_get, |
54cdfdb47 [PATCH] hrtimers:... |
102 |
.resolution = KTIME_LOW_RES, |
3c8aa39d7 [PATCH] hrtimers:... |
103 104 |
}, } |
c0a313296 [PATCH] hrtimer: ... |
105 106 107 108 |
}; /** * ktime_get_ts - get the monotonic clock in timespec format |
c0a313296 [PATCH] hrtimer: ... |
109 110 111 112 |
* @ts: pointer to timespec variable * * The function calculates the monotonic clock from the realtime * clock and the wall_to_monotonic offset and stores the result |
72fd4a35a [PATCH] Numerous ... |
113 |
* in normalized timespec format in the variable pointed to by @ts. |
c0a313296 [PATCH] hrtimer: ... |
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 |
*/ void ktime_get_ts(struct timespec *ts) { struct timespec tomono; unsigned long seq; do { seq = read_seqbegin(&xtime_lock); getnstimeofday(ts); tomono = wall_to_monotonic; } while (read_seqretry(&xtime_lock, seq)); set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, ts->tv_nsec + tomono.tv_nsec); } |
69778e325 [PATCH] Export kt... |
130 |
EXPORT_SYMBOL_GPL(ktime_get_ts); |
c0a313296 [PATCH] hrtimer: ... |
131 132 |
/* |
92127c7a4 [PATCH] hrtimers:... |
133 134 135 |
* Get the coarse grained time at the softirq based on xtime and * wall_to_monotonic. */ |
3c8aa39d7 [PATCH] hrtimers:... |
136 |
static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) |
92127c7a4 [PATCH] hrtimers:... |
137 138 |
{ ktime_t xtim, tomono; |
ad28d94ab [PATCH] hrtimer: ... |
139 |
struct timespec xts, tom; |
92127c7a4 [PATCH] hrtimers:... |
140 141 142 143 |
unsigned long seq; do { seq = read_seqbegin(&xtime_lock); |
f4304ab21 [PATCH] HZ free ntp |
144 145 146 147 148 |
#ifdef CONFIG_NO_HZ getnstimeofday(&xts); #else xts = xtime; #endif |
ad28d94ab [PATCH] hrtimer: ... |
149 |
tom = wall_to_monotonic; |
92127c7a4 [PATCH] hrtimers:... |
150 |
} while (read_seqretry(&xtime_lock, seq)); |
f4304ab21 [PATCH] HZ free ntp |
151 |
xtim = timespec_to_ktime(xts); |
ad28d94ab [PATCH] hrtimer: ... |
152 |
tomono = timespec_to_ktime(tom); |
3c8aa39d7 [PATCH] hrtimers:... |
153 154 155 |
base->clock_base[CLOCK_REALTIME].softirq_time = xtim; base->clock_base[CLOCK_MONOTONIC].softirq_time = ktime_add(xtim, tomono); |
92127c7a4 [PATCH] hrtimers:... |
156 157 158 |
} /* |
303e967ff [PATCH] hrtimers;... |
159 160 161 162 163 164 165 166 167 |
* Helper function to check, whether the timer is running the callback * function */ static inline int hrtimer_callback_running(struct hrtimer *timer) { return timer->state & HRTIMER_STATE_CALLBACK; } /* |
c0a313296 [PATCH] hrtimer: ... |
168 169 170 171 |
* Functions and macros which are different for UP/SMP systems are kept in a * single place */ #ifdef CONFIG_SMP |
c0a313296 [PATCH] hrtimer: ... |
172 173 174 175 176 177 178 179 180 181 182 183 |
/* * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock * means that all timers which are tied to this base via timer->base are * locked, and the base itself is locked too. * * So __run_timers/migrate_timers can safely modify all timers which could * be found on the lists/queues. * * When the timer's base is locked, and the timer removed from list, it is * possible to set timer->base = NULL and drop the lock: the timer remains * locked. */ |
3c8aa39d7 [PATCH] hrtimers:... |
184 185 186 |
static struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) |
c0a313296 [PATCH] hrtimer: ... |
187 |
{ |
3c8aa39d7 [PATCH] hrtimers:... |
188 |
struct hrtimer_clock_base *base; |
c0a313296 [PATCH] hrtimer: ... |
189 190 191 192 |
for (;;) { base = timer->base; if (likely(base != NULL)) { |
3c8aa39d7 [PATCH] hrtimers:... |
193 |
spin_lock_irqsave(&base->cpu_base->lock, *flags); |
c0a313296 [PATCH] hrtimer: ... |
194 195 196 |
if (likely(base == timer->base)) return base; /* The timer has migrated to another CPU: */ |
3c8aa39d7 [PATCH] hrtimers:... |
197 |
spin_unlock_irqrestore(&base->cpu_base->lock, *flags); |
c0a313296 [PATCH] hrtimer: ... |
198 199 200 201 202 203 204 205 |
} cpu_relax(); } } /* * Switch the timer base to the current CPU when possible. */ |
3c8aa39d7 [PATCH] hrtimers:... |
206 207 |
static inline struct hrtimer_clock_base * switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base) |
c0a313296 [PATCH] hrtimer: ... |
208 |
{ |
3c8aa39d7 [PATCH] hrtimers:... |
209 210 |
struct hrtimer_clock_base *new_base; struct hrtimer_cpu_base *new_cpu_base; |
c0a313296 [PATCH] hrtimer: ... |
211 |
|
3c8aa39d7 [PATCH] hrtimers:... |
212 213 |
new_cpu_base = &__get_cpu_var(hrtimer_bases); new_base = &new_cpu_base->clock_base[base->index]; |
c0a313296 [PATCH] hrtimer: ... |
214 215 216 217 218 219 220 221 222 223 224 |
if (base != new_base) { /* * We are trying to schedule the timer on the local CPU. * However we can't change timer's base while it is running, * so we keep it on the same CPU. No hassle vs. reprogramming * the event source in the high resolution case. The softirq * code will take care of this when the timer function has * completed. There is no conflict as we hold the lock until * the timer is enqueued. */ |
54cdfdb47 [PATCH] hrtimers:... |
225 |
if (unlikely(hrtimer_callback_running(timer))) |
c0a313296 [PATCH] hrtimer: ... |
226 227 228 229 |
return base; /* See the comment in lock_timer_base() */ timer->base = NULL; |
3c8aa39d7 [PATCH] hrtimers:... |
230 231 |
spin_unlock(&base->cpu_base->lock); spin_lock(&new_base->cpu_base->lock); |
c0a313296 [PATCH] hrtimer: ... |
232 233 234 235 236 237 |
timer->base = new_base; } return new_base; } #else /* CONFIG_SMP */ |
3c8aa39d7 [PATCH] hrtimers:... |
238 |
static inline struct hrtimer_clock_base * |
c0a313296 [PATCH] hrtimer: ... |
239 240 |
lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) { |
3c8aa39d7 [PATCH] hrtimers:... |
241 |
struct hrtimer_clock_base *base = timer->base; |
c0a313296 [PATCH] hrtimer: ... |
242 |
|
3c8aa39d7 [PATCH] hrtimers:... |
243 |
spin_lock_irqsave(&base->cpu_base->lock, *flags); |
c0a313296 [PATCH] hrtimer: ... |
244 245 246 |
return base; } |
54cdfdb47 [PATCH] hrtimers:... |
247 |
# define switch_hrtimer_base(t, b) (b) |
c0a313296 [PATCH] hrtimer: ... |
248 249 250 251 252 253 254 255 256 257 258 |
#endif /* !CONFIG_SMP */ /* * Functions for the union type storage format of ktime_t which are * too large for inlining: */ #if BITS_PER_LONG < 64 # ifndef CONFIG_KTIME_SCALAR /** * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable |
c0a313296 [PATCH] hrtimer: ... |
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 |
* @kt: addend * @nsec: the scalar nsec value to add * * Returns the sum of kt and nsec in ktime_t format */ ktime_t ktime_add_ns(const ktime_t kt, u64 nsec) { ktime_t tmp; if (likely(nsec < NSEC_PER_SEC)) { tmp.tv64 = nsec; } else { unsigned long rem = do_div(nsec, NSEC_PER_SEC); tmp = ktime_set((long)nsec, rem); } return ktime_add(kt, tmp); } |
b8b8fd2dc [NET]: Fix networ... |
278 279 |
EXPORT_SYMBOL_GPL(ktime_add_ns); |
c0a313296 [PATCH] hrtimer: ... |
280 281 282 283 284 |
# endif /* !CONFIG_KTIME_SCALAR */ /* * Divide a ktime value by a nanosecond value */ |
79bf2bb33 [PATCH] tick-mana... |
285 |
unsigned long ktime_divns(const ktime_t kt, s64 div) |
c0a313296 [PATCH] hrtimer: ... |
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 |
{ u64 dclc, inc, dns; int sft = 0; dclc = dns = ktime_to_ns(kt); inc = div; /* Make sure the divisor is less than 2^32: */ while (div >> 32) { sft++; div >>= 1; } dclc >>= sft; do_div(dclc, (unsigned long) div); return (unsigned long) dclc; } |
c0a313296 [PATCH] hrtimer: ... |
302 |
#endif /* BITS_PER_LONG >= 64 */ |
54cdfdb47 [PATCH] hrtimers:... |
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 |
/* High resolution timer related functions */ #ifdef CONFIG_HIGH_RES_TIMERS /* * High resolution timer enabled ? */ static int hrtimer_hres_enabled __read_mostly = 1; /* * Enable / Disable high resolution mode */ static int __init setup_hrtimer_hres(char *str) { if (!strcmp(str, "off")) hrtimer_hres_enabled = 0; else if (!strcmp(str, "on")) hrtimer_hres_enabled = 1; else return 0; return 1; } __setup("highres=", setup_hrtimer_hres); /* * hrtimer_high_res_enabled - query, if the highres mode is enabled */ static inline int hrtimer_is_hres_enabled(void) { return hrtimer_hres_enabled; } /* * Is the high resolution mode active ? */ static inline int hrtimer_hres_active(void) { return __get_cpu_var(hrtimer_bases).hres_active; } /* * Reprogram the event source with checking both queues for the * next event * Called with interrupts disabled and base->lock held */ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base) { int i; struct hrtimer_clock_base *base = cpu_base->clock_base; ktime_t expires; cpu_base->expires_next.tv64 = KTIME_MAX; for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { struct hrtimer *timer; if (!base->first) continue; timer = rb_entry(base->first, struct hrtimer, node); expires = ktime_sub(timer->expires, base->offset); if (expires.tv64 < cpu_base->expires_next.tv64) cpu_base->expires_next = expires; } if (cpu_base->expires_next.tv64 != KTIME_MAX) tick_program_event(cpu_base->expires_next, 1); } /* * Shared reprogramming for clock_realtime and clock_monotonic * * When a timer is enqueued and expires earlier than the already enqueued * timers, we have to check, whether it expires earlier than the timer for * which the clock event device was armed. * * Called with interrupts disabled and base->cpu_base.lock held */ static int hrtimer_reprogram(struct hrtimer *timer, struct hrtimer_clock_base *base) { ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next; ktime_t expires = ktime_sub(timer->expires, base->offset); int res; /* * When the callback is running, we do not reprogram the clock event * device. The timer callback is either running on a different CPU or * the callback is executed in the hrtimer_interupt context. The * reprogramming is handled either by the softirq, which called the * callback or at the end of the hrtimer_interrupt. */ if (hrtimer_callback_running(timer)) return 0; if (expires.tv64 >= expires_next->tv64) return 0; /* * Clockevents returns -ETIME, when the event was in the past. */ res = tick_program_event(expires, 0); if (!IS_ERR_VALUE(res)) *expires_next = expires; return res; } /* * Retrigger next event is called after clock was set * * Called with interrupts disabled via on_each_cpu() */ static void retrigger_next_event(void *arg) { struct hrtimer_cpu_base *base; struct timespec realtime_offset; unsigned long seq; if (!hrtimer_hres_active()) return; do { seq = read_seqbegin(&xtime_lock); set_normalized_timespec(&realtime_offset, -wall_to_monotonic.tv_sec, -wall_to_monotonic.tv_nsec); } while (read_seqretry(&xtime_lock, seq)); base = &__get_cpu_var(hrtimer_bases); /* Adjust CLOCK_REALTIME offset */ spin_lock(&base->lock); base->clock_base[CLOCK_REALTIME].offset = timespec_to_ktime(realtime_offset); hrtimer_force_reprogram(base); spin_unlock(&base->lock); } /* * Clock realtime was set * * Change the offset of the realtime clock vs. the monotonic * clock. * * We might have to reprogram the high resolution timer interrupt. On * SMP we call the architecture specific code to retrigger _all_ high * resolution timer interrupts. On UP we just disable interrupts and * call the high resolution interrupt code. */ void clock_was_set(void) { /* Retrigger the CPU local events everywhere */ on_each_cpu(retrigger_next_event, NULL, 0, 1); } /* |
995f054f2 [PATCH] high-res ... |
460 461 462 463 464 465 466 467 468 469 470 471 |
* During resume we might have to reprogram the high resolution timer * interrupt (on the local CPU): */ void hres_timers_resume(void) { WARN_ON_ONCE(num_online_cpus() > 1); /* Retrigger the CPU local events: */ retrigger_next_event(NULL); } /* |
54cdfdb47 [PATCH] hrtimers:... |
472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 |
* Check, whether the timer is on the callback pending list */ static inline int hrtimer_cb_pending(const struct hrtimer *timer) { return timer->state & HRTIMER_STATE_PENDING; } /* * Remove a timer from the callback pending list */ static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) { list_del_init(&timer->cb_entry); } /* * Initialize the high resolution related parts of cpu_base */ static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { base->expires_next.tv64 = KTIME_MAX; base->hres_active = 0; INIT_LIST_HEAD(&base->cb_pending); } /* * Initialize the high resolution related parts of a hrtimer */ static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { INIT_LIST_HEAD(&timer->cb_entry); } /* * When High resolution timers are active, try to reprogram. Note, that in case * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry * check happens. The timer gets enqueued into the rbtree. The reprogramming * and expiry check is done in the hrtimer_interrupt or in the softirq. */ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, struct hrtimer_clock_base *base) { if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { /* Timer is expired, act upon the callback mode */ switch(timer->cb_mode) { case HRTIMER_CB_IRQSAFE_NO_RESTART: /* * We can call the callback from here. No restart * happens, so no danger of recursion */ BUG_ON(timer->function(timer) != HRTIMER_NORESTART); return 1; case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ: /* * This is solely for the sched tick emulation with * dynamic tick support to ensure that we do not * restart the tick right on the edge and end up with * the tick timer in the softirq ! The calling site * takes care of this. */ return 1; case HRTIMER_CB_IRQSAFE: case HRTIMER_CB_SOFTIRQ: /* * Move everything else into the softirq pending list ! */ list_add_tail(&timer->cb_entry, &base->cpu_base->cb_pending); timer->state = HRTIMER_STATE_PENDING; raise_softirq(HRTIMER_SOFTIRQ); return 1; default: BUG(); } } return 0; } /* * Switch to high resolution mode */ |
f8953856e [PATCH] highres: ... |
554 |
static int hrtimer_switch_to_hres(void) |
54cdfdb47 [PATCH] hrtimers:... |
555 556 557 558 559 |
{ struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases); unsigned long flags; if (base->hres_active) |
f8953856e [PATCH] highres: ... |
560 |
return 1; |
54cdfdb47 [PATCH] hrtimers:... |
561 562 563 564 565 |
local_irq_save(flags); if (tick_init_highres()) { local_irq_restore(flags); |
f8953856e [PATCH] highres: ... |
566 |
return 0; |
54cdfdb47 [PATCH] hrtimers:... |
567 568 569 570 571 572 573 574 575 576 577 578 579 |
} base->hres_active = 1; base->clock_base[CLOCK_REALTIME].resolution = KTIME_HIGH_RES; base->clock_base[CLOCK_MONOTONIC].resolution = KTIME_HIGH_RES; tick_setup_sched_timer(); /* "Retrigger" the interrupt to get things going */ retrigger_next_event(NULL); local_irq_restore(flags); printk(KERN_INFO "Switched to high resolution mode on CPU %d ", smp_processor_id()); |
f8953856e [PATCH] highres: ... |
580 |
return 1; |
54cdfdb47 [PATCH] hrtimers:... |
581 582 583 584 585 586 |
} #else static inline int hrtimer_hres_active(void) { return 0; } static inline int hrtimer_is_hres_enabled(void) { return 0; } |
f8953856e [PATCH] highres: ... |
587 |
static inline int hrtimer_switch_to_hres(void) { return 0; } |
54cdfdb47 [PATCH] hrtimers:... |
588 589 590 591 592 593 594 595 596 597 598 599 |
static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { } static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, struct hrtimer_clock_base *base) { return 0; } static inline int hrtimer_cb_pending(struct hrtimer *timer) { return 0; } static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) { } static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { } #endif /* CONFIG_HIGH_RES_TIMERS */ |
82f67cd9f [PATCH] Add debug... |
600 601 602 603 604 605 606 607 608 609 610 |
#ifdef CONFIG_TIMER_STATS void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr) { if (timer->start_site) return; timer->start_site = addr; memcpy(timer->start_comm, current->comm, TASK_COMM_LEN); timer->start_pid = current->pid; } #endif |
c0a313296 [PATCH] hrtimer: ... |
611 612 613 614 615 616 |
/* * Counterpart to lock_timer_base above: */ static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) { |
3c8aa39d7 [PATCH] hrtimers:... |
617 |
spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); |
c0a313296 [PATCH] hrtimer: ... |
618 619 620 621 |
} /** * hrtimer_forward - forward the timer expiry |
c0a313296 [PATCH] hrtimer: ... |
622 |
* @timer: hrtimer to forward |
44f214755 [PATCH] hrtimers:... |
623 |
* @now: forward past this time |
c0a313296 [PATCH] hrtimer: ... |
624 625 626 |
* @interval: the interval to forward * * Forward the timer expiry so it will expire in the future. |
8dca6f33f [PATCH] hrtimer c... |
627 |
* Returns the number of overruns. |
c0a313296 [PATCH] hrtimer: ... |
628 629 |
*/ unsigned long |
44f214755 [PATCH] hrtimers:... |
630 |
hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) |
c0a313296 [PATCH] hrtimer: ... |
631 632 |
{ unsigned long orun = 1; |
44f214755 [PATCH] hrtimers:... |
633 |
ktime_t delta; |
c0a313296 [PATCH] hrtimer: ... |
634 635 636 637 638 |
delta = ktime_sub(now, timer->expires); if (delta.tv64 < 0) return 0; |
c9db4fa11 [hrtimer] Enforce... |
639 640 |
if (interval.tv64 < timer->base->resolution.tv64) interval.tv64 = timer->base->resolution.tv64; |
c0a313296 [PATCH] hrtimer: ... |
641 |
if (unlikely(delta.tv64 >= interval.tv64)) { |
df869b630 [PATCH] hrtimers:... |
642 |
s64 incr = ktime_to_ns(interval); |
c0a313296 [PATCH] hrtimer: ... |
643 644 645 646 647 648 649 650 651 652 653 654 |
orun = ktime_divns(delta, incr); timer->expires = ktime_add_ns(timer->expires, incr * orun); if (timer->expires.tv64 > now.tv64) return orun; /* * This (and the ktime_add() below) is the * correction for exact: */ orun++; } timer->expires = ktime_add(timer->expires, interval); |
13788ccc4 [PATCH] hrtimer: ... |
655 656 657 658 659 660 |
/* * Make sure, that the result did not wrap with a very large * interval. */ if (timer->expires.tv64 < 0) timer->expires = ktime_set(KTIME_SEC_MAX, 0); |
c0a313296 [PATCH] hrtimer: ... |
661 662 663 |
return orun; } |
6bdb6b620 export hrtimer_fo... |
664 |
EXPORT_SYMBOL_GPL(hrtimer_forward); |
c0a313296 [PATCH] hrtimer: ... |
665 666 667 668 669 670 671 |
/* * enqueue_hrtimer - internal function to (re)start a timer * * The timer is inserted in expiry order. Insertion into the * red black tree is O(log(n)). Must hold the base lock. */ |
3c8aa39d7 [PATCH] hrtimers:... |
672 |
static void enqueue_hrtimer(struct hrtimer *timer, |
54cdfdb47 [PATCH] hrtimers:... |
673 |
struct hrtimer_clock_base *base, int reprogram) |
c0a313296 [PATCH] hrtimer: ... |
674 675 |
{ struct rb_node **link = &base->active.rb_node; |
c0a313296 [PATCH] hrtimer: ... |
676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 |
struct rb_node *parent = NULL; struct hrtimer *entry; /* * Find the right place in the rbtree: */ while (*link) { parent = *link; entry = rb_entry(parent, struct hrtimer, node); /* * We dont care about collisions. Nodes with * the same expiry time stay together. */ if (timer->expires.tv64 < entry->expires.tv64) link = &(*link)->rb_left; |
288867ec5 [hrtimer] Remove ... |
691 |
else |
c0a313296 [PATCH] hrtimer: ... |
692 |
link = &(*link)->rb_right; |
c0a313296 [PATCH] hrtimer: ... |
693 694 695 |
} /* |
288867ec5 [hrtimer] Remove ... |
696 697 |
* Insert the timer to the rbtree and check whether it * replaces the first pending timer |
c0a313296 [PATCH] hrtimer: ... |
698 |
*/ |
54cdfdb47 [PATCH] hrtimers:... |
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 |
if (!base->first || timer->expires.tv64 < rb_entry(base->first, struct hrtimer, node)->expires.tv64) { /* * Reprogram the clock event device. When the timer is already * expired hrtimer_enqueue_reprogram has either called the * callback or added it to the pending list and raised the * softirq. * * This is a NOP for !HIGHRES */ if (reprogram && hrtimer_enqueue_reprogram(timer, base)) return; base->first = &timer->node; } |
c0a313296 [PATCH] hrtimer: ... |
714 715 |
rb_link_node(&timer->node, parent, link); rb_insert_color(&timer->node, &base->active); |
303e967ff [PATCH] hrtimers;... |
716 717 718 719 720 |
/* * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the * state of a possibly running callback. */ timer->state |= HRTIMER_STATE_ENQUEUED; |
288867ec5 [hrtimer] Remove ... |
721 |
} |
c0a313296 [PATCH] hrtimer: ... |
722 723 724 725 726 |
/* * __remove_hrtimer - internal function to remove a timer * * Caller must hold the base lock. |
54cdfdb47 [PATCH] hrtimers:... |
727 728 729 730 731 |
* * High resolution timer mode reprograms the clock event device when the * timer is the one which expires next. The caller can disable this by setting * reprogram to zero. This is useful, when the context does a reprogramming * anyway (e.g. timer interrupt) |
c0a313296 [PATCH] hrtimer: ... |
732 |
*/ |
3c8aa39d7 [PATCH] hrtimers:... |
733 |
static void __remove_hrtimer(struct hrtimer *timer, |
303e967ff [PATCH] hrtimers;... |
734 |
struct hrtimer_clock_base *base, |
54cdfdb47 [PATCH] hrtimers:... |
735 |
unsigned long newstate, int reprogram) |
c0a313296 [PATCH] hrtimer: ... |
736 |
{ |
54cdfdb47 [PATCH] hrtimers:... |
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 |
/* High res. callback list. NOP for !HIGHRES */ if (hrtimer_cb_pending(timer)) hrtimer_remove_cb_pending(timer); else { /* * Remove the timer from the rbtree and replace the * first entry pointer if necessary. */ if (base->first == &timer->node) { base->first = rb_next(&timer->node); /* Reprogram the clock event device. if enabled */ if (reprogram && hrtimer_hres_active()) hrtimer_force_reprogram(base->cpu_base); } rb_erase(&timer->node, &base->active); } |
303e967ff [PATCH] hrtimers;... |
753 |
timer->state = newstate; |
c0a313296 [PATCH] hrtimer: ... |
754 755 756 757 758 759 |
} /* * remove hrtimer, called with base lock held */ static inline int |
3c8aa39d7 [PATCH] hrtimers:... |
760 |
remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) |
c0a313296 [PATCH] hrtimer: ... |
761 |
{ |
303e967ff [PATCH] hrtimers;... |
762 |
if (hrtimer_is_queued(timer)) { |
54cdfdb47 [PATCH] hrtimers:... |
763 764 765 766 767 768 769 770 771 772 |
int reprogram; /* * Remove the timer and force reprogramming when high * resolution mode is active and the timer is on the current * CPU. If we remove a timer on another CPU, reprogramming is * skipped. The interrupt event on this CPU is fired and * reprogramming happens in the interrupt handler. This is a * rare case and less expensive than a smp call. */ |
82f67cd9f [PATCH] Add debug... |
773 |
timer_stats_hrtimer_clear_start_info(timer); |
54cdfdb47 [PATCH] hrtimers:... |
774 775 776 |
reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, reprogram); |
c0a313296 [PATCH] hrtimer: ... |
777 778 779 780 781 782 783 |
return 1; } return 0; } /** * hrtimer_start - (re)start an relative timer on the current CPU |
c0a313296 [PATCH] hrtimer: ... |
784 785 786 787 788 789 790 791 792 793 794 |
* @timer: the timer to be added * @tim: expiry time * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) * * Returns: * 0 on success * 1 when the timer was active */ int hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) { |
3c8aa39d7 [PATCH] hrtimers:... |
795 |
struct hrtimer_clock_base *base, *new_base; |
c0a313296 [PATCH] hrtimer: ... |
796 797 798 799 800 801 802 803 804 805 |
unsigned long flags; int ret; base = lock_hrtimer_base(timer, &flags); /* Remove an active timer from the queue: */ ret = remove_hrtimer(timer, base); /* Switch the timer base, if necessary: */ new_base = switch_hrtimer_base(timer, base); |
c9cb2e3d7 [PATCH] hrtimers:... |
806 |
if (mode == HRTIMER_MODE_REL) { |
c0a313296 [PATCH] hrtimer: ... |
807 |
tim = ktime_add(tim, new_base->get_time()); |
06027bdd2 [PATCH] hrtimer: ... |
808 809 810 811 812 813 814 815 816 817 818 |
/* * CONFIG_TIME_LOW_RES is a temporary way for architectures * to signal that they simply return xtime in * do_gettimeoffset(). In this case we want to round up by * resolution when starting a relative timer, to avoid short * timeouts. This will go away with the GTOD framework. */ #ifdef CONFIG_TIME_LOW_RES tim = ktime_add(tim, base->resolution); #endif } |
c0a313296 [PATCH] hrtimer: ... |
819 |
timer->expires = tim; |
82f67cd9f [PATCH] Add debug... |
820 |
timer_stats_hrtimer_set_start_info(timer); |
935c631db [PATCH] hrtimers:... |
821 822 823 824 825 826 |
/* * Only allow reprogramming if the new base is on this CPU. * (it might still be on another CPU if the timer was pending) */ enqueue_hrtimer(timer, new_base, new_base->cpu_base == &__get_cpu_var(hrtimer_bases)); |
c0a313296 [PATCH] hrtimer: ... |
827 828 829 830 831 |
unlock_hrtimer_base(timer, &flags); return ret; } |
8d16b7642 [PATCH] hrtimer: ... |
832 |
EXPORT_SYMBOL_GPL(hrtimer_start); |
c0a313296 [PATCH] hrtimer: ... |
833 834 835 |
/** * hrtimer_try_to_cancel - try to deactivate a timer |
c0a313296 [PATCH] hrtimer: ... |
836 837 838 839 840 841 |
* @timer: hrtimer to stop * * Returns: * 0 when the timer was not active * 1 when the timer was active * -1 when the timer is currently excuting the callback function and |
fa9799e33 [PATCH] ktime/hrt... |
842 |
* cannot be stopped |
c0a313296 [PATCH] hrtimer: ... |
843 844 845 |
*/ int hrtimer_try_to_cancel(struct hrtimer *timer) { |
3c8aa39d7 [PATCH] hrtimers:... |
846 |
struct hrtimer_clock_base *base; |
c0a313296 [PATCH] hrtimer: ... |
847 848 849 850 |
unsigned long flags; int ret = -1; base = lock_hrtimer_base(timer, &flags); |
303e967ff [PATCH] hrtimers;... |
851 |
if (!hrtimer_callback_running(timer)) |
c0a313296 [PATCH] hrtimer: ... |
852 853 854 855 856 857 858 |
ret = remove_hrtimer(timer, base); unlock_hrtimer_base(timer, &flags); return ret; } |
8d16b7642 [PATCH] hrtimer: ... |
859 |
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); |
c0a313296 [PATCH] hrtimer: ... |
860 861 862 |
/** * hrtimer_cancel - cancel a timer and wait for the handler to finish. |
c0a313296 [PATCH] hrtimer: ... |
863 864 865 866 867 868 869 870 871 872 873 874 875 |
* @timer: the timer to be cancelled * * Returns: * 0 when the timer was not active * 1 when the timer was active */ int hrtimer_cancel(struct hrtimer *timer) { for (;;) { int ret = hrtimer_try_to_cancel(timer); if (ret >= 0) return ret; |
5ef37b196 [PATCH] add cpu_r... |
876 |
cpu_relax(); |
c0a313296 [PATCH] hrtimer: ... |
877 878 |
} } |
8d16b7642 [PATCH] hrtimer: ... |
879 |
EXPORT_SYMBOL_GPL(hrtimer_cancel); |
c0a313296 [PATCH] hrtimer: ... |
880 881 882 |
/** * hrtimer_get_remaining - get remaining time for the timer |
c0a313296 [PATCH] hrtimer: ... |
883 884 885 886 |
* @timer: the timer to read */ ktime_t hrtimer_get_remaining(const struct hrtimer *timer) { |
3c8aa39d7 [PATCH] hrtimers:... |
887 |
struct hrtimer_clock_base *base; |
c0a313296 [PATCH] hrtimer: ... |
888 889 890 891 |
unsigned long flags; ktime_t rem; base = lock_hrtimer_base(timer, &flags); |
3c8aa39d7 [PATCH] hrtimers:... |
892 |
rem = ktime_sub(timer->expires, base->get_time()); |
c0a313296 [PATCH] hrtimer: ... |
893 894 895 896 |
unlock_hrtimer_base(timer, &flags); return rem; } |
8d16b7642 [PATCH] hrtimer: ... |
897 |
EXPORT_SYMBOL_GPL(hrtimer_get_remaining); |
c0a313296 [PATCH] hrtimer: ... |
898 |
|
fd064b9b7 [PATCH] Extend ne... |
899 |
#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ) |
69239749e [PATCH] fix next_... |
900 901 902 903 904 905 906 907 |
/** * hrtimer_get_next_event - get the time until next expiry event * * Returns the delta to the next expiry event or KTIME_MAX if no timer * is pending. */ ktime_t hrtimer_get_next_event(void) { |
3c8aa39d7 [PATCH] hrtimers:... |
908 909 |
struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); struct hrtimer_clock_base *base = cpu_base->clock_base; |
69239749e [PATCH] fix next_... |
910 911 912 |
ktime_t delta, mindelta = { .tv64 = KTIME_MAX }; unsigned long flags; int i; |
3c8aa39d7 [PATCH] hrtimers:... |
913 |
spin_lock_irqsave(&cpu_base->lock, flags); |
54cdfdb47 [PATCH] hrtimers:... |
914 915 916 |
if (!hrtimer_hres_active()) { for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { struct hrtimer *timer; |
69239749e [PATCH] fix next_... |
917 |
|
54cdfdb47 [PATCH] hrtimers:... |
918 919 |
if (!base->first) continue; |
3c8aa39d7 [PATCH] hrtimers:... |
920 |
|
54cdfdb47 [PATCH] hrtimers:... |
921 922 923 924 925 926 |
timer = rb_entry(base->first, struct hrtimer, node); delta.tv64 = timer->expires.tv64; delta = ktime_sub(delta, base->get_time()); if (delta.tv64 < mindelta.tv64) mindelta.tv64 = delta.tv64; } |
69239749e [PATCH] fix next_... |
927 |
} |
3c8aa39d7 [PATCH] hrtimers:... |
928 929 |
spin_unlock_irqrestore(&cpu_base->lock, flags); |
69239749e [PATCH] fix next_... |
930 931 932 933 934 |
if (mindelta.tv64 < 0) mindelta.tv64 = 0; return mindelta; } #endif |
c0a313296 [PATCH] hrtimer: ... |
935 |
/** |
7978672c4 [PATCH] hrtimers:... |
936 |
* hrtimer_init - initialize a timer to the given clock |
7978672c4 [PATCH] hrtimers:... |
937 |
* @timer: the timer to be initialized |
c0a313296 [PATCH] hrtimer: ... |
938 |
* @clock_id: the clock to be used |
7978672c4 [PATCH] hrtimers:... |
939 |
* @mode: timer mode abs/rel |
c0a313296 [PATCH] hrtimer: ... |
940 |
*/ |
7978672c4 [PATCH] hrtimers:... |
941 942 |
void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, enum hrtimer_mode mode) |
c0a313296 [PATCH] hrtimer: ... |
943 |
{ |
3c8aa39d7 [PATCH] hrtimers:... |
944 |
struct hrtimer_cpu_base *cpu_base; |
c0a313296 [PATCH] hrtimer: ... |
945 |
|
7978672c4 [PATCH] hrtimers:... |
946 |
memset(timer, 0, sizeof(struct hrtimer)); |
3c8aa39d7 [PATCH] hrtimers:... |
947 |
cpu_base = &__raw_get_cpu_var(hrtimer_bases); |
c0a313296 [PATCH] hrtimer: ... |
948 |
|
c9cb2e3d7 [PATCH] hrtimers:... |
949 |
if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS) |
7978672c4 [PATCH] hrtimers:... |
950 |
clock_id = CLOCK_MONOTONIC; |
3c8aa39d7 [PATCH] hrtimers:... |
951 |
timer->base = &cpu_base->clock_base[clock_id]; |
54cdfdb47 [PATCH] hrtimers:... |
952 |
hrtimer_init_timer_hres(timer); |
82f67cd9f [PATCH] Add debug... |
953 954 955 956 957 958 |
#ifdef CONFIG_TIMER_STATS timer->start_site = NULL; timer->start_pid = -1; memset(timer->start_comm, 0, TASK_COMM_LEN); #endif |
c0a313296 [PATCH] hrtimer: ... |
959 |
} |
8d16b7642 [PATCH] hrtimer: ... |
960 |
EXPORT_SYMBOL_GPL(hrtimer_init); |
c0a313296 [PATCH] hrtimer: ... |
961 962 963 |
/** * hrtimer_get_res - get the timer resolution for a clock |
c0a313296 [PATCH] hrtimer: ... |
964 965 966 |
* @which_clock: which clock to query * @tp: pointer to timespec variable to store the resolution * |
72fd4a35a [PATCH] Numerous ... |
967 968 |
* Store the resolution of the clock selected by @which_clock in the * variable pointed to by @tp. |
c0a313296 [PATCH] hrtimer: ... |
969 970 971 |
*/ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) { |
3c8aa39d7 [PATCH] hrtimers:... |
972 |
struct hrtimer_cpu_base *cpu_base; |
c0a313296 [PATCH] hrtimer: ... |
973 |
|
3c8aa39d7 [PATCH] hrtimers:... |
974 975 |
cpu_base = &__raw_get_cpu_var(hrtimer_bases); *tp = ktime_to_timespec(cpu_base->clock_base[which_clock].resolution); |
c0a313296 [PATCH] hrtimer: ... |
976 977 978 |
return 0; } |
8d16b7642 [PATCH] hrtimer: ... |
979 |
EXPORT_SYMBOL_GPL(hrtimer_get_res); |
c0a313296 [PATCH] hrtimer: ... |
980 |
|
54cdfdb47 [PATCH] hrtimers:... |
981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 |
#ifdef CONFIG_HIGH_RES_TIMERS /* * High resolution timer interrupt * Called with interrupts disabled */ void hrtimer_interrupt(struct clock_event_device *dev) { struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); struct hrtimer_clock_base *base; ktime_t expires_next, now; int i, raise = 0; BUG_ON(!cpu_base->hres_active); cpu_base->nr_events++; dev->next_event.tv64 = KTIME_MAX; retry: now = ktime_get(); expires_next.tv64 = KTIME_MAX; base = cpu_base->clock_base; for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { ktime_t basenow; struct rb_node *node; spin_lock(&cpu_base->lock); basenow = ktime_add(now, base->offset); while ((node = base->first)) { struct hrtimer *timer; timer = rb_entry(node, struct hrtimer, node); if (basenow.tv64 < timer->expires.tv64) { ktime_t expires; expires = ktime_sub(timer->expires, base->offset); if (expires.tv64 < expires_next.tv64) expires_next = expires; break; } /* Move softirq callbacks to the pending list */ if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { __remove_hrtimer(timer, base, HRTIMER_STATE_PENDING, 0); list_add_tail(&timer->cb_entry, &base->cpu_base->cb_pending); raise = 1; continue; } __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); |
82f67cd9f [PATCH] Add debug... |
1040 |
timer_stats_account_hrtimer(timer); |
54cdfdb47 [PATCH] hrtimers:... |
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 |
/* * Note: We clear the CALLBACK bit after * enqueue_hrtimer to avoid reprogramming of * the event hardware. This happens at the end * of this function anyway. */ if (timer->function(timer) != HRTIMER_NORESTART) { BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); enqueue_hrtimer(timer, base, 0); } timer->state &= ~HRTIMER_STATE_CALLBACK; } spin_unlock(&cpu_base->lock); base++; } cpu_base->expires_next = expires_next; /* Reprogramming necessary ? */ if (expires_next.tv64 != KTIME_MAX) { if (tick_program_event(expires_next, 0)) goto retry; } /* Raise softirq ? */ if (raise) raise_softirq(HRTIMER_SOFTIRQ); } static void run_hrtimer_softirq(struct softirq_action *h) { struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); spin_lock_irq(&cpu_base->lock); while (!list_empty(&cpu_base->cb_pending)) { enum hrtimer_restart (*fn)(struct hrtimer *); struct hrtimer *timer; int restart; timer = list_entry(cpu_base->cb_pending.next, struct hrtimer, cb_entry); |
82f67cd9f [PATCH] Add debug... |
1084 |
timer_stats_account_hrtimer(timer); |
54cdfdb47 [PATCH] hrtimers:... |
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 |
fn = timer->function; __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); spin_unlock_irq(&cpu_base->lock); restart = fn(timer); spin_lock_irq(&cpu_base->lock); timer->state &= ~HRTIMER_STATE_CALLBACK; if (restart == HRTIMER_RESTART) { BUG_ON(hrtimer_active(timer)); /* * Enqueue the timer, allow reprogramming of the event * device */ enqueue_hrtimer(timer, timer->base, 1); } else if (hrtimer_active(timer)) { /* * If the timer was rearmed on another CPU, reprogram * the event device. */ if (timer->base->first == &timer->node) hrtimer_reprogram(timer, timer->base); } } spin_unlock_irq(&cpu_base->lock); } #endif /* CONFIG_HIGH_RES_TIMERS */ |
c0a313296 [PATCH] hrtimer: ... |
1114 1115 1116 |
/* * Expire the per base hrtimer-queue: */ |
3c8aa39d7 [PATCH] hrtimers:... |
1117 1118 |
static inline void run_hrtimer_queue(struct hrtimer_cpu_base *cpu_base, int index) |
c0a313296 [PATCH] hrtimer: ... |
1119 |
{ |
288867ec5 [hrtimer] Remove ... |
1120 |
struct rb_node *node; |
3c8aa39d7 [PATCH] hrtimers:... |
1121 |
struct hrtimer_clock_base *base = &cpu_base->clock_base[index]; |
c0a313296 [PATCH] hrtimer: ... |
1122 |
|
3055addad [PATCH] hrtimer: ... |
1123 1124 |
if (!base->first) return; |
92127c7a4 [PATCH] hrtimers:... |
1125 1126 |
if (base->get_softirq_time) base->softirq_time = base->get_softirq_time(); |
3c8aa39d7 [PATCH] hrtimers:... |
1127 |
spin_lock_irq(&cpu_base->lock); |
c0a313296 [PATCH] hrtimer: ... |
1128 |
|
288867ec5 [hrtimer] Remove ... |
1129 |
while ((node = base->first)) { |
c0a313296 [PATCH] hrtimer: ... |
1130 |
struct hrtimer *timer; |
c9cb2e3d7 [PATCH] hrtimers:... |
1131 |
enum hrtimer_restart (*fn)(struct hrtimer *); |
c0a313296 [PATCH] hrtimer: ... |
1132 |
int restart; |
c0a313296 [PATCH] hrtimer: ... |
1133 |
|
288867ec5 [hrtimer] Remove ... |
1134 |
timer = rb_entry(node, struct hrtimer, node); |
92127c7a4 [PATCH] hrtimers:... |
1135 |
if (base->softirq_time.tv64 <= timer->expires.tv64) |
c0a313296 [PATCH] hrtimer: ... |
1136 |
break; |
f8953856e [PATCH] highres: ... |
1137 1138 1139 |
#ifdef CONFIG_HIGH_RES_TIMERS WARN_ON_ONCE(timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ); #endif |
82f67cd9f [PATCH] Add debug... |
1140 |
timer_stats_account_hrtimer(timer); |
c0a313296 [PATCH] hrtimer: ... |
1141 |
fn = timer->function; |
54cdfdb47 [PATCH] hrtimers:... |
1142 |
__remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); |
3c8aa39d7 [PATCH] hrtimers:... |
1143 |
spin_unlock_irq(&cpu_base->lock); |
c0a313296 [PATCH] hrtimer: ... |
1144 |
|
05cfb614d [PATCH] hrtimers:... |
1145 |
restart = fn(timer); |
c0a313296 [PATCH] hrtimer: ... |
1146 |
|
3c8aa39d7 [PATCH] hrtimers:... |
1147 |
spin_lock_irq(&cpu_base->lock); |
c0a313296 [PATCH] hrtimer: ... |
1148 |
|
303e967ff [PATCH] hrtimers;... |
1149 |
timer->state &= ~HRTIMER_STATE_CALLBACK; |
b75f7a51c [PATCH] hrtimers:... |
1150 1151 |
if (restart != HRTIMER_NORESTART) { BUG_ON(hrtimer_active(timer)); |
54cdfdb47 [PATCH] hrtimers:... |
1152 |
enqueue_hrtimer(timer, base, 0); |
b75f7a51c [PATCH] hrtimers:... |
1153 |
} |
c0a313296 [PATCH] hrtimer: ... |
1154 |
} |
3c8aa39d7 [PATCH] hrtimers:... |
1155 |
spin_unlock_irq(&cpu_base->lock); |
c0a313296 [PATCH] hrtimer: ... |
1156 1157 1158 1159 |
} /* * Called from timer softirq every jiffy, expire hrtimers: |
54cdfdb47 [PATCH] hrtimers:... |
1160 1161 1162 1163 |
* * For HRT its the fall back code to run the softirq in the timer * softirq context in case the hrtimer initialization failed or has * not been done yet. |
c0a313296 [PATCH] hrtimer: ... |
1164 1165 1166 |
*/ void hrtimer_run_queues(void) { |
3c8aa39d7 [PATCH] hrtimers:... |
1167 |
struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
c0a313296 [PATCH] hrtimer: ... |
1168 |
int i; |
54cdfdb47 [PATCH] hrtimers:... |
1169 1170 |
if (hrtimer_hres_active()) return; |
79bf2bb33 [PATCH] tick-mana... |
1171 1172 1173 1174 1175 1176 1177 1178 |
/* * This _is_ ugly: We have to check in the softirq context, * whether we can switch to highres and / or nohz mode. The * clocksource switch happens in the timer interrupt with * xtime_lock held. Notification from there only sets the * check bit in the tick_oneshot code, otherwise we might * deadlock vs. xtime_lock. */ |
54cdfdb47 [PATCH] hrtimers:... |
1179 |
if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) |
f8953856e [PATCH] highres: ... |
1180 1181 |
if (hrtimer_switch_to_hres()) return; |
79bf2bb33 [PATCH] tick-mana... |
1182 |
|
3c8aa39d7 [PATCH] hrtimers:... |
1183 |
hrtimer_get_softirq_time(cpu_base); |
92127c7a4 [PATCH] hrtimers:... |
1184 |
|
3c8aa39d7 [PATCH] hrtimers:... |
1185 1186 |
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) run_hrtimer_queue(cpu_base, i); |
c0a313296 [PATCH] hrtimer: ... |
1187 1188 1189 |
} /* |
10c94ec16 [PATCH] hrtimer: ... |
1190 1191 |
* Sleep related functions: */ |
c9cb2e3d7 [PATCH] hrtimers:... |
1192 |
static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) |
00362e33f [PATCH] hrtimer: ... |
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 |
{ struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); struct task_struct *task = t->task; t->task = NULL; if (task) wake_up_process(task); return HRTIMER_NORESTART; } |
36c8b5868 [PATCH] sched: cl... |
1204 |
void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) |
00362e33f [PATCH] hrtimer: ... |
1205 1206 1207 |
{ sl->timer.function = hrtimer_wakeup; sl->task = task; |
54cdfdb47 [PATCH] hrtimers:... |
1208 1209 1210 |
#ifdef CONFIG_HIGH_RES_TIMERS sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_RESTART; #endif |
00362e33f [PATCH] hrtimer: ... |
1211 |
} |
669d7868a [PATCH] hrtimer: ... |
1212 |
static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) |
432569bb9 [PATCH] hrtimers:... |
1213 |
{ |
669d7868a [PATCH] hrtimer: ... |
1214 |
hrtimer_init_sleeper(t, current); |
10c94ec16 [PATCH] hrtimer: ... |
1215 |
|
432569bb9 [PATCH] hrtimers:... |
1216 1217 1218 |
do { set_current_state(TASK_INTERRUPTIBLE); hrtimer_start(&t->timer, t->timer.expires, mode); |
54cdfdb47 [PATCH] hrtimers:... |
1219 1220 |
if (likely(t->task)) schedule(); |
432569bb9 [PATCH] hrtimers:... |
1221 |
|
669d7868a [PATCH] hrtimer: ... |
1222 |
hrtimer_cancel(&t->timer); |
c9cb2e3d7 [PATCH] hrtimers:... |
1223 |
mode = HRTIMER_MODE_ABS; |
669d7868a [PATCH] hrtimer: ... |
1224 1225 |
} while (t->task && !signal_pending(current)); |
432569bb9 [PATCH] hrtimers:... |
1226 |
|
669d7868a [PATCH] hrtimer: ... |
1227 |
return t->task == NULL; |
10c94ec16 [PATCH] hrtimer: ... |
1228 |
} |
1711ef386 [PATCH] posix-tim... |
1229 |
long __sched hrtimer_nanosleep_restart(struct restart_block *restart) |
10c94ec16 [PATCH] hrtimer: ... |
1230 |
{ |
669d7868a [PATCH] hrtimer: ... |
1231 |
struct hrtimer_sleeper t; |
ea13dbc89 [PATCH] kernel/hr... |
1232 1233 |
struct timespec __user *rmtp; struct timespec tu; |
432569bb9 [PATCH] hrtimers:... |
1234 |
ktime_t time; |
10c94ec16 [PATCH] hrtimer: ... |
1235 1236 |
restart->fn = do_no_restart_syscall; |
c9cb2e3d7 [PATCH] hrtimers:... |
1237 |
hrtimer_init(&t.timer, restart->arg0, HRTIMER_MODE_ABS); |
1711ef386 [PATCH] posix-tim... |
1238 |
t.timer.expires.tv64 = ((u64)restart->arg3 << 32) | (u64) restart->arg2; |
10c94ec16 [PATCH] hrtimer: ... |
1239 |
|
c9cb2e3d7 [PATCH] hrtimers:... |
1240 |
if (do_nanosleep(&t, HRTIMER_MODE_ABS)) |
10c94ec16 [PATCH] hrtimer: ... |
1241 |
return 0; |
1711ef386 [PATCH] posix-tim... |
1242 |
rmtp = (struct timespec __user *) restart->arg1; |
432569bb9 [PATCH] hrtimers:... |
1243 1244 1245 1246 1247 1248 1249 1250 |
if (rmtp) { time = ktime_sub(t.timer.expires, t.timer.base->get_time()); if (time.tv64 <= 0) return 0; tu = ktime_to_timespec(time); if (copy_to_user(rmtp, &tu, sizeof(tu))) return -EFAULT; } |
10c94ec16 [PATCH] hrtimer: ... |
1251 |
|
1711ef386 [PATCH] posix-tim... |
1252 |
restart->fn = hrtimer_nanosleep_restart; |
10c94ec16 [PATCH] hrtimer: ... |
1253 1254 1255 1256 |
/* The other values in restart are already filled in */ return -ERESTART_RESTARTBLOCK; } |
10c94ec16 [PATCH] hrtimer: ... |
1257 1258 1259 1260 |
long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, const enum hrtimer_mode mode, const clockid_t clockid) { struct restart_block *restart; |
669d7868a [PATCH] hrtimer: ... |
1261 |
struct hrtimer_sleeper t; |
10c94ec16 [PATCH] hrtimer: ... |
1262 1263 |
struct timespec tu; ktime_t rem; |
432569bb9 [PATCH] hrtimers:... |
1264 1265 1266 |
hrtimer_init(&t.timer, clockid, mode); t.timer.expires = timespec_to_ktime(*rqtp); if (do_nanosleep(&t, mode)) |
10c94ec16 [PATCH] hrtimer: ... |
1267 |
return 0; |
7978672c4 [PATCH] hrtimers:... |
1268 |
/* Absolute timers do not update the rmtp value and restart: */ |
c9cb2e3d7 [PATCH] hrtimers:... |
1269 |
if (mode == HRTIMER_MODE_ABS) |
10c94ec16 [PATCH] hrtimer: ... |
1270 |
return -ERESTARTNOHAND; |
432569bb9 [PATCH] hrtimers:... |
1271 1272 1273 1274 1275 1276 1277 1278 |
if (rmtp) { rem = ktime_sub(t.timer.expires, t.timer.base->get_time()); if (rem.tv64 <= 0) return 0; tu = ktime_to_timespec(rem); if (copy_to_user(rmtp, &tu, sizeof(tu))) return -EFAULT; } |
10c94ec16 [PATCH] hrtimer: ... |
1279 1280 |
restart = ¤t_thread_info()->restart_block; |
1711ef386 [PATCH] posix-tim... |
1281 1282 1283 1284 1285 |
restart->fn = hrtimer_nanosleep_restart; restart->arg0 = (unsigned long) t.timer.base->index; restart->arg1 = (unsigned long) rmtp; restart->arg2 = t.timer.expires.tv64 & 0xFFFFFFFF; restart->arg3 = t.timer.expires.tv64 >> 32; |
10c94ec16 [PATCH] hrtimer: ... |
1286 1287 1288 |
return -ERESTART_RESTARTBLOCK; } |
6ba1b9121 [PATCH] hrtimer: ... |
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 |
asmlinkage long sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp) { struct timespec tu; if (copy_from_user(&tu, rqtp, sizeof(tu))) return -EFAULT; if (!timespec_valid(&tu)) return -EINVAL; |
c9cb2e3d7 [PATCH] hrtimers:... |
1299 |
return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); |
6ba1b9121 [PATCH] hrtimer: ... |
1300 |
} |
10c94ec16 [PATCH] hrtimer: ... |
1301 |
/* |
c0a313296 [PATCH] hrtimer: ... |
1302 1303 1304 1305 |
* Functions related to boot-time initialization: */ static void __devinit init_hrtimers_cpu(int cpu) { |
3c8aa39d7 [PATCH] hrtimers:... |
1306 |
struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); |
c0a313296 [PATCH] hrtimer: ... |
1307 |
int i; |
3c8aa39d7 [PATCH] hrtimers:... |
1308 1309 1310 1311 1312 |
spin_lock_init(&cpu_base->lock); lockdep_set_class(&cpu_base->lock, &cpu_base->lock_key); for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) cpu_base->clock_base[i].cpu_base = cpu_base; |
54cdfdb47 [PATCH] hrtimers:... |
1313 |
hrtimer_init_hres(cpu_base); |
c0a313296 [PATCH] hrtimer: ... |
1314 1315 1316 |
} #ifdef CONFIG_HOTPLUG_CPU |
3c8aa39d7 [PATCH] hrtimers:... |
1317 1318 |
static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, struct hrtimer_clock_base *new_base) |
c0a313296 [PATCH] hrtimer: ... |
1319 1320 1321 1322 1323 1324 |
{ struct hrtimer *timer; struct rb_node *node; while ((node = rb_first(&old_base->active))) { timer = rb_entry(node, struct hrtimer, node); |
54cdfdb47 [PATCH] hrtimers:... |
1325 1326 |
BUG_ON(hrtimer_callback_running(timer)); __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0); |
c0a313296 [PATCH] hrtimer: ... |
1327 |
timer->base = new_base; |
54cdfdb47 [PATCH] hrtimers:... |
1328 1329 1330 1331 |
/* * Enqueue the timer. Allow reprogramming of the event device */ enqueue_hrtimer(timer, new_base, 1); |
c0a313296 [PATCH] hrtimer: ... |
1332 1333 1334 1335 1336 |
} } static void migrate_hrtimers(int cpu) { |
3c8aa39d7 [PATCH] hrtimers:... |
1337 |
struct hrtimer_cpu_base *old_base, *new_base; |
c0a313296 [PATCH] hrtimer: ... |
1338 1339 1340 |
int i; BUG_ON(cpu_online(cpu)); |
3c8aa39d7 [PATCH] hrtimers:... |
1341 1342 |
old_base = &per_cpu(hrtimer_bases, cpu); new_base = &get_cpu_var(hrtimer_bases); |
c0a313296 [PATCH] hrtimer: ... |
1343 |
|
54cdfdb47 [PATCH] hrtimers:... |
1344 |
tick_cancel_sched_timer(cpu); |
c0a313296 [PATCH] hrtimer: ... |
1345 |
local_irq_disable(); |
e81ce1f7e [PATCH] timer/hrt... |
1346 1347 |
double_spin_lock(&new_base->lock, &old_base->lock, smp_processor_id() < cpu); |
c0a313296 [PATCH] hrtimer: ... |
1348 |
|
3c8aa39d7 [PATCH] hrtimers:... |
1349 |
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
3c8aa39d7 [PATCH] hrtimers:... |
1350 1351 |
migrate_hrtimer_list(&old_base->clock_base[i], &new_base->clock_base[i]); |
c0a313296 [PATCH] hrtimer: ... |
1352 |
} |
e81ce1f7e [PATCH] timer/hrt... |
1353 1354 |
double_spin_unlock(&new_base->lock, &old_base->lock, smp_processor_id() < cpu); |
c0a313296 [PATCH] hrtimer: ... |
1355 1356 1357 1358 |
local_irq_enable(); put_cpu_var(hrtimer_bases); } #endif /* CONFIG_HOTPLUG_CPU */ |
8c78f3075 [PATCH] cpu hotpl... |
1359 |
static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, |
c0a313296 [PATCH] hrtimer: ... |
1360 1361 |
unsigned long action, void *hcpu) { |
7713a7d19 [HRTIMER] Fix cpu... |
1362 |
unsigned int cpu = (long)hcpu; |
c0a313296 [PATCH] hrtimer: ... |
1363 1364 1365 1366 |
switch (action) { case CPU_UP_PREPARE: |
8bb784428 Add suspend-relat... |
1367 |
case CPU_UP_PREPARE_FROZEN: |
c0a313296 [PATCH] hrtimer: ... |
1368 1369 1370 1371 1372 |
init_hrtimers_cpu(cpu); break; #ifdef CONFIG_HOTPLUG_CPU case CPU_DEAD: |
8bb784428 Add suspend-relat... |
1373 |
case CPU_DEAD_FROZEN: |
d316c57ff [PATCH] clockeven... |
1374 |
clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &cpu); |
c0a313296 [PATCH] hrtimer: ... |
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 |
migrate_hrtimers(cpu); break; #endif default: break; } return NOTIFY_OK; } |
8c78f3075 [PATCH] cpu hotpl... |
1385 |
static struct notifier_block __cpuinitdata hrtimers_nb = { |
c0a313296 [PATCH] hrtimer: ... |
1386 1387 1388 1389 1390 1391 1392 1393 |
.notifier_call = hrtimer_cpu_notify, }; void __init hrtimers_init(void) { hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, (void *)(long)smp_processor_id()); register_cpu_notifier(&hrtimers_nb); |
54cdfdb47 [PATCH] hrtimers:... |
1394 1395 1396 |
#ifdef CONFIG_HIGH_RES_TIMERS open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL); #endif |
c0a313296 [PATCH] hrtimer: ... |
1397 |
} |