Commit 27ec4407790d075c325e1f4da0a19c56953cce23
1 parent
018d6db4cb
Exists in
master
and in
7 other branches
sched: make cpu_clock() globally synchronous
Alexey Zaytsev reported (and bisected) that the introduction of cpu_clock() in printk made the timestamps jump back and forth. Make cpu_clock() more reliable while still keeping it fast when it's called frequently. Signed-off-by: Ingo Molnar <mingo@elte.hu>
Showing 1 changed file with 49 additions and 3 deletions Side-by-side Diff
kernel/sched.c
| ... | ... | @@ -632,12 +632,40 @@ |
| 632 | 632 | */ |
| 633 | 633 | #define RUNTIME_INF ((u64)~0ULL) |
| 634 | 634 | |
| 635 | +static const unsigned long long time_sync_thresh = 100000; | |
| 636 | + | |
| 637 | +static DEFINE_PER_CPU(unsigned long long, time_offset); | |
| 638 | +static DEFINE_PER_CPU(unsigned long long, prev_cpu_time); | |
| 639 | + | |
| 635 | 640 | /* |
| 636 | - * For kernel-internal use: high-speed (but slightly incorrect) per-cpu | |
| 637 | - * clock constructed from sched_clock(): | |
| 641 | + * Global lock which we take every now and then to synchronize | |
| 642 | + * the CPUs time. This method is not warp-safe, but it's good | |
| 643 | + * enough to synchronize slowly diverging time sources and thus | |
| 644 | + * it's good enough for tracing: | |
| 638 | 645 | */ |
| 639 | -unsigned long long cpu_clock(int cpu) | |
| 646 | +static DEFINE_SPINLOCK(time_sync_lock); | |
| 647 | +static unsigned long long prev_global_time; | |
| 648 | + | |
| 649 | +static unsigned long long __sync_cpu_clock(cycles_t time, int cpu) | |
| 640 | 650 | { |
| 651 | + unsigned long flags; | |
| 652 | + | |
| 653 | + spin_lock_irqsave(&time_sync_lock, flags); | |
| 654 | + | |
| 655 | + if (time < prev_global_time) { | |
| 656 | + per_cpu(time_offset, cpu) += prev_global_time - time; | |
| 657 | + time = prev_global_time; | |
| 658 | + } else { | |
| 659 | + prev_global_time = time; | |
| 660 | + } | |
| 661 | + | |
| 662 | + spin_unlock_irqrestore(&time_sync_lock, flags); | |
| 663 | + | |
| 664 | + return time; | |
| 665 | +} | |
| 666 | + | |
| 667 | +static unsigned long long __cpu_clock(int cpu) | |
| 668 | +{ | |
| 641 | 669 | unsigned long long now; |
| 642 | 670 | unsigned long flags; |
| 643 | 671 | struct rq *rq; |
| ... | ... | @@ -656,6 +684,24 @@ |
| 656 | 684 | local_irq_restore(flags); |
| 657 | 685 | |
| 658 | 686 | return now; |
| 687 | +} | |
| 688 | + | |
| 689 | +/* | |
| 690 | + * For kernel-internal use: high-speed (but slightly incorrect) per-cpu | |
| 691 | + * clock constructed from sched_clock(): | |
| 692 | + */ | |
| 693 | +unsigned long long cpu_clock(int cpu) | |
| 694 | +{ | |
| 695 | + unsigned long long prev_cpu_time, time, delta_time; | |
| 696 | + | |
| 697 | + prev_cpu_time = per_cpu(prev_cpu_time, cpu); | |
| 698 | + time = __cpu_clock(cpu) + per_cpu(time_offset, cpu); | |
| 699 | + delta_time = time-prev_cpu_time; | |
| 700 | + | |
| 701 | + if (unlikely(delta_time > time_sync_thresh)) | |
| 702 | + time = __sync_cpu_clock(time, cpu); | |
| 703 | + | |
| 704 | + return time; | |
| 659 | 705 | } |
| 660 | 706 | EXPORT_SYMBOL_GPL(cpu_clock); |
| 661 | 707 |