Blame view
kernel/sched_clock.c
8.01 KB
3e51f33fc sched: add option... |
1 2 3 4 5 |
/* * sched_clock for unstable cpu clocks * * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> * |
c300ba252 sched_clock: and ... |
6 7 8 |
* Updates and enhancements: * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com> * |
3e51f33fc sched: add option... |
9 10 11 12 |
* Based on code by: * Ingo Molnar <mingo@redhat.com> * Guillaume Chazarain <guichaz@gmail.com> * |
c676329ab sched_clock: Add ... |
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
* * What: * * cpu_clock(i) provides a fast (execution time) high resolution * clock with bounded drift between CPUs. The value of cpu_clock(i) * is monotonic for constant i. The timestamp returned is in nanoseconds. * * ######################### BIG FAT WARNING ########################## * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # * # go backwards !! # * #################################################################### * * There is no strict promise about the base, although it tends to start * at 0 on boot (but people really shouldn't rely on that). * * cpu_clock(i) -- can be used from any context, including NMI. * sched_clock_cpu(i) -- must be used with local IRQs disabled (implied by NMI) * local_clock() -- is cpu_clock() on the current cpu. * * How: * * The implementation either uses sched_clock() when * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the * sched_clock() is assumed to provide these properties (mostly it means * the architecture provides a globally synchronized highres time source). * * Otherwise it tries to create a semi stable clock from a mixture of other * clocks, including: * * - GTOD (clock monotomic) |
3e51f33fc sched: add option... |
43 44 45 |
* - sched_clock() * - explicit idle events * |
c676329ab sched_clock: Add ... |
46 47 48 |
* We use GTOD as base and use sched_clock() deltas to improve resolution. The * deltas are filtered to provide monotonicity and keeping it within an * expected window. |
3e51f33fc sched: add option... |
49 50 51 52 |
* * Furthermore, explicit sleep and wakeup hooks allow us to account for time * that is otherwise invisible (TSC gets stopped). * |
c676329ab sched_clock: Add ... |
53 54 55 56 57 58 59 60 61 |
* * Notes: * * The !IRQ-safetly of sched_clock() and sched_clock_cpu() comes from things * like cpufreq interrupts that can change the base clock (TSC) multiplier * and cause funny jumps in time -- although the filtering provided by * sched_clock_cpu() should mitigate serious artifacts we cannot rely on it * in general since for !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK we fully rely on * sched_clock(). |
3e51f33fc sched: add option... |
62 |
*/ |
3e51f33fc sched: add option... |
63 |
#include <linux/spinlock.h> |
6409c4da2 sched: sched_cloc... |
64 |
#include <linux/hardirq.h> |
3e51f33fc sched: add option... |
65 |
#include <linux/module.h> |
b342501cd sched: allow arch... |
66 67 68 |
#include <linux/percpu.h> #include <linux/ktime.h> #include <linux/sched.h> |
3e51f33fc sched: add option... |
69 |
|
2c3d103ba sched: move sched... |
70 71 72 73 74 75 76 |
/* * Scheduler clock - returns current time in nanosec units. * This is default implementation. * Architectures and sub-architectures can override this. */ unsigned long long __attribute__((weak)) sched_clock(void) { |
92d23f703 sched: Fix fallba... |
77 78 |
return (unsigned long long)(jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); |
2c3d103ba sched: move sched... |
79 |
} |
b6ac23af2 blkio: fix for mo... |
80 |
EXPORT_SYMBOL_GPL(sched_clock); |
3e51f33fc sched: add option... |
81 |
|
c1955a3d4 sched_clock: dela... |
82 |
static __read_mostly int sched_clock_running; |
3e51f33fc sched: add option... |
83 |
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK |
b342501cd sched: allow arch... |
84 |
__read_mostly int sched_clock_stable; |
3e51f33fc sched: add option... |
85 86 |
struct sched_clock_data { |
3e51f33fc sched: add option... |
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 |
u64 tick_raw; u64 tick_gtod; u64 clock; }; static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); static inline struct sched_clock_data *this_scd(void) { return &__get_cpu_var(sched_clock_data); } static inline struct sched_clock_data *cpu_sdc(int cpu) { return &per_cpu(sched_clock_data, cpu); } void sched_clock_init(void) { u64 ktime_now = ktime_to_ns(ktime_get()); |
3e51f33fc sched: add option... |
107 108 109 110 |
int cpu; for_each_possible_cpu(cpu) { struct sched_clock_data *scd = cpu_sdc(cpu); |
a381759d6 sched: fix sched_... |
111 |
scd->tick_raw = 0; |
3e51f33fc sched: add option... |
112 113 114 |
scd->tick_gtod = ktime_now; scd->clock = ktime_now; } |
a381759d6 sched: fix sched_... |
115 116 |
sched_clock_running = 1; |
3e51f33fc sched: add option... |
117 118 119 |
} /* |
b342501cd sched: allow arch... |
120 |
* min, max except they take wrapping into account |
354879bb9 sched_clock: fix ... |
121 122 123 124 125 126 127 128 129 130 131 132 133 |
*/ static inline u64 wrap_min(u64 x, u64 y) { return (s64)(x - y) < 0 ? x : y; } static inline u64 wrap_max(u64 x, u64 y) { return (s64)(x - y) > 0 ? x : y; } /* |
3e51f33fc sched: add option... |
134 135 136 |
* update the percpu scd from the raw @now value * * - filter out backward motion |
354879bb9 sched_clock: fix ... |
137 |
* - use the GTOD tick value to create a window to filter crazy TSC values |
3e51f33fc sched: add option... |
138 |
*/ |
def0a9b25 sched_clock: Make... |
139 |
static u64 sched_clock_local(struct sched_clock_data *scd) |
3e51f33fc sched: add option... |
140 |
{ |
def0a9b25 sched_clock: Make... |
141 142 |
u64 now, clock, old_clock, min_clock, max_clock; s64 delta; |
3e51f33fc sched: add option... |
143 |
|
def0a9b25 sched_clock: Make... |
144 145 146 |
again: now = sched_clock(); delta = now - scd->tick_raw; |
354879bb9 sched_clock: fix ... |
147 148 |
if (unlikely(delta < 0)) delta = 0; |
3e51f33fc sched: add option... |
149 |
|
def0a9b25 sched_clock: Make... |
150 |
old_clock = scd->clock; |
354879bb9 sched_clock: fix ... |
151 152 |
/* * scd->clock = clamp(scd->tick_gtod + delta, |
b342501cd sched: allow arch... |
153 154 |
* max(scd->tick_gtod, scd->clock), * scd->tick_gtod + TICK_NSEC); |
354879bb9 sched_clock: fix ... |
155 |
*/ |
3e51f33fc sched: add option... |
156 |
|
354879bb9 sched_clock: fix ... |
157 |
clock = scd->tick_gtod + delta; |
def0a9b25 sched_clock: Make... |
158 159 |
min_clock = wrap_max(scd->tick_gtod, old_clock); max_clock = wrap_max(old_clock, scd->tick_gtod + TICK_NSEC); |
3e51f33fc sched: add option... |
160 |
|
354879bb9 sched_clock: fix ... |
161 162 |
clock = wrap_max(clock, min_clock); clock = wrap_min(clock, max_clock); |
3e51f33fc sched: add option... |
163 |
|
152f9d071 sched_clock: Fix ... |
164 |
if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock) |
def0a9b25 sched_clock: Make... |
165 |
goto again; |
56b906126 sched clock: simp... |
166 |
|
def0a9b25 sched_clock: Make... |
167 |
return clock; |
3e51f33fc sched: add option... |
168 |
} |
def0a9b25 sched_clock: Make... |
169 |
static u64 sched_clock_remote(struct sched_clock_data *scd) |
3e51f33fc sched: add option... |
170 |
{ |
def0a9b25 sched_clock: Make... |
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 |
struct sched_clock_data *my_scd = this_scd(); u64 this_clock, remote_clock; u64 *ptr, old_val, val; sched_clock_local(my_scd); again: this_clock = my_scd->clock; remote_clock = scd->clock; /* * Use the opportunity that we have both locks * taken to couple the two clocks: we take the * larger time as the latest time for both * runqueues. (this creates monotonic movement) */ if (likely((s64)(remote_clock - this_clock) < 0)) { ptr = &scd->clock; old_val = remote_clock; val = this_clock; |
3e51f33fc sched: add option... |
190 |
} else { |
def0a9b25 sched_clock: Make... |
191 192 193 194 195 196 |
/* * Should be rare, but possible: */ ptr = &my_scd->clock; old_val = this_clock; val = remote_clock; |
3e51f33fc sched: add option... |
197 |
} |
def0a9b25 sched_clock: Make... |
198 |
|
152f9d071 sched_clock: Fix ... |
199 |
if (cmpxchg64(ptr, old_val, val) != old_val) |
def0a9b25 sched_clock: Make... |
200 201 202 |
goto again; return val; |
3e51f33fc sched: add option... |
203 |
} |
c676329ab sched_clock: Add ... |
204 205 206 207 208 |
/* * Similar to cpu_clock(), but requires local IRQs to be disabled. * * See cpu_clock(). */ |
3e51f33fc sched: add option... |
209 210 |
u64 sched_clock_cpu(int cpu) { |
b342501cd sched: allow arch... |
211 |
struct sched_clock_data *scd; |
def0a9b25 sched_clock: Make... |
212 213 214 |
u64 clock; WARN_ON_ONCE(!irqs_disabled()); |
3e51f33fc sched: add option... |
215 |
|
b342501cd sched: allow arch... |
216 217 |
if (sched_clock_stable) return sched_clock(); |
a381759d6 sched: fix sched_... |
218 |
|
a381759d6 sched: fix sched_... |
219 220 |
if (unlikely(!sched_clock_running)) return 0ull; |
def0a9b25 sched_clock: Make... |
221 |
scd = cpu_sdc(cpu); |
3e51f33fc sched: add option... |
222 |
|
def0a9b25 sched_clock: Make... |
223 224 225 226 |
if (cpu != smp_processor_id()) clock = sched_clock_remote(scd); else clock = sched_clock_local(scd); |
e4e4e534f sched clock: reve... |
227 |
|
3e51f33fc sched: add option... |
228 229 230 231 232 |
return clock; } void sched_clock_tick(void) { |
8325d9c09 sched_clock: clea... |
233 |
struct sched_clock_data *scd; |
3e51f33fc sched: add option... |
234 |
u64 now, now_gtod; |
8325d9c09 sched_clock: clea... |
235 236 |
if (sched_clock_stable) return; |
a381759d6 sched: fix sched_... |
237 238 |
if (unlikely(!sched_clock_running)) return; |
3e51f33fc sched: add option... |
239 |
WARN_ON_ONCE(!irqs_disabled()); |
8325d9c09 sched_clock: clea... |
240 |
scd = this_scd(); |
3e51f33fc sched: add option... |
241 |
now_gtod = ktime_to_ns(ktime_get()); |
a83bc47c3 sched_clock: reco... |
242 |
now = sched_clock(); |
3e51f33fc sched: add option... |
243 |
|
3e51f33fc sched: add option... |
244 245 |
scd->tick_raw = now; scd->tick_gtod = now_gtod; |
def0a9b25 sched_clock: Make... |
246 |
sched_clock_local(scd); |
3e51f33fc sched: add option... |
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 |
} /* * We are going deep-idle (irqs are disabled): */ void sched_clock_idle_sleep_event(void) { sched_clock_cpu(smp_processor_id()); } EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); /* * We just idled delta nanoseconds (called with irqs disabled): */ void sched_clock_idle_wakeup_event(u64 delta_ns) { |
1c5745aa3 sched_clock: prev... |
263 264 |
if (timekeeping_suspended) return; |
354879bb9 sched_clock: fix ... |
265 |
sched_clock_tick(); |
3e51f33fc sched: add option... |
266 267 268 |
touch_softlockup_watchdog(); } EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); |
c676329ab sched_clock: Add ... |
269 270 271 272 273 274 275 276 277 278 279 |
/* * As outlined at the top, provides a fast, high resolution, nanosecond * time source that is monotonic per cpu argument and has bounded drift * between cpus. * * ######################### BIG FAT WARNING ########################## * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can # * # go backwards !! # * #################################################################### */ u64 cpu_clock(int cpu) |
b9f8fcd55 sched: Fix cpu_cl... |
280 |
{ |
c676329ab sched_clock: Add ... |
281 |
u64 clock; |
b9f8fcd55 sched: Fix cpu_cl... |
282 283 284 285 286 287 288 289 |
unsigned long flags; local_irq_save(flags); clock = sched_clock_cpu(cpu); local_irq_restore(flags); return clock; } |
c676329ab sched_clock: Add ... |
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 |
/* * Similar to cpu_clock() for the current cpu. Time will only be observed * to be monotonic if care is taken to only compare timestampt taken on the * same CPU. * * See cpu_clock(). */ u64 local_clock(void) { u64 clock; unsigned long flags; local_irq_save(flags); clock = sched_clock_cpu(smp_processor_id()); local_irq_restore(flags); return clock; } |
8325d9c09 sched_clock: clea... |
308 309 310 311 312 313 314 315 316 317 318 319 320 321 |
#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ void sched_clock_init(void) { sched_clock_running = 1; } u64 sched_clock_cpu(int cpu) { if (unlikely(!sched_clock_running)) return 0; return sched_clock(); } |
c676329ab sched_clock: Add ... |
322 |
u64 cpu_clock(int cpu) |
76a2a6ee8 sched: sched_cloc... |
323 |
{ |
b9f8fcd55 sched: Fix cpu_cl... |
324 325 |
return sched_clock_cpu(cpu); } |
76a2a6ee8 sched: sched_cloc... |
326 |
|
c676329ab sched_clock: Add ... |
327 328 329 330 |
u64 local_clock(void) { return sched_clock_cpu(0); } |
b9f8fcd55 sched: Fix cpu_cl... |
331 |
#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ |
76a2a6ee8 sched: sched_cloc... |
332 |
|
4c9fe8ad8 sched: export cpu... |
333 |
EXPORT_SYMBOL_GPL(cpu_clock); |
c676329ab sched_clock: Add ... |
334 |
EXPORT_SYMBOL_GPL(local_clock); |