Commit d031476408ae0f5196e3c579f519dfdefb099b67
Committed by
Thomas Gleixner
1 parent
e490517a03
Exists in
master
and in
7 other branches
hrtimer: remove warning in hres_timers_resume
hres_timers_resume() warns if there appears to be more than one cpu online. This warning makes sense when the suspend/resume mechanism offlines all cpus but one during the suspend/resume process. However, Xen suspend does not need to offline the other cpus; it merely keeps them tied up in stop_machine() while the virtual machine is suspended. The warning hres_timers_resume issues is therefore spurious. Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> Cc: xen-devel <xen-devel@lists.xensource.com> Cc: "Rafael J. Wysocki" <rjw@sisk.pl> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Showing 1 changed file with 0 additions and 2 deletions Inline Diff
kernel/hrtimer.c
1 | /* | 1 | /* |
2 | * linux/kernel/hrtimer.c | 2 | * linux/kernel/hrtimer.c |
3 | * | 3 | * |
4 | * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> | 4 | * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> |
5 | * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar | 5 | * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar |
6 | * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner | 6 | * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner |
7 | * | 7 | * |
8 | * High-resolution kernel timers | 8 | * High-resolution kernel timers |
9 | * | 9 | * |
10 | * In contrast to the low-resolution timeout API implemented in | 10 | * In contrast to the low-resolution timeout API implemented in |
11 | * kernel/timer.c, hrtimers provide finer resolution and accuracy | 11 | * kernel/timer.c, hrtimers provide finer resolution and accuracy |
12 | * depending on system configuration and capabilities. | 12 | * depending on system configuration and capabilities. |
13 | * | 13 | * |
14 | * These timers are currently used for: | 14 | * These timers are currently used for: |
15 | * - itimers | 15 | * - itimers |
16 | * - POSIX timers | 16 | * - POSIX timers |
17 | * - nanosleep | 17 | * - nanosleep |
18 | * - precise in-kernel timing | 18 | * - precise in-kernel timing |
19 | * | 19 | * |
20 | * Started by: Thomas Gleixner and Ingo Molnar | 20 | * Started by: Thomas Gleixner and Ingo Molnar |
21 | * | 21 | * |
22 | * Credits: | 22 | * Credits: |
23 | * based on kernel/timer.c | 23 | * based on kernel/timer.c |
24 | * | 24 | * |
25 | * Help, testing, suggestions, bugfixes, improvements were | 25 | * Help, testing, suggestions, bugfixes, improvements were |
26 | * provided by: | 26 | * provided by: |
27 | * | 27 | * |
28 | * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel | 28 | * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel |
29 | * et. al. | 29 | * et. al. |
30 | * | 30 | * |
31 | * For licencing details see kernel-base/COPYING | 31 | * For licencing details see kernel-base/COPYING |
32 | */ | 32 | */ |
33 | 33 | ||
34 | #include <linux/cpu.h> | 34 | #include <linux/cpu.h> |
35 | #include <linux/irq.h> | 35 | #include <linux/irq.h> |
36 | #include <linux/module.h> | 36 | #include <linux/module.h> |
37 | #include <linux/percpu.h> | 37 | #include <linux/percpu.h> |
38 | #include <linux/hrtimer.h> | 38 | #include <linux/hrtimer.h> |
39 | #include <linux/notifier.h> | 39 | #include <linux/notifier.h> |
40 | #include <linux/syscalls.h> | 40 | #include <linux/syscalls.h> |
41 | #include <linux/kallsyms.h> | 41 | #include <linux/kallsyms.h> |
42 | #include <linux/interrupt.h> | 42 | #include <linux/interrupt.h> |
43 | #include <linux/tick.h> | 43 | #include <linux/tick.h> |
44 | #include <linux/seq_file.h> | 44 | #include <linux/seq_file.h> |
45 | #include <linux/err.h> | 45 | #include <linux/err.h> |
46 | #include <linux/debugobjects.h> | 46 | #include <linux/debugobjects.h> |
47 | 47 | ||
48 | #include <asm/uaccess.h> | 48 | #include <asm/uaccess.h> |
49 | 49 | ||
50 | /** | 50 | /** |
51 | * ktime_get - get the monotonic time in ktime_t format | 51 | * ktime_get - get the monotonic time in ktime_t format |
52 | * | 52 | * |
53 | * returns the time in ktime_t format | 53 | * returns the time in ktime_t format |
54 | */ | 54 | */ |
55 | ktime_t ktime_get(void) | 55 | ktime_t ktime_get(void) |
56 | { | 56 | { |
57 | struct timespec now; | 57 | struct timespec now; |
58 | 58 | ||
59 | ktime_get_ts(&now); | 59 | ktime_get_ts(&now); |
60 | 60 | ||
61 | return timespec_to_ktime(now); | 61 | return timespec_to_ktime(now); |
62 | } | 62 | } |
63 | EXPORT_SYMBOL_GPL(ktime_get); | 63 | EXPORT_SYMBOL_GPL(ktime_get); |
64 | 64 | ||
65 | /** | 65 | /** |
66 | * ktime_get_real - get the real (wall-) time in ktime_t format | 66 | * ktime_get_real - get the real (wall-) time in ktime_t format |
67 | * | 67 | * |
68 | * returns the time in ktime_t format | 68 | * returns the time in ktime_t format |
69 | */ | 69 | */ |
70 | ktime_t ktime_get_real(void) | 70 | ktime_t ktime_get_real(void) |
71 | { | 71 | { |
72 | struct timespec now; | 72 | struct timespec now; |
73 | 73 | ||
74 | getnstimeofday(&now); | 74 | getnstimeofday(&now); |
75 | 75 | ||
76 | return timespec_to_ktime(now); | 76 | return timespec_to_ktime(now); |
77 | } | 77 | } |
78 | 78 | ||
79 | EXPORT_SYMBOL_GPL(ktime_get_real); | 79 | EXPORT_SYMBOL_GPL(ktime_get_real); |
80 | 80 | ||
81 | /* | 81 | /* |
82 | * The timer bases: | 82 | * The timer bases: |
83 | * | 83 | * |
84 | * Note: If we want to add new timer bases, we have to skip the two | 84 | * Note: If we want to add new timer bases, we have to skip the two |
85 | * clock ids captured by the cpu-timers. We do this by holding empty | 85 | * clock ids captured by the cpu-timers. We do this by holding empty |
86 | * entries rather than doing math adjustment of the clock ids. | 86 | * entries rather than doing math adjustment of the clock ids. |
87 | * This ensures that we capture erroneous accesses to these clock ids | 87 | * This ensures that we capture erroneous accesses to these clock ids |
88 | * rather than moving them into the range of valid clock id's. | 88 | * rather than moving them into the range of valid clock id's. |
89 | */ | 89 | */ |
90 | DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = | 90 | DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = |
91 | { | 91 | { |
92 | 92 | ||
93 | .clock_base = | 93 | .clock_base = |
94 | { | 94 | { |
95 | { | 95 | { |
96 | .index = CLOCK_REALTIME, | 96 | .index = CLOCK_REALTIME, |
97 | .get_time = &ktime_get_real, | 97 | .get_time = &ktime_get_real, |
98 | .resolution = KTIME_LOW_RES, | 98 | .resolution = KTIME_LOW_RES, |
99 | }, | 99 | }, |
100 | { | 100 | { |
101 | .index = CLOCK_MONOTONIC, | 101 | .index = CLOCK_MONOTONIC, |
102 | .get_time = &ktime_get, | 102 | .get_time = &ktime_get, |
103 | .resolution = KTIME_LOW_RES, | 103 | .resolution = KTIME_LOW_RES, |
104 | }, | 104 | }, |
105 | } | 105 | } |
106 | }; | 106 | }; |
107 | 107 | ||
108 | /** | 108 | /** |
109 | * ktime_get_ts - get the monotonic clock in timespec format | 109 | * ktime_get_ts - get the monotonic clock in timespec format |
110 | * @ts: pointer to timespec variable | 110 | * @ts: pointer to timespec variable |
111 | * | 111 | * |
112 | * The function calculates the monotonic clock from the realtime | 112 | * The function calculates the monotonic clock from the realtime |
113 | * clock and the wall_to_monotonic offset and stores the result | 113 | * clock and the wall_to_monotonic offset and stores the result |
114 | * in normalized timespec format in the variable pointed to by @ts. | 114 | * in normalized timespec format in the variable pointed to by @ts. |
115 | */ | 115 | */ |
116 | void ktime_get_ts(struct timespec *ts) | 116 | void ktime_get_ts(struct timespec *ts) |
117 | { | 117 | { |
118 | struct timespec tomono; | 118 | struct timespec tomono; |
119 | unsigned long seq; | 119 | unsigned long seq; |
120 | 120 | ||
121 | do { | 121 | do { |
122 | seq = read_seqbegin(&xtime_lock); | 122 | seq = read_seqbegin(&xtime_lock); |
123 | getnstimeofday(ts); | 123 | getnstimeofday(ts); |
124 | tomono = wall_to_monotonic; | 124 | tomono = wall_to_monotonic; |
125 | 125 | ||
126 | } while (read_seqretry(&xtime_lock, seq)); | 126 | } while (read_seqretry(&xtime_lock, seq)); |
127 | 127 | ||
128 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, | 128 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, |
129 | ts->tv_nsec + tomono.tv_nsec); | 129 | ts->tv_nsec + tomono.tv_nsec); |
130 | } | 130 | } |
131 | EXPORT_SYMBOL_GPL(ktime_get_ts); | 131 | EXPORT_SYMBOL_GPL(ktime_get_ts); |
132 | 132 | ||
133 | /* | 133 | /* |
134 | * Get the coarse grained time at the softirq based on xtime and | 134 | * Get the coarse grained time at the softirq based on xtime and |
135 | * wall_to_monotonic. | 135 | * wall_to_monotonic. |
136 | */ | 136 | */ |
137 | static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) | 137 | static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) |
138 | { | 138 | { |
139 | ktime_t xtim, tomono; | 139 | ktime_t xtim, tomono; |
140 | struct timespec xts, tom; | 140 | struct timespec xts, tom; |
141 | unsigned long seq; | 141 | unsigned long seq; |
142 | 142 | ||
143 | do { | 143 | do { |
144 | seq = read_seqbegin(&xtime_lock); | 144 | seq = read_seqbegin(&xtime_lock); |
145 | xts = current_kernel_time(); | 145 | xts = current_kernel_time(); |
146 | tom = wall_to_monotonic; | 146 | tom = wall_to_monotonic; |
147 | } while (read_seqretry(&xtime_lock, seq)); | 147 | } while (read_seqretry(&xtime_lock, seq)); |
148 | 148 | ||
149 | xtim = timespec_to_ktime(xts); | 149 | xtim = timespec_to_ktime(xts); |
150 | tomono = timespec_to_ktime(tom); | 150 | tomono = timespec_to_ktime(tom); |
151 | base->clock_base[CLOCK_REALTIME].softirq_time = xtim; | 151 | base->clock_base[CLOCK_REALTIME].softirq_time = xtim; |
152 | base->clock_base[CLOCK_MONOTONIC].softirq_time = | 152 | base->clock_base[CLOCK_MONOTONIC].softirq_time = |
153 | ktime_add(xtim, tomono); | 153 | ktime_add(xtim, tomono); |
154 | } | 154 | } |
155 | 155 | ||
156 | /* | 156 | /* |
157 | * Functions and macros which are different for UP/SMP systems are kept in a | 157 | * Functions and macros which are different for UP/SMP systems are kept in a |
158 | * single place | 158 | * single place |
159 | */ | 159 | */ |
160 | #ifdef CONFIG_SMP | 160 | #ifdef CONFIG_SMP |
161 | 161 | ||
162 | /* | 162 | /* |
163 | * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock | 163 | * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock |
164 | * means that all timers which are tied to this base via timer->base are | 164 | * means that all timers which are tied to this base via timer->base are |
165 | * locked, and the base itself is locked too. | 165 | * locked, and the base itself is locked too. |
166 | * | 166 | * |
167 | * So __run_timers/migrate_timers can safely modify all timers which could | 167 | * So __run_timers/migrate_timers can safely modify all timers which could |
168 | * be found on the lists/queues. | 168 | * be found on the lists/queues. |
169 | * | 169 | * |
170 | * When the timer's base is locked, and the timer removed from list, it is | 170 | * When the timer's base is locked, and the timer removed from list, it is |
171 | * possible to set timer->base = NULL and drop the lock: the timer remains | 171 | * possible to set timer->base = NULL and drop the lock: the timer remains |
172 | * locked. | 172 | * locked. |
173 | */ | 173 | */ |
174 | static | 174 | static |
175 | struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, | 175 | struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, |
176 | unsigned long *flags) | 176 | unsigned long *flags) |
177 | { | 177 | { |
178 | struct hrtimer_clock_base *base; | 178 | struct hrtimer_clock_base *base; |
179 | 179 | ||
180 | for (;;) { | 180 | for (;;) { |
181 | base = timer->base; | 181 | base = timer->base; |
182 | if (likely(base != NULL)) { | 182 | if (likely(base != NULL)) { |
183 | spin_lock_irqsave(&base->cpu_base->lock, *flags); | 183 | spin_lock_irqsave(&base->cpu_base->lock, *flags); |
184 | if (likely(base == timer->base)) | 184 | if (likely(base == timer->base)) |
185 | return base; | 185 | return base; |
186 | /* The timer has migrated to another CPU: */ | 186 | /* The timer has migrated to another CPU: */ |
187 | spin_unlock_irqrestore(&base->cpu_base->lock, *flags); | 187 | spin_unlock_irqrestore(&base->cpu_base->lock, *flags); |
188 | } | 188 | } |
189 | cpu_relax(); | 189 | cpu_relax(); |
190 | } | 190 | } |
191 | } | 191 | } |
192 | 192 | ||
193 | /* | 193 | /* |
194 | * Switch the timer base to the current CPU when possible. | 194 | * Switch the timer base to the current CPU when possible. |
195 | */ | 195 | */ |
196 | static inline struct hrtimer_clock_base * | 196 | static inline struct hrtimer_clock_base * |
197 | switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base) | 197 | switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base) |
198 | { | 198 | { |
199 | struct hrtimer_clock_base *new_base; | 199 | struct hrtimer_clock_base *new_base; |
200 | struct hrtimer_cpu_base *new_cpu_base; | 200 | struct hrtimer_cpu_base *new_cpu_base; |
201 | 201 | ||
202 | new_cpu_base = &__get_cpu_var(hrtimer_bases); | 202 | new_cpu_base = &__get_cpu_var(hrtimer_bases); |
203 | new_base = &new_cpu_base->clock_base[base->index]; | 203 | new_base = &new_cpu_base->clock_base[base->index]; |
204 | 204 | ||
205 | if (base != new_base) { | 205 | if (base != new_base) { |
206 | /* | 206 | /* |
207 | * We are trying to schedule the timer on the local CPU. | 207 | * We are trying to schedule the timer on the local CPU. |
208 | * However we can't change timer's base while it is running, | 208 | * However we can't change timer's base while it is running, |
209 | * so we keep it on the same CPU. No hassle vs. reprogramming | 209 | * so we keep it on the same CPU. No hassle vs. reprogramming |
210 | * the event source in the high resolution case. The softirq | 210 | * the event source in the high resolution case. The softirq |
211 | * code will take care of this when the timer function has | 211 | * code will take care of this when the timer function has |
212 | * completed. There is no conflict as we hold the lock until | 212 | * completed. There is no conflict as we hold the lock until |
213 | * the timer is enqueued. | 213 | * the timer is enqueued. |
214 | */ | 214 | */ |
215 | if (unlikely(hrtimer_callback_running(timer))) | 215 | if (unlikely(hrtimer_callback_running(timer))) |
216 | return base; | 216 | return base; |
217 | 217 | ||
218 | /* See the comment in lock_timer_base() */ | 218 | /* See the comment in lock_timer_base() */ |
219 | timer->base = NULL; | 219 | timer->base = NULL; |
220 | spin_unlock(&base->cpu_base->lock); | 220 | spin_unlock(&base->cpu_base->lock); |
221 | spin_lock(&new_base->cpu_base->lock); | 221 | spin_lock(&new_base->cpu_base->lock); |
222 | timer->base = new_base; | 222 | timer->base = new_base; |
223 | } | 223 | } |
224 | return new_base; | 224 | return new_base; |
225 | } | 225 | } |
226 | 226 | ||
227 | #else /* CONFIG_SMP */ | 227 | #else /* CONFIG_SMP */ |
228 | 228 | ||
229 | static inline struct hrtimer_clock_base * | 229 | static inline struct hrtimer_clock_base * |
230 | lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | 230 | lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) |
231 | { | 231 | { |
232 | struct hrtimer_clock_base *base = timer->base; | 232 | struct hrtimer_clock_base *base = timer->base; |
233 | 233 | ||
234 | spin_lock_irqsave(&base->cpu_base->lock, *flags); | 234 | spin_lock_irqsave(&base->cpu_base->lock, *flags); |
235 | 235 | ||
236 | return base; | 236 | return base; |
237 | } | 237 | } |
238 | 238 | ||
239 | # define switch_hrtimer_base(t, b) (b) | 239 | # define switch_hrtimer_base(t, b) (b) |
240 | 240 | ||
241 | #endif /* !CONFIG_SMP */ | 241 | #endif /* !CONFIG_SMP */ |
242 | 242 | ||
243 | /* | 243 | /* |
244 | * Functions for the union type storage format of ktime_t which are | 244 | * Functions for the union type storage format of ktime_t which are |
245 | * too large for inlining: | 245 | * too large for inlining: |
246 | */ | 246 | */ |
247 | #if BITS_PER_LONG < 64 | 247 | #if BITS_PER_LONG < 64 |
248 | # ifndef CONFIG_KTIME_SCALAR | 248 | # ifndef CONFIG_KTIME_SCALAR |
249 | /** | 249 | /** |
250 | * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable | 250 | * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable |
251 | * @kt: addend | 251 | * @kt: addend |
252 | * @nsec: the scalar nsec value to add | 252 | * @nsec: the scalar nsec value to add |
253 | * | 253 | * |
254 | * Returns the sum of kt and nsec in ktime_t format | 254 | * Returns the sum of kt and nsec in ktime_t format |
255 | */ | 255 | */ |
256 | ktime_t ktime_add_ns(const ktime_t kt, u64 nsec) | 256 | ktime_t ktime_add_ns(const ktime_t kt, u64 nsec) |
257 | { | 257 | { |
258 | ktime_t tmp; | 258 | ktime_t tmp; |
259 | 259 | ||
260 | if (likely(nsec < NSEC_PER_SEC)) { | 260 | if (likely(nsec < NSEC_PER_SEC)) { |
261 | tmp.tv64 = nsec; | 261 | tmp.tv64 = nsec; |
262 | } else { | 262 | } else { |
263 | unsigned long rem = do_div(nsec, NSEC_PER_SEC); | 263 | unsigned long rem = do_div(nsec, NSEC_PER_SEC); |
264 | 264 | ||
265 | tmp = ktime_set((long)nsec, rem); | 265 | tmp = ktime_set((long)nsec, rem); |
266 | } | 266 | } |
267 | 267 | ||
268 | return ktime_add(kt, tmp); | 268 | return ktime_add(kt, tmp); |
269 | } | 269 | } |
270 | 270 | ||
271 | EXPORT_SYMBOL_GPL(ktime_add_ns); | 271 | EXPORT_SYMBOL_GPL(ktime_add_ns); |
272 | 272 | ||
273 | /** | 273 | /** |
274 | * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable | 274 | * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable |
275 | * @kt: minuend | 275 | * @kt: minuend |
276 | * @nsec: the scalar nsec value to subtract | 276 | * @nsec: the scalar nsec value to subtract |
277 | * | 277 | * |
278 | * Returns the subtraction of @nsec from @kt in ktime_t format | 278 | * Returns the subtraction of @nsec from @kt in ktime_t format |
279 | */ | 279 | */ |
280 | ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec) | 280 | ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec) |
281 | { | 281 | { |
282 | ktime_t tmp; | 282 | ktime_t tmp; |
283 | 283 | ||
284 | if (likely(nsec < NSEC_PER_SEC)) { | 284 | if (likely(nsec < NSEC_PER_SEC)) { |
285 | tmp.tv64 = nsec; | 285 | tmp.tv64 = nsec; |
286 | } else { | 286 | } else { |
287 | unsigned long rem = do_div(nsec, NSEC_PER_SEC); | 287 | unsigned long rem = do_div(nsec, NSEC_PER_SEC); |
288 | 288 | ||
289 | tmp = ktime_set((long)nsec, rem); | 289 | tmp = ktime_set((long)nsec, rem); |
290 | } | 290 | } |
291 | 291 | ||
292 | return ktime_sub(kt, tmp); | 292 | return ktime_sub(kt, tmp); |
293 | } | 293 | } |
294 | 294 | ||
295 | EXPORT_SYMBOL_GPL(ktime_sub_ns); | 295 | EXPORT_SYMBOL_GPL(ktime_sub_ns); |
296 | # endif /* !CONFIG_KTIME_SCALAR */ | 296 | # endif /* !CONFIG_KTIME_SCALAR */ |
297 | 297 | ||
298 | /* | 298 | /* |
299 | * Divide a ktime value by a nanosecond value | 299 | * Divide a ktime value by a nanosecond value |
300 | */ | 300 | */ |
301 | u64 ktime_divns(const ktime_t kt, s64 div) | 301 | u64 ktime_divns(const ktime_t kt, s64 div) |
302 | { | 302 | { |
303 | u64 dclc, inc, dns; | 303 | u64 dclc, inc, dns; |
304 | int sft = 0; | 304 | int sft = 0; |
305 | 305 | ||
306 | dclc = dns = ktime_to_ns(kt); | 306 | dclc = dns = ktime_to_ns(kt); |
307 | inc = div; | 307 | inc = div; |
308 | /* Make sure the divisor is less than 2^32: */ | 308 | /* Make sure the divisor is less than 2^32: */ |
309 | while (div >> 32) { | 309 | while (div >> 32) { |
310 | sft++; | 310 | sft++; |
311 | div >>= 1; | 311 | div >>= 1; |
312 | } | 312 | } |
313 | dclc >>= sft; | 313 | dclc >>= sft; |
314 | do_div(dclc, (unsigned long) div); | 314 | do_div(dclc, (unsigned long) div); |
315 | 315 | ||
316 | return dclc; | 316 | return dclc; |
317 | } | 317 | } |
318 | #endif /* BITS_PER_LONG >= 64 */ | 318 | #endif /* BITS_PER_LONG >= 64 */ |
319 | 319 | ||
320 | /* | 320 | /* |
321 | * Add two ktime values and do a safety check for overflow: | 321 | * Add two ktime values and do a safety check for overflow: |
322 | */ | 322 | */ |
323 | ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs) | 323 | ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs) |
324 | { | 324 | { |
325 | ktime_t res = ktime_add(lhs, rhs); | 325 | ktime_t res = ktime_add(lhs, rhs); |
326 | 326 | ||
327 | /* | 327 | /* |
328 | * We use KTIME_SEC_MAX here, the maximum timeout which we can | 328 | * We use KTIME_SEC_MAX here, the maximum timeout which we can |
329 | * return to user space in a timespec: | 329 | * return to user space in a timespec: |
330 | */ | 330 | */ |
331 | if (res.tv64 < 0 || res.tv64 < lhs.tv64 || res.tv64 < rhs.tv64) | 331 | if (res.tv64 < 0 || res.tv64 < lhs.tv64 || res.tv64 < rhs.tv64) |
332 | res = ktime_set(KTIME_SEC_MAX, 0); | 332 | res = ktime_set(KTIME_SEC_MAX, 0); |
333 | 333 | ||
334 | return res; | 334 | return res; |
335 | } | 335 | } |
336 | 336 | ||
337 | #ifdef CONFIG_DEBUG_OBJECTS_TIMERS | 337 | #ifdef CONFIG_DEBUG_OBJECTS_TIMERS |
338 | 338 | ||
339 | static struct debug_obj_descr hrtimer_debug_descr; | 339 | static struct debug_obj_descr hrtimer_debug_descr; |
340 | 340 | ||
341 | /* | 341 | /* |
342 | * fixup_init is called when: | 342 | * fixup_init is called when: |
343 | * - an active object is initialized | 343 | * - an active object is initialized |
344 | */ | 344 | */ |
345 | static int hrtimer_fixup_init(void *addr, enum debug_obj_state state) | 345 | static int hrtimer_fixup_init(void *addr, enum debug_obj_state state) |
346 | { | 346 | { |
347 | struct hrtimer *timer = addr; | 347 | struct hrtimer *timer = addr; |
348 | 348 | ||
349 | switch (state) { | 349 | switch (state) { |
350 | case ODEBUG_STATE_ACTIVE: | 350 | case ODEBUG_STATE_ACTIVE: |
351 | hrtimer_cancel(timer); | 351 | hrtimer_cancel(timer); |
352 | debug_object_init(timer, &hrtimer_debug_descr); | 352 | debug_object_init(timer, &hrtimer_debug_descr); |
353 | return 1; | 353 | return 1; |
354 | default: | 354 | default: |
355 | return 0; | 355 | return 0; |
356 | } | 356 | } |
357 | } | 357 | } |
358 | 358 | ||
359 | /* | 359 | /* |
360 | * fixup_activate is called when: | 360 | * fixup_activate is called when: |
361 | * - an active object is activated | 361 | * - an active object is activated |
362 | * - an unknown object is activated (might be a statically initialized object) | 362 | * - an unknown object is activated (might be a statically initialized object) |
363 | */ | 363 | */ |
364 | static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state) | 364 | static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state) |
365 | { | 365 | { |
366 | switch (state) { | 366 | switch (state) { |
367 | 367 | ||
368 | case ODEBUG_STATE_NOTAVAILABLE: | 368 | case ODEBUG_STATE_NOTAVAILABLE: |
369 | WARN_ON_ONCE(1); | 369 | WARN_ON_ONCE(1); |
370 | return 0; | 370 | return 0; |
371 | 371 | ||
372 | case ODEBUG_STATE_ACTIVE: | 372 | case ODEBUG_STATE_ACTIVE: |
373 | WARN_ON(1); | 373 | WARN_ON(1); |
374 | 374 | ||
375 | default: | 375 | default: |
376 | return 0; | 376 | return 0; |
377 | } | 377 | } |
378 | } | 378 | } |
379 | 379 | ||
380 | /* | 380 | /* |
381 | * fixup_free is called when: | 381 | * fixup_free is called when: |
382 | * - an active object is freed | 382 | * - an active object is freed |
383 | */ | 383 | */ |
384 | static int hrtimer_fixup_free(void *addr, enum debug_obj_state state) | 384 | static int hrtimer_fixup_free(void *addr, enum debug_obj_state state) |
385 | { | 385 | { |
386 | struct hrtimer *timer = addr; | 386 | struct hrtimer *timer = addr; |
387 | 387 | ||
388 | switch (state) { | 388 | switch (state) { |
389 | case ODEBUG_STATE_ACTIVE: | 389 | case ODEBUG_STATE_ACTIVE: |
390 | hrtimer_cancel(timer); | 390 | hrtimer_cancel(timer); |
391 | debug_object_free(timer, &hrtimer_debug_descr); | 391 | debug_object_free(timer, &hrtimer_debug_descr); |
392 | return 1; | 392 | return 1; |
393 | default: | 393 | default: |
394 | return 0; | 394 | return 0; |
395 | } | 395 | } |
396 | } | 396 | } |
397 | 397 | ||
398 | static struct debug_obj_descr hrtimer_debug_descr = { | 398 | static struct debug_obj_descr hrtimer_debug_descr = { |
399 | .name = "hrtimer", | 399 | .name = "hrtimer", |
400 | .fixup_init = hrtimer_fixup_init, | 400 | .fixup_init = hrtimer_fixup_init, |
401 | .fixup_activate = hrtimer_fixup_activate, | 401 | .fixup_activate = hrtimer_fixup_activate, |
402 | .fixup_free = hrtimer_fixup_free, | 402 | .fixup_free = hrtimer_fixup_free, |
403 | }; | 403 | }; |
404 | 404 | ||
405 | static inline void debug_hrtimer_init(struct hrtimer *timer) | 405 | static inline void debug_hrtimer_init(struct hrtimer *timer) |
406 | { | 406 | { |
407 | debug_object_init(timer, &hrtimer_debug_descr); | 407 | debug_object_init(timer, &hrtimer_debug_descr); |
408 | } | 408 | } |
409 | 409 | ||
410 | static inline void debug_hrtimer_activate(struct hrtimer *timer) | 410 | static inline void debug_hrtimer_activate(struct hrtimer *timer) |
411 | { | 411 | { |
412 | debug_object_activate(timer, &hrtimer_debug_descr); | 412 | debug_object_activate(timer, &hrtimer_debug_descr); |
413 | } | 413 | } |
414 | 414 | ||
415 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) | 415 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) |
416 | { | 416 | { |
417 | debug_object_deactivate(timer, &hrtimer_debug_descr); | 417 | debug_object_deactivate(timer, &hrtimer_debug_descr); |
418 | } | 418 | } |
419 | 419 | ||
420 | static inline void debug_hrtimer_free(struct hrtimer *timer) | 420 | static inline void debug_hrtimer_free(struct hrtimer *timer) |
421 | { | 421 | { |
422 | debug_object_free(timer, &hrtimer_debug_descr); | 422 | debug_object_free(timer, &hrtimer_debug_descr); |
423 | } | 423 | } |
424 | 424 | ||
425 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | 425 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, |
426 | enum hrtimer_mode mode); | 426 | enum hrtimer_mode mode); |
427 | 427 | ||
428 | void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, | 428 | void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, |
429 | enum hrtimer_mode mode) | 429 | enum hrtimer_mode mode) |
430 | { | 430 | { |
431 | debug_object_init_on_stack(timer, &hrtimer_debug_descr); | 431 | debug_object_init_on_stack(timer, &hrtimer_debug_descr); |
432 | __hrtimer_init(timer, clock_id, mode); | 432 | __hrtimer_init(timer, clock_id, mode); |
433 | } | 433 | } |
434 | 434 | ||
435 | void destroy_hrtimer_on_stack(struct hrtimer *timer) | 435 | void destroy_hrtimer_on_stack(struct hrtimer *timer) |
436 | { | 436 | { |
437 | debug_object_free(timer, &hrtimer_debug_descr); | 437 | debug_object_free(timer, &hrtimer_debug_descr); |
438 | } | 438 | } |
439 | 439 | ||
440 | #else | 440 | #else |
441 | static inline void debug_hrtimer_init(struct hrtimer *timer) { } | 441 | static inline void debug_hrtimer_init(struct hrtimer *timer) { } |
442 | static inline void debug_hrtimer_activate(struct hrtimer *timer) { } | 442 | static inline void debug_hrtimer_activate(struct hrtimer *timer) { } |
443 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } | 443 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } |
444 | #endif | 444 | #endif |
445 | 445 | ||
446 | /* | 446 | /* |
447 | * Check, whether the timer is on the callback pending list | 447 | * Check, whether the timer is on the callback pending list |
448 | */ | 448 | */ |
449 | static inline int hrtimer_cb_pending(const struct hrtimer *timer) | 449 | static inline int hrtimer_cb_pending(const struct hrtimer *timer) |
450 | { | 450 | { |
451 | return timer->state & HRTIMER_STATE_PENDING; | 451 | return timer->state & HRTIMER_STATE_PENDING; |
452 | } | 452 | } |
453 | 453 | ||
454 | /* | 454 | /* |
455 | * Remove a timer from the callback pending list | 455 | * Remove a timer from the callback pending list |
456 | */ | 456 | */ |
457 | static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) | 457 | static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) |
458 | { | 458 | { |
459 | list_del_init(&timer->cb_entry); | 459 | list_del_init(&timer->cb_entry); |
460 | } | 460 | } |
461 | 461 | ||
462 | /* High resolution timer related functions */ | 462 | /* High resolution timer related functions */ |
463 | #ifdef CONFIG_HIGH_RES_TIMERS | 463 | #ifdef CONFIG_HIGH_RES_TIMERS |
464 | 464 | ||
465 | /* | 465 | /* |
466 | * High resolution timer enabled ? | 466 | * High resolution timer enabled ? |
467 | */ | 467 | */ |
468 | static int hrtimer_hres_enabled __read_mostly = 1; | 468 | static int hrtimer_hres_enabled __read_mostly = 1; |
469 | 469 | ||
470 | /* | 470 | /* |
471 | * Enable / Disable high resolution mode | 471 | * Enable / Disable high resolution mode |
472 | */ | 472 | */ |
473 | static int __init setup_hrtimer_hres(char *str) | 473 | static int __init setup_hrtimer_hres(char *str) |
474 | { | 474 | { |
475 | if (!strcmp(str, "off")) | 475 | if (!strcmp(str, "off")) |
476 | hrtimer_hres_enabled = 0; | 476 | hrtimer_hres_enabled = 0; |
477 | else if (!strcmp(str, "on")) | 477 | else if (!strcmp(str, "on")) |
478 | hrtimer_hres_enabled = 1; | 478 | hrtimer_hres_enabled = 1; |
479 | else | 479 | else |
480 | return 0; | 480 | return 0; |
481 | return 1; | 481 | return 1; |
482 | } | 482 | } |
483 | 483 | ||
484 | __setup("highres=", setup_hrtimer_hres); | 484 | __setup("highres=", setup_hrtimer_hres); |
485 | 485 | ||
486 | /* | 486 | /* |
487 | * hrtimer_high_res_enabled - query, if the highres mode is enabled | 487 | * hrtimer_high_res_enabled - query, if the highres mode is enabled |
488 | */ | 488 | */ |
489 | static inline int hrtimer_is_hres_enabled(void) | 489 | static inline int hrtimer_is_hres_enabled(void) |
490 | { | 490 | { |
491 | return hrtimer_hres_enabled; | 491 | return hrtimer_hres_enabled; |
492 | } | 492 | } |
493 | 493 | ||
494 | /* | 494 | /* |
495 | * Is the high resolution mode active ? | 495 | * Is the high resolution mode active ? |
496 | */ | 496 | */ |
497 | static inline int hrtimer_hres_active(void) | 497 | static inline int hrtimer_hres_active(void) |
498 | { | 498 | { |
499 | return __get_cpu_var(hrtimer_bases).hres_active; | 499 | return __get_cpu_var(hrtimer_bases).hres_active; |
500 | } | 500 | } |
501 | 501 | ||
502 | /* | 502 | /* |
503 | * Reprogram the event source with checking both queues for the | 503 | * Reprogram the event source with checking both queues for the |
504 | * next event | 504 | * next event |
505 | * Called with interrupts disabled and base->lock held | 505 | * Called with interrupts disabled and base->lock held |
506 | */ | 506 | */ |
507 | static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base) | 507 | static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base) |
508 | { | 508 | { |
509 | int i; | 509 | int i; |
510 | struct hrtimer_clock_base *base = cpu_base->clock_base; | 510 | struct hrtimer_clock_base *base = cpu_base->clock_base; |
511 | ktime_t expires; | 511 | ktime_t expires; |
512 | 512 | ||
513 | cpu_base->expires_next.tv64 = KTIME_MAX; | 513 | cpu_base->expires_next.tv64 = KTIME_MAX; |
514 | 514 | ||
515 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { | 515 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { |
516 | struct hrtimer *timer; | 516 | struct hrtimer *timer; |
517 | 517 | ||
518 | if (!base->first) | 518 | if (!base->first) |
519 | continue; | 519 | continue; |
520 | timer = rb_entry(base->first, struct hrtimer, node); | 520 | timer = rb_entry(base->first, struct hrtimer, node); |
521 | expires = ktime_sub(timer->expires, base->offset); | 521 | expires = ktime_sub(timer->expires, base->offset); |
522 | if (expires.tv64 < cpu_base->expires_next.tv64) | 522 | if (expires.tv64 < cpu_base->expires_next.tv64) |
523 | cpu_base->expires_next = expires; | 523 | cpu_base->expires_next = expires; |
524 | } | 524 | } |
525 | 525 | ||
526 | if (cpu_base->expires_next.tv64 != KTIME_MAX) | 526 | if (cpu_base->expires_next.tv64 != KTIME_MAX) |
527 | tick_program_event(cpu_base->expires_next, 1); | 527 | tick_program_event(cpu_base->expires_next, 1); |
528 | } | 528 | } |
529 | 529 | ||
530 | /* | 530 | /* |
531 | * Shared reprogramming for clock_realtime and clock_monotonic | 531 | * Shared reprogramming for clock_realtime and clock_monotonic |
532 | * | 532 | * |
533 | * When a timer is enqueued and expires earlier than the already enqueued | 533 | * When a timer is enqueued and expires earlier than the already enqueued |
534 | * timers, we have to check, whether it expires earlier than the timer for | 534 | * timers, we have to check, whether it expires earlier than the timer for |
535 | * which the clock event device was armed. | 535 | * which the clock event device was armed. |
536 | * | 536 | * |
537 | * Called with interrupts disabled and base->cpu_base.lock held | 537 | * Called with interrupts disabled and base->cpu_base.lock held |
538 | */ | 538 | */ |
539 | static int hrtimer_reprogram(struct hrtimer *timer, | 539 | static int hrtimer_reprogram(struct hrtimer *timer, |
540 | struct hrtimer_clock_base *base) | 540 | struct hrtimer_clock_base *base) |
541 | { | 541 | { |
542 | ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next; | 542 | ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next; |
543 | ktime_t expires = ktime_sub(timer->expires, base->offset); | 543 | ktime_t expires = ktime_sub(timer->expires, base->offset); |
544 | int res; | 544 | int res; |
545 | 545 | ||
546 | WARN_ON_ONCE(timer->expires.tv64 < 0); | 546 | WARN_ON_ONCE(timer->expires.tv64 < 0); |
547 | 547 | ||
548 | /* | 548 | /* |
549 | * When the callback is running, we do not reprogram the clock event | 549 | * When the callback is running, we do not reprogram the clock event |
550 | * device. The timer callback is either running on a different CPU or | 550 | * device. The timer callback is either running on a different CPU or |
551 | * the callback is executed in the hrtimer_interrupt context. The | 551 | * the callback is executed in the hrtimer_interrupt context. The |
552 | * reprogramming is handled either by the softirq, which called the | 552 | * reprogramming is handled either by the softirq, which called the |
553 | * callback or at the end of the hrtimer_interrupt. | 553 | * callback or at the end of the hrtimer_interrupt. |
554 | */ | 554 | */ |
555 | if (hrtimer_callback_running(timer)) | 555 | if (hrtimer_callback_running(timer)) |
556 | return 0; | 556 | return 0; |
557 | 557 | ||
558 | /* | 558 | /* |
559 | * CLOCK_REALTIME timer might be requested with an absolute | 559 | * CLOCK_REALTIME timer might be requested with an absolute |
560 | * expiry time which is less than base->offset. Nothing wrong | 560 | * expiry time which is less than base->offset. Nothing wrong |
561 | * about that, just avoid to call into the tick code, which | 561 | * about that, just avoid to call into the tick code, which |
562 | * has now objections against negative expiry values. | 562 | * has now objections against negative expiry values. |
563 | */ | 563 | */ |
564 | if (expires.tv64 < 0) | 564 | if (expires.tv64 < 0) |
565 | return -ETIME; | 565 | return -ETIME; |
566 | 566 | ||
567 | if (expires.tv64 >= expires_next->tv64) | 567 | if (expires.tv64 >= expires_next->tv64) |
568 | return 0; | 568 | return 0; |
569 | 569 | ||
570 | /* | 570 | /* |
571 | * Clockevents returns -ETIME, when the event was in the past. | 571 | * Clockevents returns -ETIME, when the event was in the past. |
572 | */ | 572 | */ |
573 | res = tick_program_event(expires, 0); | 573 | res = tick_program_event(expires, 0); |
574 | if (!IS_ERR_VALUE(res)) | 574 | if (!IS_ERR_VALUE(res)) |
575 | *expires_next = expires; | 575 | *expires_next = expires; |
576 | return res; | 576 | return res; |
577 | } | 577 | } |
578 | 578 | ||
579 | 579 | ||
580 | /* | 580 | /* |
581 | * Retrigger next event is called after clock was set | 581 | * Retrigger next event is called after clock was set |
582 | * | 582 | * |
583 | * Called with interrupts disabled via on_each_cpu() | 583 | * Called with interrupts disabled via on_each_cpu() |
584 | */ | 584 | */ |
585 | static void retrigger_next_event(void *arg) | 585 | static void retrigger_next_event(void *arg) |
586 | { | 586 | { |
587 | struct hrtimer_cpu_base *base; | 587 | struct hrtimer_cpu_base *base; |
588 | struct timespec realtime_offset; | 588 | struct timespec realtime_offset; |
589 | unsigned long seq; | 589 | unsigned long seq; |
590 | 590 | ||
591 | if (!hrtimer_hres_active()) | 591 | if (!hrtimer_hres_active()) |
592 | return; | 592 | return; |
593 | 593 | ||
594 | do { | 594 | do { |
595 | seq = read_seqbegin(&xtime_lock); | 595 | seq = read_seqbegin(&xtime_lock); |
596 | set_normalized_timespec(&realtime_offset, | 596 | set_normalized_timespec(&realtime_offset, |
597 | -wall_to_monotonic.tv_sec, | 597 | -wall_to_monotonic.tv_sec, |
598 | -wall_to_monotonic.tv_nsec); | 598 | -wall_to_monotonic.tv_nsec); |
599 | } while (read_seqretry(&xtime_lock, seq)); | 599 | } while (read_seqretry(&xtime_lock, seq)); |
600 | 600 | ||
601 | base = &__get_cpu_var(hrtimer_bases); | 601 | base = &__get_cpu_var(hrtimer_bases); |
602 | 602 | ||
603 | /* Adjust CLOCK_REALTIME offset */ | 603 | /* Adjust CLOCK_REALTIME offset */ |
604 | spin_lock(&base->lock); | 604 | spin_lock(&base->lock); |
605 | base->clock_base[CLOCK_REALTIME].offset = | 605 | base->clock_base[CLOCK_REALTIME].offset = |
606 | timespec_to_ktime(realtime_offset); | 606 | timespec_to_ktime(realtime_offset); |
607 | 607 | ||
608 | hrtimer_force_reprogram(base); | 608 | hrtimer_force_reprogram(base); |
609 | spin_unlock(&base->lock); | 609 | spin_unlock(&base->lock); |
610 | } | 610 | } |
611 | 611 | ||
612 | /* | 612 | /* |
613 | * Clock realtime was set | 613 | * Clock realtime was set |
614 | * | 614 | * |
615 | * Change the offset of the realtime clock vs. the monotonic | 615 | * Change the offset of the realtime clock vs. the monotonic |
616 | * clock. | 616 | * clock. |
617 | * | 617 | * |
618 | * We might have to reprogram the high resolution timer interrupt. On | 618 | * We might have to reprogram the high resolution timer interrupt. On |
619 | * SMP we call the architecture specific code to retrigger _all_ high | 619 | * SMP we call the architecture specific code to retrigger _all_ high |
620 | * resolution timer interrupts. On UP we just disable interrupts and | 620 | * resolution timer interrupts. On UP we just disable interrupts and |
621 | * call the high resolution interrupt code. | 621 | * call the high resolution interrupt code. |
622 | */ | 622 | */ |
623 | void clock_was_set(void) | 623 | void clock_was_set(void) |
624 | { | 624 | { |
625 | /* Retrigger the CPU local events everywhere */ | 625 | /* Retrigger the CPU local events everywhere */ |
626 | on_each_cpu(retrigger_next_event, NULL, 0, 1); | 626 | on_each_cpu(retrigger_next_event, NULL, 0, 1); |
627 | } | 627 | } |
628 | 628 | ||
629 | /* | 629 | /* |
630 | * During resume we might have to reprogram the high resolution timer | 630 | * During resume we might have to reprogram the high resolution timer |
631 | * interrupt (on the local CPU): | 631 | * interrupt (on the local CPU): |
632 | */ | 632 | */ |
633 | void hres_timers_resume(void) | 633 | void hres_timers_resume(void) |
634 | { | 634 | { |
635 | WARN_ON_ONCE(num_online_cpus() > 1); | ||
636 | |||
637 | /* Retrigger the CPU local events: */ | 635 | /* Retrigger the CPU local events: */ |
638 | retrigger_next_event(NULL); | 636 | retrigger_next_event(NULL); |
639 | } | 637 | } |
640 | 638 | ||
641 | /* | 639 | /* |
642 | * Initialize the high resolution related parts of cpu_base | 640 | * Initialize the high resolution related parts of cpu_base |
643 | */ | 641 | */ |
644 | static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) | 642 | static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) |
645 | { | 643 | { |
646 | base->expires_next.tv64 = KTIME_MAX; | 644 | base->expires_next.tv64 = KTIME_MAX; |
647 | base->hres_active = 0; | 645 | base->hres_active = 0; |
648 | } | 646 | } |
649 | 647 | ||
650 | /* | 648 | /* |
651 | * Initialize the high resolution related parts of a hrtimer | 649 | * Initialize the high resolution related parts of a hrtimer |
652 | */ | 650 | */ |
653 | static inline void hrtimer_init_timer_hres(struct hrtimer *timer) | 651 | static inline void hrtimer_init_timer_hres(struct hrtimer *timer) |
654 | { | 652 | { |
655 | } | 653 | } |
656 | 654 | ||
657 | /* | 655 | /* |
658 | * When High resolution timers are active, try to reprogram. Note, that in case | 656 | * When High resolution timers are active, try to reprogram. Note, that in case |
659 | * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry | 657 | * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry |
660 | * check happens. The timer gets enqueued into the rbtree. The reprogramming | 658 | * check happens. The timer gets enqueued into the rbtree. The reprogramming |
661 | * and expiry check is done in the hrtimer_interrupt or in the softirq. | 659 | * and expiry check is done in the hrtimer_interrupt or in the softirq. |
662 | */ | 660 | */ |
663 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | 661 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, |
664 | struct hrtimer_clock_base *base) | 662 | struct hrtimer_clock_base *base) |
665 | { | 663 | { |
666 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { | 664 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { |
667 | 665 | ||
668 | /* Timer is expired, act upon the callback mode */ | 666 | /* Timer is expired, act upon the callback mode */ |
669 | switch(timer->cb_mode) { | 667 | switch(timer->cb_mode) { |
670 | case HRTIMER_CB_IRQSAFE_NO_RESTART: | 668 | case HRTIMER_CB_IRQSAFE_NO_RESTART: |
671 | debug_hrtimer_deactivate(timer); | 669 | debug_hrtimer_deactivate(timer); |
672 | /* | 670 | /* |
673 | * We can call the callback from here. No restart | 671 | * We can call the callback from here. No restart |
674 | * happens, so no danger of recursion | 672 | * happens, so no danger of recursion |
675 | */ | 673 | */ |
676 | BUG_ON(timer->function(timer) != HRTIMER_NORESTART); | 674 | BUG_ON(timer->function(timer) != HRTIMER_NORESTART); |
677 | return 1; | 675 | return 1; |
678 | case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ: | 676 | case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ: |
679 | /* | 677 | /* |
680 | * This is solely for the sched tick emulation with | 678 | * This is solely for the sched tick emulation with |
681 | * dynamic tick support to ensure that we do not | 679 | * dynamic tick support to ensure that we do not |
682 | * restart the tick right on the edge and end up with | 680 | * restart the tick right on the edge and end up with |
683 | * the tick timer in the softirq ! The calling site | 681 | * the tick timer in the softirq ! The calling site |
684 | * takes care of this. | 682 | * takes care of this. |
685 | */ | 683 | */ |
686 | debug_hrtimer_deactivate(timer); | 684 | debug_hrtimer_deactivate(timer); |
687 | return 1; | 685 | return 1; |
688 | case HRTIMER_CB_IRQSAFE: | 686 | case HRTIMER_CB_IRQSAFE: |
689 | case HRTIMER_CB_SOFTIRQ: | 687 | case HRTIMER_CB_SOFTIRQ: |
690 | /* | 688 | /* |
691 | * Move everything else into the softirq pending list ! | 689 | * Move everything else into the softirq pending list ! |
692 | */ | 690 | */ |
693 | list_add_tail(&timer->cb_entry, | 691 | list_add_tail(&timer->cb_entry, |
694 | &base->cpu_base->cb_pending); | 692 | &base->cpu_base->cb_pending); |
695 | timer->state = HRTIMER_STATE_PENDING; | 693 | timer->state = HRTIMER_STATE_PENDING; |
696 | return 1; | 694 | return 1; |
697 | default: | 695 | default: |
698 | BUG(); | 696 | BUG(); |
699 | } | 697 | } |
700 | } | 698 | } |
701 | return 0; | 699 | return 0; |
702 | } | 700 | } |
703 | 701 | ||
704 | /* | 702 | /* |
705 | * Switch to high resolution mode | 703 | * Switch to high resolution mode |
706 | */ | 704 | */ |
707 | static int hrtimer_switch_to_hres(void) | 705 | static int hrtimer_switch_to_hres(void) |
708 | { | 706 | { |
709 | int cpu = smp_processor_id(); | 707 | int cpu = smp_processor_id(); |
710 | struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); | 708 | struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); |
711 | unsigned long flags; | 709 | unsigned long flags; |
712 | 710 | ||
713 | if (base->hres_active) | 711 | if (base->hres_active) |
714 | return 1; | 712 | return 1; |
715 | 713 | ||
716 | local_irq_save(flags); | 714 | local_irq_save(flags); |
717 | 715 | ||
718 | if (tick_init_highres()) { | 716 | if (tick_init_highres()) { |
719 | local_irq_restore(flags); | 717 | local_irq_restore(flags); |
720 | printk(KERN_WARNING "Could not switch to high resolution " | 718 | printk(KERN_WARNING "Could not switch to high resolution " |
721 | "mode on CPU %d\n", cpu); | 719 | "mode on CPU %d\n", cpu); |
722 | return 0; | 720 | return 0; |
723 | } | 721 | } |
724 | base->hres_active = 1; | 722 | base->hres_active = 1; |
725 | base->clock_base[CLOCK_REALTIME].resolution = KTIME_HIGH_RES; | 723 | base->clock_base[CLOCK_REALTIME].resolution = KTIME_HIGH_RES; |
726 | base->clock_base[CLOCK_MONOTONIC].resolution = KTIME_HIGH_RES; | 724 | base->clock_base[CLOCK_MONOTONIC].resolution = KTIME_HIGH_RES; |
727 | 725 | ||
728 | tick_setup_sched_timer(); | 726 | tick_setup_sched_timer(); |
729 | 727 | ||
730 | /* "Retrigger" the interrupt to get things going */ | 728 | /* "Retrigger" the interrupt to get things going */ |
731 | retrigger_next_event(NULL); | 729 | retrigger_next_event(NULL); |
732 | local_irq_restore(flags); | 730 | local_irq_restore(flags); |
733 | printk(KERN_DEBUG "Switched to high resolution mode on CPU %d\n", | 731 | printk(KERN_DEBUG "Switched to high resolution mode on CPU %d\n", |
734 | smp_processor_id()); | 732 | smp_processor_id()); |
735 | return 1; | 733 | return 1; |
736 | } | 734 | } |
737 | 735 | ||
738 | static inline void hrtimer_raise_softirq(void) | 736 | static inline void hrtimer_raise_softirq(void) |
739 | { | 737 | { |
740 | raise_softirq(HRTIMER_SOFTIRQ); | 738 | raise_softirq(HRTIMER_SOFTIRQ); |
741 | } | 739 | } |
742 | 740 | ||
743 | #else | 741 | #else |
744 | 742 | ||
745 | static inline int hrtimer_hres_active(void) { return 0; } | 743 | static inline int hrtimer_hres_active(void) { return 0; } |
746 | static inline int hrtimer_is_hres_enabled(void) { return 0; } | 744 | static inline int hrtimer_is_hres_enabled(void) { return 0; } |
747 | static inline int hrtimer_switch_to_hres(void) { return 0; } | 745 | static inline int hrtimer_switch_to_hres(void) { return 0; } |
748 | static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { } | 746 | static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { } |
749 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | 747 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, |
750 | struct hrtimer_clock_base *base) | 748 | struct hrtimer_clock_base *base) |
751 | { | 749 | { |
752 | return 0; | 750 | return 0; |
753 | } | 751 | } |
754 | static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } | 752 | static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } |
755 | static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { } | 753 | static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { } |
756 | static inline int hrtimer_reprogram(struct hrtimer *timer, | 754 | static inline int hrtimer_reprogram(struct hrtimer *timer, |
757 | struct hrtimer_clock_base *base) | 755 | struct hrtimer_clock_base *base) |
758 | { | 756 | { |
759 | return 0; | 757 | return 0; |
760 | } | 758 | } |
761 | static inline void hrtimer_raise_softirq(void) { } | 759 | static inline void hrtimer_raise_softirq(void) { } |
762 | 760 | ||
763 | #endif /* CONFIG_HIGH_RES_TIMERS */ | 761 | #endif /* CONFIG_HIGH_RES_TIMERS */ |
764 | 762 | ||
765 | #ifdef CONFIG_TIMER_STATS | 763 | #ifdef CONFIG_TIMER_STATS |
766 | void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr) | 764 | void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr) |
767 | { | 765 | { |
768 | if (timer->start_site) | 766 | if (timer->start_site) |
769 | return; | 767 | return; |
770 | 768 | ||
771 | timer->start_site = addr; | 769 | timer->start_site = addr; |
772 | memcpy(timer->start_comm, current->comm, TASK_COMM_LEN); | 770 | memcpy(timer->start_comm, current->comm, TASK_COMM_LEN); |
773 | timer->start_pid = current->pid; | 771 | timer->start_pid = current->pid; |
774 | } | 772 | } |
775 | #endif | 773 | #endif |
776 | 774 | ||
777 | /* | 775 | /* |
778 | * Counterpart to lock_hrtimer_base above: | 776 | * Counterpart to lock_hrtimer_base above: |
779 | */ | 777 | */ |
780 | static inline | 778 | static inline |
781 | void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | 779 | void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) |
782 | { | 780 | { |
783 | spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); | 781 | spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); |
784 | } | 782 | } |
785 | 783 | ||
786 | /** | 784 | /** |
787 | * hrtimer_forward - forward the timer expiry | 785 | * hrtimer_forward - forward the timer expiry |
788 | * @timer: hrtimer to forward | 786 | * @timer: hrtimer to forward |
789 | * @now: forward past this time | 787 | * @now: forward past this time |
790 | * @interval: the interval to forward | 788 | * @interval: the interval to forward |
791 | * | 789 | * |
792 | * Forward the timer expiry so it will expire in the future. | 790 | * Forward the timer expiry so it will expire in the future. |
793 | * Returns the number of overruns. | 791 | * Returns the number of overruns. |
794 | */ | 792 | */ |
795 | u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) | 793 | u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) |
796 | { | 794 | { |
797 | u64 orun = 1; | 795 | u64 orun = 1; |
798 | ktime_t delta; | 796 | ktime_t delta; |
799 | 797 | ||
800 | delta = ktime_sub(now, timer->expires); | 798 | delta = ktime_sub(now, timer->expires); |
801 | 799 | ||
802 | if (delta.tv64 < 0) | 800 | if (delta.tv64 < 0) |
803 | return 0; | 801 | return 0; |
804 | 802 | ||
805 | if (interval.tv64 < timer->base->resolution.tv64) | 803 | if (interval.tv64 < timer->base->resolution.tv64) |
806 | interval.tv64 = timer->base->resolution.tv64; | 804 | interval.tv64 = timer->base->resolution.tv64; |
807 | 805 | ||
808 | if (unlikely(delta.tv64 >= interval.tv64)) { | 806 | if (unlikely(delta.tv64 >= interval.tv64)) { |
809 | s64 incr = ktime_to_ns(interval); | 807 | s64 incr = ktime_to_ns(interval); |
810 | 808 | ||
811 | orun = ktime_divns(delta, incr); | 809 | orun = ktime_divns(delta, incr); |
812 | timer->expires = ktime_add_ns(timer->expires, incr * orun); | 810 | timer->expires = ktime_add_ns(timer->expires, incr * orun); |
813 | if (timer->expires.tv64 > now.tv64) | 811 | if (timer->expires.tv64 > now.tv64) |
814 | return orun; | 812 | return orun; |
815 | /* | 813 | /* |
816 | * This (and the ktime_add() below) is the | 814 | * This (and the ktime_add() below) is the |
817 | * correction for exact: | 815 | * correction for exact: |
818 | */ | 816 | */ |
819 | orun++; | 817 | orun++; |
820 | } | 818 | } |
821 | timer->expires = ktime_add_safe(timer->expires, interval); | 819 | timer->expires = ktime_add_safe(timer->expires, interval); |
822 | 820 | ||
823 | return orun; | 821 | return orun; |
824 | } | 822 | } |
825 | EXPORT_SYMBOL_GPL(hrtimer_forward); | 823 | EXPORT_SYMBOL_GPL(hrtimer_forward); |
826 | 824 | ||
827 | /* | 825 | /* |
828 | * enqueue_hrtimer - internal function to (re)start a timer | 826 | * enqueue_hrtimer - internal function to (re)start a timer |
829 | * | 827 | * |
830 | * The timer is inserted in expiry order. Insertion into the | 828 | * The timer is inserted in expiry order. Insertion into the |
831 | * red black tree is O(log(n)). Must hold the base lock. | 829 | * red black tree is O(log(n)). Must hold the base lock. |
832 | */ | 830 | */ |
833 | static void enqueue_hrtimer(struct hrtimer *timer, | 831 | static void enqueue_hrtimer(struct hrtimer *timer, |
834 | struct hrtimer_clock_base *base, int reprogram) | 832 | struct hrtimer_clock_base *base, int reprogram) |
835 | { | 833 | { |
836 | struct rb_node **link = &base->active.rb_node; | 834 | struct rb_node **link = &base->active.rb_node; |
837 | struct rb_node *parent = NULL; | 835 | struct rb_node *parent = NULL; |
838 | struct hrtimer *entry; | 836 | struct hrtimer *entry; |
839 | int leftmost = 1; | 837 | int leftmost = 1; |
840 | 838 | ||
841 | debug_hrtimer_activate(timer); | 839 | debug_hrtimer_activate(timer); |
842 | 840 | ||
843 | /* | 841 | /* |
844 | * Find the right place in the rbtree: | 842 | * Find the right place in the rbtree: |
845 | */ | 843 | */ |
846 | while (*link) { | 844 | while (*link) { |
847 | parent = *link; | 845 | parent = *link; |
848 | entry = rb_entry(parent, struct hrtimer, node); | 846 | entry = rb_entry(parent, struct hrtimer, node); |
849 | /* | 847 | /* |
850 | * We dont care about collisions. Nodes with | 848 | * We dont care about collisions. Nodes with |
851 | * the same expiry time stay together. | 849 | * the same expiry time stay together. |
852 | */ | 850 | */ |
853 | if (timer->expires.tv64 < entry->expires.tv64) { | 851 | if (timer->expires.tv64 < entry->expires.tv64) { |
854 | link = &(*link)->rb_left; | 852 | link = &(*link)->rb_left; |
855 | } else { | 853 | } else { |
856 | link = &(*link)->rb_right; | 854 | link = &(*link)->rb_right; |
857 | leftmost = 0; | 855 | leftmost = 0; |
858 | } | 856 | } |
859 | } | 857 | } |
860 | 858 | ||
861 | /* | 859 | /* |
862 | * Insert the timer to the rbtree and check whether it | 860 | * Insert the timer to the rbtree and check whether it |
863 | * replaces the first pending timer | 861 | * replaces the first pending timer |
864 | */ | 862 | */ |
865 | if (leftmost) { | 863 | if (leftmost) { |
866 | /* | 864 | /* |
867 | * Reprogram the clock event device. When the timer is already | 865 | * Reprogram the clock event device. When the timer is already |
868 | * expired hrtimer_enqueue_reprogram has either called the | 866 | * expired hrtimer_enqueue_reprogram has either called the |
869 | * callback or added it to the pending list and raised the | 867 | * callback or added it to the pending list and raised the |
870 | * softirq. | 868 | * softirq. |
871 | * | 869 | * |
872 | * This is a NOP for !HIGHRES | 870 | * This is a NOP for !HIGHRES |
873 | */ | 871 | */ |
874 | if (reprogram && hrtimer_enqueue_reprogram(timer, base)) | 872 | if (reprogram && hrtimer_enqueue_reprogram(timer, base)) |
875 | return; | 873 | return; |
876 | 874 | ||
877 | base->first = &timer->node; | 875 | base->first = &timer->node; |
878 | } | 876 | } |
879 | 877 | ||
880 | rb_link_node(&timer->node, parent, link); | 878 | rb_link_node(&timer->node, parent, link); |
881 | rb_insert_color(&timer->node, &base->active); | 879 | rb_insert_color(&timer->node, &base->active); |
882 | /* | 880 | /* |
883 | * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the | 881 | * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the |
884 | * state of a possibly running callback. | 882 | * state of a possibly running callback. |
885 | */ | 883 | */ |
886 | timer->state |= HRTIMER_STATE_ENQUEUED; | 884 | timer->state |= HRTIMER_STATE_ENQUEUED; |
887 | } | 885 | } |
888 | 886 | ||
889 | /* | 887 | /* |
890 | * __remove_hrtimer - internal function to remove a timer | 888 | * __remove_hrtimer - internal function to remove a timer |
891 | * | 889 | * |
892 | * Caller must hold the base lock. | 890 | * Caller must hold the base lock. |
893 | * | 891 | * |
894 | * High resolution timer mode reprograms the clock event device when the | 892 | * High resolution timer mode reprograms the clock event device when the |
895 | * timer is the one which expires next. The caller can disable this by setting | 893 | * timer is the one which expires next. The caller can disable this by setting |
896 | * reprogram to zero. This is useful, when the context does a reprogramming | 894 | * reprogram to zero. This is useful, when the context does a reprogramming |
897 | * anyway (e.g. timer interrupt) | 895 | * anyway (e.g. timer interrupt) |
898 | */ | 896 | */ |
899 | static void __remove_hrtimer(struct hrtimer *timer, | 897 | static void __remove_hrtimer(struct hrtimer *timer, |
900 | struct hrtimer_clock_base *base, | 898 | struct hrtimer_clock_base *base, |
901 | unsigned long newstate, int reprogram) | 899 | unsigned long newstate, int reprogram) |
902 | { | 900 | { |
903 | /* High res. callback list. NOP for !HIGHRES */ | 901 | /* High res. callback list. NOP for !HIGHRES */ |
904 | if (hrtimer_cb_pending(timer)) | 902 | if (hrtimer_cb_pending(timer)) |
905 | hrtimer_remove_cb_pending(timer); | 903 | hrtimer_remove_cb_pending(timer); |
906 | else { | 904 | else { |
907 | /* | 905 | /* |
908 | * Remove the timer from the rbtree and replace the | 906 | * Remove the timer from the rbtree and replace the |
909 | * first entry pointer if necessary. | 907 | * first entry pointer if necessary. |
910 | */ | 908 | */ |
911 | if (base->first == &timer->node) { | 909 | if (base->first == &timer->node) { |
912 | base->first = rb_next(&timer->node); | 910 | base->first = rb_next(&timer->node); |
913 | /* Reprogram the clock event device. if enabled */ | 911 | /* Reprogram the clock event device. if enabled */ |
914 | if (reprogram && hrtimer_hres_active()) | 912 | if (reprogram && hrtimer_hres_active()) |
915 | hrtimer_force_reprogram(base->cpu_base); | 913 | hrtimer_force_reprogram(base->cpu_base); |
916 | } | 914 | } |
917 | rb_erase(&timer->node, &base->active); | 915 | rb_erase(&timer->node, &base->active); |
918 | } | 916 | } |
919 | timer->state = newstate; | 917 | timer->state = newstate; |
920 | } | 918 | } |
921 | 919 | ||
922 | /* | 920 | /* |
923 | * remove hrtimer, called with base lock held | 921 | * remove hrtimer, called with base lock held |
924 | */ | 922 | */ |
925 | static inline int | 923 | static inline int |
926 | remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) | 924 | remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) |
927 | { | 925 | { |
928 | if (hrtimer_is_queued(timer)) { | 926 | if (hrtimer_is_queued(timer)) { |
929 | int reprogram; | 927 | int reprogram; |
930 | 928 | ||
931 | /* | 929 | /* |
932 | * Remove the timer and force reprogramming when high | 930 | * Remove the timer and force reprogramming when high |
933 | * resolution mode is active and the timer is on the current | 931 | * resolution mode is active and the timer is on the current |
934 | * CPU. If we remove a timer on another CPU, reprogramming is | 932 | * CPU. If we remove a timer on another CPU, reprogramming is |
935 | * skipped. The interrupt event on this CPU is fired and | 933 | * skipped. The interrupt event on this CPU is fired and |
936 | * reprogramming happens in the interrupt handler. This is a | 934 | * reprogramming happens in the interrupt handler. This is a |
937 | * rare case and less expensive than a smp call. | 935 | * rare case and less expensive than a smp call. |
938 | */ | 936 | */ |
939 | debug_hrtimer_deactivate(timer); | 937 | debug_hrtimer_deactivate(timer); |
940 | timer_stats_hrtimer_clear_start_info(timer); | 938 | timer_stats_hrtimer_clear_start_info(timer); |
941 | reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); | 939 | reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); |
942 | __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, | 940 | __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, |
943 | reprogram); | 941 | reprogram); |
944 | return 1; | 942 | return 1; |
945 | } | 943 | } |
946 | return 0; | 944 | return 0; |
947 | } | 945 | } |
948 | 946 | ||
949 | /** | 947 | /** |
950 | * hrtimer_start - (re)start an relative timer on the current CPU | 948 | * hrtimer_start - (re)start an relative timer on the current CPU |
951 | * @timer: the timer to be added | 949 | * @timer: the timer to be added |
952 | * @tim: expiry time | 950 | * @tim: expiry time |
953 | * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) | 951 | * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) |
954 | * | 952 | * |
955 | * Returns: | 953 | * Returns: |
956 | * 0 on success | 954 | * 0 on success |
957 | * 1 when the timer was active | 955 | * 1 when the timer was active |
958 | */ | 956 | */ |
959 | int | 957 | int |
960 | hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | 958 | hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) |
961 | { | 959 | { |
962 | struct hrtimer_clock_base *base, *new_base; | 960 | struct hrtimer_clock_base *base, *new_base; |
963 | unsigned long flags; | 961 | unsigned long flags; |
964 | int ret, raise; | 962 | int ret, raise; |
965 | 963 | ||
966 | base = lock_hrtimer_base(timer, &flags); | 964 | base = lock_hrtimer_base(timer, &flags); |
967 | 965 | ||
968 | /* Remove an active timer from the queue: */ | 966 | /* Remove an active timer from the queue: */ |
969 | ret = remove_hrtimer(timer, base); | 967 | ret = remove_hrtimer(timer, base); |
970 | 968 | ||
971 | /* Switch the timer base, if necessary: */ | 969 | /* Switch the timer base, if necessary: */ |
972 | new_base = switch_hrtimer_base(timer, base); | 970 | new_base = switch_hrtimer_base(timer, base); |
973 | 971 | ||
974 | if (mode == HRTIMER_MODE_REL) { | 972 | if (mode == HRTIMER_MODE_REL) { |
975 | tim = ktime_add_safe(tim, new_base->get_time()); | 973 | tim = ktime_add_safe(tim, new_base->get_time()); |
976 | /* | 974 | /* |
977 | * CONFIG_TIME_LOW_RES is a temporary way for architectures | 975 | * CONFIG_TIME_LOW_RES is a temporary way for architectures |
978 | * to signal that they simply return xtime in | 976 | * to signal that they simply return xtime in |
979 | * do_gettimeoffset(). In this case we want to round up by | 977 | * do_gettimeoffset(). In this case we want to round up by |
980 | * resolution when starting a relative timer, to avoid short | 978 | * resolution when starting a relative timer, to avoid short |
981 | * timeouts. This will go away with the GTOD framework. | 979 | * timeouts. This will go away with the GTOD framework. |
982 | */ | 980 | */ |
983 | #ifdef CONFIG_TIME_LOW_RES | 981 | #ifdef CONFIG_TIME_LOW_RES |
984 | tim = ktime_add_safe(tim, base->resolution); | 982 | tim = ktime_add_safe(tim, base->resolution); |
985 | #endif | 983 | #endif |
986 | } | 984 | } |
987 | 985 | ||
988 | timer->expires = tim; | 986 | timer->expires = tim; |
989 | 987 | ||
990 | timer_stats_hrtimer_set_start_info(timer); | 988 | timer_stats_hrtimer_set_start_info(timer); |
991 | 989 | ||
992 | /* | 990 | /* |
993 | * Only allow reprogramming if the new base is on this CPU. | 991 | * Only allow reprogramming if the new base is on this CPU. |
994 | * (it might still be on another CPU if the timer was pending) | 992 | * (it might still be on another CPU if the timer was pending) |
995 | */ | 993 | */ |
996 | enqueue_hrtimer(timer, new_base, | 994 | enqueue_hrtimer(timer, new_base, |
997 | new_base->cpu_base == &__get_cpu_var(hrtimer_bases)); | 995 | new_base->cpu_base == &__get_cpu_var(hrtimer_bases)); |
998 | 996 | ||
999 | /* | 997 | /* |
1000 | * The timer may be expired and moved to the cb_pending | 998 | * The timer may be expired and moved to the cb_pending |
1001 | * list. We can not raise the softirq with base lock held due | 999 | * list. We can not raise the softirq with base lock held due |
1002 | * to a possible deadlock with runqueue lock. | 1000 | * to a possible deadlock with runqueue lock. |
1003 | */ | 1001 | */ |
1004 | raise = timer->state == HRTIMER_STATE_PENDING; | 1002 | raise = timer->state == HRTIMER_STATE_PENDING; |
1005 | 1003 | ||
1006 | unlock_hrtimer_base(timer, &flags); | 1004 | unlock_hrtimer_base(timer, &flags); |
1007 | 1005 | ||
1008 | if (raise) | 1006 | if (raise) |
1009 | hrtimer_raise_softirq(); | 1007 | hrtimer_raise_softirq(); |
1010 | 1008 | ||
1011 | return ret; | 1009 | return ret; |
1012 | } | 1010 | } |
1013 | EXPORT_SYMBOL_GPL(hrtimer_start); | 1011 | EXPORT_SYMBOL_GPL(hrtimer_start); |
1014 | 1012 | ||
1015 | /** | 1013 | /** |
1016 | * hrtimer_try_to_cancel - try to deactivate a timer | 1014 | * hrtimer_try_to_cancel - try to deactivate a timer |
1017 | * @timer: hrtimer to stop | 1015 | * @timer: hrtimer to stop |
1018 | * | 1016 | * |
1019 | * Returns: | 1017 | * Returns: |
1020 | * 0 when the timer was not active | 1018 | * 0 when the timer was not active |
1021 | * 1 when the timer was active | 1019 | * 1 when the timer was active |
1022 | * -1 when the timer is currently excuting the callback function and | 1020 | * -1 when the timer is currently excuting the callback function and |
1023 | * cannot be stopped | 1021 | * cannot be stopped |
1024 | */ | 1022 | */ |
1025 | int hrtimer_try_to_cancel(struct hrtimer *timer) | 1023 | int hrtimer_try_to_cancel(struct hrtimer *timer) |
1026 | { | 1024 | { |
1027 | struct hrtimer_clock_base *base; | 1025 | struct hrtimer_clock_base *base; |
1028 | unsigned long flags; | 1026 | unsigned long flags; |
1029 | int ret = -1; | 1027 | int ret = -1; |
1030 | 1028 | ||
1031 | base = lock_hrtimer_base(timer, &flags); | 1029 | base = lock_hrtimer_base(timer, &flags); |
1032 | 1030 | ||
1033 | if (!hrtimer_callback_running(timer)) | 1031 | if (!hrtimer_callback_running(timer)) |
1034 | ret = remove_hrtimer(timer, base); | 1032 | ret = remove_hrtimer(timer, base); |
1035 | 1033 | ||
1036 | unlock_hrtimer_base(timer, &flags); | 1034 | unlock_hrtimer_base(timer, &flags); |
1037 | 1035 | ||
1038 | return ret; | 1036 | return ret; |
1039 | 1037 | ||
1040 | } | 1038 | } |
1041 | EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); | 1039 | EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); |
1042 | 1040 | ||
1043 | /** | 1041 | /** |
1044 | * hrtimer_cancel - cancel a timer and wait for the handler to finish. | 1042 | * hrtimer_cancel - cancel a timer and wait for the handler to finish. |
1045 | * @timer: the timer to be cancelled | 1043 | * @timer: the timer to be cancelled |
1046 | * | 1044 | * |
1047 | * Returns: | 1045 | * Returns: |
1048 | * 0 when the timer was not active | 1046 | * 0 when the timer was not active |
1049 | * 1 when the timer was active | 1047 | * 1 when the timer was active |
1050 | */ | 1048 | */ |
1051 | int hrtimer_cancel(struct hrtimer *timer) | 1049 | int hrtimer_cancel(struct hrtimer *timer) |
1052 | { | 1050 | { |
1053 | for (;;) { | 1051 | for (;;) { |
1054 | int ret = hrtimer_try_to_cancel(timer); | 1052 | int ret = hrtimer_try_to_cancel(timer); |
1055 | 1053 | ||
1056 | if (ret >= 0) | 1054 | if (ret >= 0) |
1057 | return ret; | 1055 | return ret; |
1058 | cpu_relax(); | 1056 | cpu_relax(); |
1059 | } | 1057 | } |
1060 | } | 1058 | } |
1061 | EXPORT_SYMBOL_GPL(hrtimer_cancel); | 1059 | EXPORT_SYMBOL_GPL(hrtimer_cancel); |
1062 | 1060 | ||
1063 | /** | 1061 | /** |
1064 | * hrtimer_get_remaining - get remaining time for the timer | 1062 | * hrtimer_get_remaining - get remaining time for the timer |
1065 | * @timer: the timer to read | 1063 | * @timer: the timer to read |
1066 | */ | 1064 | */ |
1067 | ktime_t hrtimer_get_remaining(const struct hrtimer *timer) | 1065 | ktime_t hrtimer_get_remaining(const struct hrtimer *timer) |
1068 | { | 1066 | { |
1069 | struct hrtimer_clock_base *base; | 1067 | struct hrtimer_clock_base *base; |
1070 | unsigned long flags; | 1068 | unsigned long flags; |
1071 | ktime_t rem; | 1069 | ktime_t rem; |
1072 | 1070 | ||
1073 | base = lock_hrtimer_base(timer, &flags); | 1071 | base = lock_hrtimer_base(timer, &flags); |
1074 | rem = ktime_sub(timer->expires, base->get_time()); | 1072 | rem = ktime_sub(timer->expires, base->get_time()); |
1075 | unlock_hrtimer_base(timer, &flags); | 1073 | unlock_hrtimer_base(timer, &flags); |
1076 | 1074 | ||
1077 | return rem; | 1075 | return rem; |
1078 | } | 1076 | } |
1079 | EXPORT_SYMBOL_GPL(hrtimer_get_remaining); | 1077 | EXPORT_SYMBOL_GPL(hrtimer_get_remaining); |
1080 | 1078 | ||
1081 | #if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ) | 1079 | #if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ) |
1082 | /** | 1080 | /** |
1083 | * hrtimer_get_next_event - get the time until next expiry event | 1081 | * hrtimer_get_next_event - get the time until next expiry event |
1084 | * | 1082 | * |
1085 | * Returns the delta to the next expiry event or KTIME_MAX if no timer | 1083 | * Returns the delta to the next expiry event or KTIME_MAX if no timer |
1086 | * is pending. | 1084 | * is pending. |
1087 | */ | 1085 | */ |
1088 | ktime_t hrtimer_get_next_event(void) | 1086 | ktime_t hrtimer_get_next_event(void) |
1089 | { | 1087 | { |
1090 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | 1088 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
1091 | struct hrtimer_clock_base *base = cpu_base->clock_base; | 1089 | struct hrtimer_clock_base *base = cpu_base->clock_base; |
1092 | ktime_t delta, mindelta = { .tv64 = KTIME_MAX }; | 1090 | ktime_t delta, mindelta = { .tv64 = KTIME_MAX }; |
1093 | unsigned long flags; | 1091 | unsigned long flags; |
1094 | int i; | 1092 | int i; |
1095 | 1093 | ||
1096 | spin_lock_irqsave(&cpu_base->lock, flags); | 1094 | spin_lock_irqsave(&cpu_base->lock, flags); |
1097 | 1095 | ||
1098 | if (!hrtimer_hres_active()) { | 1096 | if (!hrtimer_hres_active()) { |
1099 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { | 1097 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { |
1100 | struct hrtimer *timer; | 1098 | struct hrtimer *timer; |
1101 | 1099 | ||
1102 | if (!base->first) | 1100 | if (!base->first) |
1103 | continue; | 1101 | continue; |
1104 | 1102 | ||
1105 | timer = rb_entry(base->first, struct hrtimer, node); | 1103 | timer = rb_entry(base->first, struct hrtimer, node); |
1106 | delta.tv64 = timer->expires.tv64; | 1104 | delta.tv64 = timer->expires.tv64; |
1107 | delta = ktime_sub(delta, base->get_time()); | 1105 | delta = ktime_sub(delta, base->get_time()); |
1108 | if (delta.tv64 < mindelta.tv64) | 1106 | if (delta.tv64 < mindelta.tv64) |
1109 | mindelta.tv64 = delta.tv64; | 1107 | mindelta.tv64 = delta.tv64; |
1110 | } | 1108 | } |
1111 | } | 1109 | } |
1112 | 1110 | ||
1113 | spin_unlock_irqrestore(&cpu_base->lock, flags); | 1111 | spin_unlock_irqrestore(&cpu_base->lock, flags); |
1114 | 1112 | ||
1115 | if (mindelta.tv64 < 0) | 1113 | if (mindelta.tv64 < 0) |
1116 | mindelta.tv64 = 0; | 1114 | mindelta.tv64 = 0; |
1117 | return mindelta; | 1115 | return mindelta; |
1118 | } | 1116 | } |
1119 | #endif | 1117 | #endif |
1120 | 1118 | ||
1121 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | 1119 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, |
1122 | enum hrtimer_mode mode) | 1120 | enum hrtimer_mode mode) |
1123 | { | 1121 | { |
1124 | struct hrtimer_cpu_base *cpu_base; | 1122 | struct hrtimer_cpu_base *cpu_base; |
1125 | 1123 | ||
1126 | memset(timer, 0, sizeof(struct hrtimer)); | 1124 | memset(timer, 0, sizeof(struct hrtimer)); |
1127 | 1125 | ||
1128 | cpu_base = &__raw_get_cpu_var(hrtimer_bases); | 1126 | cpu_base = &__raw_get_cpu_var(hrtimer_bases); |
1129 | 1127 | ||
1130 | if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS) | 1128 | if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS) |
1131 | clock_id = CLOCK_MONOTONIC; | 1129 | clock_id = CLOCK_MONOTONIC; |
1132 | 1130 | ||
1133 | timer->base = &cpu_base->clock_base[clock_id]; | 1131 | timer->base = &cpu_base->clock_base[clock_id]; |
1134 | INIT_LIST_HEAD(&timer->cb_entry); | 1132 | INIT_LIST_HEAD(&timer->cb_entry); |
1135 | hrtimer_init_timer_hres(timer); | 1133 | hrtimer_init_timer_hres(timer); |
1136 | 1134 | ||
1137 | #ifdef CONFIG_TIMER_STATS | 1135 | #ifdef CONFIG_TIMER_STATS |
1138 | timer->start_site = NULL; | 1136 | timer->start_site = NULL; |
1139 | timer->start_pid = -1; | 1137 | timer->start_pid = -1; |
1140 | memset(timer->start_comm, 0, TASK_COMM_LEN); | 1138 | memset(timer->start_comm, 0, TASK_COMM_LEN); |
1141 | #endif | 1139 | #endif |
1142 | } | 1140 | } |
1143 | 1141 | ||
1144 | /** | 1142 | /** |
1145 | * hrtimer_init - initialize a timer to the given clock | 1143 | * hrtimer_init - initialize a timer to the given clock |
1146 | * @timer: the timer to be initialized | 1144 | * @timer: the timer to be initialized |
1147 | * @clock_id: the clock to be used | 1145 | * @clock_id: the clock to be used |
1148 | * @mode: timer mode abs/rel | 1146 | * @mode: timer mode abs/rel |
1149 | */ | 1147 | */ |
1150 | void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | 1148 | void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, |
1151 | enum hrtimer_mode mode) | 1149 | enum hrtimer_mode mode) |
1152 | { | 1150 | { |
1153 | debug_hrtimer_init(timer); | 1151 | debug_hrtimer_init(timer); |
1154 | __hrtimer_init(timer, clock_id, mode); | 1152 | __hrtimer_init(timer, clock_id, mode); |
1155 | } | 1153 | } |
1156 | EXPORT_SYMBOL_GPL(hrtimer_init); | 1154 | EXPORT_SYMBOL_GPL(hrtimer_init); |
1157 | 1155 | ||
1158 | /** | 1156 | /** |
1159 | * hrtimer_get_res - get the timer resolution for a clock | 1157 | * hrtimer_get_res - get the timer resolution for a clock |
1160 | * @which_clock: which clock to query | 1158 | * @which_clock: which clock to query |
1161 | * @tp: pointer to timespec variable to store the resolution | 1159 | * @tp: pointer to timespec variable to store the resolution |
1162 | * | 1160 | * |
1163 | * Store the resolution of the clock selected by @which_clock in the | 1161 | * Store the resolution of the clock selected by @which_clock in the |
1164 | * variable pointed to by @tp. | 1162 | * variable pointed to by @tp. |
1165 | */ | 1163 | */ |
1166 | int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) | 1164 | int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) |
1167 | { | 1165 | { |
1168 | struct hrtimer_cpu_base *cpu_base; | 1166 | struct hrtimer_cpu_base *cpu_base; |
1169 | 1167 | ||
1170 | cpu_base = &__raw_get_cpu_var(hrtimer_bases); | 1168 | cpu_base = &__raw_get_cpu_var(hrtimer_bases); |
1171 | *tp = ktime_to_timespec(cpu_base->clock_base[which_clock].resolution); | 1169 | *tp = ktime_to_timespec(cpu_base->clock_base[which_clock].resolution); |
1172 | 1170 | ||
1173 | return 0; | 1171 | return 0; |
1174 | } | 1172 | } |
1175 | EXPORT_SYMBOL_GPL(hrtimer_get_res); | 1173 | EXPORT_SYMBOL_GPL(hrtimer_get_res); |
1176 | 1174 | ||
1177 | static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) | 1175 | static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) |
1178 | { | 1176 | { |
1179 | spin_lock_irq(&cpu_base->lock); | 1177 | spin_lock_irq(&cpu_base->lock); |
1180 | 1178 | ||
1181 | while (!list_empty(&cpu_base->cb_pending)) { | 1179 | while (!list_empty(&cpu_base->cb_pending)) { |
1182 | enum hrtimer_restart (*fn)(struct hrtimer *); | 1180 | enum hrtimer_restart (*fn)(struct hrtimer *); |
1183 | struct hrtimer *timer; | 1181 | struct hrtimer *timer; |
1184 | int restart; | 1182 | int restart; |
1185 | 1183 | ||
1186 | timer = list_entry(cpu_base->cb_pending.next, | 1184 | timer = list_entry(cpu_base->cb_pending.next, |
1187 | struct hrtimer, cb_entry); | 1185 | struct hrtimer, cb_entry); |
1188 | 1186 | ||
1189 | debug_hrtimer_deactivate(timer); | 1187 | debug_hrtimer_deactivate(timer); |
1190 | timer_stats_account_hrtimer(timer); | 1188 | timer_stats_account_hrtimer(timer); |
1191 | 1189 | ||
1192 | fn = timer->function; | 1190 | fn = timer->function; |
1193 | __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); | 1191 | __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); |
1194 | spin_unlock_irq(&cpu_base->lock); | 1192 | spin_unlock_irq(&cpu_base->lock); |
1195 | 1193 | ||
1196 | restart = fn(timer); | 1194 | restart = fn(timer); |
1197 | 1195 | ||
1198 | spin_lock_irq(&cpu_base->lock); | 1196 | spin_lock_irq(&cpu_base->lock); |
1199 | 1197 | ||
1200 | timer->state &= ~HRTIMER_STATE_CALLBACK; | 1198 | timer->state &= ~HRTIMER_STATE_CALLBACK; |
1201 | if (restart == HRTIMER_RESTART) { | 1199 | if (restart == HRTIMER_RESTART) { |
1202 | BUG_ON(hrtimer_active(timer)); | 1200 | BUG_ON(hrtimer_active(timer)); |
1203 | /* | 1201 | /* |
1204 | * Enqueue the timer, allow reprogramming of the event | 1202 | * Enqueue the timer, allow reprogramming of the event |
1205 | * device | 1203 | * device |
1206 | */ | 1204 | */ |
1207 | enqueue_hrtimer(timer, timer->base, 1); | 1205 | enqueue_hrtimer(timer, timer->base, 1); |
1208 | } else if (hrtimer_active(timer)) { | 1206 | } else if (hrtimer_active(timer)) { |
1209 | /* | 1207 | /* |
1210 | * If the timer was rearmed on another CPU, reprogram | 1208 | * If the timer was rearmed on another CPU, reprogram |
1211 | * the event device. | 1209 | * the event device. |
1212 | */ | 1210 | */ |
1213 | struct hrtimer_clock_base *base = timer->base; | 1211 | struct hrtimer_clock_base *base = timer->base; |
1214 | 1212 | ||
1215 | if (base->first == &timer->node && | 1213 | if (base->first == &timer->node && |
1216 | hrtimer_reprogram(timer, base)) { | 1214 | hrtimer_reprogram(timer, base)) { |
1217 | /* | 1215 | /* |
1218 | * Timer is expired. Thus move it from tree to | 1216 | * Timer is expired. Thus move it from tree to |
1219 | * pending list again. | 1217 | * pending list again. |
1220 | */ | 1218 | */ |
1221 | __remove_hrtimer(timer, base, | 1219 | __remove_hrtimer(timer, base, |
1222 | HRTIMER_STATE_PENDING, 0); | 1220 | HRTIMER_STATE_PENDING, 0); |
1223 | list_add_tail(&timer->cb_entry, | 1221 | list_add_tail(&timer->cb_entry, |
1224 | &base->cpu_base->cb_pending); | 1222 | &base->cpu_base->cb_pending); |
1225 | } | 1223 | } |
1226 | } | 1224 | } |
1227 | } | 1225 | } |
1228 | spin_unlock_irq(&cpu_base->lock); | 1226 | spin_unlock_irq(&cpu_base->lock); |
1229 | } | 1227 | } |
1230 | 1228 | ||
1231 | static void __run_hrtimer(struct hrtimer *timer) | 1229 | static void __run_hrtimer(struct hrtimer *timer) |
1232 | { | 1230 | { |
1233 | struct hrtimer_clock_base *base = timer->base; | 1231 | struct hrtimer_clock_base *base = timer->base; |
1234 | struct hrtimer_cpu_base *cpu_base = base->cpu_base; | 1232 | struct hrtimer_cpu_base *cpu_base = base->cpu_base; |
1235 | enum hrtimer_restart (*fn)(struct hrtimer *); | 1233 | enum hrtimer_restart (*fn)(struct hrtimer *); |
1236 | int restart; | 1234 | int restart; |
1237 | 1235 | ||
1238 | debug_hrtimer_deactivate(timer); | 1236 | debug_hrtimer_deactivate(timer); |
1239 | __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); | 1237 | __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); |
1240 | timer_stats_account_hrtimer(timer); | 1238 | timer_stats_account_hrtimer(timer); |
1241 | 1239 | ||
1242 | fn = timer->function; | 1240 | fn = timer->function; |
1243 | if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) { | 1241 | if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) { |
1244 | /* | 1242 | /* |
1245 | * Used for scheduler timers, avoid lock inversion with | 1243 | * Used for scheduler timers, avoid lock inversion with |
1246 | * rq->lock and tasklist_lock. | 1244 | * rq->lock and tasklist_lock. |
1247 | * | 1245 | * |
1248 | * These timers are required to deal with enqueue expiry | 1246 | * These timers are required to deal with enqueue expiry |
1249 | * themselves and are not allowed to migrate. | 1247 | * themselves and are not allowed to migrate. |
1250 | */ | 1248 | */ |
1251 | spin_unlock(&cpu_base->lock); | 1249 | spin_unlock(&cpu_base->lock); |
1252 | restart = fn(timer); | 1250 | restart = fn(timer); |
1253 | spin_lock(&cpu_base->lock); | 1251 | spin_lock(&cpu_base->lock); |
1254 | } else | 1252 | } else |
1255 | restart = fn(timer); | 1253 | restart = fn(timer); |
1256 | 1254 | ||
1257 | /* | 1255 | /* |
1258 | * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid | 1256 | * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid |
1259 | * reprogramming of the event hardware. This happens at the end of this | 1257 | * reprogramming of the event hardware. This happens at the end of this |
1260 | * function anyway. | 1258 | * function anyway. |
1261 | */ | 1259 | */ |
1262 | if (restart != HRTIMER_NORESTART) { | 1260 | if (restart != HRTIMER_NORESTART) { |
1263 | BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); | 1261 | BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); |
1264 | enqueue_hrtimer(timer, base, 0); | 1262 | enqueue_hrtimer(timer, base, 0); |
1265 | } | 1263 | } |
1266 | timer->state &= ~HRTIMER_STATE_CALLBACK; | 1264 | timer->state &= ~HRTIMER_STATE_CALLBACK; |
1267 | } | 1265 | } |
1268 | 1266 | ||
1269 | #ifdef CONFIG_HIGH_RES_TIMERS | 1267 | #ifdef CONFIG_HIGH_RES_TIMERS |
1270 | 1268 | ||
1271 | /* | 1269 | /* |
1272 | * High resolution timer interrupt | 1270 | * High resolution timer interrupt |
1273 | * Called with interrupts disabled | 1271 | * Called with interrupts disabled |
1274 | */ | 1272 | */ |
1275 | void hrtimer_interrupt(struct clock_event_device *dev) | 1273 | void hrtimer_interrupt(struct clock_event_device *dev) |
1276 | { | 1274 | { |
1277 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | 1275 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
1278 | struct hrtimer_clock_base *base; | 1276 | struct hrtimer_clock_base *base; |
1279 | ktime_t expires_next, now; | 1277 | ktime_t expires_next, now; |
1280 | int i, raise = 0; | 1278 | int i, raise = 0; |
1281 | 1279 | ||
1282 | BUG_ON(!cpu_base->hres_active); | 1280 | BUG_ON(!cpu_base->hres_active); |
1283 | cpu_base->nr_events++; | 1281 | cpu_base->nr_events++; |
1284 | dev->next_event.tv64 = KTIME_MAX; | 1282 | dev->next_event.tv64 = KTIME_MAX; |
1285 | 1283 | ||
1286 | retry: | 1284 | retry: |
1287 | now = ktime_get(); | 1285 | now = ktime_get(); |
1288 | 1286 | ||
1289 | expires_next.tv64 = KTIME_MAX; | 1287 | expires_next.tv64 = KTIME_MAX; |
1290 | 1288 | ||
1291 | base = cpu_base->clock_base; | 1289 | base = cpu_base->clock_base; |
1292 | 1290 | ||
1293 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 1291 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
1294 | ktime_t basenow; | 1292 | ktime_t basenow; |
1295 | struct rb_node *node; | 1293 | struct rb_node *node; |
1296 | 1294 | ||
1297 | spin_lock(&cpu_base->lock); | 1295 | spin_lock(&cpu_base->lock); |
1298 | 1296 | ||
1299 | basenow = ktime_add(now, base->offset); | 1297 | basenow = ktime_add(now, base->offset); |
1300 | 1298 | ||
1301 | while ((node = base->first)) { | 1299 | while ((node = base->first)) { |
1302 | struct hrtimer *timer; | 1300 | struct hrtimer *timer; |
1303 | 1301 | ||
1304 | timer = rb_entry(node, struct hrtimer, node); | 1302 | timer = rb_entry(node, struct hrtimer, node); |
1305 | 1303 | ||
1306 | if (basenow.tv64 < timer->expires.tv64) { | 1304 | if (basenow.tv64 < timer->expires.tv64) { |
1307 | ktime_t expires; | 1305 | ktime_t expires; |
1308 | 1306 | ||
1309 | expires = ktime_sub(timer->expires, | 1307 | expires = ktime_sub(timer->expires, |
1310 | base->offset); | 1308 | base->offset); |
1311 | if (expires.tv64 < expires_next.tv64) | 1309 | if (expires.tv64 < expires_next.tv64) |
1312 | expires_next = expires; | 1310 | expires_next = expires; |
1313 | break; | 1311 | break; |
1314 | } | 1312 | } |
1315 | 1313 | ||
1316 | /* Move softirq callbacks to the pending list */ | 1314 | /* Move softirq callbacks to the pending list */ |
1317 | if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { | 1315 | if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { |
1318 | __remove_hrtimer(timer, base, | 1316 | __remove_hrtimer(timer, base, |
1319 | HRTIMER_STATE_PENDING, 0); | 1317 | HRTIMER_STATE_PENDING, 0); |
1320 | list_add_tail(&timer->cb_entry, | 1318 | list_add_tail(&timer->cb_entry, |
1321 | &base->cpu_base->cb_pending); | 1319 | &base->cpu_base->cb_pending); |
1322 | raise = 1; | 1320 | raise = 1; |
1323 | continue; | 1321 | continue; |
1324 | } | 1322 | } |
1325 | 1323 | ||
1326 | __run_hrtimer(timer); | 1324 | __run_hrtimer(timer); |
1327 | } | 1325 | } |
1328 | spin_unlock(&cpu_base->lock); | 1326 | spin_unlock(&cpu_base->lock); |
1329 | base++; | 1327 | base++; |
1330 | } | 1328 | } |
1331 | 1329 | ||
1332 | cpu_base->expires_next = expires_next; | 1330 | cpu_base->expires_next = expires_next; |
1333 | 1331 | ||
1334 | /* Reprogramming necessary ? */ | 1332 | /* Reprogramming necessary ? */ |
1335 | if (expires_next.tv64 != KTIME_MAX) { | 1333 | if (expires_next.tv64 != KTIME_MAX) { |
1336 | if (tick_program_event(expires_next, 0)) | 1334 | if (tick_program_event(expires_next, 0)) |
1337 | goto retry; | 1335 | goto retry; |
1338 | } | 1336 | } |
1339 | 1337 | ||
1340 | /* Raise softirq ? */ | 1338 | /* Raise softirq ? */ |
1341 | if (raise) | 1339 | if (raise) |
1342 | raise_softirq(HRTIMER_SOFTIRQ); | 1340 | raise_softirq(HRTIMER_SOFTIRQ); |
1343 | } | 1341 | } |
1344 | 1342 | ||
1345 | static void run_hrtimer_softirq(struct softirq_action *h) | 1343 | static void run_hrtimer_softirq(struct softirq_action *h) |
1346 | { | 1344 | { |
1347 | run_hrtimer_pending(&__get_cpu_var(hrtimer_bases)); | 1345 | run_hrtimer_pending(&__get_cpu_var(hrtimer_bases)); |
1348 | } | 1346 | } |
1349 | 1347 | ||
1350 | #endif /* CONFIG_HIGH_RES_TIMERS */ | 1348 | #endif /* CONFIG_HIGH_RES_TIMERS */ |
1351 | 1349 | ||
1352 | /* | 1350 | /* |
1353 | * Called from timer softirq every jiffy, expire hrtimers: | 1351 | * Called from timer softirq every jiffy, expire hrtimers: |
1354 | * | 1352 | * |
1355 | * For HRT its the fall back code to run the softirq in the timer | 1353 | * For HRT its the fall back code to run the softirq in the timer |
1356 | * softirq context in case the hrtimer initialization failed or has | 1354 | * softirq context in case the hrtimer initialization failed or has |
1357 | * not been done yet. | 1355 | * not been done yet. |
1358 | */ | 1356 | */ |
1359 | void hrtimer_run_pending(void) | 1357 | void hrtimer_run_pending(void) |
1360 | { | 1358 | { |
1361 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | 1359 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
1362 | 1360 | ||
1363 | if (hrtimer_hres_active()) | 1361 | if (hrtimer_hres_active()) |
1364 | return; | 1362 | return; |
1365 | 1363 | ||
1366 | /* | 1364 | /* |
1367 | * This _is_ ugly: We have to check in the softirq context, | 1365 | * This _is_ ugly: We have to check in the softirq context, |
1368 | * whether we can switch to highres and / or nohz mode. The | 1366 | * whether we can switch to highres and / or nohz mode. The |
1369 | * clocksource switch happens in the timer interrupt with | 1367 | * clocksource switch happens in the timer interrupt with |
1370 | * xtime_lock held. Notification from there only sets the | 1368 | * xtime_lock held. Notification from there only sets the |
1371 | * check bit in the tick_oneshot code, otherwise we might | 1369 | * check bit in the tick_oneshot code, otherwise we might |
1372 | * deadlock vs. xtime_lock. | 1370 | * deadlock vs. xtime_lock. |
1373 | */ | 1371 | */ |
1374 | if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) | 1372 | if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) |
1375 | hrtimer_switch_to_hres(); | 1373 | hrtimer_switch_to_hres(); |
1376 | 1374 | ||
1377 | run_hrtimer_pending(cpu_base); | 1375 | run_hrtimer_pending(cpu_base); |
1378 | } | 1376 | } |
1379 | 1377 | ||
1380 | /* | 1378 | /* |
1381 | * Called from hardirq context every jiffy | 1379 | * Called from hardirq context every jiffy |
1382 | */ | 1380 | */ |
1383 | void hrtimer_run_queues(void) | 1381 | void hrtimer_run_queues(void) |
1384 | { | 1382 | { |
1385 | struct rb_node *node; | 1383 | struct rb_node *node; |
1386 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | 1384 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
1387 | struct hrtimer_clock_base *base; | 1385 | struct hrtimer_clock_base *base; |
1388 | int index, gettime = 1; | 1386 | int index, gettime = 1; |
1389 | 1387 | ||
1390 | if (hrtimer_hres_active()) | 1388 | if (hrtimer_hres_active()) |
1391 | return; | 1389 | return; |
1392 | 1390 | ||
1393 | for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { | 1391 | for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { |
1394 | base = &cpu_base->clock_base[index]; | 1392 | base = &cpu_base->clock_base[index]; |
1395 | 1393 | ||
1396 | if (!base->first) | 1394 | if (!base->first) |
1397 | continue; | 1395 | continue; |
1398 | 1396 | ||
1399 | if (base->get_softirq_time) | 1397 | if (base->get_softirq_time) |
1400 | base->softirq_time = base->get_softirq_time(); | 1398 | base->softirq_time = base->get_softirq_time(); |
1401 | else if (gettime) { | 1399 | else if (gettime) { |
1402 | hrtimer_get_softirq_time(cpu_base); | 1400 | hrtimer_get_softirq_time(cpu_base); |
1403 | gettime = 0; | 1401 | gettime = 0; |
1404 | } | 1402 | } |
1405 | 1403 | ||
1406 | spin_lock(&cpu_base->lock); | 1404 | spin_lock(&cpu_base->lock); |
1407 | 1405 | ||
1408 | while ((node = base->first)) { | 1406 | while ((node = base->first)) { |
1409 | struct hrtimer *timer; | 1407 | struct hrtimer *timer; |
1410 | 1408 | ||
1411 | timer = rb_entry(node, struct hrtimer, node); | 1409 | timer = rb_entry(node, struct hrtimer, node); |
1412 | if (base->softirq_time.tv64 <= timer->expires.tv64) | 1410 | if (base->softirq_time.tv64 <= timer->expires.tv64) |
1413 | break; | 1411 | break; |
1414 | 1412 | ||
1415 | if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { | 1413 | if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { |
1416 | __remove_hrtimer(timer, base, | 1414 | __remove_hrtimer(timer, base, |
1417 | HRTIMER_STATE_PENDING, 0); | 1415 | HRTIMER_STATE_PENDING, 0); |
1418 | list_add_tail(&timer->cb_entry, | 1416 | list_add_tail(&timer->cb_entry, |
1419 | &base->cpu_base->cb_pending); | 1417 | &base->cpu_base->cb_pending); |
1420 | continue; | 1418 | continue; |
1421 | } | 1419 | } |
1422 | 1420 | ||
1423 | __run_hrtimer(timer); | 1421 | __run_hrtimer(timer); |
1424 | } | 1422 | } |
1425 | spin_unlock(&cpu_base->lock); | 1423 | spin_unlock(&cpu_base->lock); |
1426 | } | 1424 | } |
1427 | } | 1425 | } |
1428 | 1426 | ||
1429 | /* | 1427 | /* |
1430 | * Sleep related functions: | 1428 | * Sleep related functions: |
1431 | */ | 1429 | */ |
1432 | static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) | 1430 | static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) |
1433 | { | 1431 | { |
1434 | struct hrtimer_sleeper *t = | 1432 | struct hrtimer_sleeper *t = |
1435 | container_of(timer, struct hrtimer_sleeper, timer); | 1433 | container_of(timer, struct hrtimer_sleeper, timer); |
1436 | struct task_struct *task = t->task; | 1434 | struct task_struct *task = t->task; |
1437 | 1435 | ||
1438 | t->task = NULL; | 1436 | t->task = NULL; |
1439 | if (task) | 1437 | if (task) |
1440 | wake_up_process(task); | 1438 | wake_up_process(task); |
1441 | 1439 | ||
1442 | return HRTIMER_NORESTART; | 1440 | return HRTIMER_NORESTART; |
1443 | } | 1441 | } |
1444 | 1442 | ||
1445 | void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) | 1443 | void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) |
1446 | { | 1444 | { |
1447 | sl->timer.function = hrtimer_wakeup; | 1445 | sl->timer.function = hrtimer_wakeup; |
1448 | sl->task = task; | 1446 | sl->task = task; |
1449 | #ifdef CONFIG_HIGH_RES_TIMERS | 1447 | #ifdef CONFIG_HIGH_RES_TIMERS |
1450 | sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; | 1448 | sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ; |
1451 | #endif | 1449 | #endif |
1452 | } | 1450 | } |
1453 | 1451 | ||
1454 | static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) | 1452 | static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) |
1455 | { | 1453 | { |
1456 | hrtimer_init_sleeper(t, current); | 1454 | hrtimer_init_sleeper(t, current); |
1457 | 1455 | ||
1458 | do { | 1456 | do { |
1459 | set_current_state(TASK_INTERRUPTIBLE); | 1457 | set_current_state(TASK_INTERRUPTIBLE); |
1460 | hrtimer_start(&t->timer, t->timer.expires, mode); | 1458 | hrtimer_start(&t->timer, t->timer.expires, mode); |
1461 | if (!hrtimer_active(&t->timer)) | 1459 | if (!hrtimer_active(&t->timer)) |
1462 | t->task = NULL; | 1460 | t->task = NULL; |
1463 | 1461 | ||
1464 | if (likely(t->task)) | 1462 | if (likely(t->task)) |
1465 | schedule(); | 1463 | schedule(); |
1466 | 1464 | ||
1467 | hrtimer_cancel(&t->timer); | 1465 | hrtimer_cancel(&t->timer); |
1468 | mode = HRTIMER_MODE_ABS; | 1466 | mode = HRTIMER_MODE_ABS; |
1469 | 1467 | ||
1470 | } while (t->task && !signal_pending(current)); | 1468 | } while (t->task && !signal_pending(current)); |
1471 | 1469 | ||
1472 | __set_current_state(TASK_RUNNING); | 1470 | __set_current_state(TASK_RUNNING); |
1473 | 1471 | ||
1474 | return t->task == NULL; | 1472 | return t->task == NULL; |
1475 | } | 1473 | } |
1476 | 1474 | ||
1477 | static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp) | 1475 | static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp) |
1478 | { | 1476 | { |
1479 | struct timespec rmt; | 1477 | struct timespec rmt; |
1480 | ktime_t rem; | 1478 | ktime_t rem; |
1481 | 1479 | ||
1482 | rem = ktime_sub(timer->expires, timer->base->get_time()); | 1480 | rem = ktime_sub(timer->expires, timer->base->get_time()); |
1483 | if (rem.tv64 <= 0) | 1481 | if (rem.tv64 <= 0) |
1484 | return 0; | 1482 | return 0; |
1485 | rmt = ktime_to_timespec(rem); | 1483 | rmt = ktime_to_timespec(rem); |
1486 | 1484 | ||
1487 | if (copy_to_user(rmtp, &rmt, sizeof(*rmtp))) | 1485 | if (copy_to_user(rmtp, &rmt, sizeof(*rmtp))) |
1488 | return -EFAULT; | 1486 | return -EFAULT; |
1489 | 1487 | ||
1490 | return 1; | 1488 | return 1; |
1491 | } | 1489 | } |
1492 | 1490 | ||
1493 | long __sched hrtimer_nanosleep_restart(struct restart_block *restart) | 1491 | long __sched hrtimer_nanosleep_restart(struct restart_block *restart) |
1494 | { | 1492 | { |
1495 | struct hrtimer_sleeper t; | 1493 | struct hrtimer_sleeper t; |
1496 | struct timespec __user *rmtp; | 1494 | struct timespec __user *rmtp; |
1497 | int ret = 0; | 1495 | int ret = 0; |
1498 | 1496 | ||
1499 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.index, | 1497 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.index, |
1500 | HRTIMER_MODE_ABS); | 1498 | HRTIMER_MODE_ABS); |
1501 | t.timer.expires.tv64 = restart->nanosleep.expires; | 1499 | t.timer.expires.tv64 = restart->nanosleep.expires; |
1502 | 1500 | ||
1503 | if (do_nanosleep(&t, HRTIMER_MODE_ABS)) | 1501 | if (do_nanosleep(&t, HRTIMER_MODE_ABS)) |
1504 | goto out; | 1502 | goto out; |
1505 | 1503 | ||
1506 | rmtp = restart->nanosleep.rmtp; | 1504 | rmtp = restart->nanosleep.rmtp; |
1507 | if (rmtp) { | 1505 | if (rmtp) { |
1508 | ret = update_rmtp(&t.timer, rmtp); | 1506 | ret = update_rmtp(&t.timer, rmtp); |
1509 | if (ret <= 0) | 1507 | if (ret <= 0) |
1510 | goto out; | 1508 | goto out; |
1511 | } | 1509 | } |
1512 | 1510 | ||
1513 | /* The other values in restart are already filled in */ | 1511 | /* The other values in restart are already filled in */ |
1514 | ret = -ERESTART_RESTARTBLOCK; | 1512 | ret = -ERESTART_RESTARTBLOCK; |
1515 | out: | 1513 | out: |
1516 | destroy_hrtimer_on_stack(&t.timer); | 1514 | destroy_hrtimer_on_stack(&t.timer); |
1517 | return ret; | 1515 | return ret; |
1518 | } | 1516 | } |
1519 | 1517 | ||
1520 | long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | 1518 | long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, |
1521 | const enum hrtimer_mode mode, const clockid_t clockid) | 1519 | const enum hrtimer_mode mode, const clockid_t clockid) |
1522 | { | 1520 | { |
1523 | struct restart_block *restart; | 1521 | struct restart_block *restart; |
1524 | struct hrtimer_sleeper t; | 1522 | struct hrtimer_sleeper t; |
1525 | int ret = 0; | 1523 | int ret = 0; |
1526 | 1524 | ||
1527 | hrtimer_init_on_stack(&t.timer, clockid, mode); | 1525 | hrtimer_init_on_stack(&t.timer, clockid, mode); |
1528 | t.timer.expires = timespec_to_ktime(*rqtp); | 1526 | t.timer.expires = timespec_to_ktime(*rqtp); |
1529 | if (do_nanosleep(&t, mode)) | 1527 | if (do_nanosleep(&t, mode)) |
1530 | goto out; | 1528 | goto out; |
1531 | 1529 | ||
1532 | /* Absolute timers do not update the rmtp value and restart: */ | 1530 | /* Absolute timers do not update the rmtp value and restart: */ |
1533 | if (mode == HRTIMER_MODE_ABS) { | 1531 | if (mode == HRTIMER_MODE_ABS) { |
1534 | ret = -ERESTARTNOHAND; | 1532 | ret = -ERESTARTNOHAND; |
1535 | goto out; | 1533 | goto out; |
1536 | } | 1534 | } |
1537 | 1535 | ||
1538 | if (rmtp) { | 1536 | if (rmtp) { |
1539 | ret = update_rmtp(&t.timer, rmtp); | 1537 | ret = update_rmtp(&t.timer, rmtp); |
1540 | if (ret <= 0) | 1538 | if (ret <= 0) |
1541 | goto out; | 1539 | goto out; |
1542 | } | 1540 | } |
1543 | 1541 | ||
1544 | restart = ¤t_thread_info()->restart_block; | 1542 | restart = ¤t_thread_info()->restart_block; |
1545 | restart->fn = hrtimer_nanosleep_restart; | 1543 | restart->fn = hrtimer_nanosleep_restart; |
1546 | restart->nanosleep.index = t.timer.base->index; | 1544 | restart->nanosleep.index = t.timer.base->index; |
1547 | restart->nanosleep.rmtp = rmtp; | 1545 | restart->nanosleep.rmtp = rmtp; |
1548 | restart->nanosleep.expires = t.timer.expires.tv64; | 1546 | restart->nanosleep.expires = t.timer.expires.tv64; |
1549 | 1547 | ||
1550 | ret = -ERESTART_RESTARTBLOCK; | 1548 | ret = -ERESTART_RESTARTBLOCK; |
1551 | out: | 1549 | out: |
1552 | destroy_hrtimer_on_stack(&t.timer); | 1550 | destroy_hrtimer_on_stack(&t.timer); |
1553 | return ret; | 1551 | return ret; |
1554 | } | 1552 | } |
1555 | 1553 | ||
1556 | asmlinkage long | 1554 | asmlinkage long |
1557 | sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp) | 1555 | sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp) |
1558 | { | 1556 | { |
1559 | struct timespec tu; | 1557 | struct timespec tu; |
1560 | 1558 | ||
1561 | if (copy_from_user(&tu, rqtp, sizeof(tu))) | 1559 | if (copy_from_user(&tu, rqtp, sizeof(tu))) |
1562 | return -EFAULT; | 1560 | return -EFAULT; |
1563 | 1561 | ||
1564 | if (!timespec_valid(&tu)) | 1562 | if (!timespec_valid(&tu)) |
1565 | return -EINVAL; | 1563 | return -EINVAL; |
1566 | 1564 | ||
1567 | return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); | 1565 | return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); |
1568 | } | 1566 | } |
1569 | 1567 | ||
1570 | /* | 1568 | /* |
1571 | * Functions related to boot-time initialization: | 1569 | * Functions related to boot-time initialization: |
1572 | */ | 1570 | */ |
1573 | static void __cpuinit init_hrtimers_cpu(int cpu) | 1571 | static void __cpuinit init_hrtimers_cpu(int cpu) |
1574 | { | 1572 | { |
1575 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); | 1573 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); |
1576 | int i; | 1574 | int i; |
1577 | 1575 | ||
1578 | spin_lock_init(&cpu_base->lock); | 1576 | spin_lock_init(&cpu_base->lock); |
1579 | 1577 | ||
1580 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) | 1578 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) |
1581 | cpu_base->clock_base[i].cpu_base = cpu_base; | 1579 | cpu_base->clock_base[i].cpu_base = cpu_base; |
1582 | 1580 | ||
1583 | INIT_LIST_HEAD(&cpu_base->cb_pending); | 1581 | INIT_LIST_HEAD(&cpu_base->cb_pending); |
1584 | hrtimer_init_hres(cpu_base); | 1582 | hrtimer_init_hres(cpu_base); |
1585 | } | 1583 | } |
1586 | 1584 | ||
1587 | #ifdef CONFIG_HOTPLUG_CPU | 1585 | #ifdef CONFIG_HOTPLUG_CPU |
1588 | 1586 | ||
1589 | static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, | 1587 | static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, |
1590 | struct hrtimer_clock_base *new_base) | 1588 | struct hrtimer_clock_base *new_base) |
1591 | { | 1589 | { |
1592 | struct hrtimer *timer; | 1590 | struct hrtimer *timer; |
1593 | struct rb_node *node; | 1591 | struct rb_node *node; |
1594 | 1592 | ||
1595 | while ((node = rb_first(&old_base->active))) { | 1593 | while ((node = rb_first(&old_base->active))) { |
1596 | timer = rb_entry(node, struct hrtimer, node); | 1594 | timer = rb_entry(node, struct hrtimer, node); |
1597 | BUG_ON(hrtimer_callback_running(timer)); | 1595 | BUG_ON(hrtimer_callback_running(timer)); |
1598 | debug_hrtimer_deactivate(timer); | 1596 | debug_hrtimer_deactivate(timer); |
1599 | __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0); | 1597 | __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0); |
1600 | timer->base = new_base; | 1598 | timer->base = new_base; |
1601 | /* | 1599 | /* |
1602 | * Enqueue the timer. Allow reprogramming of the event device | 1600 | * Enqueue the timer. Allow reprogramming of the event device |
1603 | */ | 1601 | */ |
1604 | enqueue_hrtimer(timer, new_base, 1); | 1602 | enqueue_hrtimer(timer, new_base, 1); |
1605 | } | 1603 | } |
1606 | } | 1604 | } |
1607 | 1605 | ||
1608 | static void migrate_hrtimers(int cpu) | 1606 | static void migrate_hrtimers(int cpu) |
1609 | { | 1607 | { |
1610 | struct hrtimer_cpu_base *old_base, *new_base; | 1608 | struct hrtimer_cpu_base *old_base, *new_base; |
1611 | int i; | 1609 | int i; |
1612 | 1610 | ||
1613 | BUG_ON(cpu_online(cpu)); | 1611 | BUG_ON(cpu_online(cpu)); |
1614 | old_base = &per_cpu(hrtimer_bases, cpu); | 1612 | old_base = &per_cpu(hrtimer_bases, cpu); |
1615 | new_base = &get_cpu_var(hrtimer_bases); | 1613 | new_base = &get_cpu_var(hrtimer_bases); |
1616 | 1614 | ||
1617 | tick_cancel_sched_timer(cpu); | 1615 | tick_cancel_sched_timer(cpu); |
1618 | 1616 | ||
1619 | local_irq_disable(); | 1617 | local_irq_disable(); |
1620 | spin_lock(&new_base->lock); | 1618 | spin_lock(&new_base->lock); |
1621 | spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); | 1619 | spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); |
1622 | 1620 | ||
1623 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 1621 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
1624 | migrate_hrtimer_list(&old_base->clock_base[i], | 1622 | migrate_hrtimer_list(&old_base->clock_base[i], |
1625 | &new_base->clock_base[i]); | 1623 | &new_base->clock_base[i]); |
1626 | } | 1624 | } |
1627 | 1625 | ||
1628 | spin_unlock(&old_base->lock); | 1626 | spin_unlock(&old_base->lock); |
1629 | spin_unlock(&new_base->lock); | 1627 | spin_unlock(&new_base->lock); |
1630 | local_irq_enable(); | 1628 | local_irq_enable(); |
1631 | put_cpu_var(hrtimer_bases); | 1629 | put_cpu_var(hrtimer_bases); |
1632 | } | 1630 | } |
1633 | #endif /* CONFIG_HOTPLUG_CPU */ | 1631 | #endif /* CONFIG_HOTPLUG_CPU */ |
1634 | 1632 | ||
1635 | static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, | 1633 | static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, |
1636 | unsigned long action, void *hcpu) | 1634 | unsigned long action, void *hcpu) |
1637 | { | 1635 | { |
1638 | unsigned int cpu = (long)hcpu; | 1636 | unsigned int cpu = (long)hcpu; |
1639 | 1637 | ||
1640 | switch (action) { | 1638 | switch (action) { |
1641 | 1639 | ||
1642 | case CPU_UP_PREPARE: | 1640 | case CPU_UP_PREPARE: |
1643 | case CPU_UP_PREPARE_FROZEN: | 1641 | case CPU_UP_PREPARE_FROZEN: |
1644 | init_hrtimers_cpu(cpu); | 1642 | init_hrtimers_cpu(cpu); |
1645 | break; | 1643 | break; |
1646 | 1644 | ||
1647 | #ifdef CONFIG_HOTPLUG_CPU | 1645 | #ifdef CONFIG_HOTPLUG_CPU |
1648 | case CPU_DEAD: | 1646 | case CPU_DEAD: |
1649 | case CPU_DEAD_FROZEN: | 1647 | case CPU_DEAD_FROZEN: |
1650 | clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &cpu); | 1648 | clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &cpu); |
1651 | migrate_hrtimers(cpu); | 1649 | migrate_hrtimers(cpu); |
1652 | break; | 1650 | break; |
1653 | #endif | 1651 | #endif |
1654 | 1652 | ||
1655 | default: | 1653 | default: |
1656 | break; | 1654 | break; |
1657 | } | 1655 | } |
1658 | 1656 | ||
1659 | return NOTIFY_OK; | 1657 | return NOTIFY_OK; |
1660 | } | 1658 | } |
1661 | 1659 | ||
1662 | static struct notifier_block __cpuinitdata hrtimers_nb = { | 1660 | static struct notifier_block __cpuinitdata hrtimers_nb = { |
1663 | .notifier_call = hrtimer_cpu_notify, | 1661 | .notifier_call = hrtimer_cpu_notify, |
1664 | }; | 1662 | }; |
1665 | 1663 | ||
1666 | void __init hrtimers_init(void) | 1664 | void __init hrtimers_init(void) |
1667 | { | 1665 | { |
1668 | hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, | 1666 | hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, |
1669 | (void *)(long)smp_processor_id()); | 1667 | (void *)(long)smp_processor_id()); |
1670 | register_cpu_notifier(&hrtimers_nb); | 1668 | register_cpu_notifier(&hrtimers_nb); |
1671 | #ifdef CONFIG_HIGH_RES_TIMERS | 1669 | #ifdef CONFIG_HIGH_RES_TIMERS |
1672 | open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL); | 1670 | open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL); |
1673 | #endif | 1671 | #endif |
1674 | } | 1672 | } |
1675 | 1673 | ||
1676 | 1674 |