Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 288867ec5c377db82933b64460ce050e5c998ee9

Authored by Thomas Gleixner
1 parent 593195f9b2
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

[hrtimer] Remove listhead from hrtimer struct 

The list_head in the hrtimer structure was introduced for easy access
to the first timer with the further extensions of real high resolution
timers in mind, but it turned out in the course of development that
it is not necessary for the standard use case. Remove the list head
and access the first expiry timer by a datafield in the timer base.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

Showing 2 changed files with 16 additions and 17 deletions Inline Diff

include/linux/hrtimer.h
Diff comments   View file @ 288867e
1 /* 1 /*
2 * include/linux/hrtimer.h 2 * include/linux/hrtimer.h
3 * 3 *
4 * hrtimers - High-resolution kernel timers 4 * hrtimers - High-resolution kernel timers
5 * 5 *
6 * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de> 6 * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
7 * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar 7 * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
8 * 8 *
9 * data type definitions, declarations, prototypes 9 * data type definitions, declarations, prototypes
10 * 10 *
11 * Started by: Thomas Gleixner and Ingo Molnar 11 * Started by: Thomas Gleixner and Ingo Molnar
12 * 12 *
13 * For licencing details see kernel-base/COPYING 13 * For licencing details see kernel-base/COPYING
14 */ 14 */
15 #ifndef _LINUX_HRTIMER_H 15 #ifndef _LINUX_HRTIMER_H
16 #define _LINUX_HRTIMER_H 16 #define _LINUX_HRTIMER_H
17 17
18 #include <linux/rbtree.h> 18 #include <linux/rbtree.h>
19 #include <linux/ktime.h> 19 #include <linux/ktime.h>
20 #include <linux/init.h> 20 #include <linux/init.h>
21 #include <linux/list.h> 21 #include <linux/list.h>
22 #include <linux/wait.h> 22 #include <linux/wait.h>
23 23
24 /* 24 /*
25 * Mode arguments of xxx_hrtimer functions: 25 * Mode arguments of xxx_hrtimer functions:
26 */ 26 */
27 enum hrtimer_mode { 27 enum hrtimer_mode {
28 HRTIMER_ABS, /* Time value is absolute */ 28 HRTIMER_ABS, /* Time value is absolute */
29 HRTIMER_REL, /* Time value is relative to now */ 29 HRTIMER_REL, /* Time value is relative to now */
30 }; 30 };
31 31
32 enum hrtimer_restart { 32 enum hrtimer_restart {
33 HRTIMER_NORESTART, 33 HRTIMER_NORESTART,
34 HRTIMER_RESTART, 34 HRTIMER_RESTART,
35 }; 35 };
36 36
37 /* 37 /*
38 * Timer states: 38 * Timer states:
39 */ 39 */
40 enum hrtimer_state { 40 enum hrtimer_state {
41 HRTIMER_INACTIVE, /* Timer is inactive */ 41 HRTIMER_INACTIVE, /* Timer is inactive */
42 HRTIMER_EXPIRED, /* Timer is expired */ 42 HRTIMER_EXPIRED, /* Timer is expired */
43 HRTIMER_PENDING, /* Timer is pending */ 43 HRTIMER_PENDING, /* Timer is pending */
44 }; 44 };
45 45
46 struct hrtimer_base; 46 struct hrtimer_base;
47 47
48 /** 48 /**
49 * struct hrtimer - the basic hrtimer structure 49 * struct hrtimer - the basic hrtimer structure
50 * 50 *
51 * @node: red black tree node for time ordered insertion 51 * @node: red black tree node for time ordered insertion
52 * @list: list head for easier access to the time ordered list,
53 * without walking the red black tree.
54 * @expires: the absolute expiry time in the hrtimers internal 52 * @expires: the absolute expiry time in the hrtimers internal
55 * representation. The time is related to the clock on 53 * representation. The time is related to the clock on
56 * which the timer is based. 54 * which the timer is based.
57 * @state: state of the timer 55 * @state: state of the timer
58 * @function: timer expiry callback function 56 * @function: timer expiry callback function
59 * @data: argument for the callback function 57 * @data: argument for the callback function
60 * @base: pointer to the timer base (per cpu and per clock) 58 * @base: pointer to the timer base (per cpu and per clock)
61 * 59 *
62 * The hrtimer structure must be initialized by init_hrtimer_#CLOCKTYPE() 60 * The hrtimer structure must be initialized by init_hrtimer_#CLOCKTYPE()
63 */ 61 */
64 struct hrtimer { 62 struct hrtimer {
65 struct rb_node node; 63 struct rb_node node;
66 struct list_head list;
67 ktime_t expires; 64 ktime_t expires;
68 enum hrtimer_state state; 65 enum hrtimer_state state;
69 int (*function)(void *); 66 int (*function)(void *);
70 void *data; 67 void *data;
71 struct hrtimer_base *base; 68 struct hrtimer_base *base;
72 }; 69 };
73 70
74 /** 71 /**
75 * struct hrtimer_base - the timer base for a specific clock 72 * struct hrtimer_base - the timer base for a specific clock
76 * 73 *
77 * @index: clock type index for per_cpu support when moving a timer 74 * @index: clock type index for per_cpu support when moving a timer
78 * to a base on another cpu. 75 * to a base on another cpu.
79 * @lock: lock protecting the base and associated timers 76 * @lock: lock protecting the base and associated timers
80 * @active: red black tree root node for the active timers 77 * @active: red black tree root node for the active timers
81 * @pending: list of pending timers for simple time ordered access 78 * @first: pointer to the timer node which expires first
82 * @resolution: the resolution of the clock, in nanoseconds 79 * @resolution: the resolution of the clock, in nanoseconds
83 * @get_time: function to retrieve the current time of the clock 80 * @get_time: function to retrieve the current time of the clock
84 * @curr_timer: the timer which is executing a callback right now 81 * @curr_timer: the timer which is executing a callback right now
85 */ 82 */
86 struct hrtimer_base { 83 struct hrtimer_base {
87 clockid_t index; 84 clockid_t index;
88 spinlock_t lock; 85 spinlock_t lock;
89 struct rb_root active; 86 struct rb_root active;
90 struct list_head pending; 87 struct rb_node *first;
91 unsigned long resolution; 88 unsigned long resolution;
92 ktime_t (*get_time)(void); 89 ktime_t (*get_time)(void);
93 struct hrtimer *curr_timer; 90 struct hrtimer *curr_timer;
94 }; 91 };
95 92
96 /* 93 /*
97 * clock_was_set() is a NOP for non- high-resolution systems. The 94 * clock_was_set() is a NOP for non- high-resolution systems. The
98 * time-sorted order guarantees that a timer does not expire early and 95 * time-sorted order guarantees that a timer does not expire early and
99 * is expired in the next softirq when the clock was advanced. 96 * is expired in the next softirq when the clock was advanced.
100 */ 97 */
101 #define clock_was_set() do { } while (0) 98 #define clock_was_set() do { } while (0)
102 99
103 /* Exported timer functions: */ 100 /* Exported timer functions: */
104 101
105 /* Initialize timers: */ 102 /* Initialize timers: */
106 extern void hrtimer_init(struct hrtimer *timer, const clockid_t which_clock); 103 extern void hrtimer_init(struct hrtimer *timer, const clockid_t which_clock);
107 extern void hrtimer_rebase(struct hrtimer *timer, const clockid_t which_clock); 104 extern void hrtimer_rebase(struct hrtimer *timer, const clockid_t which_clock);
108 105
109 106
110 /* Basic timer operations: */ 107 /* Basic timer operations: */
111 extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, 108 extern int hrtimer_start(struct hrtimer *timer, ktime_t tim,
112 const enum hrtimer_mode mode); 109 const enum hrtimer_mode mode);
113 extern int hrtimer_cancel(struct hrtimer *timer); 110 extern int hrtimer_cancel(struct hrtimer *timer);
114 extern int hrtimer_try_to_cancel(struct hrtimer *timer); 111 extern int hrtimer_try_to_cancel(struct hrtimer *timer);
115 112
116 #define hrtimer_restart(timer) hrtimer_start((timer), (timer)->expires, HRTIMER_ABS) 113 #define hrtimer_restart(timer) hrtimer_start((timer), (timer)->expires, HRTIMER_ABS)
117 114
118 /* Query timers: */ 115 /* Query timers: */
119 extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer); 116 extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer);
120 extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp); 117 extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp);
121 118
122 static inline int hrtimer_active(const struct hrtimer *timer) 119 static inline int hrtimer_active(const struct hrtimer *timer)
123 { 120 {
124 return timer->state == HRTIMER_PENDING; 121 return timer->state == HRTIMER_PENDING;
125 } 122 }
126 123
127 /* Forward a hrtimer so it expires after now: */ 124 /* Forward a hrtimer so it expires after now: */
128 extern unsigned long hrtimer_forward(struct hrtimer *timer, 125 extern unsigned long hrtimer_forward(struct hrtimer *timer,
129 const ktime_t interval); 126 const ktime_t interval);
130 127
131 /* Precise sleep: */ 128 /* Precise sleep: */
132 extern long hrtimer_nanosleep(struct timespec *rqtp, 129 extern long hrtimer_nanosleep(struct timespec *rqtp,
133 struct timespec __user *rmtp, 130 struct timespec __user *rmtp,
134 const enum hrtimer_mode mode, 131 const enum hrtimer_mode mode,
135 const clockid_t clockid); 132 const clockid_t clockid);
136 133
137 /* Soft interrupt function to run the hrtimer queues: */ 134 /* Soft interrupt function to run the hrtimer queues: */
138 extern void hrtimer_run_queues(void); 135 extern void hrtimer_run_queues(void);
139 136
140 /* Bootup initialization: */ 137 /* Bootup initialization: */
141 extern void __init hrtimers_init(void); 138 extern void __init hrtimers_init(void);
142 139
143 #endif 140 #endif
144 141
1 /* 1 /*
2 * linux/kernel/hrtimer.c 2 * linux/kernel/hrtimer.c
3 * 3 *
4 * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de> 4 * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar 5 * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
6 * 6 *
7 * High-resolution kernel timers 7 * High-resolution kernel timers
8 * 8 *
9 * In contrast to the low-resolution timeout API implemented in 9 * In contrast to the low-resolution timeout API implemented in
10 * kernel/timer.c, hrtimers provide finer resolution and accuracy 10 * kernel/timer.c, hrtimers provide finer resolution and accuracy
11 * depending on system configuration and capabilities. 11 * depending on system configuration and capabilities.
12 * 12 *
13 * These timers are currently used for: 13 * These timers are currently used for:
14 * - itimers 14 * - itimers
15 * - POSIX timers 15 * - POSIX timers
16 * - nanosleep 16 * - nanosleep
17 * - precise in-kernel timing 17 * - precise in-kernel timing
18 * 18 *
19 * Started by: Thomas Gleixner and Ingo Molnar 19 * Started by: Thomas Gleixner and Ingo Molnar
20 * 20 *
21 * Credits: 21 * Credits:
22 * based on kernel/timer.c 22 * based on kernel/timer.c
23 * 23 *
24 * For licencing details see kernel-base/COPYING 24 * For licencing details see kernel-base/COPYING
25 */ 25 */
26 26
27 #include <linux/cpu.h> 27 #include <linux/cpu.h>
28 #include <linux/module.h> 28 #include <linux/module.h>
29 #include <linux/percpu.h> 29 #include <linux/percpu.h>
30 #include <linux/hrtimer.h> 30 #include <linux/hrtimer.h>
31 #include <linux/notifier.h> 31 #include <linux/notifier.h>
32 #include <linux/syscalls.h> 32 #include <linux/syscalls.h>
33 #include <linux/interrupt.h> 33 #include <linux/interrupt.h>
34 34
35 #include <asm/uaccess.h> 35 #include <asm/uaccess.h>
36 36
37 /** 37 /**
38 * ktime_get - get the monotonic time in ktime_t format 38 * ktime_get - get the monotonic time in ktime_t format
39 * 39 *
40 * returns the time in ktime_t format 40 * returns the time in ktime_t format
41 */ 41 */
42 static ktime_t ktime_get(void) 42 static ktime_t ktime_get(void)
43 { 43 {
44 struct timespec now; 44 struct timespec now;
45 45
46 ktime_get_ts(&now); 46 ktime_get_ts(&now);
47 47
48 return timespec_to_ktime(now); 48 return timespec_to_ktime(now);
49 } 49 }
50 50
51 /** 51 /**
52 * ktime_get_real - get the real (wall-) time in ktime_t format 52 * ktime_get_real - get the real (wall-) time in ktime_t format
53 * 53 *
54 * returns the time in ktime_t format 54 * returns the time in ktime_t format
55 */ 55 */
56 static ktime_t ktime_get_real(void) 56 static ktime_t ktime_get_real(void)
57 { 57 {
58 struct timespec now; 58 struct timespec now;
59 59
60 getnstimeofday(&now); 60 getnstimeofday(&now);
61 61
62 return timespec_to_ktime(now); 62 return timespec_to_ktime(now);
63 } 63 }
64 64
65 EXPORT_SYMBOL_GPL(ktime_get_real); 65 EXPORT_SYMBOL_GPL(ktime_get_real);
66 66
67 /* 67 /*
68 * The timer bases: 68 * The timer bases:
69 */ 69 */
70 70
71 #define MAX_HRTIMER_BASES 2 71 #define MAX_HRTIMER_BASES 2
72 72
73 static DEFINE_PER_CPU(struct hrtimer_base, hrtimer_bases[MAX_HRTIMER_BASES]) = 73 static DEFINE_PER_CPU(struct hrtimer_base, hrtimer_bases[MAX_HRTIMER_BASES]) =
74 { 74 {
75 { 75 {
76 .index = CLOCK_REALTIME, 76 .index = CLOCK_REALTIME,
77 .get_time = &ktime_get_real, 77 .get_time = &ktime_get_real,
78 .resolution = KTIME_REALTIME_RES, 78 .resolution = KTIME_REALTIME_RES,
79 }, 79 },
80 { 80 {
81 .index = CLOCK_MONOTONIC, 81 .index = CLOCK_MONOTONIC,
82 .get_time = &ktime_get, 82 .get_time = &ktime_get,
83 .resolution = KTIME_MONOTONIC_RES, 83 .resolution = KTIME_MONOTONIC_RES,
84 }, 84 },
85 }; 85 };
86 86
87 /** 87 /**
88 * ktime_get_ts - get the monotonic clock in timespec format 88 * ktime_get_ts - get the monotonic clock in timespec format
89 * 89 *
90 * @ts: pointer to timespec variable 90 * @ts: pointer to timespec variable
91 * 91 *
92 * The function calculates the monotonic clock from the realtime 92 * The function calculates the monotonic clock from the realtime
93 * clock and the wall_to_monotonic offset and stores the result 93 * clock and the wall_to_monotonic offset and stores the result
94 * in normalized timespec format in the variable pointed to by ts. 94 * in normalized timespec format in the variable pointed to by ts.
95 */ 95 */
96 void ktime_get_ts(struct timespec *ts) 96 void ktime_get_ts(struct timespec *ts)
97 { 97 {
98 struct timespec tomono; 98 struct timespec tomono;
99 unsigned long seq; 99 unsigned long seq;
100 100
101 do { 101 do {
102 seq = read_seqbegin(&xtime_lock); 102 seq = read_seqbegin(&xtime_lock);
103 getnstimeofday(ts); 103 getnstimeofday(ts);
104 tomono = wall_to_monotonic; 104 tomono = wall_to_monotonic;
105 105
106 } while (read_seqretry(&xtime_lock, seq)); 106 } while (read_seqretry(&xtime_lock, seq));
107 107
108 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, 108 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
109 ts->tv_nsec + tomono.tv_nsec); 109 ts->tv_nsec + tomono.tv_nsec);
110 } 110 }
111 EXPORT_SYMBOL_GPL(ktime_get_ts); 111 EXPORT_SYMBOL_GPL(ktime_get_ts);
112 112
113 /* 113 /*
114 * Functions and macros which are different for UP/SMP systems are kept in a 114 * Functions and macros which are different for UP/SMP systems are kept in a
115 * single place 115 * single place
116 */ 116 */
117 #ifdef CONFIG_SMP 117 #ifdef CONFIG_SMP
118 118
119 #define set_curr_timer(b, t) do { (b)->curr_timer = (t); } while (0) 119 #define set_curr_timer(b, t) do { (b)->curr_timer = (t); } while (0)
120 120
121 /* 121 /*
122 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock 122 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
123 * means that all timers which are tied to this base via timer->base are 123 * means that all timers which are tied to this base via timer->base are
124 * locked, and the base itself is locked too. 124 * locked, and the base itself is locked too.
125 * 125 *
126 * So __run_timers/migrate_timers can safely modify all timers which could 126 * So __run_timers/migrate_timers can safely modify all timers which could
127 * be found on the lists/queues. 127 * be found on the lists/queues.
128 * 128 *
129 * When the timer's base is locked, and the timer removed from list, it is 129 * When the timer's base is locked, and the timer removed from list, it is
130 * possible to set timer->base = NULL and drop the lock: the timer remains 130 * possible to set timer->base = NULL and drop the lock: the timer remains
131 * locked. 131 * locked.
132 */ 132 */
133 static struct hrtimer_base *lock_hrtimer_base(const struct hrtimer *timer, 133 static struct hrtimer_base *lock_hrtimer_base(const struct hrtimer *timer,
134 unsigned long *flags) 134 unsigned long *flags)
135 { 135 {
136 struct hrtimer_base *base; 136 struct hrtimer_base *base;
137 137
138 for (;;) { 138 for (;;) {
139 base = timer->base; 139 base = timer->base;
140 if (likely(base != NULL)) { 140 if (likely(base != NULL)) {
141 spin_lock_irqsave(&base->lock, *flags); 141 spin_lock_irqsave(&base->lock, *flags);
142 if (likely(base == timer->base)) 142 if (likely(base == timer->base))
143 return base; 143 return base;
144 /* The timer has migrated to another CPU: */ 144 /* The timer has migrated to another CPU: */
145 spin_unlock_irqrestore(&base->lock, *flags); 145 spin_unlock_irqrestore(&base->lock, *flags);
146 } 146 }
147 cpu_relax(); 147 cpu_relax();
148 } 148 }
149 } 149 }
150 150
151 /* 151 /*
152 * Switch the timer base to the current CPU when possible. 152 * Switch the timer base to the current CPU when possible.
153 */ 153 */
154 static inline struct hrtimer_base * 154 static inline struct hrtimer_base *
155 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_base *base) 155 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_base *base)
156 { 156 {
157 struct hrtimer_base *new_base; 157 struct hrtimer_base *new_base;
158 158
159 new_base = &__get_cpu_var(hrtimer_bases[base->index]); 159 new_base = &__get_cpu_var(hrtimer_bases[base->index]);
160 160
161 if (base != new_base) { 161 if (base != new_base) {
162 /* 162 /*
163 * We are trying to schedule the timer on the local CPU. 163 * We are trying to schedule the timer on the local CPU.
164 * However we can't change timer's base while it is running, 164 * However we can't change timer's base while it is running,
165 * so we keep it on the same CPU. No hassle vs. reprogramming 165 * so we keep it on the same CPU. No hassle vs. reprogramming
166 * the event source in the high resolution case. The softirq 166 * the event source in the high resolution case. The softirq
167 * code will take care of this when the timer function has 167 * code will take care of this when the timer function has
168 * completed. There is no conflict as we hold the lock until 168 * completed. There is no conflict as we hold the lock until
169 * the timer is enqueued. 169 * the timer is enqueued.
170 */ 170 */
171 if (unlikely(base->curr_timer == timer)) 171 if (unlikely(base->curr_timer == timer))
172 return base; 172 return base;
173 173
174 /* See the comment in lock_timer_base() */ 174 /* See the comment in lock_timer_base() */
175 timer->base = NULL; 175 timer->base = NULL;
176 spin_unlock(&base->lock); 176 spin_unlock(&base->lock);
177 spin_lock(&new_base->lock); 177 spin_lock(&new_base->lock);
178 timer->base = new_base; 178 timer->base = new_base;
179 } 179 }
180 return new_base; 180 return new_base;
181 } 181 }
182 182
183 #else /* CONFIG_SMP */ 183 #else /* CONFIG_SMP */
184 184
185 #define set_curr_timer(b, t) do { } while (0) 185 #define set_curr_timer(b, t) do { } while (0)
186 186
187 static inline struct hrtimer_base * 187 static inline struct hrtimer_base *
188 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) 188 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
189 { 189 {
190 struct hrtimer_base *base = timer->base; 190 struct hrtimer_base *base = timer->base;
191 191
192 spin_lock_irqsave(&base->lock, *flags); 192 spin_lock_irqsave(&base->lock, *flags);
193 193
194 return base; 194 return base;
195 } 195 }
196 196
197 #define switch_hrtimer_base(t, b) (b) 197 #define switch_hrtimer_base(t, b) (b)
198 198
199 #endif /* !CONFIG_SMP */ 199 #endif /* !CONFIG_SMP */
200 200
201 /* 201 /*
202 * Functions for the union type storage format of ktime_t which are 202 * Functions for the union type storage format of ktime_t which are
203 * too large for inlining: 203 * too large for inlining:
204 */ 204 */
205 #if BITS_PER_LONG < 64 205 #if BITS_PER_LONG < 64
206 # ifndef CONFIG_KTIME_SCALAR 206 # ifndef CONFIG_KTIME_SCALAR
207 /** 207 /**
208 * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable 208 * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
209 * 209 *
210 * @kt: addend 210 * @kt: addend
211 * @nsec: the scalar nsec value to add 211 * @nsec: the scalar nsec value to add
212 * 212 *
213 * Returns the sum of kt and nsec in ktime_t format 213 * Returns the sum of kt and nsec in ktime_t format
214 */ 214 */
215 ktime_t ktime_add_ns(const ktime_t kt, u64 nsec) 215 ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
216 { 216 {
217 ktime_t tmp; 217 ktime_t tmp;
218 218
219 if (likely(nsec < NSEC_PER_SEC)) { 219 if (likely(nsec < NSEC_PER_SEC)) {
220 tmp.tv64 = nsec; 220 tmp.tv64 = nsec;
221 } else { 221 } else {
222 unsigned long rem = do_div(nsec, NSEC_PER_SEC); 222 unsigned long rem = do_div(nsec, NSEC_PER_SEC);
223 223
224 tmp = ktime_set((long)nsec, rem); 224 tmp = ktime_set((long)nsec, rem);
225 } 225 }
226 226
227 return ktime_add(kt, tmp); 227 return ktime_add(kt, tmp);
228 } 228 }
229 229
230 #else /* CONFIG_KTIME_SCALAR */ 230 #else /* CONFIG_KTIME_SCALAR */
231 231
232 # endif /* !CONFIG_KTIME_SCALAR */ 232 # endif /* !CONFIG_KTIME_SCALAR */
233 233
234 /* 234 /*
235 * Divide a ktime value by a nanosecond value 235 * Divide a ktime value by a nanosecond value
236 */ 236 */
237 static unsigned long ktime_divns(const ktime_t kt, nsec_t div) 237 static unsigned long ktime_divns(const ktime_t kt, nsec_t div)
238 { 238 {
239 u64 dclc, inc, dns; 239 u64 dclc, inc, dns;
240 int sft = 0; 240 int sft = 0;
241 241
242 dclc = dns = ktime_to_ns(kt); 242 dclc = dns = ktime_to_ns(kt);
243 inc = div; 243 inc = div;
244 /* Make sure the divisor is less than 2^32: */ 244 /* Make sure the divisor is less than 2^32: */
245 while (div >> 32) { 245 while (div >> 32) {
246 sft++; 246 sft++;
247 div >>= 1; 247 div >>= 1;
248 } 248 }
249 dclc >>= sft; 249 dclc >>= sft;
250 do_div(dclc, (unsigned long) div); 250 do_div(dclc, (unsigned long) div);
251 251
252 return (unsigned long) dclc; 252 return (unsigned long) dclc;
253 } 253 }
254 254
255 #else /* BITS_PER_LONG < 64 */ 255 #else /* BITS_PER_LONG < 64 */
256 # define ktime_divns(kt, div) (unsigned long)((kt).tv64 / (div)) 256 # define ktime_divns(kt, div) (unsigned long)((kt).tv64 / (div))
257 #endif /* BITS_PER_LONG >= 64 */ 257 #endif /* BITS_PER_LONG >= 64 */
258 258
259 /* 259 /*
260 * Counterpart to lock_timer_base above: 260 * Counterpart to lock_timer_base above:
261 */ 261 */
262 static inline 262 static inline
263 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) 263 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
264 { 264 {
265 spin_unlock_irqrestore(&timer->base->lock, *flags); 265 spin_unlock_irqrestore(&timer->base->lock, *flags);
266 } 266 }
267 267
268 /** 268 /**
269 * hrtimer_forward - forward the timer expiry 269 * hrtimer_forward - forward the timer expiry
270 * 270 *
271 * @timer: hrtimer to forward 271 * @timer: hrtimer to forward
272 * @interval: the interval to forward 272 * @interval: the interval to forward
273 * 273 *
274 * Forward the timer expiry so it will expire in the future. 274 * Forward the timer expiry so it will expire in the future.
275 * The number of overruns is added to the overrun field. 275 * The number of overruns is added to the overrun field.
276 */ 276 */
277 unsigned long 277 unsigned long
278 hrtimer_forward(struct hrtimer *timer, const ktime_t interval) 278 hrtimer_forward(struct hrtimer *timer, const ktime_t interval)
279 { 279 {
280 unsigned long orun = 1; 280 unsigned long orun = 1;
281 ktime_t delta, now; 281 ktime_t delta, now;
282 282
283 now = timer->base->get_time(); 283 now = timer->base->get_time();
284 284
285 delta = ktime_sub(now, timer->expires); 285 delta = ktime_sub(now, timer->expires);
286 286
287 if (delta.tv64 < 0) 287 if (delta.tv64 < 0)
288 return 0; 288 return 0;
289 289
290 if (unlikely(delta.tv64 >= interval.tv64)) { 290 if (unlikely(delta.tv64 >= interval.tv64)) {
291 nsec_t incr = ktime_to_ns(interval); 291 nsec_t incr = ktime_to_ns(interval);
292 292
293 orun = ktime_divns(delta, incr); 293 orun = ktime_divns(delta, incr);
294 timer->expires = ktime_add_ns(timer->expires, incr * orun); 294 timer->expires = ktime_add_ns(timer->expires, incr * orun);
295 if (timer->expires.tv64 > now.tv64) 295 if (timer->expires.tv64 > now.tv64)
296 return orun; 296 return orun;
297 /* 297 /*
298 * This (and the ktime_add() below) is the 298 * This (and the ktime_add() below) is the
299 * correction for exact: 299 * correction for exact:
300 */ 300 */
301 orun++; 301 orun++;
302 } 302 }
303 timer->expires = ktime_add(timer->expires, interval); 303 timer->expires = ktime_add(timer->expires, interval);
304 304
305 return orun; 305 return orun;
306 } 306 }
307 307
308 /* 308 /*
309 * enqueue_hrtimer - internal function to (re)start a timer 309 * enqueue_hrtimer - internal function to (re)start a timer
310 * 310 *
311 * The timer is inserted in expiry order. Insertion into the 311 * The timer is inserted in expiry order. Insertion into the
312 * red black tree is O(log(n)). Must hold the base lock. 312 * red black tree is O(log(n)). Must hold the base lock.
313 */ 313 */
314 static void enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_base *base) 314 static void enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
315 { 315 {
316 struct rb_node **link = &base->active.rb_node; 316 struct rb_node **link = &base->active.rb_node;
317 struct list_head *prev = &base->pending;
318 struct rb_node *parent = NULL; 317 struct rb_node *parent = NULL;
319 struct hrtimer *entry; 318 struct hrtimer *entry;
320 319
321 /* 320 /*
322 * Find the right place in the rbtree: 321 * Find the right place in the rbtree:
323 */ 322 */
324 while (*link) { 323 while (*link) {
325 parent = *link; 324 parent = *link;
326 entry = rb_entry(parent, struct hrtimer, node); 325 entry = rb_entry(parent, struct hrtimer, node);
327 /* 326 /*
328 * We dont care about collisions. Nodes with 327 * We dont care about collisions. Nodes with
329 * the same expiry time stay together. 328 * the same expiry time stay together.
330 */ 329 */
331 if (timer->expires.tv64 < entry->expires.tv64) 330 if (timer->expires.tv64 < entry->expires.tv64)
332 link = &(*link)->rb_left; 331 link = &(*link)->rb_left;
333 else { 332 else
334 link = &(*link)->rb_right; 333 link = &(*link)->rb_right;
335 prev = &entry->list;
336 }
337 } 334 }
338 335
339 /* 336 /*
340 * Insert the timer to the rbtree and to the sorted list: 337 * Insert the timer to the rbtree and check whether it
338 * replaces the first pending timer
341 */ 339 */
342 rb_link_node(&timer->node, parent, link); 340 rb_link_node(&timer->node, parent, link);
343 rb_insert_color(&timer->node, &base->active); 341 rb_insert_color(&timer->node, &base->active);
344 list_add(&timer->list, prev);
345 342
346 timer->state = HRTIMER_PENDING; 343 timer->state = HRTIMER_PENDING;
344
345 if (!base->first || timer->expires.tv64 <
346 rb_entry(base->first, struct hrtimer, node)->expires.tv64)
347 base->first = &timer->node;
347 } 348 }
348 349
349
350 /* 350 /*
351 * __remove_hrtimer - internal function to remove a timer 351 * __remove_hrtimer - internal function to remove a timer
352 * 352 *
353 * Caller must hold the base lock. 353 * Caller must hold the base lock.
354 */ 354 */
355 static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base) 355 static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
356 { 356 {
357 /* 357 /*
358 * Remove the timer from the sorted list and from the rbtree: 358 * Remove the timer from the rbtree and replace the
359 * first entry pointer if necessary.
359 */ 360 */
360 list_del(&timer->list); 361 if (base->first == &timer->node)
362 base->first = rb_next(&timer->node);
361 rb_erase(&timer->node, &base->active); 363 rb_erase(&timer->node, &base->active);
362 } 364 }
363 365
364 /* 366 /*
365 * remove hrtimer, called with base lock held 367 * remove hrtimer, called with base lock held
366 */ 368 */
367 static inline int 369 static inline int
368 remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base) 370 remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
369 { 371 {
370 if (hrtimer_active(timer)) { 372 if (hrtimer_active(timer)) {
371 __remove_hrtimer(timer, base); 373 __remove_hrtimer(timer, base);
372 timer->state = HRTIMER_INACTIVE; 374 timer->state = HRTIMER_INACTIVE;
373 return 1; 375 return 1;
374 } 376 }
375 return 0; 377 return 0;
376 } 378 }
377 379
378 /** 380 /**
379 * hrtimer_start - (re)start an relative timer on the current CPU 381 * hrtimer_start - (re)start an relative timer on the current CPU
380 * 382 *
381 * @timer: the timer to be added 383 * @timer: the timer to be added
382 * @tim: expiry time 384 * @tim: expiry time
383 * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) 385 * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
384 * 386 *
385 * Returns: 387 * Returns:
386 * 0 on success 388 * 0 on success
387 * 1 when the timer was active 389 * 1 when the timer was active
388 */ 390 */
389 int 391 int
390 hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) 392 hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
391 { 393 {
392 struct hrtimer_base *base, *new_base; 394 struct hrtimer_base *base, *new_base;
393 unsigned long flags; 395 unsigned long flags;
394 int ret; 396 int ret;
395 397
396 base = lock_hrtimer_base(timer, &flags); 398 base = lock_hrtimer_base(timer, &flags);
397 399
398 /* Remove an active timer from the queue: */ 400 /* Remove an active timer from the queue: */
399 ret = remove_hrtimer(timer, base); 401 ret = remove_hrtimer(timer, base);
400 402
401 /* Switch the timer base, if necessary: */ 403 /* Switch the timer base, if necessary: */
402 new_base = switch_hrtimer_base(timer, base); 404 new_base = switch_hrtimer_base(timer, base);
403 405
404 if (mode == HRTIMER_REL) 406 if (mode == HRTIMER_REL)
405 tim = ktime_add(tim, new_base->get_time()); 407 tim = ktime_add(tim, new_base->get_time());
406 timer->expires = tim; 408 timer->expires = tim;
407 409
408 enqueue_hrtimer(timer, new_base); 410 enqueue_hrtimer(timer, new_base);
409 411
410 unlock_hrtimer_base(timer, &flags); 412 unlock_hrtimer_base(timer, &flags);
411 413
412 return ret; 414 return ret;
413 } 415 }
414 416
415 /** 417 /**
416 * hrtimer_try_to_cancel - try to deactivate a timer 418 * hrtimer_try_to_cancel - try to deactivate a timer
417 * 419 *
418 * @timer: hrtimer to stop 420 * @timer: hrtimer to stop
419 * 421 *
420 * Returns: 422 * Returns:
421 * 0 when the timer was not active 423 * 0 when the timer was not active
422 * 1 when the timer was active 424 * 1 when the timer was active
423 * -1 when the timer is currently excuting the callback function and 425 * -1 when the timer is currently excuting the callback function and
424 * can not be stopped 426 * can not be stopped
425 */ 427 */
426 int hrtimer_try_to_cancel(struct hrtimer *timer) 428 int hrtimer_try_to_cancel(struct hrtimer *timer)
427 { 429 {
428 struct hrtimer_base *base; 430 struct hrtimer_base *base;
429 unsigned long flags; 431 unsigned long flags;
430 int ret = -1; 432 int ret = -1;
431 433
432 base = lock_hrtimer_base(timer, &flags); 434 base = lock_hrtimer_base(timer, &flags);
433 435
434 if (base->curr_timer != timer) 436 if (base->curr_timer != timer)
435 ret = remove_hrtimer(timer, base); 437 ret = remove_hrtimer(timer, base);
436 438
437 unlock_hrtimer_base(timer, &flags); 439 unlock_hrtimer_base(timer, &flags);
438 440
439 return ret; 441 return ret;
440 442
441 } 443 }
442 444
443 /** 445 /**
444 * hrtimer_cancel - cancel a timer and wait for the handler to finish. 446 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
445 * 447 *
446 * @timer: the timer to be cancelled 448 * @timer: the timer to be cancelled
447 * 449 *
448 * Returns: 450 * Returns:
449 * 0 when the timer was not active 451 * 0 when the timer was not active
450 * 1 when the timer was active 452 * 1 when the timer was active
451 */ 453 */
452 int hrtimer_cancel(struct hrtimer *timer) 454 int hrtimer_cancel(struct hrtimer *timer)
453 { 455 {
454 for (;;) { 456 for (;;) {
455 int ret = hrtimer_try_to_cancel(timer); 457 int ret = hrtimer_try_to_cancel(timer);
456 458
457 if (ret >= 0) 459 if (ret >= 0)
458 return ret; 460 return ret;
459 } 461 }
460 } 462 }
461 463
462 /** 464 /**
463 * hrtimer_get_remaining - get remaining time for the timer 465 * hrtimer_get_remaining - get remaining time for the timer
464 * 466 *
465 * @timer: the timer to read 467 * @timer: the timer to read
466 */ 468 */
467 ktime_t hrtimer_get_remaining(const struct hrtimer *timer) 469 ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
468 { 470 {
469 struct hrtimer_base *base; 471 struct hrtimer_base *base;
470 unsigned long flags; 472 unsigned long flags;
471 ktime_t rem; 473 ktime_t rem;
472 474
473 base = lock_hrtimer_base(timer, &flags); 475 base = lock_hrtimer_base(timer, &flags);
474 rem = ktime_sub(timer->expires, timer->base->get_time()); 476 rem = ktime_sub(timer->expires, timer->base->get_time());
475 unlock_hrtimer_base(timer, &flags); 477 unlock_hrtimer_base(timer, &flags);
476 478
477 return rem; 479 return rem;
478 } 480 }
479 481
480 /** 482 /**
481 * hrtimer_rebase - rebase an initialized hrtimer to a different base 483 * hrtimer_rebase - rebase an initialized hrtimer to a different base
482 * 484 *
483 * @timer: the timer to be rebased 485 * @timer: the timer to be rebased
484 * @clock_id: the clock to be used 486 * @clock_id: the clock to be used
485 */ 487 */
486 void hrtimer_rebase(struct hrtimer *timer, const clockid_t clock_id) 488 void hrtimer_rebase(struct hrtimer *timer, const clockid_t clock_id)
487 { 489 {
488 struct hrtimer_base *bases; 490 struct hrtimer_base *bases;
489 491
490 bases = per_cpu(hrtimer_bases, raw_smp_processor_id()); 492 bases = per_cpu(hrtimer_bases, raw_smp_processor_id());
491 timer->base = &bases[clock_id]; 493 timer->base = &bases[clock_id];
492 } 494 }
493 495
494 /** 496 /**
495 * hrtimer_init - initialize a timer to the given clock 497 * hrtimer_init - initialize a timer to the given clock
496 * 498 *
497 * @timer: the timer to be initialized 499 * @timer: the timer to be initialized
498 * @clock_id: the clock to be used 500 * @clock_id: the clock to be used
499 */ 501 */
500 void hrtimer_init(struct hrtimer *timer, const clockid_t clock_id) 502 void hrtimer_init(struct hrtimer *timer, const clockid_t clock_id)
501 { 503 {
502 memset(timer, 0, sizeof(struct hrtimer)); 504 memset(timer, 0, sizeof(struct hrtimer));
503 hrtimer_rebase(timer, clock_id); 505 hrtimer_rebase(timer, clock_id);
504 } 506 }
505 507
506 /** 508 /**
507 * hrtimer_get_res - get the timer resolution for a clock 509 * hrtimer_get_res - get the timer resolution for a clock
508 * 510 *
509 * @which_clock: which clock to query 511 * @which_clock: which clock to query
510 * @tp: pointer to timespec variable to store the resolution 512 * @tp: pointer to timespec variable to store the resolution
511 * 513 *
512 * Store the resolution of the clock selected by which_clock in the 514 * Store the resolution of the clock selected by which_clock in the
513 * variable pointed to by tp. 515 * variable pointed to by tp.
514 */ 516 */
515 int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) 517 int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
516 { 518 {
517 struct hrtimer_base *bases; 519 struct hrtimer_base *bases;
518 520
519 tp->tv_sec = 0; 521 tp->tv_sec = 0;
520 bases = per_cpu(hrtimer_bases, raw_smp_processor_id()); 522 bases = per_cpu(hrtimer_bases, raw_smp_processor_id());
521 tp->tv_nsec = bases[which_clock].resolution; 523 tp->tv_nsec = bases[which_clock].resolution;
522 524
523 return 0; 525 return 0;
524 } 526 }
525 527
526 /* 528 /*
527 * Expire the per base hrtimer-queue: 529 * Expire the per base hrtimer-queue:
528 */ 530 */
529 static inline void run_hrtimer_queue(struct hrtimer_base *base) 531 static inline void run_hrtimer_queue(struct hrtimer_base *base)
530 { 532 {
531 ktime_t now = base->get_time(); 533 ktime_t now = base->get_time();
534 struct rb_node *node;
532 535
533 spin_lock_irq(&base->lock); 536 spin_lock_irq(&base->lock);
534 537
535 while (!list_empty(&base->pending)) { 538 while ((node = base->first)) {
536 struct hrtimer *timer; 539 struct hrtimer *timer;
537 int (*fn)(void *); 540 int (*fn)(void *);
538 int restart; 541 int restart;
539 void *data; 542 void *data;
540 543
541 timer = list_entry(base->pending.next, struct hrtimer, list); 544 timer = rb_entry(node, struct hrtimer, node);
542 if (now.tv64 <= timer->expires.tv64) 545 if (now.tv64 <= timer->expires.tv64)
543 break; 546 break;
544 547
545 fn = timer->function; 548 fn = timer->function;
546 data = timer->data; 549 data = timer->data;
547 set_curr_timer(base, timer); 550 set_curr_timer(base, timer);
548 __remove_hrtimer(timer, base); 551 __remove_hrtimer(timer, base);
549 spin_unlock_irq(&base->lock); 552 spin_unlock_irq(&base->lock);
550 553
551 /* 554 /*
552 * fn == NULL is special case for the simplest timer 555 * fn == NULL is special case for the simplest timer
553 * variant - wake up process and do not restart: 556 * variant - wake up process and do not restart:
554 */ 557 */
555 if (!fn) { 558 if (!fn) {
556 wake_up_process(data); 559 wake_up_process(data);
557 restart = HRTIMER_NORESTART; 560 restart = HRTIMER_NORESTART;
558 } else 561 } else
559 restart = fn(data); 562 restart = fn(data);
560 563
561 spin_lock_irq(&base->lock); 564 spin_lock_irq(&base->lock);
562 565
563 if (restart == HRTIMER_RESTART) 566 if (restart == HRTIMER_RESTART)
564 enqueue_hrtimer(timer, base); 567 enqueue_hrtimer(timer, base);
565 else 568 else
566 timer->state = HRTIMER_EXPIRED; 569 timer->state = HRTIMER_EXPIRED;
567 } 570 }
568 set_curr_timer(base, NULL); 571 set_curr_timer(base, NULL);
569 spin_unlock_irq(&base->lock); 572 spin_unlock_irq(&base->lock);
570 } 573 }
571 574
572 /* 575 /*
573 * Called from timer softirq every jiffy, expire hrtimers: 576 * Called from timer softirq every jiffy, expire hrtimers:
574 */ 577 */
575 void hrtimer_run_queues(void) 578 void hrtimer_run_queues(void)
576 { 579 {
577 struct hrtimer_base *base = __get_cpu_var(hrtimer_bases); 580 struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
578 int i; 581 int i;
579 582
580 for (i = 0; i < MAX_HRTIMER_BASES; i++) 583 for (i = 0; i < MAX_HRTIMER_BASES; i++)
581 run_hrtimer_queue(&base[i]); 584 run_hrtimer_queue(&base[i]);
582 } 585 }
583 586
584 /* 587 /*
585 * Sleep related functions: 588 * Sleep related functions:
586 */ 589 */
587 590
588 /** 591 /**
589 * schedule_hrtimer - sleep until timeout 592 * schedule_hrtimer - sleep until timeout
590 * 593 *
591 * @timer: hrtimer variable initialized with the correct clock base 594 * @timer: hrtimer variable initialized with the correct clock base
592 * @mode: timeout value is abs/rel 595 * @mode: timeout value is abs/rel
593 * 596 *
594 * Make the current task sleep until @timeout is 597 * Make the current task sleep until @timeout is
595 * elapsed. 598 * elapsed.
596 * 599 *
597 * You can set the task state as follows - 600 * You can set the task state as follows -
598 * 601 *
599 * %TASK_UNINTERRUPTIBLE - at least @timeout is guaranteed to 602 * %TASK_UNINTERRUPTIBLE - at least @timeout is guaranteed to
600 * pass before the routine returns. The routine will return 0 603 * pass before the routine returns. The routine will return 0
601 * 604 *
602 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is 605 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
603 * delivered to the current task. In this case the remaining time 606 * delivered to the current task. In this case the remaining time
604 * will be returned 607 * will be returned
605 * 608 *
606 * The current task state is guaranteed to be TASK_RUNNING when this 609 * The current task state is guaranteed to be TASK_RUNNING when this
607 * routine returns. 610 * routine returns.
608 */ 611 */
609 static ktime_t __sched 612 static ktime_t __sched
610 schedule_hrtimer(struct hrtimer *timer, const enum hrtimer_mode mode) 613 schedule_hrtimer(struct hrtimer *timer, const enum hrtimer_mode mode)
611 { 614 {
612 /* fn stays NULL, meaning single-shot wakeup: */ 615 /* fn stays NULL, meaning single-shot wakeup: */
613 timer->data = current; 616 timer->data = current;
614 617
615 hrtimer_start(timer, timer->expires, mode); 618 hrtimer_start(timer, timer->expires, mode);
616 619
617 schedule(); 620 schedule();
618 hrtimer_cancel(timer); 621 hrtimer_cancel(timer);
619 622
620 /* Return the remaining time: */ 623 /* Return the remaining time: */
621 if (timer->state != HRTIMER_EXPIRED) 624 if (timer->state != HRTIMER_EXPIRED)
622 return ktime_sub(timer->expires, timer->base->get_time()); 625 return ktime_sub(timer->expires, timer->base->get_time());
623 else 626 else
624 return (ktime_t) {.tv64 = 0 }; 627 return (ktime_t) {.tv64 = 0 };
625 } 628 }
626 629
627 static inline ktime_t __sched 630 static inline ktime_t __sched
628 schedule_hrtimer_interruptible(struct hrtimer *timer, 631 schedule_hrtimer_interruptible(struct hrtimer *timer,
629 const enum hrtimer_mode mode) 632 const enum hrtimer_mode mode)
630 { 633 {
631 set_current_state(TASK_INTERRUPTIBLE); 634 set_current_state(TASK_INTERRUPTIBLE);
632 635
633 return schedule_hrtimer(timer, mode); 636 return schedule_hrtimer(timer, mode);
634 } 637 }
635 638
636 static long __sched 639 static long __sched
637 nanosleep_restart(struct restart_block *restart, clockid_t clockid) 640 nanosleep_restart(struct restart_block *restart, clockid_t clockid)
638 { 641 {
639 struct timespec __user *rmtp, tu; 642 struct timespec __user *rmtp, tu;
640 void *rfn_save = restart->fn; 643 void *rfn_save = restart->fn;
641 struct hrtimer timer; 644 struct hrtimer timer;
642 ktime_t rem; 645 ktime_t rem;
643 646
644 restart->fn = do_no_restart_syscall; 647 restart->fn = do_no_restart_syscall;
645 648
646 hrtimer_init(&timer, clockid); 649 hrtimer_init(&timer, clockid);
647 650
648 timer.expires.tv64 = ((u64)restart->arg1 << 32) | (u64) restart->arg0; 651 timer.expires.tv64 = ((u64)restart->arg1 << 32) | (u64) restart->arg0;
649 652
650 rem = schedule_hrtimer_interruptible(&timer, HRTIMER_ABS); 653 rem = schedule_hrtimer_interruptible(&timer, HRTIMER_ABS);
651 654
652 if (rem.tv64 <= 0) 655 if (rem.tv64 <= 0)
653 return 0; 656 return 0;
654 657
655 rmtp = (struct timespec __user *) restart->arg2; 658 rmtp = (struct timespec __user *) restart->arg2;
656 tu = ktime_to_timespec(rem); 659 tu = ktime_to_timespec(rem);
657 if (rmtp && copy_to_user(rmtp, &tu, sizeof(tu))) 660 if (rmtp && copy_to_user(rmtp, &tu, sizeof(tu)))
658 return -EFAULT; 661 return -EFAULT;
659 662
660 restart->fn = rfn_save; 663 restart->fn = rfn_save;
661 664
662 /* The other values in restart are already filled in */ 665 /* The other values in restart are already filled in */
663 return -ERESTART_RESTARTBLOCK; 666 return -ERESTART_RESTARTBLOCK;
664 } 667 }
665 668
666 static long __sched nanosleep_restart_mono(struct restart_block *restart) 669 static long __sched nanosleep_restart_mono(struct restart_block *restart)
667 { 670 {
668 return nanosleep_restart(restart, CLOCK_MONOTONIC); 671 return nanosleep_restart(restart, CLOCK_MONOTONIC);
669 } 672 }
670 673
671 static long __sched nanosleep_restart_real(struct restart_block *restart) 674 static long __sched nanosleep_restart_real(struct restart_block *restart)
672 { 675 {
673 return nanosleep_restart(restart, CLOCK_REALTIME); 676 return nanosleep_restart(restart, CLOCK_REALTIME);
674 } 677 }
675 678
676 long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, 679 long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
677 const enum hrtimer_mode mode, const clockid_t clockid) 680 const enum hrtimer_mode mode, const clockid_t clockid)
678 { 681 {
679 struct restart_block *restart; 682 struct restart_block *restart;
680 struct hrtimer timer; 683 struct hrtimer timer;
681 struct timespec tu; 684 struct timespec tu;
682 ktime_t rem; 685 ktime_t rem;
683 686
684 hrtimer_init(&timer, clockid); 687 hrtimer_init(&timer, clockid);
685 688
686 timer.expires = timespec_to_ktime(*rqtp); 689 timer.expires = timespec_to_ktime(*rqtp);
687 690
688 rem = schedule_hrtimer_interruptible(&timer, mode); 691 rem = schedule_hrtimer_interruptible(&timer, mode);
689 if (rem.tv64 <= 0) 692 if (rem.tv64 <= 0)
690 return 0; 693 return 0;
691 694
692 /* Absolute timers do not update the rmtp value: */ 695 /* Absolute timers do not update the rmtp value: */
693 if (mode == HRTIMER_ABS) 696 if (mode == HRTIMER_ABS)
694 return -ERESTARTNOHAND; 697 return -ERESTARTNOHAND;
695 698
696 tu = ktime_to_timespec(rem); 699 tu = ktime_to_timespec(rem);
697 700
698 if (rmtp && copy_to_user(rmtp, &tu, sizeof(tu))) 701 if (rmtp && copy_to_user(rmtp, &tu, sizeof(tu)))
699 return -EFAULT; 702 return -EFAULT;
700 703
701 restart = &current_thread_info()->restart_block; 704 restart = &current_thread_info()->restart_block;
702 restart->fn = (clockid == CLOCK_MONOTONIC) ? 705 restart->fn = (clockid == CLOCK_MONOTONIC) ?
703 nanosleep_restart_mono : nanosleep_restart_real; 706 nanosleep_restart_mono : nanosleep_restart_real;
704 restart->arg0 = timer.expires.tv64 & 0xFFFFFFFF; 707 restart->arg0 = timer.expires.tv64 & 0xFFFFFFFF;
705 restart->arg1 = timer.expires.tv64 >> 32; 708 restart->arg1 = timer.expires.tv64 >> 32;
706 restart->arg2 = (unsigned long) rmtp; 709 restart->arg2 = (unsigned long) rmtp;
707 710
708 return -ERESTART_RESTARTBLOCK; 711 return -ERESTART_RESTARTBLOCK;
709 } 712 }
710 713
711 asmlinkage long 714 asmlinkage long
712 sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp) 715 sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp)
713 { 716 {
714 struct timespec tu; 717 struct timespec tu;
715 718
716 if (copy_from_user(&tu, rqtp, sizeof(tu))) 719 if (copy_from_user(&tu, rqtp, sizeof(tu)))
717 return -EFAULT; 720 return -EFAULT;
718 721
719 if (!timespec_valid(&tu)) 722 if (!timespec_valid(&tu))
720 return -EINVAL; 723 return -EINVAL;
721 724
722 return hrtimer_nanosleep(&tu, rmtp, HRTIMER_REL, CLOCK_MONOTONIC); 725 return hrtimer_nanosleep(&tu, rmtp, HRTIMER_REL, CLOCK_MONOTONIC);
723 } 726 }
724 727
725 /* 728 /*
726 * Functions related to boot-time initialization: 729 * Functions related to boot-time initialization:
727 */ 730 */
728 static void __devinit init_hrtimers_cpu(int cpu) 731 static void __devinit init_hrtimers_cpu(int cpu)
729 { 732 {
730 struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu); 733 struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu);
731 int i; 734 int i;
732 735
733 for (i = 0; i < MAX_HRTIMER_BASES; i++) { 736 for (i = 0; i < MAX_HRTIMER_BASES; i++) {
734 spin_lock_init(&base->lock); 737 spin_lock_init(&base->lock);
735 INIT_LIST_HEAD(&base->pending);
736 base++; 738 base++;
737 } 739 }
738 } 740 }
739 741
740 #ifdef CONFIG_HOTPLUG_CPU 742 #ifdef CONFIG_HOTPLUG_CPU
741 743
742 static void migrate_hrtimer_list(struct hrtimer_base *old_base, 744 static void migrate_hrtimer_list(struct hrtimer_base *old_base,
743 struct hrtimer_base *new_base) 745 struct hrtimer_base *new_base)
744 { 746 {
745 struct hrtimer *timer; 747 struct hrtimer *timer;
746 struct rb_node *node; 748 struct rb_node *node;
747 749
748 while ((node = rb_first(&old_base->active))) { 750 while ((node = rb_first(&old_base->active))) {
749 timer = rb_entry(node, struct hrtimer, node); 751 timer = rb_entry(node, struct hrtimer, node);
750 __remove_hrtimer(timer, old_base); 752 __remove_hrtimer(timer, old_base);
751 timer->base = new_base; 753 timer->base = new_base;
752 enqueue_hrtimer(timer, new_base); 754 enqueue_hrtimer(timer, new_base);
753 } 755 }
754 } 756 }
755 757
756 static void migrate_hrtimers(int cpu) 758 static void migrate_hrtimers(int cpu)
757 { 759 {
758 struct hrtimer_base *old_base, *new_base; 760 struct hrtimer_base *old_base, *new_base;
759 int i; 761 int i;
760 762
761 BUG_ON(cpu_online(cpu)); 763 BUG_ON(cpu_online(cpu));
762 old_base = per_cpu(hrtimer_bases, cpu); 764 old_base = per_cpu(hrtimer_bases, cpu);
763 new_base = get_cpu_var(hrtimer_bases); 765 new_base = get_cpu_var(hrtimer_bases);
764 766
765 local_irq_disable(); 767 local_irq_disable();
766 768
767 for (i = 0; i < MAX_HRTIMER_BASES; i++) { 769 for (i = 0; i < MAX_HRTIMER_BASES; i++) {
768 770
769 spin_lock(&new_base->lock); 771 spin_lock(&new_base->lock);
770 spin_lock(&old_base->lock); 772 spin_lock(&old_base->lock);
771 773
772 BUG_ON(old_base->curr_timer); 774 BUG_ON(old_base->curr_timer);
773 775
774 migrate_hrtimer_list(old_base, new_base); 776 migrate_hrtimer_list(old_base, new_base);
775 777
776 spin_unlock(&old_base->lock); 778 spin_unlock(&old_base->lock);
777 spin_unlock(&new_base->lock); 779 spin_unlock(&new_base->lock);
778 old_base++; 780 old_base++;
779 new_base++; 781 new_base++;
780 } 782 }
781 783
782 local_irq_enable(); 784 local_irq_enable();
783 put_cpu_var(hrtimer_bases); 785 put_cpu_var(hrtimer_bases);
784 } 786 }
785 #endif /* CONFIG_HOTPLUG_CPU */ 787 #endif /* CONFIG_HOTPLUG_CPU */
786 788
787 static int __devinit hrtimer_cpu_notify(struct notifier_block *self, 789 static int __devinit hrtimer_cpu_notify(struct notifier_block *self,
788 unsigned long action, void *hcpu) 790 unsigned long action, void *hcpu)
789 { 791 {
790 long cpu = (long)hcpu; 792 long cpu = (long)hcpu;
791 793
792 switch (action) { 794 switch (action) {
793 795
794 case CPU_UP_PREPARE: 796 case CPU_UP_PREPARE:
795 init_hrtimers_cpu(cpu); 797 init_hrtimers_cpu(cpu);
796 break; 798 break;
797 799
798 #ifdef CONFIG_HOTPLUG_CPU 800 #ifdef CONFIG_HOTPLUG_CPU
799 case CPU_DEAD: 801 case CPU_DEAD:
800 migrate_hrtimers(cpu); 802 migrate_hrtimers(cpu);
801 break; 803 break;
802 #endif 804 #endif
803 805
804 default: 806 default:
805 break; 807 break;
806 } 808 }
807 809
808 return NOTIFY_OK; 810 return NOTIFY_OK;
809 } 811 }
810 812
811 static struct notifier_block __devinitdata hrtimers_nb = { 813 static struct notifier_block __devinitdata hrtimers_nb = {
812 .notifier_call = hrtimer_cpu_notify, 814 .notifier_call = hrtimer_cpu_notify,
813 }; 815 };
814 816
815 void __init hrtimers_init(void) 817 void __init hrtimers_init(void)
816 { 818 {