Eric Lee / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit dd6414b50fa2b1cd247a8aa8f8bd42414b7453e1

Authored by Phil Carmody
Committed by Thomas Gleixner
1 parent 2bf1c05e3c
Exists in master and in 39 other branches 8mp-imx_5.4.70_2.3.0, 8qm-imx_5.4.70_2.3.0, emb_imx_lf-5.15.y, emb_lf-6.1.y, imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, imx_4.1.15_1.0.0_ga, pitx_8mp_lf-5.10.y, rt-smarc-imx_4.1.15_1.0.0_ga, rt_linux_5.15.71, smarc-8m-android-11.0.0_2.0.0, smarc-imx6_4.14.98_2.0.0_ga, smarc-imx6_4.9.88_2.0.0_ga, smarc-imx7_4.14.98_2.0.0_ga, smarc-imx7_4.9.11_1.0.0_ga, smarc-imx7_4.9.88_2.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-imx_4.1.15_1.0.0_ga, smarc-imx_4.9.11_1.0.0_ga, smarc-imx_4.9.51_imx8m_ga, smarc-imx_4.9.88_2.0.0_ga, smarc-m6.0.1_2.1.0-ga, smarc-n7.1.2_2.0.0-ga, smarc-rel_imx_4.1.15_1.2.0_ga, smarc_8m_00d0_imx_4.14.98_2.0.0_ga, smarc_8m_imx_4.14.78_1.0.0_ga, smarc_8m_imx_4.14.98_2.0.0_ga, smarc_8m_imx_4.19.35_1.1.0, smarc_8mm_imx_4.14.78_1.0.0_ga, smarc_8mm_imx_4.14.98_2.0.0_ga, smarc_8mm_imx_4.19.35_1.1.0, smarc_8mm_imx_5.4.24_2.1.0, smarc_8mp_lf-5.10.y, smarc_8mq_imx_5.4.24_2.1.0, smarc_8mq_lf-5.10.y, smarc_imx_lf-5.15.y

timer: Permit statically-declared work with deferrable timers 

Currently, you have to just define a delayed_work uninitialised, and then
initialise it before first use.  That's a tad clumsy.  At risk of playing
mind-games with the compiler, fooling it into doing pointer arithmetic
with compile-time-constants, this lets clients properly initialise delayed
work with deferrable timers statically.

This patch was inspired by the issues which lead Artem Bityutskiy to
commit 8eab945c5616fc984 ("sunrpc: make the cache cleaner workqueue
deferrable").

Signed-off-by: Phil Carmody <ext-phil.2.carmody@nokia.com>
Acked-by: Artem Bityutskiy <Artem.Bityutskiy@nokia.com>
Cc: Arjan van de Ven <arjan@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

Showing 3 changed files with 34 additions and 14 deletions Inline Diff

include/linux/timer.h
Diff comments   View file @ dd6414b
1 #ifndef _LINUX_TIMER_H 1 #ifndef _LINUX_TIMER_H
2 #define _LINUX_TIMER_H 2 #define _LINUX_TIMER_H
3 3
4 #include <linux/list.h> 4 #include <linux/list.h>
5 #include <linux/ktime.h> 5 #include <linux/ktime.h>
6 #include <linux/stddef.h> 6 #include <linux/stddef.h>
7 #include <linux/debugobjects.h> 7 #include <linux/debugobjects.h>
8 #include <linux/stringify.h> 8 #include <linux/stringify.h>
9 9
10 struct tvec_base; 10 struct tvec_base;
11 11
12 struct timer_list { 12 struct timer_list {
13 /* 13 /*
14 * All fields that change during normal runtime grouped to the 14 * All fields that change during normal runtime grouped to the
15 * same cacheline 15 * same cacheline
16 */ 16 */
17 struct list_head entry; 17 struct list_head entry;
18 unsigned long expires; 18 unsigned long expires;
19 struct tvec_base *base; 19 struct tvec_base *base;
20 20
21 void (*function)(unsigned long); 21 void (*function)(unsigned long);
22 unsigned long data; 22 unsigned long data;
23 23
24 int slack; 24 int slack;
25 25
26 #ifdef CONFIG_TIMER_STATS 26 #ifdef CONFIG_TIMER_STATS
27 int start_pid; 27 int start_pid;
28 void *start_site; 28 void *start_site;
29 char start_comm[16]; 29 char start_comm[16];
30 #endif 30 #endif
31 #ifdef CONFIG_LOCKDEP 31 #ifdef CONFIG_LOCKDEP
32 struct lockdep_map lockdep_map; 32 struct lockdep_map lockdep_map;
33 #endif 33 #endif
34 }; 34 };
35 35
36 extern struct tvec_base boot_tvec_bases; 36 extern struct tvec_base boot_tvec_bases;
37 37
38 #ifdef CONFIG_LOCKDEP 38 #ifdef CONFIG_LOCKDEP
39 /* 39 /*
40 * NB: because we have to copy the lockdep_map, setting the lockdep_map key 40 * NB: because we have to copy the lockdep_map, setting the lockdep_map key
41 * (second argument) here is required, otherwise it could be initialised to 41 * (second argument) here is required, otherwise it could be initialised to
42 * the copy of the lockdep_map later! We use the pointer to and the string 42 * the copy of the lockdep_map later! We use the pointer to and the string
43 * "<file>:<line>" as the key resp. the name of the lockdep_map. 43 * "<file>:<line>" as the key resp. the name of the lockdep_map.
44 */ 44 */
45 #define __TIMER_LOCKDEP_MAP_INITIALIZER(_kn) \ 45 #define __TIMER_LOCKDEP_MAP_INITIALIZER(_kn) \
46 .lockdep_map = STATIC_LOCKDEP_MAP_INIT(_kn, &_kn), 46 .lockdep_map = STATIC_LOCKDEP_MAP_INIT(_kn, &_kn),
47 #else 47 #else
48 #define __TIMER_LOCKDEP_MAP_INITIALIZER(_kn) 48 #define __TIMER_LOCKDEP_MAP_INITIALIZER(_kn)
49 #endif 49 #endif
50 50
51 /*
52 * Note that all tvec_bases are 2 byte aligned and lower bit of
53 * base in timer_list is guaranteed to be zero. Use the LSB to
54 * indicate whether the timer is deferrable.
55 *
56 * A deferrable timer will work normally when the system is busy, but
57 * will not cause a CPU to come out of idle just to service it; instead,
58 * the timer will be serviced when the CPU eventually wakes up with a
59 * subsequent non-deferrable timer.
60 */
61 #define TBASE_DEFERRABLE_FLAG (0x1)
62
51 #define TIMER_INITIALIZER(_function, _expires, _data) { \ 63 #define TIMER_INITIALIZER(_function, _expires, _data) { \
52 .entry = { .prev = TIMER_ENTRY_STATIC }, \ 64 .entry = { .prev = TIMER_ENTRY_STATIC }, \
53 .function = (_function), \ 65 .function = (_function), \
54 .expires = (_expires), \ 66 .expires = (_expires), \
55 .data = (_data), \ 67 .data = (_data), \
56 .base = &boot_tvec_bases, \ 68 .base = &boot_tvec_bases, \
57 .slack = -1, \ 69 .slack = -1, \
70 __TIMER_LOCKDEP_MAP_INITIALIZER( \
71 __FILE__ ":" __stringify(__LINE__)) \
72 }
73
74 #define TBASE_MAKE_DEFERRED(ptr) ((struct tvec_base *) \
75 ((unsigned char *)(ptr) + TBASE_DEFERRABLE_FLAG))
76
77 #define TIMER_DEFERRED_INITIALIZER(_function, _expires, _data) {\
78 .entry = { .prev = TIMER_ENTRY_STATIC }, \
79 .function = (_function), \
80 .expires = (_expires), \
81 .data = (_data), \
82 .base = TBASE_MAKE_DEFERRED(&boot_tvec_bases), \
58 __TIMER_LOCKDEP_MAP_INITIALIZER( \ 83 __TIMER_LOCKDEP_MAP_INITIALIZER( \
59 __FILE__ ":" __stringify(__LINE__)) \ 84 __FILE__ ":" __stringify(__LINE__)) \
60 } 85 }
61 86
62 #define DEFINE_TIMER(_name, _function, _expires, _data) \ 87 #define DEFINE_TIMER(_name, _function, _expires, _data) \
63 struct timer_list _name = \ 88 struct timer_list _name = \
64 TIMER_INITIALIZER(_function, _expires, _data) 89 TIMER_INITIALIZER(_function, _expires, _data)
65 90
66 void init_timer_key(struct timer_list *timer, 91 void init_timer_key(struct timer_list *timer,
67 const char *name, 92 const char *name,
68 struct lock_class_key *key); 93 struct lock_class_key *key);
69 void init_timer_deferrable_key(struct timer_list *timer, 94 void init_timer_deferrable_key(struct timer_list *timer,
70 const char *name, 95 const char *name,
71 struct lock_class_key *key); 96 struct lock_class_key *key);
72 97
73 #ifdef CONFIG_LOCKDEP 98 #ifdef CONFIG_LOCKDEP
74 #define init_timer(timer) \ 99 #define init_timer(timer) \
75 do { \ 100 do { \
76 static struct lock_class_key __key; \ 101 static struct lock_class_key __key; \
77 init_timer_key((timer), #timer, &__key); \ 102 init_timer_key((timer), #timer, &__key); \
78 } while (0) 103 } while (0)
79 104
80 #define init_timer_deferrable(timer) \ 105 #define init_timer_deferrable(timer) \
81 do { \ 106 do { \
82 static struct lock_class_key __key; \ 107 static struct lock_class_key __key; \
83 init_timer_deferrable_key((timer), #timer, &__key); \ 108 init_timer_deferrable_key((timer), #timer, &__key); \
84 } while (0) 109 } while (0)
85 110
86 #define init_timer_on_stack(timer) \ 111 #define init_timer_on_stack(timer) \
87 do { \ 112 do { \
88 static struct lock_class_key __key; \ 113 static struct lock_class_key __key; \
89 init_timer_on_stack_key((timer), #timer, &__key); \ 114 init_timer_on_stack_key((timer), #timer, &__key); \
90 } while (0) 115 } while (0)
91 116
92 #define setup_timer(timer, fn, data) \ 117 #define setup_timer(timer, fn, data) \
93 do { \ 118 do { \
94 static struct lock_class_key __key; \ 119 static struct lock_class_key __key; \
95 setup_timer_key((timer), #timer, &__key, (fn), (data));\ 120 setup_timer_key((timer), #timer, &__key, (fn), (data));\
96 } while (0) 121 } while (0)
97 122
98 #define setup_timer_on_stack(timer, fn, data) \ 123 #define setup_timer_on_stack(timer, fn, data) \
99 do { \ 124 do { \
100 static struct lock_class_key __key; \ 125 static struct lock_class_key __key; \
101 setup_timer_on_stack_key((timer), #timer, &__key, \ 126 setup_timer_on_stack_key((timer), #timer, &__key, \
102 (fn), (data)); \ 127 (fn), (data)); \
103 } while (0) 128 } while (0)
104 #define setup_deferrable_timer_on_stack(timer, fn, data) \ 129 #define setup_deferrable_timer_on_stack(timer, fn, data) \
105 do { \ 130 do { \
106 static struct lock_class_key __key; \ 131 static struct lock_class_key __key; \
107 setup_deferrable_timer_on_stack_key((timer), #timer, \ 132 setup_deferrable_timer_on_stack_key((timer), #timer, \
108 &__key, (fn), \ 133 &__key, (fn), \
109 (data)); \ 134 (data)); \
110 } while (0) 135 } while (0)
111 #else 136 #else
112 #define init_timer(timer)\ 137 #define init_timer(timer)\
113 init_timer_key((timer), NULL, NULL) 138 init_timer_key((timer), NULL, NULL)
114 #define init_timer_deferrable(timer)\ 139 #define init_timer_deferrable(timer)\
115 init_timer_deferrable_key((timer), NULL, NULL) 140 init_timer_deferrable_key((timer), NULL, NULL)
116 #define init_timer_on_stack(timer)\ 141 #define init_timer_on_stack(timer)\
117 init_timer_on_stack_key((timer), NULL, NULL) 142 init_timer_on_stack_key((timer), NULL, NULL)
118 #define setup_timer(timer, fn, data)\ 143 #define setup_timer(timer, fn, data)\
119 setup_timer_key((timer), NULL, NULL, (fn), (data)) 144 setup_timer_key((timer), NULL, NULL, (fn), (data))
120 #define setup_timer_on_stack(timer, fn, data)\ 145 #define setup_timer_on_stack(timer, fn, data)\
121 setup_timer_on_stack_key((timer), NULL, NULL, (fn), (data)) 146 setup_timer_on_stack_key((timer), NULL, NULL, (fn), (data))
122 #define setup_deferrable_timer_on_stack(timer, fn, data)\ 147 #define setup_deferrable_timer_on_stack(timer, fn, data)\
123 setup_deferrable_timer_on_stack_key((timer), NULL, NULL, (fn), (data)) 148 setup_deferrable_timer_on_stack_key((timer), NULL, NULL, (fn), (data))
124 #endif 149 #endif
125 150
126 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS 151 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
127 extern void init_timer_on_stack_key(struct timer_list *timer, 152 extern void init_timer_on_stack_key(struct timer_list *timer,
128 const char *name, 153 const char *name,
129 struct lock_class_key *key); 154 struct lock_class_key *key);
130 extern void destroy_timer_on_stack(struct timer_list *timer); 155 extern void destroy_timer_on_stack(struct timer_list *timer);
131 #else 156 #else
132 static inline void destroy_timer_on_stack(struct timer_list *timer) { } 157 static inline void destroy_timer_on_stack(struct timer_list *timer) { }
133 static inline void init_timer_on_stack_key(struct timer_list *timer, 158 static inline void init_timer_on_stack_key(struct timer_list *timer,
134 const char *name, 159 const char *name,
135 struct lock_class_key *key) 160 struct lock_class_key *key)
136 { 161 {
137 init_timer_key(timer, name, key); 162 init_timer_key(timer, name, key);
138 } 163 }
139 #endif 164 #endif
140 165
141 static inline void setup_timer_key(struct timer_list * timer, 166 static inline void setup_timer_key(struct timer_list * timer,
142 const char *name, 167 const char *name,
143 struct lock_class_key *key, 168 struct lock_class_key *key,
144 void (*function)(unsigned long), 169 void (*function)(unsigned long),
145 unsigned long data) 170 unsigned long data)
146 { 171 {
147 timer->function = function; 172 timer->function = function;
148 timer->data = data; 173 timer->data = data;
149 init_timer_key(timer, name, key); 174 init_timer_key(timer, name, key);
150 } 175 }
151 176
152 static inline void setup_timer_on_stack_key(struct timer_list *timer, 177 static inline void setup_timer_on_stack_key(struct timer_list *timer,
153 const char *name, 178 const char *name,
154 struct lock_class_key *key, 179 struct lock_class_key *key,
155 void (*function)(unsigned long), 180 void (*function)(unsigned long),
156 unsigned long data) 181 unsigned long data)
157 { 182 {
158 timer->function = function; 183 timer->function = function;
159 timer->data = data; 184 timer->data = data;
160 init_timer_on_stack_key(timer, name, key); 185 init_timer_on_stack_key(timer, name, key);
161 } 186 }
162 187
163 extern void setup_deferrable_timer_on_stack_key(struct timer_list *timer, 188 extern void setup_deferrable_timer_on_stack_key(struct timer_list *timer,
164 const char *name, 189 const char *name,
165 struct lock_class_key *key, 190 struct lock_class_key *key,
166 void (*function)(unsigned long), 191 void (*function)(unsigned long),
167 unsigned long data); 192 unsigned long data);
168 193
169 /** 194 /**
170 * timer_pending - is a timer pending? 195 * timer_pending - is a timer pending?
171 * @timer: the timer in question 196 * @timer: the timer in question
172 * 197 *
173 * timer_pending will tell whether a given timer is currently pending, 198 * timer_pending will tell whether a given timer is currently pending,
174 * or not. Callers must ensure serialization wrt. other operations done 199 * or not. Callers must ensure serialization wrt. other operations done
175 * to this timer, eg. interrupt contexts, or other CPUs on SMP. 200 * to this timer, eg. interrupt contexts, or other CPUs on SMP.
176 * 201 *
177 * return value: 1 if the timer is pending, 0 if not. 202 * return value: 1 if the timer is pending, 0 if not.
178 */ 203 */
179 static inline int timer_pending(const struct timer_list * timer) 204 static inline int timer_pending(const struct timer_list * timer)
180 { 205 {
181 return timer->entry.next != NULL; 206 return timer->entry.next != NULL;
182 } 207 }
183 208
184 extern void add_timer_on(struct timer_list *timer, int cpu); 209 extern void add_timer_on(struct timer_list *timer, int cpu);
185 extern int del_timer(struct timer_list * timer); 210 extern int del_timer(struct timer_list * timer);
186 extern int mod_timer(struct timer_list *timer, unsigned long expires); 211 extern int mod_timer(struct timer_list *timer, unsigned long expires);
187 extern int mod_timer_pending(struct timer_list *timer, unsigned long expires); 212 extern int mod_timer_pending(struct timer_list *timer, unsigned long expires);
188 extern int mod_timer_pinned(struct timer_list *timer, unsigned long expires); 213 extern int mod_timer_pinned(struct timer_list *timer, unsigned long expires);
189 214
190 extern void set_timer_slack(struct timer_list *time, int slack_hz); 215 extern void set_timer_slack(struct timer_list *time, int slack_hz);
191 216
192 #define TIMER_NOT_PINNED 0 217 #define TIMER_NOT_PINNED 0
193 #define TIMER_PINNED 1 218 #define TIMER_PINNED 1
194 /* 219 /*
195 * The jiffies value which is added to now, when there is no timer 220 * The jiffies value which is added to now, when there is no timer
196 * in the timer wheel: 221 * in the timer wheel:
197 */ 222 */
198 #define NEXT_TIMER_MAX_DELTA ((1UL << 30) - 1) 223 #define NEXT_TIMER_MAX_DELTA ((1UL << 30) - 1)
199 224
200 /* 225 /*
201 * Return when the next timer-wheel timeout occurs (in absolute jiffies), 226 * Return when the next timer-wheel timeout occurs (in absolute jiffies),
202 * locks the timer base and does the comparison against the given 227 * locks the timer base and does the comparison against the given
203 * jiffie. 228 * jiffie.
204 */ 229 */
205 extern unsigned long get_next_timer_interrupt(unsigned long now); 230 extern unsigned long get_next_timer_interrupt(unsigned long now);
206 231
207 /* 232 /*
208 * Timer-statistics info: 233 * Timer-statistics info:
209 */ 234 */
210 #ifdef CONFIG_TIMER_STATS 235 #ifdef CONFIG_TIMER_STATS
211 236
212 extern int timer_stats_active; 237 extern int timer_stats_active;
213 238
214 #define TIMER_STATS_FLAG_DEFERRABLE 0x1 239 #define TIMER_STATS_FLAG_DEFERRABLE 0x1
215 240
216 extern void init_timer_stats(void); 241 extern void init_timer_stats(void);
217 242
218 extern void timer_stats_update_stats(void *timer, pid_t pid, void *startf, 243 extern void timer_stats_update_stats(void *timer, pid_t pid, void *startf,
219 void *timerf, char *comm, 244 void *timerf, char *comm,
220 unsigned int timer_flag); 245 unsigned int timer_flag);
221 246
222 extern void __timer_stats_timer_set_start_info(struct timer_list *timer, 247 extern void __timer_stats_timer_set_start_info(struct timer_list *timer,
223 void *addr); 248 void *addr);
224 249
225 static inline void timer_stats_timer_set_start_info(struct timer_list *timer) 250 static inline void timer_stats_timer_set_start_info(struct timer_list *timer)
226 { 251 {
227 if (likely(!timer_stats_active)) 252 if (likely(!timer_stats_active))
228 return; 253 return;
229 __timer_stats_timer_set_start_info(timer, __builtin_return_address(0)); 254 __timer_stats_timer_set_start_info(timer, __builtin_return_address(0));
230 } 255 }
231 256
232 static inline void timer_stats_timer_clear_start_info(struct timer_list *timer) 257 static inline void timer_stats_timer_clear_start_info(struct timer_list *timer)
233 { 258 {
234 timer->start_site = NULL; 259 timer->start_site = NULL;
235 } 260 }
236 #else 261 #else
237 static inline void init_timer_stats(void) 262 static inline void init_timer_stats(void)
238 { 263 {
239 } 264 }
240 265
241 static inline void timer_stats_timer_set_start_info(struct timer_list *timer) 266 static inline void timer_stats_timer_set_start_info(struct timer_list *timer)
242 { 267 {
243 } 268 }
244 269
245 static inline void timer_stats_timer_clear_start_info(struct timer_list *timer) 270 static inline void timer_stats_timer_clear_start_info(struct timer_list *timer)
246 { 271 {
247 } 272 }
248 #endif 273 #endif
249 274
250 extern void add_timer(struct timer_list *timer); 275 extern void add_timer(struct timer_list *timer);
251 276
252 #ifdef CONFIG_SMP 277 #ifdef CONFIG_SMP
253 extern int try_to_del_timer_sync(struct timer_list *timer); 278 extern int try_to_del_timer_sync(struct timer_list *timer);
254 extern int del_timer_sync(struct timer_list *timer); 279 extern int del_timer_sync(struct timer_list *timer);
255 #else 280 #else
256 # define try_to_del_timer_sync(t) del_timer(t) 281 # define try_to_del_timer_sync(t) del_timer(t)
257 # define del_timer_sync(t) del_timer(t) 282 # define del_timer_sync(t) del_timer(t)
258 #endif 283 #endif
259 284
260 #define del_singleshot_timer_sync(t) del_timer_sync(t) 285 #define del_singleshot_timer_sync(t) del_timer_sync(t)
261 286
262 extern void init_timers(void); 287 extern void init_timers(void);
263 extern void run_local_timers(void); 288 extern void run_local_timers(void);
264 struct hrtimer; 289 struct hrtimer;
265 extern enum hrtimer_restart it_real_fn(struct hrtimer *); 290 extern enum hrtimer_restart it_real_fn(struct hrtimer *);
266 291
267 unsigned long __round_jiffies(unsigned long j, int cpu); 292 unsigned long __round_jiffies(unsigned long j, int cpu);
268 unsigned long __round_jiffies_relative(unsigned long j, int cpu); 293 unsigned long __round_jiffies_relative(unsigned long j, int cpu);
269 unsigned long round_jiffies(unsigned long j); 294 unsigned long round_jiffies(unsigned long j);
270 unsigned long round_jiffies_relative(unsigned long j); 295 unsigned long round_jiffies_relative(unsigned long j);
271 296
272 unsigned long __round_jiffies_up(unsigned long j, int cpu); 297 unsigned long __round_jiffies_up(unsigned long j, int cpu);
273 unsigned long __round_jiffies_up_relative(unsigned long j, int cpu); 298 unsigned long __round_jiffies_up_relative(unsigned long j, int cpu);
274 unsigned long round_jiffies_up(unsigned long j); 299 unsigned long round_jiffies_up(unsigned long j);
275 unsigned long round_jiffies_up_relative(unsigned long j); 300 unsigned long round_jiffies_up_relative(unsigned long j);
276 301
277 #endif 302 #endif
278 303
include/linux/workqueue.h
Diff comments   View file @ dd6414b
1 /* 1 /*
2 * workqueue.h --- work queue handling for Linux. 2 * workqueue.h --- work queue handling for Linux.
3 */ 3 */
4 4
5 #ifndef _LINUX_WORKQUEUE_H 5 #ifndef _LINUX_WORKQUEUE_H
6 #define _LINUX_WORKQUEUE_H 6 #define _LINUX_WORKQUEUE_H
7 7
8 #include <linux/timer.h> 8 #include <linux/timer.h>
9 #include <linux/linkage.h> 9 #include <linux/linkage.h>
10 #include <linux/bitops.h> 10 #include <linux/bitops.h>
11 #include <linux/lockdep.h> 11 #include <linux/lockdep.h>
12 #include <linux/threads.h> 12 #include <linux/threads.h>
13 #include <asm/atomic.h> 13 #include <asm/atomic.h>
14 14
15 struct workqueue_struct; 15 struct workqueue_struct;
16 16
17 struct work_struct; 17 struct work_struct;
18 typedef void (*work_func_t)(struct work_struct *work); 18 typedef void (*work_func_t)(struct work_struct *work);
19 19
20 /* 20 /*
21 * The first word is the work queue pointer and the flags rolled into 21 * The first word is the work queue pointer and the flags rolled into
22 * one 22 * one
23 */ 23 */
24 #define work_data_bits(work) ((unsigned long *)(&(work)->data)) 24 #define work_data_bits(work) ((unsigned long *)(&(work)->data))
25 25
26 enum { 26 enum {
27 WORK_STRUCT_PENDING_BIT = 0, /* work item is pending execution */ 27 WORK_STRUCT_PENDING_BIT = 0, /* work item is pending execution */
28 WORK_STRUCT_DELAYED_BIT = 1, /* work item is delayed */ 28 WORK_STRUCT_DELAYED_BIT = 1, /* work item is delayed */
29 WORK_STRUCT_CWQ_BIT = 2, /* data points to cwq */ 29 WORK_STRUCT_CWQ_BIT = 2, /* data points to cwq */
30 WORK_STRUCT_LINKED_BIT = 3, /* next work is linked to this one */ 30 WORK_STRUCT_LINKED_BIT = 3, /* next work is linked to this one */
31 #ifdef CONFIG_DEBUG_OBJECTS_WORK 31 #ifdef CONFIG_DEBUG_OBJECTS_WORK
32 WORK_STRUCT_STATIC_BIT = 4, /* static initializer (debugobjects) */ 32 WORK_STRUCT_STATIC_BIT = 4, /* static initializer (debugobjects) */
33 WORK_STRUCT_COLOR_SHIFT = 5, /* color for workqueue flushing */ 33 WORK_STRUCT_COLOR_SHIFT = 5, /* color for workqueue flushing */
34 #else 34 #else
35 WORK_STRUCT_COLOR_SHIFT = 4, /* color for workqueue flushing */ 35 WORK_STRUCT_COLOR_SHIFT = 4, /* color for workqueue flushing */
36 #endif 36 #endif
37 37
38 WORK_STRUCT_COLOR_BITS = 4, 38 WORK_STRUCT_COLOR_BITS = 4,
39 39
40 WORK_STRUCT_PENDING = 1 << WORK_STRUCT_PENDING_BIT, 40 WORK_STRUCT_PENDING = 1 << WORK_STRUCT_PENDING_BIT,
41 WORK_STRUCT_DELAYED = 1 << WORK_STRUCT_DELAYED_BIT, 41 WORK_STRUCT_DELAYED = 1 << WORK_STRUCT_DELAYED_BIT,
42 WORK_STRUCT_CWQ = 1 << WORK_STRUCT_CWQ_BIT, 42 WORK_STRUCT_CWQ = 1 << WORK_STRUCT_CWQ_BIT,
43 WORK_STRUCT_LINKED = 1 << WORK_STRUCT_LINKED_BIT, 43 WORK_STRUCT_LINKED = 1 << WORK_STRUCT_LINKED_BIT,
44 #ifdef CONFIG_DEBUG_OBJECTS_WORK 44 #ifdef CONFIG_DEBUG_OBJECTS_WORK
45 WORK_STRUCT_STATIC = 1 << WORK_STRUCT_STATIC_BIT, 45 WORK_STRUCT_STATIC = 1 << WORK_STRUCT_STATIC_BIT,
46 #else 46 #else
47 WORK_STRUCT_STATIC = 0, 47 WORK_STRUCT_STATIC = 0,
48 #endif 48 #endif
49 49
50 /* 50 /*
51 * The last color is no color used for works which don't 51 * The last color is no color used for works which don't
52 * participate in workqueue flushing. 52 * participate in workqueue flushing.
53 */ 53 */
54 WORK_NR_COLORS = (1 << WORK_STRUCT_COLOR_BITS) - 1, 54 WORK_NR_COLORS = (1 << WORK_STRUCT_COLOR_BITS) - 1,
55 WORK_NO_COLOR = WORK_NR_COLORS, 55 WORK_NO_COLOR = WORK_NR_COLORS,
56 56
57 /* special cpu IDs */ 57 /* special cpu IDs */
58 WORK_CPU_UNBOUND = NR_CPUS, 58 WORK_CPU_UNBOUND = NR_CPUS,
59 WORK_CPU_NONE = NR_CPUS + 1, 59 WORK_CPU_NONE = NR_CPUS + 1,
60 WORK_CPU_LAST = WORK_CPU_NONE, 60 WORK_CPU_LAST = WORK_CPU_NONE,
61 61
62 /* 62 /*
63 * Reserve 7 bits off of cwq pointer w/ debugobjects turned 63 * Reserve 7 bits off of cwq pointer w/ debugobjects turned
64 * off. This makes cwqs aligned to 256 bytes and allows 15 64 * off. This makes cwqs aligned to 256 bytes and allows 15
65 * workqueue flush colors. 65 * workqueue flush colors.
66 */ 66 */
67 WORK_STRUCT_FLAG_BITS = WORK_STRUCT_COLOR_SHIFT + 67 WORK_STRUCT_FLAG_BITS = WORK_STRUCT_COLOR_SHIFT +
68 WORK_STRUCT_COLOR_BITS, 68 WORK_STRUCT_COLOR_BITS,
69 69
70 WORK_STRUCT_FLAG_MASK = (1UL << WORK_STRUCT_FLAG_BITS) - 1, 70 WORK_STRUCT_FLAG_MASK = (1UL << WORK_STRUCT_FLAG_BITS) - 1,
71 WORK_STRUCT_WQ_DATA_MASK = ~WORK_STRUCT_FLAG_MASK, 71 WORK_STRUCT_WQ_DATA_MASK = ~WORK_STRUCT_FLAG_MASK,
72 WORK_STRUCT_NO_CPU = WORK_CPU_NONE << WORK_STRUCT_FLAG_BITS, 72 WORK_STRUCT_NO_CPU = WORK_CPU_NONE << WORK_STRUCT_FLAG_BITS,
73 73
74 /* bit mask for work_busy() return values */ 74 /* bit mask for work_busy() return values */
75 WORK_BUSY_PENDING = 1 << 0, 75 WORK_BUSY_PENDING = 1 << 0,
76 WORK_BUSY_RUNNING = 1 << 1, 76 WORK_BUSY_RUNNING = 1 << 1,
77 }; 77 };
78 78
79 struct work_struct { 79 struct work_struct {
80 atomic_long_t data; 80 atomic_long_t data;
81 struct list_head entry; 81 struct list_head entry;
82 work_func_t func; 82 work_func_t func;
83 #ifdef CONFIG_LOCKDEP 83 #ifdef CONFIG_LOCKDEP
84 struct lockdep_map lockdep_map; 84 struct lockdep_map lockdep_map;
85 #endif 85 #endif
86 }; 86 };
87 87
88 #define WORK_DATA_INIT() ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU) 88 #define WORK_DATA_INIT() ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU)
89 #define WORK_DATA_STATIC_INIT() \ 89 #define WORK_DATA_STATIC_INIT() \
90 ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU | WORK_STRUCT_STATIC) 90 ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU | WORK_STRUCT_STATIC)
91 91
92 struct delayed_work { 92 struct delayed_work {
93 struct work_struct work; 93 struct work_struct work;
94 struct timer_list timer; 94 struct timer_list timer;
95 }; 95 };
96 96
97 static inline struct delayed_work *to_delayed_work(struct work_struct *work) 97 static inline struct delayed_work *to_delayed_work(struct work_struct *work)
98 { 98 {
99 return container_of(work, struct delayed_work, work); 99 return container_of(work, struct delayed_work, work);
100 } 100 }
101 101
102 struct execute_work { 102 struct execute_work {
103 struct work_struct work; 103 struct work_struct work;
104 }; 104 };
105 105
106 #ifdef CONFIG_LOCKDEP 106 #ifdef CONFIG_LOCKDEP
107 /* 107 /*
108 * NB: because we have to copy the lockdep_map, setting _key 108 * NB: because we have to copy the lockdep_map, setting _key
109 * here is required, otherwise it could get initialised to the 109 * here is required, otherwise it could get initialised to the
110 * copy of the lockdep_map! 110 * copy of the lockdep_map!
111 */ 111 */
112 #define __WORK_INIT_LOCKDEP_MAP(n, k) \ 112 #define __WORK_INIT_LOCKDEP_MAP(n, k) \
113 .lockdep_map = STATIC_LOCKDEP_MAP_INIT(n, k), 113 .lockdep_map = STATIC_LOCKDEP_MAP_INIT(n, k),
114 #else 114 #else
115 #define __WORK_INIT_LOCKDEP_MAP(n, k) 115 #define __WORK_INIT_LOCKDEP_MAP(n, k)
116 #endif 116 #endif
117 117
118 #define __WORK_INITIALIZER(n, f) { \ 118 #define __WORK_INITIALIZER(n, f) { \
119 .data = WORK_DATA_STATIC_INIT(), \ 119 .data = WORK_DATA_STATIC_INIT(), \
120 .entry = { &(n).entry, &(n).entry }, \ 120 .entry = { &(n).entry, &(n).entry }, \
121 .func = (f), \ 121 .func = (f), \
122 __WORK_INIT_LOCKDEP_MAP(#n, &(n)) \ 122 __WORK_INIT_LOCKDEP_MAP(#n, &(n)) \
123 } 123 }
124 124
125 #define __DELAYED_WORK_INITIALIZER(n, f) { \ 125 #define __DELAYED_WORK_INITIALIZER(n, f) { \
126 .work = __WORK_INITIALIZER((n).work, (f)), \ 126 .work = __WORK_INITIALIZER((n).work, (f)), \
127 .timer = TIMER_INITIALIZER(NULL, 0, 0), \ 127 .timer = TIMER_INITIALIZER(NULL, 0, 0), \
128 } 128 }
129 129
130 #define __DEFERRED_WORK_INITIALIZER(n, f) { \
131 .work = __WORK_INITIALIZER((n).work, (f)), \
132 .timer = TIMER_DEFERRED_INITIALIZER(NULL, 0, 0), \
133 }
134
130 #define DECLARE_WORK(n, f) \ 135 #define DECLARE_WORK(n, f) \
131 struct work_struct n = __WORK_INITIALIZER(n, f) 136 struct work_struct n = __WORK_INITIALIZER(n, f)
132 137
133 #define DECLARE_DELAYED_WORK(n, f) \ 138 #define DECLARE_DELAYED_WORK(n, f) \
134 struct delayed_work n = __DELAYED_WORK_INITIALIZER(n, f) 139 struct delayed_work n = __DELAYED_WORK_INITIALIZER(n, f)
140
141 #define DECLARE_DEFERRED_WORK(n, f) \
142 struct delayed_work n = __DEFERRED_WORK_INITIALIZER(n, f)
135 143
136 /* 144 /*
137 * initialize a work item's function pointer 145 * initialize a work item's function pointer
138 */ 146 */
139 #define PREPARE_WORK(_work, _func) \ 147 #define PREPARE_WORK(_work, _func) \
140 do { \ 148 do { \
141 (_work)->func = (_func); \ 149 (_work)->func = (_func); \
142 } while (0) 150 } while (0)
143 151
144 #define PREPARE_DELAYED_WORK(_work, _func) \ 152 #define PREPARE_DELAYED_WORK(_work, _func) \
145 PREPARE_WORK(&(_work)->work, (_func)) 153 PREPARE_WORK(&(_work)->work, (_func))
146 154
147 #ifdef CONFIG_DEBUG_OBJECTS_WORK 155 #ifdef CONFIG_DEBUG_OBJECTS_WORK
148 extern void __init_work(struct work_struct *work, int onstack); 156 extern void __init_work(struct work_struct *work, int onstack);
149 extern void destroy_work_on_stack(struct work_struct *work); 157 extern void destroy_work_on_stack(struct work_struct *work);
150 static inline unsigned int work_static(struct work_struct *work) 158 static inline unsigned int work_static(struct work_struct *work)
151 { 159 {
152 return *work_data_bits(work) & WORK_STRUCT_STATIC; 160 return *work_data_bits(work) & WORK_STRUCT_STATIC;
153 } 161 }
154 #else 162 #else
155 static inline void __init_work(struct work_struct *work, int onstack) { } 163 static inline void __init_work(struct work_struct *work, int onstack) { }
156 static inline void destroy_work_on_stack(struct work_struct *work) { } 164 static inline void destroy_work_on_stack(struct work_struct *work) { }
157 static inline unsigned int work_static(struct work_struct *work) { return 0; } 165 static inline unsigned int work_static(struct work_struct *work) { return 0; }
158 #endif 166 #endif
159 167
160 /* 168 /*
161 * initialize all of a work item in one go 169 * initialize all of a work item in one go
162 * 170 *
163 * NOTE! No point in using "atomic_long_set()": using a direct 171 * NOTE! No point in using "atomic_long_set()": using a direct
164 * assignment of the work data initializer allows the compiler 172 * assignment of the work data initializer allows the compiler
165 * to generate better code. 173 * to generate better code.
166 */ 174 */
167 #ifdef CONFIG_LOCKDEP 175 #ifdef CONFIG_LOCKDEP
168 #define __INIT_WORK(_work, _func, _onstack) \ 176 #define __INIT_WORK(_work, _func, _onstack) \
169 do { \ 177 do { \
170 static struct lock_class_key __key; \ 178 static struct lock_class_key __key; \
171 \ 179 \
172 __init_work((_work), _onstack); \ 180 __init_work((_work), _onstack); \
173 (_work)->data = (atomic_long_t) WORK_DATA_INIT(); \ 181 (_work)->data = (atomic_long_t) WORK_DATA_INIT(); \
174 lockdep_init_map(&(_work)->lockdep_map, #_work, &__key, 0);\ 182 lockdep_init_map(&(_work)->lockdep_map, #_work, &__key, 0);\
175 INIT_LIST_HEAD(&(_work)->entry); \ 183 INIT_LIST_HEAD(&(_work)->entry); \
176 PREPARE_WORK((_work), (_func)); \ 184 PREPARE_WORK((_work), (_func)); \
177 } while (0) 185 } while (0)
178 #else 186 #else
179 #define __INIT_WORK(_work, _func, _onstack) \ 187 #define __INIT_WORK(_work, _func, _onstack) \
180 do { \ 188 do { \
181 __init_work((_work), _onstack); \ 189 __init_work((_work), _onstack); \
182 (_work)->data = (atomic_long_t) WORK_DATA_INIT(); \ 190 (_work)->data = (atomic_long_t) WORK_DATA_INIT(); \
183 INIT_LIST_HEAD(&(_work)->entry); \ 191 INIT_LIST_HEAD(&(_work)->entry); \
184 PREPARE_WORK((_work), (_func)); \ 192 PREPARE_WORK((_work), (_func)); \
185 } while (0) 193 } while (0)
186 #endif 194 #endif
187 195
188 #define INIT_WORK(_work, _func) \ 196 #define INIT_WORK(_work, _func) \
189 do { \ 197 do { \
190 __INIT_WORK((_work), (_func), 0); \ 198 __INIT_WORK((_work), (_func), 0); \
191 } while (0) 199 } while (0)
192 200
193 #define INIT_WORK_ON_STACK(_work, _func) \ 201 #define INIT_WORK_ON_STACK(_work, _func) \
194 do { \ 202 do { \
195 __INIT_WORK((_work), (_func), 1); \ 203 __INIT_WORK((_work), (_func), 1); \
196 } while (0) 204 } while (0)
197 205
198 #define INIT_DELAYED_WORK(_work, _func) \ 206 #define INIT_DELAYED_WORK(_work, _func) \
199 do { \ 207 do { \
200 INIT_WORK(&(_work)->work, (_func)); \ 208 INIT_WORK(&(_work)->work, (_func)); \
201 init_timer(&(_work)->timer); \ 209 init_timer(&(_work)->timer); \
202 } while (0) 210 } while (0)
203 211
204 #define INIT_DELAYED_WORK_ON_STACK(_work, _func) \ 212 #define INIT_DELAYED_WORK_ON_STACK(_work, _func) \
205 do { \ 213 do { \
206 INIT_WORK_ON_STACK(&(_work)->work, (_func)); \ 214 INIT_WORK_ON_STACK(&(_work)->work, (_func)); \
207 init_timer_on_stack(&(_work)->timer); \ 215 init_timer_on_stack(&(_work)->timer); \
208 } while (0) 216 } while (0)
209 217
210 #define INIT_DELAYED_WORK_DEFERRABLE(_work, _func) \ 218 #define INIT_DELAYED_WORK_DEFERRABLE(_work, _func) \
211 do { \ 219 do { \
212 INIT_WORK(&(_work)->work, (_func)); \ 220 INIT_WORK(&(_work)->work, (_func)); \
213 init_timer_deferrable(&(_work)->timer); \ 221 init_timer_deferrable(&(_work)->timer); \
214 } while (0) 222 } while (0)
215 223
216 /** 224 /**
217 * work_pending - Find out whether a work item is currently pending 225 * work_pending - Find out whether a work item is currently pending
218 * @work: The work item in question 226 * @work: The work item in question
219 */ 227 */
220 #define work_pending(work) \ 228 #define work_pending(work) \
221 test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) 229 test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))
222 230
223 /** 231 /**
224 * delayed_work_pending - Find out whether a delayable work item is currently 232 * delayed_work_pending - Find out whether a delayable work item is currently
225 * pending 233 * pending
226 * @work: The work item in question 234 * @work: The work item in question
227 */ 235 */
228 #define delayed_work_pending(w) \ 236 #define delayed_work_pending(w) \
229 work_pending(&(w)->work) 237 work_pending(&(w)->work)
230 238
231 /** 239 /**
232 * work_clear_pending - for internal use only, mark a work item as not pending 240 * work_clear_pending - for internal use only, mark a work item as not pending
233 * @work: The work item in question 241 * @work: The work item in question
234 */ 242 */
235 #define work_clear_pending(work) \ 243 #define work_clear_pending(work) \
236 clear_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) 244 clear_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))
237 245
238 enum { 246 enum {
239 WQ_NON_REENTRANT = 1 << 0, /* guarantee non-reentrance */ 247 WQ_NON_REENTRANT = 1 << 0, /* guarantee non-reentrance */
240 WQ_UNBOUND = 1 << 1, /* not bound to any cpu */ 248 WQ_UNBOUND = 1 << 1, /* not bound to any cpu */
241 WQ_FREEZEABLE = 1 << 2, /* freeze during suspend */ 249 WQ_FREEZEABLE = 1 << 2, /* freeze during suspend */
242 WQ_RESCUER = 1 << 3, /* has an rescue worker */ 250 WQ_RESCUER = 1 << 3, /* has an rescue worker */
243 WQ_HIGHPRI = 1 << 4, /* high priority */ 251 WQ_HIGHPRI = 1 << 4, /* high priority */
244 WQ_CPU_INTENSIVE = 1 << 5, /* cpu instensive workqueue */ 252 WQ_CPU_INTENSIVE = 1 << 5, /* cpu instensive workqueue */
245 253
246 WQ_DYING = 1 << 6, /* internal: workqueue is dying */ 254 WQ_DYING = 1 << 6, /* internal: workqueue is dying */
247 255
248 WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */ 256 WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */
249 WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */ 257 WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */
250 WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2, 258 WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2,
251 }; 259 };
252 260
253 /* unbound wq's aren't per-cpu, scale max_active according to #cpus */ 261 /* unbound wq's aren't per-cpu, scale max_active according to #cpus */
254 #define WQ_UNBOUND_MAX_ACTIVE \ 262 #define WQ_UNBOUND_MAX_ACTIVE \
255 max_t(int, WQ_MAX_ACTIVE, num_possible_cpus() * WQ_MAX_UNBOUND_PER_CPU) 263 max_t(int, WQ_MAX_ACTIVE, num_possible_cpus() * WQ_MAX_UNBOUND_PER_CPU)
256 264
257 /* 265 /*
258 * System-wide workqueues which are always present. 266 * System-wide workqueues which are always present.
259 * 267 *
260 * system_wq is the one used by schedule[_delayed]_work[_on](). 268 * system_wq is the one used by schedule[_delayed]_work[_on]().
261 * Multi-CPU multi-threaded. There are users which expect relatively 269 * Multi-CPU multi-threaded. There are users which expect relatively
262 * short queue flush time. Don't queue works which can run for too 270 * short queue flush time. Don't queue works which can run for too
263 * long. 271 * long.
264 * 272 *
265 * system_long_wq is similar to system_wq but may host long running 273 * system_long_wq is similar to system_wq but may host long running
266 * works. Queue flushing might take relatively long. 274 * works. Queue flushing might take relatively long.
267 * 275 *
268 * system_nrt_wq is non-reentrant and guarantees that any given work 276 * system_nrt_wq is non-reentrant and guarantees that any given work
269 * item is never executed in parallel by multiple CPUs. Queue 277 * item is never executed in parallel by multiple CPUs. Queue
270 * flushing might take relatively long. 278 * flushing might take relatively long.
271 * 279 *
272 * system_unbound_wq is unbound workqueue. Workers are not bound to 280 * system_unbound_wq is unbound workqueue. Workers are not bound to
273 * any specific CPU, not concurrency managed, and all queued works are 281 * any specific CPU, not concurrency managed, and all queued works are
274 * executed immediately as long as max_active limit is not reached and 282 * executed immediately as long as max_active limit is not reached and
275 * resources are available. 283 * resources are available.
276 */ 284 */
277 extern struct workqueue_struct *system_wq; 285 extern struct workqueue_struct *system_wq;
278 extern struct workqueue_struct *system_long_wq; 286 extern struct workqueue_struct *system_long_wq;
279 extern struct workqueue_struct *system_nrt_wq; 287 extern struct workqueue_struct *system_nrt_wq;
280 extern struct workqueue_struct *system_unbound_wq; 288 extern struct workqueue_struct *system_unbound_wq;
281 289
282 extern struct workqueue_struct * 290 extern struct workqueue_struct *
283 __alloc_workqueue_key(const char *name, unsigned int flags, int max_active, 291 __alloc_workqueue_key(const char *name, unsigned int flags, int max_active,
284 struct lock_class_key *key, const char *lock_name); 292 struct lock_class_key *key, const char *lock_name);
285 293
286 #ifdef CONFIG_LOCKDEP 294 #ifdef CONFIG_LOCKDEP
287 #define alloc_workqueue(name, flags, max_active) \ 295 #define alloc_workqueue(name, flags, max_active) \
288 ({ \ 296 ({ \
289 static struct lock_class_key __key; \ 297 static struct lock_class_key __key; \
290 const char *__lock_name; \ 298 const char *__lock_name; \
291 \ 299 \
292 if (__builtin_constant_p(name)) \ 300 if (__builtin_constant_p(name)) \
293 __lock_name = (name); \ 301 __lock_name = (name); \
294 else \ 302 else \
295 __lock_name = #name; \ 303 __lock_name = #name; \
296 \ 304 \
297 __alloc_workqueue_key((name), (flags), (max_active), \ 305 __alloc_workqueue_key((name), (flags), (max_active), \
298 &__key, __lock_name); \ 306 &__key, __lock_name); \
299 }) 307 })
300 #else 308 #else
301 #define alloc_workqueue(name, flags, max_active) \ 309 #define alloc_workqueue(name, flags, max_active) \
302 __alloc_workqueue_key((name), (flags), (max_active), NULL, NULL) 310 __alloc_workqueue_key((name), (flags), (max_active), NULL, NULL)
303 #endif 311 #endif
304 312
305 #define create_workqueue(name) \ 313 #define create_workqueue(name) \
306 alloc_workqueue((name), WQ_RESCUER, 1) 314 alloc_workqueue((name), WQ_RESCUER, 1)
307 #define create_freezeable_workqueue(name) \ 315 #define create_freezeable_workqueue(name) \
308 alloc_workqueue((name), WQ_FREEZEABLE | WQ_UNBOUND | WQ_RESCUER, 1) 316 alloc_workqueue((name), WQ_FREEZEABLE | WQ_UNBOUND | WQ_RESCUER, 1)
309 #define create_singlethread_workqueue(name) \ 317 #define create_singlethread_workqueue(name) \
310 alloc_workqueue((name), WQ_UNBOUND | WQ_RESCUER, 1) 318 alloc_workqueue((name), WQ_UNBOUND | WQ_RESCUER, 1)
311 319
312 extern void destroy_workqueue(struct workqueue_struct *wq); 320 extern void destroy_workqueue(struct workqueue_struct *wq);
313 321
314 extern int queue_work(struct workqueue_struct *wq, struct work_struct *work); 322 extern int queue_work(struct workqueue_struct *wq, struct work_struct *work);
315 extern int queue_work_on(int cpu, struct workqueue_struct *wq, 323 extern int queue_work_on(int cpu, struct workqueue_struct *wq,
316 struct work_struct *work); 324 struct work_struct *work);
317 extern int queue_delayed_work(struct workqueue_struct *wq, 325 extern int queue_delayed_work(struct workqueue_struct *wq,
318 struct delayed_work *work, unsigned long delay); 326 struct delayed_work *work, unsigned long delay);
319 extern int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, 327 extern int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
320 struct delayed_work *work, unsigned long delay); 328 struct delayed_work *work, unsigned long delay);
321 329
322 extern void flush_workqueue(struct workqueue_struct *wq); 330 extern void flush_workqueue(struct workqueue_struct *wq);
323 extern void flush_scheduled_work(void); 331 extern void flush_scheduled_work(void);
324 extern void flush_delayed_work(struct delayed_work *work); 332 extern void flush_delayed_work(struct delayed_work *work);
325 333
326 extern int schedule_work(struct work_struct *work); 334 extern int schedule_work(struct work_struct *work);
327 extern int schedule_work_on(int cpu, struct work_struct *work); 335 extern int schedule_work_on(int cpu, struct work_struct *work);
328 extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay); 336 extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay);
329 extern int schedule_delayed_work_on(int cpu, struct delayed_work *work, 337 extern int schedule_delayed_work_on(int cpu, struct delayed_work *work,
330 unsigned long delay); 338 unsigned long delay);
331 extern int schedule_on_each_cpu(work_func_t func); 339 extern int schedule_on_each_cpu(work_func_t func);
332 extern int keventd_up(void); 340 extern int keventd_up(void);
333 341
334 int execute_in_process_context(work_func_t fn, struct execute_work *); 342 int execute_in_process_context(work_func_t fn, struct execute_work *);
335 343
336 extern int flush_work(struct work_struct *work); 344 extern int flush_work(struct work_struct *work);
337 extern int cancel_work_sync(struct work_struct *work); 345 extern int cancel_work_sync(struct work_struct *work);
338 346
339 extern void workqueue_set_max_active(struct workqueue_struct *wq, 347 extern void workqueue_set_max_active(struct workqueue_struct *wq,
340 int max_active); 348 int max_active);
341 extern bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq); 349 extern bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq);
342 extern unsigned int work_cpu(struct work_struct *work); 350 extern unsigned int work_cpu(struct work_struct *work);
343 extern unsigned int work_busy(struct work_struct *work); 351 extern unsigned int work_busy(struct work_struct *work);
344 352
345 /* 353 /*
346 * Kill off a pending schedule_delayed_work(). Note that the work callback 354 * Kill off a pending schedule_delayed_work(). Note that the work callback
347 * function may still be running on return from cancel_delayed_work(), unless 355 * function may still be running on return from cancel_delayed_work(), unless
348 * it returns 1 and the work doesn't re-arm itself. Run flush_workqueue() or 356 * it returns 1 and the work doesn't re-arm itself. Run flush_workqueue() or
349 * cancel_work_sync() to wait on it. 357 * cancel_work_sync() to wait on it.
350 */ 358 */
351 static inline int cancel_delayed_work(struct delayed_work *work) 359 static inline int cancel_delayed_work(struct delayed_work *work)
352 { 360 {
353 int ret; 361 int ret;
354 362
355 ret = del_timer_sync(&work->timer); 363 ret = del_timer_sync(&work->timer);
356 if (ret) 364 if (ret)
357 work_clear_pending(&work->work); 365 work_clear_pending(&work->work);
358 return ret; 366 return ret;
359 } 367 }
360 368
361 /* 369 /*
362 * Like above, but uses del_timer() instead of del_timer_sync(). This means, 370 * Like above, but uses del_timer() instead of del_timer_sync(). This means,
363 * if it returns 0 the timer function may be running and the queueing is in 371 * if it returns 0 the timer function may be running and the queueing is in
364 * progress. 372 * progress.
365 */ 373 */
366 static inline int __cancel_delayed_work(struct delayed_work *work) 374 static inline int __cancel_delayed_work(struct delayed_work *work)
367 { 375 {
368 int ret; 376 int ret;
369 377
370 ret = del_timer(&work->timer); 378 ret = del_timer(&work->timer);
371 if (ret) 379 if (ret)
372 work_clear_pending(&work->work); 380 work_clear_pending(&work->work);
373 return ret; 381 return ret;
374 } 382 }
375 383
376 extern int cancel_delayed_work_sync(struct delayed_work *work); 384 extern int cancel_delayed_work_sync(struct delayed_work *work);
377 385
378 /* Obsolete. use cancel_delayed_work_sync() */ 386 /* Obsolete. use cancel_delayed_work_sync() */
379 static inline 387 static inline
380 void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq, 388 void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq,
381 struct delayed_work *work) 389 struct delayed_work *work)
382 { 390 {
383 cancel_delayed_work_sync(work); 391 cancel_delayed_work_sync(work);
384 } 392 }
385 393
386 /* Obsolete. use cancel_delayed_work_sync() */ 394 /* Obsolete. use cancel_delayed_work_sync() */
387 static inline 395 static inline
388 void cancel_rearming_delayed_work(struct delayed_work *work) 396 void cancel_rearming_delayed_work(struct delayed_work *work)
389 { 397 {
390 cancel_delayed_work_sync(work); 398 cancel_delayed_work_sync(work);
391 } 399 }
392 400
393 #ifndef CONFIG_SMP 401 #ifndef CONFIG_SMP
394 static inline long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg) 402 static inline long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg)
395 { 403 {
396 return fn(arg); 404 return fn(arg);
397 } 405 }
398 #else 406 #else
399 long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg); 407 long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg);
400 #endif /* CONFIG_SMP */ 408 #endif /* CONFIG_SMP */
401 409
402 #ifdef CONFIG_FREEZER 410 #ifdef CONFIG_FREEZER
403 extern void freeze_workqueues_begin(void); 411 extern void freeze_workqueues_begin(void);
404 extern bool freeze_workqueues_busy(void); 412 extern bool freeze_workqueues_busy(void);
405 extern void thaw_workqueues(void); 413 extern void thaw_workqueues(void);
406 #endif /* CONFIG_FREEZER */ 414 #endif /* CONFIG_FREEZER */
407 415
408 #ifdef CONFIG_LOCKDEP 416 #ifdef CONFIG_LOCKDEP
409 int in_workqueue_context(struct workqueue_struct *wq); 417 int in_workqueue_context(struct workqueue_struct *wq);
410 #endif 418 #endif
411 419
412 #endif 420 #endif
413 421
1 /* 1 /*
2 * linux/kernel/timer.c 2 * linux/kernel/timer.c
3 * 3 *
4 * Kernel internal timers, basic process system calls 4 * Kernel internal timers, basic process system calls
5 * 5 *
6 * Copyright (C) 1991, 1992 Linus Torvalds 6 * Copyright (C) 1991, 1992 Linus Torvalds
7 * 7 *
8 * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better. 8 * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
9 * 9 *
10 * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 10 * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
11 * "A Kernel Model for Precision Timekeeping" by Dave Mills 11 * "A Kernel Model for Precision Timekeeping" by Dave Mills
12 * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to 12 * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
13 * serialize accesses to xtime/lost_ticks). 13 * serialize accesses to xtime/lost_ticks).
14 * Copyright (C) 1998 Andrea Arcangeli 14 * Copyright (C) 1998 Andrea Arcangeli
15 * 1999-03-10 Improved NTP compatibility by Ulrich Windl 15 * 1999-03-10 Improved NTP compatibility by Ulrich Windl
16 * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love 16 * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
17 * 2000-10-05 Implemented scalable SMP per-CPU timer handling. 17 * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
18 * Copyright (C) 2000, 2001, 2002 Ingo Molnar 18 * Copyright (C) 2000, 2001, 2002 Ingo Molnar
19 * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar 19 * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
20 */ 20 */
21 21
22 #include <linux/kernel_stat.h> 22 #include <linux/kernel_stat.h>
23 #include <linux/module.h> 23 #include <linux/module.h>
24 #include <linux/interrupt.h> 24 #include <linux/interrupt.h>
25 #include <linux/percpu.h> 25 #include <linux/percpu.h>
26 #include <linux/init.h> 26 #include <linux/init.h>
27 #include <linux/mm.h> 27 #include <linux/mm.h>
28 #include <linux/swap.h> 28 #include <linux/swap.h>
29 #include <linux/pid_namespace.h> 29 #include <linux/pid_namespace.h>
30 #include <linux/notifier.h> 30 #include <linux/notifier.h>
31 #include <linux/thread_info.h> 31 #include <linux/thread_info.h>
32 #include <linux/time.h> 32 #include <linux/time.h>
33 #include <linux/jiffies.h> 33 #include <linux/jiffies.h>
34 #include <linux/posix-timers.h> 34 #include <linux/posix-timers.h>
35 #include <linux/cpu.h> 35 #include <linux/cpu.h>
36 #include <linux/syscalls.h> 36 #include <linux/syscalls.h>
37 #include <linux/delay.h> 37 #include <linux/delay.h>
38 #include <linux/tick.h> 38 #include <linux/tick.h>
39 #include <linux/kallsyms.h> 39 #include <linux/kallsyms.h>
40 #include <linux/perf_event.h> 40 #include <linux/perf_event.h>
41 #include <linux/sched.h> 41 #include <linux/sched.h>
42 #include <linux/slab.h> 42 #include <linux/slab.h>
43 43
44 #include <asm/uaccess.h> 44 #include <asm/uaccess.h>
45 #include <asm/unistd.h> 45 #include <asm/unistd.h>
46 #include <asm/div64.h> 46 #include <asm/div64.h>
47 #include <asm/timex.h> 47 #include <asm/timex.h>
48 #include <asm/io.h> 48 #include <asm/io.h>
49 49
50 #define CREATE_TRACE_POINTS 50 #define CREATE_TRACE_POINTS
51 #include <trace/events/timer.h> 51 #include <trace/events/timer.h>
52 52
53 u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; 53 u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
54 54
55 EXPORT_SYMBOL(jiffies_64); 55 EXPORT_SYMBOL(jiffies_64);
56 56
57 /* 57 /*
58 * per-CPU timer vector definitions: 58 * per-CPU timer vector definitions:
59 */ 59 */
60 #define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6) 60 #define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)
61 #define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8) 61 #define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)
62 #define TVN_SIZE (1 << TVN_BITS) 62 #define TVN_SIZE (1 << TVN_BITS)
63 #define TVR_SIZE (1 << TVR_BITS) 63 #define TVR_SIZE (1 << TVR_BITS)
64 #define TVN_MASK (TVN_SIZE - 1) 64 #define TVN_MASK (TVN_SIZE - 1)
65 #define TVR_MASK (TVR_SIZE - 1) 65 #define TVR_MASK (TVR_SIZE - 1)
66 66
67 struct tvec { 67 struct tvec {
68 struct list_head vec[TVN_SIZE]; 68 struct list_head vec[TVN_SIZE];
69 }; 69 };
70 70
71 struct tvec_root { 71 struct tvec_root {
72 struct list_head vec[TVR_SIZE]; 72 struct list_head vec[TVR_SIZE];
73 }; 73 };
74 74
75 struct tvec_base { 75 struct tvec_base {
76 spinlock_t lock; 76 spinlock_t lock;
77 struct timer_list *running_timer; 77 struct timer_list *running_timer;
78 unsigned long timer_jiffies; 78 unsigned long timer_jiffies;
79 unsigned long next_timer; 79 unsigned long next_timer;
80 struct tvec_root tv1; 80 struct tvec_root tv1;
81 struct tvec tv2; 81 struct tvec tv2;
82 struct tvec tv3; 82 struct tvec tv3;
83 struct tvec tv4; 83 struct tvec tv4;
84 struct tvec tv5; 84 struct tvec tv5;
85 } ____cacheline_aligned; 85 } ____cacheline_aligned;
86 86
87 struct tvec_base boot_tvec_bases; 87 struct tvec_base boot_tvec_bases;
88 EXPORT_SYMBOL(boot_tvec_bases); 88 EXPORT_SYMBOL(boot_tvec_bases);
89 static DEFINE_PER_CPU(struct tvec_base *, tvec_bases) = &boot_tvec_bases; 89 static DEFINE_PER_CPU(struct tvec_base *, tvec_bases) = &boot_tvec_bases;
90 90
91 /*
92 * Note that all tvec_bases are 2 byte aligned and lower bit of
93 * base in timer_list is guaranteed to be zero. Use the LSB to
94 * indicate whether the timer is deferrable.
95 *
96 * A deferrable timer will work normally when the system is busy, but
97 * will not cause a CPU to come out of idle just to service it; instead,
98 * the timer will be serviced when the CPU eventually wakes up with a
99 * subsequent non-deferrable timer.
100 */
101 #define TBASE_DEFERRABLE_FLAG (0x1)
102
103 /* Functions below help us manage 'deferrable' flag */ 91 /* Functions below help us manage 'deferrable' flag */
104 static inline unsigned int tbase_get_deferrable(struct tvec_base *base) 92 static inline unsigned int tbase_get_deferrable(struct tvec_base *base)
105 { 93 {
106 return ((unsigned int)(unsigned long)base & TBASE_DEFERRABLE_FLAG); 94 return ((unsigned int)(unsigned long)base & TBASE_DEFERRABLE_FLAG);
107 } 95 }
108 96
109 static inline struct tvec_base *tbase_get_base(struct tvec_base *base) 97 static inline struct tvec_base *tbase_get_base(struct tvec_base *base)
110 { 98 {
111 return ((struct tvec_base *)((unsigned long)base & ~TBASE_DEFERRABLE_FLAG)); 99 return ((struct tvec_base *)((unsigned long)base & ~TBASE_DEFERRABLE_FLAG));
112 } 100 }
113 101
114 static inline void timer_set_deferrable(struct timer_list *timer) 102 static inline void timer_set_deferrable(struct timer_list *timer)
115 { 103 {
116 timer->base = ((struct tvec_base *)((unsigned long)(timer->base) | 104 timer->base = TBASE_MAKE_DEFERRED(timer->base);
117 TBASE_DEFERRABLE_FLAG));
118 } 105 }
119 106
120 static inline void 107 static inline void
121 timer_set_base(struct timer_list *timer, struct tvec_base *new_base) 108 timer_set_base(struct timer_list *timer, struct tvec_base *new_base)
122 { 109 {
123 timer->base = (struct tvec_base *)((unsigned long)(new_base) | 110 timer->base = (struct tvec_base *)((unsigned long)(new_base) |
124 tbase_get_deferrable(timer->base)); 111 tbase_get_deferrable(timer->base));
125 } 112 }
126 113
127 static unsigned long round_jiffies_common(unsigned long j, int cpu, 114 static unsigned long round_jiffies_common(unsigned long j, int cpu,
128 bool force_up) 115 bool force_up)
129 { 116 {
130 int rem; 117 int rem;
131 unsigned long original = j; 118 unsigned long original = j;
132 119
133 /* 120 /*
134 * We don't want all cpus firing their timers at once hitting the 121 * We don't want all cpus firing their timers at once hitting the
135 * same lock or cachelines, so we skew each extra cpu with an extra 122 * same lock or cachelines, so we skew each extra cpu with an extra
136 * 3 jiffies. This 3 jiffies came originally from the mm/ code which 123 * 3 jiffies. This 3 jiffies came originally from the mm/ code which
137 * already did this. 124 * already did this.
138 * The skew is done by adding 3*cpunr, then round, then subtract this 125 * The skew is done by adding 3*cpunr, then round, then subtract this
139 * extra offset again. 126 * extra offset again.
140 */ 127 */
141 j += cpu * 3; 128 j += cpu * 3;
142 129
143 rem = j % HZ; 130 rem = j % HZ;
144 131
145 /* 132 /*
146 * If the target jiffie is just after a whole second (which can happen 133 * If the target jiffie is just after a whole second (which can happen
147 * due to delays of the timer irq, long irq off times etc etc) then 134 * due to delays of the timer irq, long irq off times etc etc) then
148 * we should round down to the whole second, not up. Use 1/4th second 135 * we should round down to the whole second, not up. Use 1/4th second
149 * as cutoff for this rounding as an extreme upper bound for this. 136 * as cutoff for this rounding as an extreme upper bound for this.
150 * But never round down if @force_up is set. 137 * But never round down if @force_up is set.
151 */ 138 */
152 if (rem < HZ/4 && !force_up) /* round down */ 139 if (rem < HZ/4 && !force_up) /* round down */
153 j = j - rem; 140 j = j - rem;
154 else /* round up */ 141 else /* round up */
155 j = j - rem + HZ; 142 j = j - rem + HZ;
156 143
157 /* now that we have rounded, subtract the extra skew again */ 144 /* now that we have rounded, subtract the extra skew again */
158 j -= cpu * 3; 145 j -= cpu * 3;
159 146
160 if (j <= jiffies) /* rounding ate our timeout entirely; */ 147 if (j <= jiffies) /* rounding ate our timeout entirely; */
161 return original; 148 return original;
162 return j; 149 return j;
163 } 150 }
164 151
165 /** 152 /**
166 * __round_jiffies - function to round jiffies to a full second 153 * __round_jiffies - function to round jiffies to a full second
167 * @j: the time in (absolute) jiffies that should be rounded 154 * @j: the time in (absolute) jiffies that should be rounded
168 * @cpu: the processor number on which the timeout will happen 155 * @cpu: the processor number on which the timeout will happen
169 * 156 *
170 * __round_jiffies() rounds an absolute time in the future (in jiffies) 157 * __round_jiffies() rounds an absolute time in the future (in jiffies)
171 * up or down to (approximately) full seconds. This is useful for timers 158 * up or down to (approximately) full seconds. This is useful for timers
172 * for which the exact time they fire does not matter too much, as long as 159 * for which the exact time they fire does not matter too much, as long as
173 * they fire approximately every X seconds. 160 * they fire approximately every X seconds.
174 * 161 *
175 * By rounding these timers to whole seconds, all such timers will fire 162 * By rounding these timers to whole seconds, all such timers will fire
176 * at the same time, rather than at various times spread out. The goal 163 * at the same time, rather than at various times spread out. The goal
177 * of this is to have the CPU wake up less, which saves power. 164 * of this is to have the CPU wake up less, which saves power.
178 * 165 *
179 * The exact rounding is skewed for each processor to avoid all 166 * The exact rounding is skewed for each processor to avoid all
180 * processors firing at the exact same time, which could lead 167 * processors firing at the exact same time, which could lead
181 * to lock contention or spurious cache line bouncing. 168 * to lock contention or spurious cache line bouncing.
182 * 169 *
183 * The return value is the rounded version of the @j parameter. 170 * The return value is the rounded version of the @j parameter.
184 */ 171 */
185 unsigned long __round_jiffies(unsigned long j, int cpu) 172 unsigned long __round_jiffies(unsigned long j, int cpu)
186 { 173 {
187 return round_jiffies_common(j, cpu, false); 174 return round_jiffies_common(j, cpu, false);
188 } 175 }
189 EXPORT_SYMBOL_GPL(__round_jiffies); 176 EXPORT_SYMBOL_GPL(__round_jiffies);
190 177
191 /** 178 /**
192 * __round_jiffies_relative - function to round jiffies to a full second 179 * __round_jiffies_relative - function to round jiffies to a full second
193 * @j: the time in (relative) jiffies that should be rounded 180 * @j: the time in (relative) jiffies that should be rounded
194 * @cpu: the processor number on which the timeout will happen 181 * @cpu: the processor number on which the timeout will happen
195 * 182 *
196 * __round_jiffies_relative() rounds a time delta in the future (in jiffies) 183 * __round_jiffies_relative() rounds a time delta in the future (in jiffies)
197 * up or down to (approximately) full seconds. This is useful for timers 184 * up or down to (approximately) full seconds. This is useful for timers
198 * for which the exact time they fire does not matter too much, as long as 185 * for which the exact time they fire does not matter too much, as long as
199 * they fire approximately every X seconds. 186 * they fire approximately every X seconds.
200 * 187 *
201 * By rounding these timers to whole seconds, all such timers will fire 188 * By rounding these timers to whole seconds, all such timers will fire
202 * at the same time, rather than at various times spread out. The goal 189 * at the same time, rather than at various times spread out. The goal
203 * of this is to have the CPU wake up less, which saves power. 190 * of this is to have the CPU wake up less, which saves power.
204 * 191 *
205 * The exact rounding is skewed for each processor to avoid all 192 * The exact rounding is skewed for each processor to avoid all
206 * processors firing at the exact same time, which could lead 193 * processors firing at the exact same time, which could lead
207 * to lock contention or spurious cache line bouncing. 194 * to lock contention or spurious cache line bouncing.
208 * 195 *
209 * The return value is the rounded version of the @j parameter. 196 * The return value is the rounded version of the @j parameter.
210 */ 197 */
211 unsigned long __round_jiffies_relative(unsigned long j, int cpu) 198 unsigned long __round_jiffies_relative(unsigned long j, int cpu)
212 { 199 {
213 unsigned long j0 = jiffies; 200 unsigned long j0 = jiffies;
214 201
215 /* Use j0 because jiffies might change while we run */ 202 /* Use j0 because jiffies might change while we run */
216 return round_jiffies_common(j + j0, cpu, false) - j0; 203 return round_jiffies_common(j + j0, cpu, false) - j0;
217 } 204 }
218 EXPORT_SYMBOL_GPL(__round_jiffies_relative); 205 EXPORT_SYMBOL_GPL(__round_jiffies_relative);
219 206
220 /** 207 /**
221 * round_jiffies - function to round jiffies to a full second 208 * round_jiffies - function to round jiffies to a full second
222 * @j: the time in (absolute) jiffies that should be rounded 209 * @j: the time in (absolute) jiffies that should be rounded
223 * 210 *
224 * round_jiffies() rounds an absolute time in the future (in jiffies) 211 * round_jiffies() rounds an absolute time in the future (in jiffies)
225 * up or down to (approximately) full seconds. This is useful for timers 212 * up or down to (approximately) full seconds. This is useful for timers
226 * for which the exact time they fire does not matter too much, as long as 213 * for which the exact time they fire does not matter too much, as long as
227 * they fire approximately every X seconds. 214 * they fire approximately every X seconds.
228 * 215 *
229 * By rounding these timers to whole seconds, all such timers will fire 216 * By rounding these timers to whole seconds, all such timers will fire
230 * at the same time, rather than at various times spread out. The goal 217 * at the same time, rather than at various times spread out. The goal
231 * of this is to have the CPU wake up less, which saves power. 218 * of this is to have the CPU wake up less, which saves power.
232 * 219 *
233 * The return value is the rounded version of the @j parameter. 220 * The return value is the rounded version of the @j parameter.
234 */ 221 */
235 unsigned long round_jiffies(unsigned long j) 222 unsigned long round_jiffies(unsigned long j)
236 { 223 {
237 return round_jiffies_common(j, raw_smp_processor_id(), false); 224 return round_jiffies_common(j, raw_smp_processor_id(), false);
238 } 225 }
239 EXPORT_SYMBOL_GPL(round_jiffies); 226 EXPORT_SYMBOL_GPL(round_jiffies);
240 227
241 /** 228 /**
242 * round_jiffies_relative - function to round jiffies to a full second 229 * round_jiffies_relative - function to round jiffies to a full second
243 * @j: the time in (relative) jiffies that should be rounded 230 * @j: the time in (relative) jiffies that should be rounded
244 * 231 *
245 * round_jiffies_relative() rounds a time delta in the future (in jiffies) 232 * round_jiffies_relative() rounds a time delta in the future (in jiffies)
246 * up or down to (approximately) full seconds. This is useful for timers 233 * up or down to (approximately) full seconds. This is useful for timers
247 * for which the exact time they fire does not matter too much, as long as 234 * for which the exact time they fire does not matter too much, as long as
248 * they fire approximately every X seconds. 235 * they fire approximately every X seconds.
249 * 236 *
250 * By rounding these timers to whole seconds, all such timers will fire 237 * By rounding these timers to whole seconds, all such timers will fire
251 * at the same time, rather than at various times spread out. The goal 238 * at the same time, rather than at various times spread out. The goal
252 * of this is to have the CPU wake up less, which saves power. 239 * of this is to have the CPU wake up less, which saves power.
253 * 240 *
254 * The return value is the rounded version of the @j parameter. 241 * The return value is the rounded version of the @j parameter.
255 */ 242 */
256 unsigned long round_jiffies_relative(unsigned long j) 243 unsigned long round_jiffies_relative(unsigned long j)
257 { 244 {
258 return __round_jiffies_relative(j, raw_smp_processor_id()); 245 return __round_jiffies_relative(j, raw_smp_processor_id());
259 } 246 }
260 EXPORT_SYMBOL_GPL(round_jiffies_relative); 247 EXPORT_SYMBOL_GPL(round_jiffies_relative);
261 248
262 /** 249 /**
263 * __round_jiffies_up - function to round jiffies up to a full second 250 * __round_jiffies_up - function to round jiffies up to a full second
264 * @j: the time in (absolute) jiffies that should be rounded 251 * @j: the time in (absolute) jiffies that should be rounded
265 * @cpu: the processor number on which the timeout will happen 252 * @cpu: the processor number on which the timeout will happen
266 * 253 *
267 * This is the same as __round_jiffies() except that it will never 254 * This is the same as __round_jiffies() except that it will never
268 * round down. This is useful for timeouts for which the exact time 255 * round down. This is useful for timeouts for which the exact time
269 * of firing does not matter too much, as long as they don't fire too 256 * of firing does not matter too much, as long as they don't fire too
270 * early. 257 * early.
271 */ 258 */
272 unsigned long __round_jiffies_up(unsigned long j, int cpu) 259 unsigned long __round_jiffies_up(unsigned long j, int cpu)
273 { 260 {
274 return round_jiffies_common(j, cpu, true); 261 return round_jiffies_common(j, cpu, true);
275 } 262 }
276 EXPORT_SYMBOL_GPL(__round_jiffies_up); 263 EXPORT_SYMBOL_GPL(__round_jiffies_up);
277 264
278 /** 265 /**
279 * __round_jiffies_up_relative - function to round jiffies up to a full second 266 * __round_jiffies_up_relative - function to round jiffies up to a full second
280 * @j: the time in (relative) jiffies that should be rounded 267 * @j: the time in (relative) jiffies that should be rounded
281 * @cpu: the processor number on which the timeout will happen 268 * @cpu: the processor number on which the timeout will happen
282 * 269 *
283 * This is the same as __round_jiffies_relative() except that it will never 270 * This is the same as __round_jiffies_relative() except that it will never
284 * round down. This is useful for timeouts for which the exact time 271 * round down. This is useful for timeouts for which the exact time
285 * of firing does not matter too much, as long as they don't fire too 272 * of firing does not matter too much, as long as they don't fire too
286 * early. 273 * early.
287 */ 274 */
288 unsigned long __round_jiffies_up_relative(unsigned long j, int cpu) 275 unsigned long __round_jiffies_up_relative(unsigned long j, int cpu)
289 { 276 {
290 unsigned long j0 = jiffies; 277 unsigned long j0 = jiffies;
291 278
292 /* Use j0 because jiffies might change while we run */ 279 /* Use j0 because jiffies might change while we run */
293 return round_jiffies_common(j + j0, cpu, true) - j0; 280 return round_jiffies_common(j + j0, cpu, true) - j0;
294 } 281 }
295 EXPORT_SYMBOL_GPL(__round_jiffies_up_relative); 282 EXPORT_SYMBOL_GPL(__round_jiffies_up_relative);
296 283
297 /** 284 /**
298 * round_jiffies_up - function to round jiffies up to a full second 285 * round_jiffies_up - function to round jiffies up to a full second
299 * @j: the time in (absolute) jiffies that should be rounded 286 * @j: the time in (absolute) jiffies that should be rounded
300 * 287 *
301 * This is the same as round_jiffies() except that it will never 288 * This is the same as round_jiffies() except that it will never
302 * round down. This is useful for timeouts for which the exact time 289 * round down. This is useful for timeouts for which the exact time
303 * of firing does not matter too much, as long as they don't fire too 290 * of firing does not matter too much, as long as they don't fire too
304 * early. 291 * early.
305 */ 292 */
306 unsigned long round_jiffies_up(unsigned long j) 293 unsigned long round_jiffies_up(unsigned long j)
307 { 294 {
308 return round_jiffies_common(j, raw_smp_processor_id(), true); 295 return round_jiffies_common(j, raw_smp_processor_id(), true);
309 } 296 }
310 EXPORT_SYMBOL_GPL(round_jiffies_up); 297 EXPORT_SYMBOL_GPL(round_jiffies_up);
311 298
312 /** 299 /**
313 * round_jiffies_up_relative - function to round jiffies up to a full second 300 * round_jiffies_up_relative - function to round jiffies up to a full second
314 * @j: the time in (relative) jiffies that should be rounded 301 * @j: the time in (relative) jiffies that should be rounded
315 * 302 *
316 * This is the same as round_jiffies_relative() except that it will never 303 * This is the same as round_jiffies_relative() except that it will never
317 * round down. This is useful for timeouts for which the exact time 304 * round down. This is useful for timeouts for which the exact time
318 * of firing does not matter too much, as long as they don't fire too 305 * of firing does not matter too much, as long as they don't fire too
319 * early. 306 * early.
320 */ 307 */
321 unsigned long round_jiffies_up_relative(unsigned long j) 308 unsigned long round_jiffies_up_relative(unsigned long j)
322 { 309 {
323 return __round_jiffies_up_relative(j, raw_smp_processor_id()); 310 return __round_jiffies_up_relative(j, raw_smp_processor_id());
324 } 311 }
325 EXPORT_SYMBOL_GPL(round_jiffies_up_relative); 312 EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
326 313
327 /** 314 /**
328 * set_timer_slack - set the allowed slack for a timer 315 * set_timer_slack - set the allowed slack for a timer
329 * @timer: the timer to be modified 316 * @timer: the timer to be modified
330 * @slack_hz: the amount of time (in jiffies) allowed for rounding 317 * @slack_hz: the amount of time (in jiffies) allowed for rounding
331 * 318 *
332 * Set the amount of time, in jiffies, that a certain timer has 319 * Set the amount of time, in jiffies, that a certain timer has
333 * in terms of slack. By setting this value, the timer subsystem 320 * in terms of slack. By setting this value, the timer subsystem
334 * will schedule the actual timer somewhere between 321 * will schedule the actual timer somewhere between
335 * the time mod_timer() asks for, and that time plus the slack. 322 * the time mod_timer() asks for, and that time plus the slack.
336 * 323 *
337 * By setting the slack to -1, a percentage of the delay is used 324 * By setting the slack to -1, a percentage of the delay is used
338 * instead. 325 * instead.
339 */ 326 */
340 void set_timer_slack(struct timer_list *timer, int slack_hz) 327 void set_timer_slack(struct timer_list *timer, int slack_hz)
341 { 328 {
342 timer->slack = slack_hz; 329 timer->slack = slack_hz;
343 } 330 }
344 EXPORT_SYMBOL_GPL(set_timer_slack); 331 EXPORT_SYMBOL_GPL(set_timer_slack);
345 332
346 333
347 static inline void set_running_timer(struct tvec_base *base, 334 static inline void set_running_timer(struct tvec_base *base,
348 struct timer_list *timer) 335 struct timer_list *timer)
349 { 336 {
350 #ifdef CONFIG_SMP 337 #ifdef CONFIG_SMP
351 base->running_timer = timer; 338 base->running_timer = timer;
352 #endif 339 #endif
353 } 340 }
354 341
355 static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) 342 static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
356 { 343 {
357 unsigned long expires = timer->expires; 344 unsigned long expires = timer->expires;
358 unsigned long idx = expires - base->timer_jiffies; 345 unsigned long idx = expires - base->timer_jiffies;
359 struct list_head *vec; 346 struct list_head *vec;
360 347
361 if (idx < TVR_SIZE) { 348 if (idx < TVR_SIZE) {
362 int i = expires & TVR_MASK; 349 int i = expires & TVR_MASK;
363 vec = base->tv1.vec + i; 350 vec = base->tv1.vec + i;
364 } else if (idx < 1 << (TVR_BITS + TVN_BITS)) { 351 } else if (idx < 1 << (TVR_BITS + TVN_BITS)) {
365 int i = (expires >> TVR_BITS) & TVN_MASK; 352 int i = (expires >> TVR_BITS) & TVN_MASK;
366 vec = base->tv2.vec + i; 353 vec = base->tv2.vec + i;
367 } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) { 354 } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {
368 int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK; 355 int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;
369 vec = base->tv3.vec + i; 356 vec = base->tv3.vec + i;
370 } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) { 357 } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {
371 int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK; 358 int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;
372 vec = base->tv4.vec + i; 359 vec = base->tv4.vec + i;
373 } else if ((signed long) idx < 0) { 360 } else if ((signed long) idx < 0) {
374 /* 361 /*
375 * Can happen if you add a timer with expires == jiffies, 362 * Can happen if you add a timer with expires == jiffies,
376 * or you set a timer to go off in the past 363 * or you set a timer to go off in the past
377 */ 364 */
378 vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK); 365 vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);
379 } else { 366 } else {
380 int i; 367 int i;
381 /* If the timeout is larger than 0xffffffff on 64-bit 368 /* If the timeout is larger than 0xffffffff on 64-bit
382 * architectures then we use the maximum timeout: 369 * architectures then we use the maximum timeout:
383 */ 370 */
384 if (idx > 0xffffffffUL) { 371 if (idx > 0xffffffffUL) {
385 idx = 0xffffffffUL; 372 idx = 0xffffffffUL;
386 expires = idx + base->timer_jiffies; 373 expires = idx + base->timer_jiffies;
387 } 374 }
388 i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK; 375 i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;
389 vec = base->tv5.vec + i; 376 vec = base->tv5.vec + i;
390 } 377 }
391 /* 378 /*
392 * Timers are FIFO: 379 * Timers are FIFO:
393 */ 380 */
394 list_add_tail(&timer->entry, vec); 381 list_add_tail(&timer->entry, vec);
395 } 382 }
396 383
397 #ifdef CONFIG_TIMER_STATS 384 #ifdef CONFIG_TIMER_STATS
398 void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr) 385 void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr)
399 { 386 {
400 if (timer->start_site) 387 if (timer->start_site)
401 return; 388 return;
402 389
403 timer->start_site = addr; 390 timer->start_site = addr;
404 memcpy(timer->start_comm, current->comm, TASK_COMM_LEN); 391 memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
405 timer->start_pid = current->pid; 392 timer->start_pid = current->pid;
406 } 393 }
407 394
408 static void timer_stats_account_timer(struct timer_list *timer) 395 static void timer_stats_account_timer(struct timer_list *timer)
409 { 396 {
410 unsigned int flag = 0; 397 unsigned int flag = 0;
411 398
412 if (likely(!timer->start_site)) 399 if (likely(!timer->start_site))
413 return; 400 return;
414 if (unlikely(tbase_get_deferrable(timer->base))) 401 if (unlikely(tbase_get_deferrable(timer->base)))
415 flag |= TIMER_STATS_FLAG_DEFERRABLE; 402 flag |= TIMER_STATS_FLAG_DEFERRABLE;
416 403
417 timer_stats_update_stats(timer, timer->start_pid, timer->start_site, 404 timer_stats_update_stats(timer, timer->start_pid, timer->start_site,
418 timer->function, timer->start_comm, flag); 405 timer->function, timer->start_comm, flag);
419 } 406 }
420 407
421 #else 408 #else
422 static void timer_stats_account_timer(struct timer_list *timer) {} 409 static void timer_stats_account_timer(struct timer_list *timer) {}
423 #endif 410 #endif
424 411
425 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS 412 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
426 413
427 static struct debug_obj_descr timer_debug_descr; 414 static struct debug_obj_descr timer_debug_descr;
428 415
429 /* 416 /*
430 * fixup_init is called when: 417 * fixup_init is called when:
431 * - an active object is initialized 418 * - an active object is initialized
432 */ 419 */
433 static int timer_fixup_init(void *addr, enum debug_obj_state state) 420 static int timer_fixup_init(void *addr, enum debug_obj_state state)
434 { 421 {
435 struct timer_list *timer = addr; 422 struct timer_list *timer = addr;
436 423
437 switch (state) { 424 switch (state) {
438 case ODEBUG_STATE_ACTIVE: 425 case ODEBUG_STATE_ACTIVE:
439 del_timer_sync(timer); 426 del_timer_sync(timer);
440 debug_object_init(timer, &timer_debug_descr); 427 debug_object_init(timer, &timer_debug_descr);
441 return 1; 428 return 1;
442 default: 429 default:
443 return 0; 430 return 0;
444 } 431 }
445 } 432 }
446 433
447 /* 434 /*
448 * fixup_activate is called when: 435 * fixup_activate is called when:
449 * - an active object is activated 436 * - an active object is activated
450 * - an unknown object is activated (might be a statically initialized object) 437 * - an unknown object is activated (might be a statically initialized object)
451 */ 438 */
452 static int timer_fixup_activate(void *addr, enum debug_obj_state state) 439 static int timer_fixup_activate(void *addr, enum debug_obj_state state)
453 { 440 {
454 struct timer_list *timer = addr; 441 struct timer_list *timer = addr;
455 442
456 switch (state) { 443 switch (state) {
457 444
458 case ODEBUG_STATE_NOTAVAILABLE: 445 case ODEBUG_STATE_NOTAVAILABLE:
459 /* 446 /*
460 * This is not really a fixup. The timer was 447 * This is not really a fixup. The timer was
461 * statically initialized. We just make sure that it 448 * statically initialized. We just make sure that it
462 * is tracked in the object tracker. 449 * is tracked in the object tracker.
463 */ 450 */
464 if (timer->entry.next == NULL && 451 if (timer->entry.next == NULL &&
465 timer->entry.prev == TIMER_ENTRY_STATIC) { 452 timer->entry.prev == TIMER_ENTRY_STATIC) {
466 debug_object_init(timer, &timer_debug_descr); 453 debug_object_init(timer, &timer_debug_descr);
467 debug_object_activate(timer, &timer_debug_descr); 454 debug_object_activate(timer, &timer_debug_descr);
468 return 0; 455 return 0;
469 } else { 456 } else {
470 WARN_ON_ONCE(1); 457 WARN_ON_ONCE(1);
471 } 458 }
472 return 0; 459 return 0;
473 460
474 case ODEBUG_STATE_ACTIVE: 461 case ODEBUG_STATE_ACTIVE:
475 WARN_ON(1); 462 WARN_ON(1);
476 463
477 default: 464 default:
478 return 0; 465 return 0;
479 } 466 }
480 } 467 }
481 468
482 /* 469 /*
483 * fixup_free is called when: 470 * fixup_free is called when:
484 * - an active object is freed 471 * - an active object is freed
485 */ 472 */
486 static int timer_fixup_free(void *addr, enum debug_obj_state state) 473 static int timer_fixup_free(void *addr, enum debug_obj_state state)
487 { 474 {
488 struct timer_list *timer = addr; 475 struct timer_list *timer = addr;
489 476
490 switch (state) { 477 switch (state) {
491 case ODEBUG_STATE_ACTIVE: 478 case ODEBUG_STATE_ACTIVE:
492 del_timer_sync(timer); 479 del_timer_sync(timer);
493 debug_object_free(timer, &timer_debug_descr); 480 debug_object_free(timer, &timer_debug_descr);
494 return 1; 481 return 1;
495 default: 482 default:
496 return 0; 483 return 0;
497 } 484 }
498 } 485 }
499 486
500 static struct debug_obj_descr timer_debug_descr = { 487 static struct debug_obj_descr timer_debug_descr = {
501 .name = "timer_list", 488 .name = "timer_list",
502 .fixup_init = timer_fixup_init, 489 .fixup_init = timer_fixup_init,
503 .fixup_activate = timer_fixup_activate, 490 .fixup_activate = timer_fixup_activate,
504 .fixup_free = timer_fixup_free, 491 .fixup_free = timer_fixup_free,
505 }; 492 };
506 493
507 static inline void debug_timer_init(struct timer_list *timer) 494 static inline void debug_timer_init(struct timer_list *timer)
508 { 495 {
509 debug_object_init(timer, &timer_debug_descr); 496 debug_object_init(timer, &timer_debug_descr);
510 } 497 }
511 498
512 static inline void debug_timer_activate(struct timer_list *timer) 499 static inline void debug_timer_activate(struct timer_list *timer)
513 { 500 {
514 debug_object_activate(timer, &timer_debug_descr); 501 debug_object_activate(timer, &timer_debug_descr);
515 } 502 }
516 503
517 static inline void debug_timer_deactivate(struct timer_list *timer) 504 static inline void debug_timer_deactivate(struct timer_list *timer)
518 { 505 {
519 debug_object_deactivate(timer, &timer_debug_descr); 506 debug_object_deactivate(timer, &timer_debug_descr);
520 } 507 }
521 508
522 static inline void debug_timer_free(struct timer_list *timer) 509 static inline void debug_timer_free(struct timer_list *timer)
523 { 510 {
524 debug_object_free(timer, &timer_debug_descr); 511 debug_object_free(timer, &timer_debug_descr);
525 } 512 }
526 513
527 static void __init_timer(struct timer_list *timer, 514 static void __init_timer(struct timer_list *timer,
528 const char *name, 515 const char *name,
529 struct lock_class_key *key); 516 struct lock_class_key *key);
530 517
531 void init_timer_on_stack_key(struct timer_list *timer, 518 void init_timer_on_stack_key(struct timer_list *timer,
532 const char *name, 519 const char *name,
533 struct lock_class_key *key) 520 struct lock_class_key *key)
534 { 521 {
535 debug_object_init_on_stack(timer, &timer_debug_descr); 522 debug_object_init_on_stack(timer, &timer_debug_descr);
536 __init_timer(timer, name, key); 523 __init_timer(timer, name, key);
537 } 524 }
538 EXPORT_SYMBOL_GPL(init_timer_on_stack_key); 525 EXPORT_SYMBOL_GPL(init_timer_on_stack_key);
539 526
540 void destroy_timer_on_stack(struct timer_list *timer) 527 void destroy_timer_on_stack(struct timer_list *timer)
541 { 528 {
542 debug_object_free(timer, &timer_debug_descr); 529 debug_object_free(timer, &timer_debug_descr);
543 } 530 }
544 EXPORT_SYMBOL_GPL(destroy_timer_on_stack); 531 EXPORT_SYMBOL_GPL(destroy_timer_on_stack);
545 532
546 #else 533 #else
547 static inline void debug_timer_init(struct timer_list *timer) { } 534 static inline void debug_timer_init(struct timer_list *timer) { }
548 static inline void debug_timer_activate(struct timer_list *timer) { } 535 static inline void debug_timer_activate(struct timer_list *timer) { }
549 static inline void debug_timer_deactivate(struct timer_list *timer) { } 536 static inline void debug_timer_deactivate(struct timer_list *timer) { }
550 #endif 537 #endif
551 538
552 static inline void debug_init(struct timer_list *timer) 539 static inline void debug_init(struct timer_list *timer)
553 { 540 {
554 debug_timer_init(timer); 541 debug_timer_init(timer);
555 trace_timer_init(timer); 542 trace_timer_init(timer);
556 } 543 }
557 544
558 static inline void 545 static inline void
559 debug_activate(struct timer_list *timer, unsigned long expires) 546 debug_activate(struct timer_list *timer, unsigned long expires)
560 { 547 {
561 debug_timer_activate(timer); 548 debug_timer_activate(timer);
562 trace_timer_start(timer, expires); 549 trace_timer_start(timer, expires);
563 } 550 }
564 551
565 static inline void debug_deactivate(struct timer_list *timer) 552 static inline void debug_deactivate(struct timer_list *timer)
566 { 553 {
567 debug_timer_deactivate(timer); 554 debug_timer_deactivate(timer);
568 trace_timer_cancel(timer); 555 trace_timer_cancel(timer);
569 } 556 }
570 557
571 static void __init_timer(struct timer_list *timer, 558 static void __init_timer(struct timer_list *timer,
572 const char *name, 559 const char *name,
573 struct lock_class_key *key) 560 struct lock_class_key *key)
574 { 561 {
575 timer->entry.next = NULL; 562 timer->entry.next = NULL;
576 timer->base = __raw_get_cpu_var(tvec_bases); 563 timer->base = __raw_get_cpu_var(tvec_bases);
577 timer->slack = -1; 564 timer->slack = -1;
578 #ifdef CONFIG_TIMER_STATS 565 #ifdef CONFIG_TIMER_STATS
579 timer->start_site = NULL; 566 timer->start_site = NULL;
580 timer->start_pid = -1; 567 timer->start_pid = -1;
581 memset(timer->start_comm, 0, TASK_COMM_LEN); 568 memset(timer->start_comm, 0, TASK_COMM_LEN);
582 #endif 569 #endif
583 lockdep_init_map(&timer->lockdep_map, name, key, 0); 570 lockdep_init_map(&timer->lockdep_map, name, key, 0);
584 } 571 }
585 572
586 void setup_deferrable_timer_on_stack_key(struct timer_list *timer, 573 void setup_deferrable_timer_on_stack_key(struct timer_list *timer,
587 const char *name, 574 const char *name,
588 struct lock_class_key *key, 575 struct lock_class_key *key,
589 void (*function)(unsigned long), 576 void (*function)(unsigned long),
590 unsigned long data) 577 unsigned long data)
591 { 578 {
592 timer->function = function; 579 timer->function = function;
593 timer->data = data; 580 timer->data = data;
594 init_timer_on_stack_key(timer, name, key); 581 init_timer_on_stack_key(timer, name, key);
595 timer_set_deferrable(timer); 582 timer_set_deferrable(timer);
596 } 583 }
597 EXPORT_SYMBOL_GPL(setup_deferrable_timer_on_stack_key); 584 EXPORT_SYMBOL_GPL(setup_deferrable_timer_on_stack_key);
598 585
599 /** 586 /**
600 * init_timer_key - initialize a timer 587 * init_timer_key - initialize a timer
601 * @timer: the timer to be initialized 588 * @timer: the timer to be initialized
602 * @name: name of the timer 589 * @name: name of the timer
603 * @key: lockdep class key of the fake lock used for tracking timer 590 * @key: lockdep class key of the fake lock used for tracking timer
604 * sync lock dependencies 591 * sync lock dependencies
605 * 592 *
606 * init_timer_key() must be done to a timer prior calling *any* of the 593 * init_timer_key() must be done to a timer prior calling *any* of the
607 * other timer functions. 594 * other timer functions.
608 */ 595 */
609 void init_timer_key(struct timer_list *timer, 596 void init_timer_key(struct timer_list *timer,
610 const char *name, 597 const char *name,
611 struct lock_class_key *key) 598 struct lock_class_key *key)
612 { 599 {
613 debug_init(timer); 600 debug_init(timer);
614 __init_timer(timer, name, key); 601 __init_timer(timer, name, key);
615 } 602 }
616 EXPORT_SYMBOL(init_timer_key); 603 EXPORT_SYMBOL(init_timer_key);
617 604
618 void init_timer_deferrable_key(struct timer_list *timer, 605 void init_timer_deferrable_key(struct timer_list *timer,
619 const char *name, 606 const char *name,
620 struct lock_class_key *key) 607 struct lock_class_key *key)
621 { 608 {
622 init_timer_key(timer, name, key); 609 init_timer_key(timer, name, key);
623 timer_set_deferrable(timer); 610 timer_set_deferrable(timer);
624 } 611 }
625 EXPORT_SYMBOL(init_timer_deferrable_key); 612 EXPORT_SYMBOL(init_timer_deferrable_key);
626 613
627 static inline void detach_timer(struct timer_list *timer, 614 static inline void detach_timer(struct timer_list *timer,
628 int clear_pending) 615 int clear_pending)
629 { 616 {
630 struct list_head *entry = &timer->entry; 617 struct list_head *entry = &timer->entry;
631 618
632 debug_deactivate(timer); 619 debug_deactivate(timer);
633 620
634 __list_del(entry->prev, entry->next); 621 __list_del(entry->prev, entry->next);
635 if (clear_pending) 622 if (clear_pending)
636 entry->next = NULL; 623 entry->next = NULL;
637 entry->prev = LIST_POISON2; 624 entry->prev = LIST_POISON2;
638 } 625 }
639 626
640 /* 627 /*
641 * We are using hashed locking: holding per_cpu(tvec_bases).lock 628 * We are using hashed locking: holding per_cpu(tvec_bases).lock
642 * means that all timers which are tied to this base via timer->base are 629 * means that all timers which are tied to this base via timer->base are
643 * locked, and the base itself is locked too. 630 * locked, and the base itself is locked too.
644 * 631 *
645 * So __run_timers/migrate_timers can safely modify all timers which could 632 * So __run_timers/migrate_timers can safely modify all timers which could
646 * be found on ->tvX lists. 633 * be found on ->tvX lists.
647 * 634 *
648 * When the timer's base is locked, and the timer removed from list, it is 635 * When the timer's base is locked, and the timer removed from list, it is
649 * possible to set timer->base = NULL and drop the lock: the timer remains 636 * possible to set timer->base = NULL and drop the lock: the timer remains
650 * locked. 637 * locked.
651 */ 638 */
652 static struct tvec_base *lock_timer_base(struct timer_list *timer, 639 static struct tvec_base *lock_timer_base(struct timer_list *timer,
653 unsigned long *flags) 640 unsigned long *flags)
654 __acquires(timer->base->lock) 641 __acquires(timer->base->lock)
655 { 642 {
656 struct tvec_base *base; 643 struct tvec_base *base;
657 644
658 for (;;) { 645 for (;;) {
659 struct tvec_base *prelock_base = timer->base; 646 struct tvec_base *prelock_base = timer->base;
660 base = tbase_get_base(prelock_base); 647 base = tbase_get_base(prelock_base);
661 if (likely(base != NULL)) { 648 if (likely(base != NULL)) {
662 spin_lock_irqsave(&base->lock, *flags); 649 spin_lock_irqsave(&base->lock, *flags);
663 if (likely(prelock_base == timer->base)) 650 if (likely(prelock_base == timer->base))
664 return base; 651 return base;
665 /* The timer has migrated to another CPU */ 652 /* The timer has migrated to another CPU */
666 spin_unlock_irqrestore(&base->lock, *flags); 653 spin_unlock_irqrestore(&base->lock, *flags);
667 } 654 }
668 cpu_relax(); 655 cpu_relax();
669 } 656 }
670 } 657 }
671 658
672 static inline int 659 static inline int
673 __mod_timer(struct timer_list *timer, unsigned long expires, 660 __mod_timer(struct timer_list *timer, unsigned long expires,
674 bool pending_only, int pinned) 661 bool pending_only, int pinned)
675 { 662 {
676 struct tvec_base *base, *new_base; 663 struct tvec_base *base, *new_base;
677 unsigned long flags; 664 unsigned long flags;
678 int ret = 0 , cpu; 665 int ret = 0 , cpu;
679 666
680 timer_stats_timer_set_start_info(timer); 667 timer_stats_timer_set_start_info(timer);
681 BUG_ON(!timer->function); 668 BUG_ON(!timer->function);
682 669
683 base = lock_timer_base(timer, &flags); 670 base = lock_timer_base(timer, &flags);
684 671
685 if (timer_pending(timer)) { 672 if (timer_pending(timer)) {
686 detach_timer(timer, 0); 673 detach_timer(timer, 0);
687 if (timer->expires == base->next_timer && 674 if (timer->expires == base->next_timer &&
688 !tbase_get_deferrable(timer->base)) 675 !tbase_get_deferrable(timer->base))
689 base->next_timer = base->timer_jiffies; 676 base->next_timer = base->timer_jiffies;
690 ret = 1; 677 ret = 1;
691 } else { 678 } else {
692 if (pending_only) 679 if (pending_only)
693 goto out_unlock; 680 goto out_unlock;
694 } 681 }
695 682
696 debug_activate(timer, expires); 683 debug_activate(timer, expires);
697 684
698 cpu = smp_processor_id(); 685 cpu = smp_processor_id();
699 686
700 #if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP) 687 #if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
701 if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) 688 if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu))
702 cpu = get_nohz_timer_target(); 689 cpu = get_nohz_timer_target();
703 #endif 690 #endif
704 new_base = per_cpu(tvec_bases, cpu); 691 new_base = per_cpu(tvec_bases, cpu);
705 692
706 if (base != new_base) { 693 if (base != new_base) {
707 /* 694 /*
708 * We are trying to schedule the timer on the local CPU. 695 * We are trying to schedule the timer on the local CPU.
709 * However we can't change timer's base while it is running, 696 * However we can't change timer's base while it is running,
710 * otherwise del_timer_sync() can't detect that the timer's 697 * otherwise del_timer_sync() can't detect that the timer's
711 * handler yet has not finished. This also guarantees that 698 * handler yet has not finished. This also guarantees that
712 * the timer is serialized wrt itself. 699 * the timer is serialized wrt itself.
713 */ 700 */
714 if (likely(base->running_timer != timer)) { 701 if (likely(base->running_timer != timer)) {
715 /* See the comment in lock_timer_base() */ 702 /* See the comment in lock_timer_base() */
716 timer_set_base(timer, NULL); 703 timer_set_base(timer, NULL);
717 spin_unlock(&base->lock); 704 spin_unlock(&base->lock);
718 base = new_base; 705 base = new_base;
719 spin_lock(&base->lock); 706 spin_lock(&base->lock);
720 timer_set_base(timer, base); 707 timer_set_base(timer, base);
721 } 708 }
722 } 709 }
723 710
724 timer->expires = expires; 711 timer->expires = expires;
725 if (time_before(timer->expires, base->next_timer) && 712 if (time_before(timer->expires, base->next_timer) &&
726 !tbase_get_deferrable(timer->base)) 713 !tbase_get_deferrable(timer->base))
727 base->next_timer = timer->expires; 714 base->next_timer = timer->expires;
728 internal_add_timer(base, timer); 715 internal_add_timer(base, timer);
729 716
730 out_unlock: 717 out_unlock:
731 spin_unlock_irqrestore(&base->lock, flags); 718 spin_unlock_irqrestore(&base->lock, flags);
732 719
733 return ret; 720 return ret;
734 } 721 }
735 722
736 /** 723 /**
737 * mod_timer_pending - modify a pending timer's timeout 724 * mod_timer_pending - modify a pending timer's timeout
738 * @timer: the pending timer to be modified 725 * @timer: the pending timer to be modified
739 * @expires: new timeout in jiffies 726 * @expires: new timeout in jiffies
740 * 727 *
741 * mod_timer_pending() is the same for pending timers as mod_timer(), 728 * mod_timer_pending() is the same for pending timers as mod_timer(),
742 * but will not re-activate and modify already deleted timers. 729 * but will not re-activate and modify already deleted timers.
743 * 730 *
744 * It is useful for unserialized use of timers. 731 * It is useful for unserialized use of timers.
745 */ 732 */
746 int mod_timer_pending(struct timer_list *timer, unsigned long expires) 733 int mod_timer_pending(struct timer_list *timer, unsigned long expires)
747 { 734 {
748 return __mod_timer(timer, expires, true, TIMER_NOT_PINNED); 735 return __mod_timer(timer, expires, true, TIMER_NOT_PINNED);
749 } 736 }
750 EXPORT_SYMBOL(mod_timer_pending); 737 EXPORT_SYMBOL(mod_timer_pending);
751 738
752 /* 739 /*
753 * Decide where to put the timer while taking the slack into account 740 * Decide where to put the timer while taking the slack into account
754 * 741 *
755 * Algorithm: 742 * Algorithm:
756 * 1) calculate the maximum (absolute) time 743 * 1) calculate the maximum (absolute) time
757 * 2) calculate the highest bit where the expires and new max are different 744 * 2) calculate the highest bit where the expires and new max are different
758 * 3) use this bit to make a mask 745 * 3) use this bit to make a mask
759 * 4) use the bitmask to round down the maximum time, so that all last 746 * 4) use the bitmask to round down the maximum time, so that all last
760 * bits are zeros 747 * bits are zeros
761 */ 748 */
762 static inline 749 static inline
763 unsigned long apply_slack(struct timer_list *timer, unsigned long expires) 750 unsigned long apply_slack(struct timer_list *timer, unsigned long expires)
764 { 751 {
765 unsigned long expires_limit, mask; 752 unsigned long expires_limit, mask;
766 int bit; 753 int bit;
767 754
768 expires_limit = expires; 755 expires_limit = expires;
769 756
770 if (timer->slack >= 0) { 757 if (timer->slack >= 0) {
771 expires_limit = expires + timer->slack; 758 expires_limit = expires + timer->slack;
772 } else { 759 } else {
773 unsigned long now = jiffies; 760 unsigned long now = jiffies;
774 761
775 /* No slack, if already expired else auto slack 0.4% */ 762 /* No slack, if already expired else auto slack 0.4% */
776 if (time_after(expires, now)) 763 if (time_after(expires, now))
777 expires_limit = expires + (expires - now)/256; 764 expires_limit = expires + (expires - now)/256;
778 } 765 }
779 mask = expires ^ expires_limit; 766 mask = expires ^ expires_limit;
780 if (mask == 0) 767 if (mask == 0)
781 return expires; 768 return expires;
782 769
783 bit = find_last_bit(&mask, BITS_PER_LONG); 770 bit = find_last_bit(&mask, BITS_PER_LONG);
784 771
785 mask = (1 << bit) - 1; 772 mask = (1 << bit) - 1;
786 773
787 expires_limit = expires_limit & ~(mask); 774 expires_limit = expires_limit & ~(mask);
788 775
789 return expires_limit; 776 return expires_limit;
790 } 777 }
791 778
792 /** 779 /**
793 * mod_timer - modify a timer's timeout 780 * mod_timer - modify a timer's timeout
794 * @timer: the timer to be modified 781 * @timer: the timer to be modified
795 * @expires: new timeout in jiffies 782 * @expires: new timeout in jiffies
796 * 783 *
797 * mod_timer() is a more efficient way to update the expire field of an 784 * mod_timer() is a more efficient way to update the expire field of an
798 * active timer (if the timer is inactive it will be activated) 785 * active timer (if the timer is inactive it will be activated)
799 * 786 *
800 * mod_timer(timer, expires) is equivalent to: 787 * mod_timer(timer, expires) is equivalent to:
801 * 788 *
802 * del_timer(timer); timer->expires = expires; add_timer(timer); 789 * del_timer(timer); timer->expires = expires; add_timer(timer);
803 * 790 *
804 * Note that if there are multiple unserialized concurrent users of the 791 * Note that if there are multiple unserialized concurrent users of the
805 * same timer, then mod_timer() is the only safe way to modify the timeout, 792 * same timer, then mod_timer() is the only safe way to modify the timeout,
806 * since add_timer() cannot modify an already running timer. 793 * since add_timer() cannot modify an already running timer.
807 * 794 *
808 * The function returns whether it has modified a pending timer or not. 795 * The function returns whether it has modified a pending timer or not.
809 * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an 796 * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an
810 * active timer returns 1.) 797 * active timer returns 1.)
811 */ 798 */
812 int mod_timer(struct timer_list *timer, unsigned long expires) 799 int mod_timer(struct timer_list *timer, unsigned long expires)
813 { 800 {
814 /* 801 /*
815 * This is a common optimization triggered by the 802 * This is a common optimization triggered by the
816 * networking code - if the timer is re-modified 803 * networking code - if the timer is re-modified
817 * to be the same thing then just return: 804 * to be the same thing then just return:
818 */ 805 */
819 if (timer_pending(timer) && timer->expires == expires) 806 if (timer_pending(timer) && timer->expires == expires)
820 return 1; 807 return 1;
821 808
822 expires = apply_slack(timer, expires); 809 expires = apply_slack(timer, expires);
823 810
824 return __mod_timer(timer, expires, false, TIMER_NOT_PINNED); 811 return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);
825 } 812 }
826 EXPORT_SYMBOL(mod_timer); 813 EXPORT_SYMBOL(mod_timer);
827 814
828 /** 815 /**
829 * mod_timer_pinned - modify a timer's timeout 816 * mod_timer_pinned - modify a timer's timeout
830 * @timer: the timer to be modified 817 * @timer: the timer to be modified
831 * @expires: new timeout in jiffies 818 * @expires: new timeout in jiffies
832 * 819 *
833 * mod_timer_pinned() is a way to update the expire field of an 820 * mod_timer_pinned() is a way to update the expire field of an
834 * active timer (if the timer is inactive it will be activated) 821 * active timer (if the timer is inactive it will be activated)
835 * and not allow the timer to be migrated to a different CPU. 822 * and not allow the timer to be migrated to a different CPU.
836 * 823 *
837 * mod_timer_pinned(timer, expires) is equivalent to: 824 * mod_timer_pinned(timer, expires) is equivalent to:
838 * 825 *
839 * del_timer(timer); timer->expires = expires; add_timer(timer); 826 * del_timer(timer); timer->expires = expires; add_timer(timer);
840 */ 827 */
841 int mod_timer_pinned(struct timer_list *timer, unsigned long expires) 828 int mod_timer_pinned(struct timer_list *timer, unsigned long expires)
842 { 829 {
843 if (timer->expires == expires && timer_pending(timer)) 830 if (timer->expires == expires && timer_pending(timer))
844 return 1; 831 return 1;
845 832
846 return __mod_timer(timer, expires, false, TIMER_PINNED); 833 return __mod_timer(timer, expires, false, TIMER_PINNED);
847 } 834 }
848 EXPORT_SYMBOL(mod_timer_pinned); 835 EXPORT_SYMBOL(mod_timer_pinned);
849 836
850 /** 837 /**
851 * add_timer - start a timer 838 * add_timer - start a timer
852 * @timer: the timer to be added 839 * @timer: the timer to be added
853 * 840 *
854 * The kernel will do a ->function(->data) callback from the 841 * The kernel will do a ->function(->data) callback from the
855 * timer interrupt at the ->expires point in the future. The 842 * timer interrupt at the ->expires point in the future. The
856 * current time is 'jiffies'. 843 * current time is 'jiffies'.
857 * 844 *
858 * The timer's ->expires, ->function (and if the handler uses it, ->data) 845 * The timer's ->expires, ->function (and if the handler uses it, ->data)
859 * fields must be set prior calling this function. 846 * fields must be set prior calling this function.
860 * 847 *
861 * Timers with an ->expires field in the past will be executed in the next 848 * Timers with an ->expires field in the past will be executed in the next
862 * timer tick. 849 * timer tick.
863 */ 850 */
864 void add_timer(struct timer_list *timer) 851 void add_timer(struct timer_list *timer)
865 { 852 {
866 BUG_ON(timer_pending(timer)); 853 BUG_ON(timer_pending(timer));
867 mod_timer(timer, timer->expires); 854 mod_timer(timer, timer->expires);
868 } 855 }
869 EXPORT_SYMBOL(add_timer); 856 EXPORT_SYMBOL(add_timer);
870 857
871 /** 858 /**
872 * add_timer_on - start a timer on a particular CPU 859 * add_timer_on - start a timer on a particular CPU
873 * @timer: the timer to be added 860 * @timer: the timer to be added
874 * @cpu: the CPU to start it on 861 * @cpu: the CPU to start it on
875 * 862 *
876 * This is not very scalable on SMP. Double adds are not possible. 863 * This is not very scalable on SMP. Double adds are not possible.
877 */ 864 */
878 void add_timer_on(struct timer_list *timer, int cpu) 865 void add_timer_on(struct timer_list *timer, int cpu)
879 { 866 {
880 struct tvec_base *base = per_cpu(tvec_bases, cpu); 867 struct tvec_base *base = per_cpu(tvec_bases, cpu);
881 unsigned long flags; 868 unsigned long flags;
882 869
883 timer_stats_timer_set_start_info(timer); 870 timer_stats_timer_set_start_info(timer);
884 BUG_ON(timer_pending(timer) || !timer->function); 871 BUG_ON(timer_pending(timer) || !timer->function);
885 spin_lock_irqsave(&base->lock, flags); 872 spin_lock_irqsave(&base->lock, flags);
886 timer_set_base(timer, base); 873 timer_set_base(timer, base);
887 debug_activate(timer, timer->expires); 874 debug_activate(timer, timer->expires);
888 if (time_before(timer->expires, base->next_timer) && 875 if (time_before(timer->expires, base->next_timer) &&
889 !tbase_get_deferrable(timer->base)) 876 !tbase_get_deferrable(timer->base))
890 base->next_timer = timer->expires; 877 base->next_timer = timer->expires;
891 internal_add_timer(base, timer); 878 internal_add_timer(base, timer);
892 /* 879 /*
893 * Check whether the other CPU is idle and needs to be 880 * Check whether the other CPU is idle and needs to be
894 * triggered to reevaluate the timer wheel when nohz is 881 * triggered to reevaluate the timer wheel when nohz is
895 * active. We are protected against the other CPU fiddling 882 * active. We are protected against the other CPU fiddling
896 * with the timer by holding the timer base lock. This also 883 * with the timer by holding the timer base lock. This also
897 * makes sure that a CPU on the way to idle can not evaluate 884 * makes sure that a CPU on the way to idle can not evaluate
898 * the timer wheel. 885 * the timer wheel.
899 */ 886 */
900 wake_up_idle_cpu(cpu); 887 wake_up_idle_cpu(cpu);
901 spin_unlock_irqrestore(&base->lock, flags); 888 spin_unlock_irqrestore(&base->lock, flags);
902 } 889 }
903 EXPORT_SYMBOL_GPL(add_timer_on); 890 EXPORT_SYMBOL_GPL(add_timer_on);
904 891
905 /** 892 /**
906 * del_timer - deactive a timer. 893 * del_timer - deactive a timer.
907 * @timer: the timer to be deactivated 894 * @timer: the timer to be deactivated
908 * 895 *
909 * del_timer() deactivates a timer - this works on both active and inactive 896 * del_timer() deactivates a timer - this works on both active and inactive
910 * timers. 897 * timers.
911 * 898 *
912 * The function returns whether it has deactivated a pending timer or not. 899 * The function returns whether it has deactivated a pending timer or not.
913 * (ie. del_timer() of an inactive timer returns 0, del_timer() of an 900 * (ie. del_timer() of an inactive timer returns 0, del_timer() of an
914 * active timer returns 1.) 901 * active timer returns 1.)
915 */ 902 */
916 int del_timer(struct timer_list *timer) 903 int del_timer(struct timer_list *timer)
917 { 904 {
918 struct tvec_base *base; 905 struct tvec_base *base;
919 unsigned long flags; 906 unsigned long flags;
920 int ret = 0; 907 int ret = 0;
921 908
922 timer_stats_timer_clear_start_info(timer); 909 timer_stats_timer_clear_start_info(timer);
923 if (timer_pending(timer)) { 910 if (timer_pending(timer)) {
924 base = lock_timer_base(timer, &flags); 911 base = lock_timer_base(timer, &flags);
925 if (timer_pending(timer)) { 912 if (timer_pending(timer)) {
926 detach_timer(timer, 1); 913 detach_timer(timer, 1);
927 if (timer->expires == base->next_timer && 914 if (timer->expires == base->next_timer &&
928 !tbase_get_deferrable(timer->base)) 915 !tbase_get_deferrable(timer->base))
929 base->next_timer = base->timer_jiffies; 916 base->next_timer = base->timer_jiffies;
930 ret = 1; 917 ret = 1;
931 } 918 }
932 spin_unlock_irqrestore(&base->lock, flags); 919 spin_unlock_irqrestore(&base->lock, flags);
933 } 920 }
934 921
935 return ret; 922 return ret;
936 } 923 }
937 EXPORT_SYMBOL(del_timer); 924 EXPORT_SYMBOL(del_timer);
938 925
939 #ifdef CONFIG_SMP 926 #ifdef CONFIG_SMP
940 /** 927 /**
941 * try_to_del_timer_sync - Try to deactivate a timer 928 * try_to_del_timer_sync - Try to deactivate a timer
942 * @timer: timer do del 929 * @timer: timer do del
943 * 930 *
944 * This function tries to deactivate a timer. Upon successful (ret >= 0) 931 * This function tries to deactivate a timer. Upon successful (ret >= 0)
945 * exit the timer is not queued and the handler is not running on any CPU. 932 * exit the timer is not queued and the handler is not running on any CPU.
946 * 933 *
947 * It must not be called from interrupt contexts. 934 * It must not be called from interrupt contexts.
948 */ 935 */
949 int try_to_del_timer_sync(struct timer_list *timer) 936 int try_to_del_timer_sync(struct timer_list *timer)
950 { 937 {
951 struct tvec_base *base; 938 struct tvec_base *base;
952 unsigned long flags; 939 unsigned long flags;
953 int ret = -1; 940 int ret = -1;
954 941
955 base = lock_timer_base(timer, &flags); 942 base = lock_timer_base(timer, &flags);
956 943
957 if (base->running_timer == timer) 944 if (base->running_timer == timer)
958 goto out; 945 goto out;
959 946
960 timer_stats_timer_clear_start_info(timer); 947 timer_stats_timer_clear_start_info(timer);
961 ret = 0; 948 ret = 0;
962 if (timer_pending(timer)) { 949 if (timer_pending(timer)) {
963 detach_timer(timer, 1); 950 detach_timer(timer, 1);
964 if (timer->expires == base->next_timer && 951 if (timer->expires == base->next_timer &&
965 !tbase_get_deferrable(timer->base)) 952 !tbase_get_deferrable(timer->base))
966 base->next_timer = base->timer_jiffies; 953 base->next_timer = base->timer_jiffies;
967 ret = 1; 954 ret = 1;
968 } 955 }
969 out: 956 out:
970 spin_unlock_irqrestore(&base->lock, flags); 957 spin_unlock_irqrestore(&base->lock, flags);
971 958
972 return ret; 959 return ret;
973 } 960 }
974 EXPORT_SYMBOL(try_to_del_timer_sync); 961 EXPORT_SYMBOL(try_to_del_timer_sync);
975 962
976 /** 963 /**
977 * del_timer_sync - deactivate a timer and wait for the handler to finish. 964 * del_timer_sync - deactivate a timer and wait for the handler to finish.
978 * @timer: the timer to be deactivated 965 * @timer: the timer to be deactivated
979 * 966 *
980 * This function only differs from del_timer() on SMP: besides deactivating 967 * This function only differs from del_timer() on SMP: besides deactivating
981 * the timer it also makes sure the handler has finished executing on other 968 * the timer it also makes sure the handler has finished executing on other
982 * CPUs. 969 * CPUs.
983 * 970 *
984 * Synchronization rules: Callers must prevent restarting of the timer, 971 * Synchronization rules: Callers must prevent restarting of the timer,
985 * otherwise this function is meaningless. It must not be called from 972 * otherwise this function is meaningless. It must not be called from
986 * interrupt contexts. The caller must not hold locks which would prevent 973 * interrupt contexts. The caller must not hold locks which would prevent
987 * completion of the timer's handler. The timer's handler must not call 974 * completion of the timer's handler. The timer's handler must not call
988 * add_timer_on(). Upon exit the timer is not queued and the handler is 975 * add_timer_on(). Upon exit the timer is not queued and the handler is
989 * not running on any CPU. 976 * not running on any CPU.
990 * 977 *
991 * The function returns whether it has deactivated a pending timer or not. 978 * The function returns whether it has deactivated a pending timer or not.
992 */ 979 */
993 int del_timer_sync(struct timer_list *timer) 980 int del_timer_sync(struct timer_list *timer)
994 { 981 {
995 #ifdef CONFIG_LOCKDEP 982 #ifdef CONFIG_LOCKDEP
996 unsigned long flags; 983 unsigned long flags;
997 984
998 local_irq_save(flags); 985 local_irq_save(flags);
999 lock_map_acquire(&timer->lockdep_map); 986 lock_map_acquire(&timer->lockdep_map);
1000 lock_map_release(&timer->lockdep_map); 987 lock_map_release(&timer->lockdep_map);
1001 local_irq_restore(flags); 988 local_irq_restore(flags);
1002 #endif 989 #endif
1003 990
1004 for (;;) { 991 for (;;) {
1005 int ret = try_to_del_timer_sync(timer); 992 int ret = try_to_del_timer_sync(timer);
1006 if (ret >= 0) 993 if (ret >= 0)
1007 return ret; 994 return ret;
1008 cpu_relax(); 995 cpu_relax();
1009 } 996 }
1010 } 997 }
1011 EXPORT_SYMBOL(del_timer_sync); 998 EXPORT_SYMBOL(del_timer_sync);
1012 #endif 999 #endif
1013 1000
1014 static int cascade(struct tvec_base *base, struct tvec *tv, int index) 1001 static int cascade(struct tvec_base *base, struct tvec *tv, int index)
1015 { 1002 {
1016 /* cascade all the timers from tv up one level */ 1003 /* cascade all the timers from tv up one level */
1017 struct timer_list *timer, *tmp; 1004 struct timer_list *timer, *tmp;
1018 struct list_head tv_list; 1005 struct list_head tv_list;
1019 1006
1020 list_replace_init(tv->vec + index, &tv_list); 1007 list_replace_init(tv->vec + index, &tv_list);
1021 1008
1022 /* 1009 /*
1023 * We are removing _all_ timers from the list, so we 1010 * We are removing _all_ timers from the list, so we
1024 * don't have to detach them individually. 1011 * don't have to detach them individually.
1025 */ 1012 */
1026 list_for_each_entry_safe(timer, tmp, &tv_list, entry) { 1013 list_for_each_entry_safe(timer, tmp, &tv_list, entry) {
1027 BUG_ON(tbase_get_base(timer->base) != base); 1014 BUG_ON(tbase_get_base(timer->base) != base);
1028 internal_add_timer(base, timer); 1015 internal_add_timer(base, timer);
1029 } 1016 }
1030 1017
1031 return index; 1018 return index;
1032 } 1019 }
1033 1020
1034 static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long), 1021 static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long),
1035 unsigned long data) 1022 unsigned long data)
1036 { 1023 {
1037 int preempt_count = preempt_count(); 1024 int preempt_count = preempt_count();
1038 1025
1039 #ifdef CONFIG_LOCKDEP 1026 #ifdef CONFIG_LOCKDEP
1040 /* 1027 /*
1041 * It is permissible to free the timer from inside the 1028 * It is permissible to free the timer from inside the
1042 * function that is called from it, this we need to take into 1029 * function that is called from it, this we need to take into
1043 * account for lockdep too. To avoid bogus "held lock freed" 1030 * account for lockdep too. To avoid bogus "held lock freed"
1044 * warnings as well as problems when looking into 1031 * warnings as well as problems when looking into
1045 * timer->lockdep_map, make a copy and use that here. 1032 * timer->lockdep_map, make a copy and use that here.
1046 */ 1033 */
1047 struct lockdep_map lockdep_map = timer->lockdep_map; 1034 struct lockdep_map lockdep_map = timer->lockdep_map;
1048 #endif 1035 #endif
1049 /* 1036 /*
1050 * Couple the lock chain with the lock chain at 1037 * Couple the lock chain with the lock chain at
1051 * del_timer_sync() by acquiring the lock_map around the fn() 1038 * del_timer_sync() by acquiring the lock_map around the fn()
1052 * call here and in del_timer_sync(). 1039 * call here and in del_timer_sync().
1053 */ 1040 */
1054 lock_map_acquire(&lockdep_map); 1041 lock_map_acquire(&lockdep_map);
1055 1042
1056 trace_timer_expire_entry(timer); 1043 trace_timer_expire_entry(timer);
1057 fn(data); 1044 fn(data);
1058 trace_timer_expire_exit(timer); 1045 trace_timer_expire_exit(timer);
1059 1046
1060 lock_map_release(&lockdep_map); 1047 lock_map_release(&lockdep_map);
1061 1048
1062 if (preempt_count != preempt_count()) { 1049 if (preempt_count != preempt_count()) {
1063 WARN_ONCE(1, "timer: %pF preempt leak: %08x -> %08x\n", 1050 WARN_ONCE(1, "timer: %pF preempt leak: %08x -> %08x\n",
1064 fn, preempt_count, preempt_count()); 1051 fn, preempt_count, preempt_count());
1065 /* 1052 /*
1066 * Restore the preempt count. That gives us a decent 1053 * Restore the preempt count. That gives us a decent
1067 * chance to survive and extract information. If the 1054 * chance to survive and extract information. If the
1068 * callback kept a lock held, bad luck, but not worse 1055 * callback kept a lock held, bad luck, but not worse
1069 * than the BUG() we had. 1056 * than the BUG() we had.
1070 */ 1057 */
1071 preempt_count() = preempt_count; 1058 preempt_count() = preempt_count;
1072 } 1059 }
1073 } 1060 }
1074 1061
1075 #define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK) 1062 #define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
1076 1063
1077 /** 1064 /**
1078 * __run_timers - run all expired timers (if any) on this CPU. 1065 * __run_timers - run all expired timers (if any) on this CPU.
1079 * @base: the timer vector to be processed. 1066 * @base: the timer vector to be processed.
1080 * 1067 *
1081 * This function cascades all vectors and executes all expired timer 1068 * This function cascades all vectors and executes all expired timer
1082 * vectors. 1069 * vectors.
1083 */ 1070 */
1084 static inline void __run_timers(struct tvec_base *base) 1071 static inline void __run_timers(struct tvec_base *base)
1085 { 1072 {
1086 struct timer_list *timer; 1073 struct timer_list *timer;
1087 1074
1088 spin_lock_irq(&base->lock); 1075 spin_lock_irq(&base->lock);
1089 while (time_after_eq(jiffies, base->timer_jiffies)) { 1076 while (time_after_eq(jiffies, base->timer_jiffies)) {
1090 struct list_head work_list; 1077 struct list_head work_list;
1091 struct list_head *head = &work_list; 1078 struct list_head *head = &work_list;
1092 int index = base->timer_jiffies & TVR_MASK; 1079 int index = base->timer_jiffies & TVR_MASK;
1093 1080
1094 /* 1081 /*
1095 * Cascade timers: 1082 * Cascade timers:
1096 */ 1083 */
1097 if (!index && 1084 if (!index &&
1098 (!cascade(base, &base->tv2, INDEX(0))) && 1085 (!cascade(base, &base->tv2, INDEX(0))) &&
1099 (!cascade(base, &base->tv3, INDEX(1))) && 1086 (!cascade(base, &base->tv3, INDEX(1))) &&
1100 !cascade(base, &base->tv4, INDEX(2))) 1087 !cascade(base, &base->tv4, INDEX(2)))
1101 cascade(base, &base->tv5, INDEX(3)); 1088 cascade(base, &base->tv5, INDEX(3));
1102 ++base->timer_jiffies; 1089 ++base->timer_jiffies;
1103 list_replace_init(base->tv1.vec + index, &work_list); 1090 list_replace_init(base->tv1.vec + index, &work_list);
1104 while (!list_empty(head)) { 1091 while (!list_empty(head)) {
1105 void (*fn)(unsigned long); 1092 void (*fn)(unsigned long);
1106 unsigned long data; 1093 unsigned long data;
1107 1094
1108 timer = list_first_entry(head, struct timer_list,entry); 1095 timer = list_first_entry(head, struct timer_list,entry);
1109 fn = timer->function; 1096 fn = timer->function;
1110 data = timer->data; 1097 data = timer->data;
1111 1098
1112 timer_stats_account_timer(timer); 1099 timer_stats_account_timer(timer);
1113 1100
1114 set_running_timer(base, timer); 1101 set_running_timer(base, timer);
1115 detach_timer(timer, 1); 1102 detach_timer(timer, 1);
1116 1103
1117 spin_unlock_irq(&base->lock); 1104 spin_unlock_irq(&base->lock);
1118 call_timer_fn(timer, fn, data); 1105 call_timer_fn(timer, fn, data);
1119 spin_lock_irq(&base->lock); 1106 spin_lock_irq(&base->lock);
1120 } 1107 }
1121 } 1108 }
1122 set_running_timer(base, NULL); 1109 set_running_timer(base, NULL);
1123 spin_unlock_irq(&base->lock); 1110 spin_unlock_irq(&base->lock);
1124 } 1111 }
1125 1112
1126 #ifdef CONFIG_NO_HZ 1113 #ifdef CONFIG_NO_HZ
1127 /* 1114 /*
1128 * Find out when the next timer event is due to happen. This 1115 * Find out when the next timer event is due to happen. This
1129 * is used on S/390 to stop all activity when a CPU is idle. 1116 * is used on S/390 to stop all activity when a CPU is idle.
1130 * This function needs to be called with interrupts disabled. 1117 * This function needs to be called with interrupts disabled.
1131 */ 1118 */
1132 static unsigned long __next_timer_interrupt(struct tvec_base *base) 1119 static unsigned long __next_timer_interrupt(struct tvec_base *base)
1133 { 1120 {
1134 unsigned long timer_jiffies = base->timer_jiffies; 1121 unsigned long timer_jiffies = base->timer_jiffies;
1135 unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA; 1122 unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA;
1136 int index, slot, array, found = 0; 1123 int index, slot, array, found = 0;
1137 struct timer_list *nte; 1124 struct timer_list *nte;
1138 struct tvec *varray[4]; 1125 struct tvec *varray[4];
1139 1126
1140 /* Look for timer events in tv1. */ 1127 /* Look for timer events in tv1. */
1141 index = slot = timer_jiffies & TVR_MASK; 1128 index = slot = timer_jiffies & TVR_MASK;
1142 do { 1129 do {
1143 list_for_each_entry(nte, base->tv1.vec + slot, entry) { 1130 list_for_each_entry(nte, base->tv1.vec + slot, entry) {
1144 if (tbase_get_deferrable(nte->base)) 1131 if (tbase_get_deferrable(nte->base))
1145 continue; 1132 continue;
1146 1133
1147 found = 1; 1134 found = 1;
1148 expires = nte->expires; 1135 expires = nte->expires;
1149 /* Look at the cascade bucket(s)? */ 1136 /* Look at the cascade bucket(s)? */
1150 if (!index || slot < index) 1137 if (!index || slot < index)
1151 goto cascade; 1138 goto cascade;
1152 return expires; 1139 return expires;
1153 } 1140 }
1154 slot = (slot + 1) & TVR_MASK; 1141 slot = (slot + 1) & TVR_MASK;
1155 } while (slot != index); 1142 } while (slot != index);
1156 1143
1157 cascade: 1144 cascade:
1158 /* Calculate the next cascade event */ 1145 /* Calculate the next cascade event */
1159 if (index) 1146 if (index)
1160 timer_jiffies += TVR_SIZE - index; 1147 timer_jiffies += TVR_SIZE - index;
1161 timer_jiffies >>= TVR_BITS; 1148 timer_jiffies >>= TVR_BITS;
1162 1149
1163 /* Check tv2-tv5. */ 1150 /* Check tv2-tv5. */
1164 varray[0] = &base->tv2; 1151 varray[0] = &base->tv2;
1165 varray[1] = &base->tv3; 1152 varray[1] = &base->tv3;
1166 varray[2] = &base->tv4; 1153 varray[2] = &base->tv4;
1167 varray[3] = &base->tv5; 1154 varray[3] = &base->tv5;
1168 1155
1169 for (array = 0; array < 4; array++) { 1156 for (array = 0; array < 4; array++) {
1170 struct tvec *varp = varray[array]; 1157 struct tvec *varp = varray[array];
1171 1158
1172 index = slot = timer_jiffies & TVN_MASK; 1159 index = slot = timer_jiffies & TVN_MASK;
1173 do { 1160 do {
1174 list_for_each_entry(nte, varp->vec + slot, entry) { 1161 list_for_each_entry(nte, varp->vec + slot, entry) {
1175 if (tbase_get_deferrable(nte->base)) 1162 if (tbase_get_deferrable(nte->base))
1176 continue; 1163 continue;
1177 1164
1178 found = 1; 1165 found = 1;
1179 if (time_before(nte->expires, expires)) 1166 if (time_before(nte->expires, expires))
1180 expires = nte->expires; 1167 expires = nte->expires;
1181 } 1168 }
1182 /* 1169 /*
1183 * Do we still search for the first timer or are 1170 * Do we still search for the first timer or are
1184 * we looking up the cascade buckets ? 1171 * we looking up the cascade buckets ?
1185 */ 1172 */
1186 if (found) { 1173 if (found) {
1187 /* Look at the cascade bucket(s)? */ 1174 /* Look at the cascade bucket(s)? */
1188 if (!index || slot < index) 1175 if (!index || slot < index)
1189 break; 1176 break;
1190 return expires; 1177 return expires;
1191 } 1178 }
1192 slot = (slot + 1) & TVN_MASK; 1179 slot = (slot + 1) & TVN_MASK;
1193 } while (slot != index); 1180 } while (slot != index);
1194 1181
1195 if (index) 1182 if (index)
1196 timer_jiffies += TVN_SIZE - index; 1183 timer_jiffies += TVN_SIZE - index;
1197 timer_jiffies >>= TVN_BITS; 1184 timer_jiffies >>= TVN_BITS;
1198 } 1185 }
1199 return expires; 1186 return expires;
1200 } 1187 }
1201 1188
1202 /* 1189 /*
1203 * Check, if the next hrtimer event is before the next timer wheel 1190 * Check, if the next hrtimer event is before the next timer wheel
1204 * event: 1191 * event:
1205 */ 1192 */
1206 static unsigned long cmp_next_hrtimer_event(unsigned long now, 1193 static unsigned long cmp_next_hrtimer_event(unsigned long now,
1207 unsigned long expires) 1194 unsigned long expires)
1208 { 1195 {
1209 ktime_t hr_delta = hrtimer_get_next_event(); 1196 ktime_t hr_delta = hrtimer_get_next_event();
1210 struct timespec tsdelta; 1197 struct timespec tsdelta;
1211 unsigned long delta; 1198 unsigned long delta;
1212 1199
1213 if (hr_delta.tv64 == KTIME_MAX) 1200 if (hr_delta.tv64 == KTIME_MAX)
1214 return expires; 1201 return expires;
1215 1202
1216 /* 1203 /*
1217 * Expired timer available, let it expire in the next tick 1204 * Expired timer available, let it expire in the next tick
1218 */ 1205 */
1219 if (hr_delta.tv64 <= 0) 1206 if (hr_delta.tv64 <= 0)
1220 return now + 1; 1207 return now + 1;
1221 1208
1222 tsdelta = ktime_to_timespec(hr_delta); 1209 tsdelta = ktime_to_timespec(hr_delta);
1223 delta = timespec_to_jiffies(&tsdelta); 1210 delta = timespec_to_jiffies(&tsdelta);
1224 1211
1225 /* 1212 /*
1226 * Limit the delta to the max value, which is checked in 1213 * Limit the delta to the max value, which is checked in
1227 * tick_nohz_stop_sched_tick(): 1214 * tick_nohz_stop_sched_tick():
1228 */ 1215 */
1229 if (delta > NEXT_TIMER_MAX_DELTA) 1216 if (delta > NEXT_TIMER_MAX_DELTA)
1230 delta = NEXT_TIMER_MAX_DELTA; 1217 delta = NEXT_TIMER_MAX_DELTA;
1231 1218
1232 /* 1219 /*
1233 * Take rounding errors in to account and make sure, that it 1220 * Take rounding errors in to account and make sure, that it
1234 * expires in the next tick. Otherwise we go into an endless 1221 * expires in the next tick. Otherwise we go into an endless
1235 * ping pong due to tick_nohz_stop_sched_tick() retriggering 1222 * ping pong due to tick_nohz_stop_sched_tick() retriggering
1236 * the timer softirq 1223 * the timer softirq
1237 */ 1224 */
1238 if (delta < 1) 1225 if (delta < 1)
1239 delta = 1; 1226 delta = 1;
1240 now += delta; 1227 now += delta;
1241 if (time_before(now, expires)) 1228 if (time_before(now, expires))
1242 return now; 1229 return now;
1243 return expires; 1230 return expires;
1244 } 1231 }
1245 1232
1246 /** 1233 /**
1247 * get_next_timer_interrupt - return the jiffy of the next pending timer 1234 * get_next_timer_interrupt - return the jiffy of the next pending timer
1248 * @now: current time (in jiffies) 1235 * @now: current time (in jiffies)
1249 */ 1236 */
1250 unsigned long get_next_timer_interrupt(unsigned long now) 1237 unsigned long get_next_timer_interrupt(unsigned long now)
1251 { 1238 {
1252 struct tvec_base *base = __get_cpu_var(tvec_bases); 1239 struct tvec_base *base = __get_cpu_var(tvec_bases);
1253 unsigned long expires; 1240 unsigned long expires;
1254 1241
1255 spin_lock(&base->lock); 1242 spin_lock(&base->lock);
1256 if (time_before_eq(base->next_timer, base->timer_jiffies)) 1243 if (time_before_eq(base->next_timer, base->timer_jiffies))
1257 base->next_timer = __next_timer_interrupt(base); 1244 base->next_timer = __next_timer_interrupt(base);
1258 expires = base->next_timer; 1245 expires = base->next_timer;
1259 spin_unlock(&base->lock); 1246 spin_unlock(&base->lock);
1260 1247
1261 if (time_before_eq(expires, now)) 1248 if (time_before_eq(expires, now))
1262 return now; 1249 return now;
1263 1250
1264 return cmp_next_hrtimer_event(now, expires); 1251 return cmp_next_hrtimer_event(now, expires);
1265 } 1252 }
1266 #endif 1253 #endif
1267 1254
1268 /* 1255 /*
1269 * Called from the timer interrupt handler to charge one tick to the current 1256 * Called from the timer interrupt handler to charge one tick to the current
1270 * process. user_tick is 1 if the tick is user time, 0 for system. 1257 * process. user_tick is 1 if the tick is user time, 0 for system.
1271 */ 1258 */
1272 void update_process_times(int user_tick) 1259 void update_process_times(int user_tick)
1273 { 1260 {
1274 struct task_struct *p = current; 1261 struct task_struct *p = current;
1275 int cpu = smp_processor_id(); 1262 int cpu = smp_processor_id();
1276 1263
1277 /* Note: this timer irq context must be accounted for as well. */ 1264 /* Note: this timer irq context must be accounted for as well. */
1278 account_process_tick(p, user_tick); 1265 account_process_tick(p, user_tick);
1279 run_local_timers(); 1266 run_local_timers();
1280 rcu_check_callbacks(cpu, user_tick); 1267 rcu_check_callbacks(cpu, user_tick);
1281 printk_tick(); 1268 printk_tick();
1282 perf_event_do_pending(); 1269 perf_event_do_pending();
1283 scheduler_tick(); 1270 scheduler_tick();
1284 run_posix_cpu_timers(p); 1271 run_posix_cpu_timers(p);
1285 } 1272 }
1286 1273
1287 /* 1274 /*
1288 * This function runs timers and the timer-tq in bottom half context. 1275 * This function runs timers and the timer-tq in bottom half context.
1289 */ 1276 */
1290 static void run_timer_softirq(struct softirq_action *h) 1277 static void run_timer_softirq(struct softirq_action *h)
1291 { 1278 {
1292 struct tvec_base *base = __get_cpu_var(tvec_bases); 1279 struct tvec_base *base = __get_cpu_var(tvec_bases);
1293 1280
1294 hrtimer_run_pending(); 1281 hrtimer_run_pending();
1295 1282
1296 if (time_after_eq(jiffies, base->timer_jiffies)) 1283 if (time_after_eq(jiffies, base->timer_jiffies))
1297 __run_timers(base); 1284 __run_timers(base);
1298 } 1285 }
1299 1286
1300 /* 1287 /*
1301 * Called by the local, per-CPU timer interrupt on SMP. 1288 * Called by the local, per-CPU timer interrupt on SMP.
1302 */ 1289 */
1303 void run_local_timers(void) 1290 void run_local_timers(void)
1304 { 1291 {
1305 hrtimer_run_queues(); 1292 hrtimer_run_queues();
1306 raise_softirq(TIMER_SOFTIRQ); 1293 raise_softirq(TIMER_SOFTIRQ);
1307 } 1294 }
1308 1295
1309 /* 1296 /*
1310 * The 64-bit jiffies value is not atomic - you MUST NOT read it 1297 * The 64-bit jiffies value is not atomic - you MUST NOT read it
1311 * without sampling the sequence number in xtime_lock. 1298 * without sampling the sequence number in xtime_lock.
1312 * jiffies is defined in the linker script... 1299 * jiffies is defined in the linker script...
1313 */ 1300 */
1314 1301
1315 void do_timer(unsigned long ticks) 1302 void do_timer(unsigned long ticks)
1316 { 1303 {
1317 jiffies_64 += ticks; 1304 jiffies_64 += ticks;
1318 update_wall_time(); 1305 update_wall_time();
1319 calc_global_load(); 1306 calc_global_load();
1320 } 1307 }
1321 1308
1322 #ifdef __ARCH_WANT_SYS_ALARM 1309 #ifdef __ARCH_WANT_SYS_ALARM
1323 1310
1324 /* 1311 /*
1325 * For backwards compatibility? This can be done in libc so Alpha 1312 * For backwards compatibility? This can be done in libc so Alpha
1326 * and all newer ports shouldn't need it. 1313 * and all newer ports shouldn't need it.
1327 */ 1314 */
1328 SYSCALL_DEFINE1(alarm, unsigned int, seconds) 1315 SYSCALL_DEFINE1(alarm, unsigned int, seconds)
1329 { 1316 {
1330 return alarm_setitimer(seconds); 1317 return alarm_setitimer(seconds);
1331 } 1318 }
1332 1319
1333 #endif 1320 #endif
1334 1321
1335 #ifndef __alpha__ 1322 #ifndef __alpha__
1336 1323
1337 /* 1324 /*
1338 * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this 1325 * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this
1339 * should be moved into arch/i386 instead? 1326 * should be moved into arch/i386 instead?
1340 */ 1327 */
1341 1328
1342 /** 1329 /**
1343 * sys_getpid - return the thread group id of the current process 1330 * sys_getpid - return the thread group id of the current process
1344 * 1331 *
1345 * Note, despite the name, this returns the tgid not the pid. The tgid and 1332 * Note, despite the name, this returns the tgid not the pid. The tgid and
1346 * the pid are identical unless CLONE_THREAD was specified on clone() in 1333 * the pid are identical unless CLONE_THREAD was specified on clone() in
1347 * which case the tgid is the same in all threads of the same group. 1334 * which case the tgid is the same in all threads of the same group.
1348 * 1335 *
1349 * This is SMP safe as current->tgid does not change. 1336 * This is SMP safe as current->tgid does not change.
1350 */ 1337 */
1351 SYSCALL_DEFINE0(getpid) 1338 SYSCALL_DEFINE0(getpid)
1352 { 1339 {
1353 return task_tgid_vnr(current); 1340 return task_tgid_vnr(current);
1354 } 1341 }
1355 1342
1356 /* 1343 /*
1357 * Accessing ->real_parent is not SMP-safe, it could 1344 * Accessing ->real_parent is not SMP-safe, it could
1358 * change from under us. However, we can use a stale 1345 * change from under us. However, we can use a stale
1359 * value of ->real_parent under rcu_read_lock(), see 1346 * value of ->real_parent under rcu_read_lock(), see
1360 * release_task()->call_rcu(delayed_put_task_struct). 1347 * release_task()->call_rcu(delayed_put_task_struct).
1361 */ 1348 */
1362 SYSCALL_DEFINE0(getppid) 1349 SYSCALL_DEFINE0(getppid)
1363 { 1350 {
1364 int pid; 1351 int pid;
1365 1352
1366 rcu_read_lock(); 1353 rcu_read_lock();
1367 pid = task_tgid_vnr(current->real_parent); 1354 pid = task_tgid_vnr(current->real_parent);
1368 rcu_read_unlock(); 1355 rcu_read_unlock();
1369 1356
1370 return pid; 1357 return pid;
1371 } 1358 }
1372 1359
1373 SYSCALL_DEFINE0(getuid) 1360 SYSCALL_DEFINE0(getuid)
1374 { 1361 {
1375 /* Only we change this so SMP safe */ 1362 /* Only we change this so SMP safe */
1376 return current_uid(); 1363 return current_uid();
1377 } 1364 }
1378 1365
1379 SYSCALL_DEFINE0(geteuid) 1366 SYSCALL_DEFINE0(geteuid)
1380 { 1367 {
1381 /* Only we change this so SMP safe */ 1368 /* Only we change this so SMP safe */
1382 return current_euid(); 1369 return current_euid();
1383 } 1370 }
1384 1371
1385 SYSCALL_DEFINE0(getgid) 1372 SYSCALL_DEFINE0(getgid)
1386 { 1373 {
1387 /* Only we change this so SMP safe */ 1374 /* Only we change this so SMP safe */
1388 return current_gid(); 1375 return current_gid();
1389 } 1376 }
1390 1377
1391 SYSCALL_DEFINE0(getegid) 1378 SYSCALL_DEFINE0(getegid)
1392 { 1379 {
1393 /* Only we change this so SMP safe */ 1380 /* Only we change this so SMP safe */
1394 return current_egid(); 1381 return current_egid();
1395 } 1382 }
1396 1383
1397 #endif 1384 #endif
1398 1385
1399 static void process_timeout(unsigned long __data) 1386 static void process_timeout(unsigned long __data)
1400 { 1387 {
1401 wake_up_process((struct task_struct *)__data); 1388 wake_up_process((struct task_struct *)__data);
1402 } 1389 }
1403 1390
1404 /** 1391 /**
1405 * schedule_timeout - sleep until timeout 1392 * schedule_timeout - sleep until timeout
1406 * @timeout: timeout value in jiffies 1393 * @timeout: timeout value in jiffies
1407 * 1394 *
1408 * Make the current task sleep until @timeout jiffies have 1395 * Make the current task sleep until @timeout jiffies have
1409 * elapsed. The routine will return immediately unless 1396 * elapsed. The routine will return immediately unless
1410 * the current task state has been set (see set_current_state()). 1397 * the current task state has been set (see set_current_state()).
1411 * 1398 *
1412 * You can set the task state as follows - 1399 * You can set the task state as follows -
1413 * 1400 *
1414 * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to 1401 * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
1415 * pass before the routine returns. The routine will return 0 1402 * pass before the routine returns. The routine will return 0
1416 * 1403 *
1417 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is 1404 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1418 * delivered to the current task. In this case the remaining time 1405 * delivered to the current task. In this case the remaining time
1419 * in jiffies will be returned, or 0 if the timer expired in time 1406 * in jiffies will be returned, or 0 if the timer expired in time
1420 * 1407 *
1421 * The current task state is guaranteed to be TASK_RUNNING when this 1408 * The current task state is guaranteed to be TASK_RUNNING when this
1422 * routine returns. 1409 * routine returns.
1423 * 1410 *
1424 * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule 1411 * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
1425 * the CPU away without a bound on the timeout. In this case the return 1412 * the CPU away without a bound on the timeout. In this case the return
1426 * value will be %MAX_SCHEDULE_TIMEOUT. 1413 * value will be %MAX_SCHEDULE_TIMEOUT.
1427 * 1414 *
1428 * In all cases the return value is guaranteed to be non-negative. 1415 * In all cases the return value is guaranteed to be non-negative.
1429 */ 1416 */
1430 signed long __sched schedule_timeout(signed long timeout) 1417 signed long __sched schedule_timeout(signed long timeout)
1431 { 1418 {
1432 struct timer_list timer; 1419 struct timer_list timer;
1433 unsigned long expire; 1420 unsigned long expire;
1434 1421
1435 switch (timeout) 1422 switch (timeout)
1436 { 1423 {
1437 case MAX_SCHEDULE_TIMEOUT: 1424 case MAX_SCHEDULE_TIMEOUT:
1438 /* 1425 /*
1439 * These two special cases are useful to be comfortable 1426 * These two special cases are useful to be comfortable
1440 * in the caller. Nothing more. We could take 1427 * in the caller. Nothing more. We could take
1441 * MAX_SCHEDULE_TIMEOUT from one of the negative value 1428 * MAX_SCHEDULE_TIMEOUT from one of the negative value
1442 * but I' d like to return a valid offset (>=0) to allow 1429 * but I' d like to return a valid offset (>=0) to allow
1443 * the caller to do everything it want with the retval. 1430 * the caller to do everything it want with the retval.
1444 */ 1431 */
1445 schedule(); 1432 schedule();
1446 goto out; 1433 goto out;
1447 default: 1434 default:
1448 /* 1435 /*
1449 * Another bit of PARANOID. Note that the retval will be 1436 * Another bit of PARANOID. Note that the retval will be
1450 * 0 since no piece of kernel is supposed to do a check 1437 * 0 since no piece of kernel is supposed to do a check
1451 * for a negative retval of schedule_timeout() (since it 1438 * for a negative retval of schedule_timeout() (since it
1452 * should never happens anyway). You just have the printk() 1439 * should never happens anyway). You just have the printk()
1453 * that will tell you if something is gone wrong and where. 1440 * that will tell you if something is gone wrong and where.
1454 */ 1441 */
1455 if (timeout < 0) { 1442 if (timeout < 0) {
1456 printk(KERN_ERR "schedule_timeout: wrong timeout " 1443 printk(KERN_ERR "schedule_timeout: wrong timeout "
1457 "value %lx\n", timeout); 1444 "value %lx\n", timeout);
1458 dump_stack(); 1445 dump_stack();
1459 current->state = TASK_RUNNING; 1446 current->state = TASK_RUNNING;
1460 goto out; 1447 goto out;
1461 } 1448 }
1462 } 1449 }
1463 1450
1464 expire = timeout + jiffies; 1451 expire = timeout + jiffies;
1465 1452
1466 setup_timer_on_stack(&timer, process_timeout, (unsigned long)current); 1453 setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);
1467 __mod_timer(&timer, expire, false, TIMER_NOT_PINNED); 1454 __mod_timer(&timer, expire, false, TIMER_NOT_PINNED);
1468 schedule(); 1455 schedule();
1469 del_singleshot_timer_sync(&timer); 1456 del_singleshot_timer_sync(&timer);
1470 1457
1471 /* Remove the timer from the object tracker */ 1458 /* Remove the timer from the object tracker */
1472 destroy_timer_on_stack(&timer); 1459 destroy_timer_on_stack(&timer);
1473 1460
1474 timeout = expire - jiffies; 1461 timeout = expire - jiffies;
1475 1462
1476 out: 1463 out:
1477 return timeout < 0 ? 0 : timeout; 1464 return timeout < 0 ? 0 : timeout;
1478 } 1465 }
1479 EXPORT_SYMBOL(schedule_timeout); 1466 EXPORT_SYMBOL(schedule_timeout);
1480 1467
1481 /* 1468 /*
1482 * We can use __set_current_state() here because schedule_timeout() calls 1469 * We can use __set_current_state() here because schedule_timeout() calls
1483 * schedule() unconditionally. 1470 * schedule() unconditionally.
1484 */ 1471 */
1485 signed long __sched schedule_timeout_interruptible(signed long timeout) 1472 signed long __sched schedule_timeout_interruptible(signed long timeout)
1486 { 1473 {
1487 __set_current_state(TASK_INTERRUPTIBLE); 1474 __set_current_state(TASK_INTERRUPTIBLE);
1488 return schedule_timeout(timeout); 1475 return schedule_timeout(timeout);
1489 } 1476 }
1490 EXPORT_SYMBOL(schedule_timeout_interruptible); 1477 EXPORT_SYMBOL(schedule_timeout_interruptible);
1491 1478
1492 signed long __sched schedule_timeout_killable(signed long timeout) 1479 signed long __sched schedule_timeout_killable(signed long timeout)
1493 { 1480 {
1494 __set_current_state(TASK_KILLABLE); 1481 __set_current_state(TASK_KILLABLE);
1495 return schedule_timeout(timeout); 1482 return schedule_timeout(timeout);
1496 } 1483 }
1497 EXPORT_SYMBOL(schedule_timeout_killable); 1484 EXPORT_SYMBOL(schedule_timeout_killable);
1498 1485
1499 signed long __sched schedule_timeout_uninterruptible(signed long timeout) 1486 signed long __sched schedule_timeout_uninterruptible(signed long timeout)
1500 { 1487 {
1501 __set_current_state(TASK_UNINTERRUPTIBLE); 1488 __set_current_state(TASK_UNINTERRUPTIBLE);
1502 return schedule_timeout(timeout); 1489 return schedule_timeout(timeout);
1503 } 1490 }
1504 EXPORT_SYMBOL(schedule_timeout_uninterruptible); 1491 EXPORT_SYMBOL(schedule_timeout_uninterruptible);
1505 1492
1506 /* Thread ID - the internal kernel "pid" */ 1493 /* Thread ID - the internal kernel "pid" */
1507 SYSCALL_DEFINE0(gettid) 1494 SYSCALL_DEFINE0(gettid)
1508 { 1495 {
1509 return task_pid_vnr(current); 1496 return task_pid_vnr(current);
1510 } 1497 }
1511 1498
1512 /** 1499 /**
1513 * do_sysinfo - fill in sysinfo struct 1500 * do_sysinfo - fill in sysinfo struct
1514 * @info: pointer to buffer to fill 1501 * @info: pointer to buffer to fill
1515 */ 1502 */
1516 int do_sysinfo(struct sysinfo *info) 1503 int do_sysinfo(struct sysinfo *info)
1517 { 1504 {
1518 unsigned long mem_total, sav_total; 1505 unsigned long mem_total, sav_total;
1519 unsigned int mem_unit, bitcount; 1506 unsigned int mem_unit, bitcount;
1520 struct timespec tp; 1507 struct timespec tp;
1521 1508
1522 memset(info, 0, sizeof(struct sysinfo)); 1509 memset(info, 0, sizeof(struct sysinfo));
1523 1510
1524 ktime_get_ts(&tp); 1511 ktime_get_ts(&tp);
1525 monotonic_to_bootbased(&tp); 1512 monotonic_to_bootbased(&tp);
1526 info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0); 1513 info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
1527 1514
1528 get_avenrun(info->loads, 0, SI_LOAD_SHIFT - FSHIFT); 1515 get_avenrun(info->loads, 0, SI_LOAD_SHIFT - FSHIFT);
1529 1516
1530 info->procs = nr_threads; 1517 info->procs = nr_threads;
1531 1518
1532 si_meminfo(info); 1519 si_meminfo(info);
1533 si_swapinfo(info); 1520 si_swapinfo(info);
1534 1521
1535 /* 1522 /*
1536 * If the sum of all the available memory (i.e. ram + swap) 1523 * If the sum of all the available memory (i.e. ram + swap)
1537 * is less than can be stored in a 32 bit unsigned long then 1524 * is less than can be stored in a 32 bit unsigned long then
1538 * we can be binary compatible with 2.2.x kernels. If not, 1525 * we can be binary compatible with 2.2.x kernels. If not,
1539 * well, in that case 2.2.x was broken anyways... 1526 * well, in that case 2.2.x was broken anyways...
1540 * 1527 *
1541 * -Erik Andersen <andersee@debian.org> 1528 * -Erik Andersen <andersee@debian.org>
1542 */ 1529 */
1543 1530
1544 mem_total = info->totalram + info->totalswap; 1531 mem_total = info->totalram + info->totalswap;
1545 if (mem_total < info->totalram || mem_total < info->totalswap) 1532 if (mem_total < info->totalram || mem_total < info->totalswap)
1546 goto out; 1533 goto out;
1547 bitcount = 0; 1534 bitcount = 0;
1548 mem_unit = info->mem_unit; 1535 mem_unit = info->mem_unit;
1549 while (mem_unit > 1) { 1536 while (mem_unit > 1) {
1550 bitcount++; 1537 bitcount++;
1551 mem_unit >>= 1; 1538 mem_unit >>= 1;
1552 sav_total = mem_total; 1539 sav_total = mem_total;
1553 mem_total <<= 1; 1540 mem_total <<= 1;
1554 if (mem_total < sav_total) 1541 if (mem_total < sav_total)
1555 goto out; 1542 goto out;
1556 } 1543 }
1557 1544
1558 /* 1545 /*
1559 * If mem_total did not overflow, multiply all memory values by 1546 * If mem_total did not overflow, multiply all memory values by
1560 * info->mem_unit and set it to 1. This leaves things compatible 1547 * info->mem_unit and set it to 1. This leaves things compatible
1561 * with 2.2.x, and also retains compatibility with earlier 2.4.x 1548 * with 2.2.x, and also retains compatibility with earlier 2.4.x
1562 * kernels... 1549 * kernels...
1563 */ 1550 */
1564 1551
1565 info->mem_unit = 1; 1552 info->mem_unit = 1;
1566 info->totalram <<= bitcount; 1553 info->totalram <<= bitcount;
1567 info->freeram <<= bitcount; 1554 info->freeram <<= bitcount;
1568 info->sharedram <<= bitcount; 1555 info->sharedram <<= bitcount;
1569 info->bufferram <<= bitcount; 1556 info->bufferram <<= bitcount;
1570 info->totalswap <<= bitcount; 1557 info->totalswap <<= bitcount;
1571 info->freeswap <<= bitcount; 1558 info->freeswap <<= bitcount;
1572 info->totalhigh <<= bitcount; 1559 info->totalhigh <<= bitcount;
1573 info->freehigh <<= bitcount; 1560 info->freehigh <<= bitcount;
1574 1561
1575 out: 1562 out:
1576 return 0; 1563 return 0;
1577 } 1564 }
1578 1565
1579 SYSCALL_DEFINE1(sysinfo, struct sysinfo __user *, info) 1566 SYSCALL_DEFINE1(sysinfo, struct sysinfo __user *, info)
1580 { 1567 {
1581 struct sysinfo val; 1568 struct sysinfo val;
1582 1569
1583 do_sysinfo(&val); 1570 do_sysinfo(&val);
1584 1571
1585 if (copy_to_user(info, &val, sizeof(struct sysinfo))) 1572 if (copy_to_user(info, &val, sizeof(struct sysinfo)))
1586 return -EFAULT; 1573 return -EFAULT;
1587 1574
1588 return 0; 1575 return 0;
1589 } 1576 }
1590 1577
1591 static int __cpuinit init_timers_cpu(int cpu) 1578 static int __cpuinit init_timers_cpu(int cpu)
1592 { 1579 {
1593 int j; 1580 int j;
1594 struct tvec_base *base; 1581 struct tvec_base *base;
1595 static char __cpuinitdata tvec_base_done[NR_CPUS]; 1582 static char __cpuinitdata tvec_base_done[NR_CPUS];
1596 1583
1597 if (!tvec_base_done[cpu]) { 1584 if (!tvec_base_done[cpu]) {
1598 static char boot_done; 1585 static char boot_done;
1599 1586
1600 if (boot_done) { 1587 if (boot_done) {
1601 /* 1588 /*
1602 * The APs use this path later in boot 1589 * The APs use this path later in boot
1603 */ 1590 */
1604 base = kmalloc_node(sizeof(*base), 1591 base = kmalloc_node(sizeof(*base),
1605 GFP_KERNEL | __GFP_ZERO, 1592 GFP_KERNEL | __GFP_ZERO,
1606 cpu_to_node(cpu)); 1593 cpu_to_node(cpu));
1607 if (!base) 1594 if (!base)
1608 return -ENOMEM; 1595 return -ENOMEM;
1609 1596
1610 /* Make sure that tvec_base is 2 byte aligned */ 1597 /* Make sure that tvec_base is 2 byte aligned */
1611 if (tbase_get_deferrable(base)) { 1598 if (tbase_get_deferrable(base)) {
1612 WARN_ON(1); 1599 WARN_ON(1);
1613 kfree(base); 1600 kfree(base);
1614 return -ENOMEM; 1601 return -ENOMEM;
1615 } 1602 }
1616 per_cpu(tvec_bases, cpu) = base; 1603 per_cpu(tvec_bases, cpu) = base;
1617 } else { 1604 } else {
1618 /* 1605 /*
1619 * This is for the boot CPU - we use compile-time 1606 * This is for the boot CPU - we use compile-time
1620 * static initialisation because per-cpu memory isn't 1607 * static initialisation because per-cpu memory isn't
1621 * ready yet and because the memory allocators are not 1608 * ready yet and because the memory allocators are not
1622 * initialised either. 1609 * initialised either.
1623 */ 1610 */
1624 boot_done = 1; 1611 boot_done = 1;
1625 base = &boot_tvec_bases; 1612 base = &boot_tvec_bases;
1626 } 1613 }
1627 tvec_base_done[cpu] = 1; 1614 tvec_base_done[cpu] = 1;
1628 } else { 1615 } else {
1629 base = per_cpu(tvec_bases, cpu); 1616 base = per_cpu(tvec_bases, cpu);
1630 } 1617 }
1631 1618
1632 spin_lock_init(&base->lock); 1619 spin_lock_init(&base->lock);
1633 1620
1634 for (j = 0; j < TVN_SIZE; j++) { 1621 for (j = 0; j < TVN_SIZE; j++) {
1635 INIT_LIST_HEAD(base->tv5.vec + j); 1622 INIT_LIST_HEAD(base->tv5.vec + j);
1636 INIT_LIST_HEAD(base->tv4.vec + j); 1623 INIT_LIST_HEAD(base->tv4.vec + j);
1637 INIT_LIST_HEAD(base->tv3.vec + j); 1624 INIT_LIST_HEAD(base->tv3.vec + j);
1638 INIT_LIST_HEAD(base->tv2.vec + j); 1625 INIT_LIST_HEAD(base->tv2.vec + j);
1639 } 1626 }
1640 for (j = 0; j < TVR_SIZE; j++) 1627 for (j = 0; j < TVR_SIZE; j++)
1641 INIT_LIST_HEAD(base->tv1.vec + j); 1628 INIT_LIST_HEAD(base->tv1.vec + j);
1642 1629
1643 base->timer_jiffies = jiffies; 1630 base->timer_jiffies = jiffies;
1644 base->next_timer = base->timer_jiffies; 1631 base->next_timer = base->timer_jiffies;
1645 return 0; 1632 return 0;
1646 } 1633 }
1647 1634
1648 #ifdef CONFIG_HOTPLUG_CPU 1635 #ifdef CONFIG_HOTPLUG_CPU
1649 static void migrate_timer_list(struct tvec_base *new_base, struct list_head *head) 1636 static void migrate_timer_list(struct tvec_base *new_base, struct list_head *head)
1650 { 1637 {
1651 struct timer_list *timer; 1638 struct timer_list *timer;
1652 1639
1653 while (!list_empty(head)) { 1640 while (!list_empty(head)) {
1654 timer = list_first_entry(head, struct timer_list, entry); 1641 timer = list_first_entry(head, struct timer_list, entry);
1655 detach_timer(timer, 0); 1642 detach_timer(timer, 0);
1656 timer_set_base(timer, new_base); 1643 timer_set_base(timer, new_base);
1657 if (time_before(timer->expires, new_base->next_timer) && 1644 if (time_before(timer->expires, new_base->next_timer) &&
1658 !tbase_get_deferrable(timer->base)) 1645 !tbase_get_deferrable(timer->base))
1659 new_base->next_timer = timer->expires; 1646 new_base->next_timer = timer->expires;
1660 internal_add_timer(new_base, timer); 1647 internal_add_timer(new_base, timer);
1661 } 1648 }
1662 } 1649 }
1663 1650
1664 static void __cpuinit migrate_timers(int cpu) 1651 static void __cpuinit migrate_timers(int cpu)
1665 { 1652 {
1666 struct tvec_base *old_base; 1653 struct tvec_base *old_base;
1667 struct tvec_base *new_base; 1654 struct tvec_base *new_base;
1668 int i; 1655 int i;
1669 1656
1670 BUG_ON(cpu_online(cpu)); 1657 BUG_ON(cpu_online(cpu));
1671 old_base = per_cpu(tvec_bases, cpu); 1658 old_base = per_cpu(tvec_bases, cpu);
1672 new_base = get_cpu_var(tvec_bases); 1659 new_base = get_cpu_var(tvec_bases);
1673 /* 1660 /*
1674 * The caller is globally serialized and nobody else 1661 * The caller is globally serialized and nobody else
1675 * takes two locks at once, deadlock is not possible. 1662 * takes two locks at once, deadlock is not possible.
1676 */ 1663 */
1677 spin_lock_irq(&new_base->lock); 1664 spin_lock_irq(&new_base->lock);
1678 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); 1665 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
1679 1666
1680 BUG_ON(old_base->running_timer); 1667 BUG_ON(old_base->running_timer);
1681 1668
1682 for (i = 0; i < TVR_SIZE; i++) 1669 for (i = 0; i < TVR_SIZE; i++)
1683 migrate_timer_list(new_base, old_base->tv1.vec + i); 1670 migrate_timer_list(new_base, old_base->tv1.vec + i);
1684 for (i = 0; i < TVN_SIZE; i++) { 1671 for (i = 0; i < TVN_SIZE; i++) {
1685 migrate_timer_list(new_base, old_base->tv2.vec + i); 1672 migrate_timer_list(new_base, old_base->tv2.vec + i);
1686 migrate_timer_list(new_base, old_base->tv3.vec + i); 1673 migrate_timer_list(new_base, old_base->tv3.vec + i);
1687 migrate_timer_list(new_base, old_base->tv4.vec + i); 1674 migrate_timer_list(new_base, old_base->tv4.vec + i);
1688 migrate_timer_list(new_base, old_base->tv5.vec + i); 1675 migrate_timer_list(new_base, old_base->tv5.vec + i);
1689 } 1676 }
1690 1677
1691 spin_unlock(&old_base->lock); 1678 spin_unlock(&old_base->lock);
1692 spin_unlock_irq(&new_base->lock); 1679 spin_unlock_irq(&new_base->lock);
1693 put_cpu_var(tvec_bases); 1680 put_cpu_var(tvec_bases);
1694 } 1681 }
1695 #endif /* CONFIG_HOTPLUG_CPU */ 1682 #endif /* CONFIG_HOTPLUG_CPU */
1696 1683
1697 static int __cpuinit timer_cpu_notify(struct notifier_block *self, 1684 static int __cpuinit timer_cpu_notify(struct notifier_block *self,
1698 unsigned long action, void *hcpu) 1685 unsigned long action, void *hcpu)
1699 { 1686 {
1700 long cpu = (long)hcpu; 1687 long cpu = (long)hcpu;
1701 int err; 1688 int err;
1702 1689
1703 switch(action) { 1690 switch(action) {
1704 case CPU_UP_PREPARE: 1691 case CPU_UP_PREPARE:
1705 case CPU_UP_PREPARE_FROZEN: 1692 case CPU_UP_PREPARE_FROZEN:
1706 err = init_timers_cpu(cpu); 1693 err = init_timers_cpu(cpu);
1707 if (err < 0) 1694 if (err < 0)
1708 return notifier_from_errno(err); 1695 return notifier_from_errno(err);
1709 break; 1696 break;
1710 #ifdef CONFIG_HOTPLUG_CPU 1697 #ifdef CONFIG_HOTPLUG_CPU
1711 case CPU_DEAD: 1698 case CPU_DEAD:
1712 case CPU_DEAD_FROZEN: 1699 case CPU_DEAD_FROZEN:
1713 migrate_timers(cpu); 1700 migrate_timers(cpu);
1714 break; 1701 break;
1715 #endif 1702 #endif
1716 default: 1703 default:
1717 break; 1704 break;
1718 } 1705 }
1719 return NOTIFY_OK; 1706 return NOTIFY_OK;
1720 } 1707 }
1721 1708
1722 static struct notifier_block __cpuinitdata timers_nb = { 1709 static struct notifier_block __cpuinitdata timers_nb = {
1723 .notifier_call = timer_cpu_notify, 1710 .notifier_call = timer_cpu_notify,
1724 }; 1711 };
1725 1712
1726 1713
1727 void __init init_timers(void) 1714 void __init init_timers(void)
1728 { 1715 {
1729 int err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE, 1716 int err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE,
1730 (void *)(long)smp_processor_id()); 1717 (void *)(long)smp_processor_id());
1731 1718
1732 init_timer_stats(); 1719 init_timer_stats();
1733 1720
1734 BUG_ON(err != NOTIFY_OK); 1721 BUG_ON(err != NOTIFY_OK);
1735 register_cpu_notifier(&timers_nb); 1722 register_cpu_notifier(&timers_nb);
1736 open_softirq(TIMER_SOFTIRQ, run_timer_softirq); 1723 open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
1737 } 1724 }
1738 1725
1739 /** 1726 /**
1740 * msleep - sleep safely even with waitqueue interruptions 1727 * msleep - sleep safely even with waitqueue interruptions
1741 * @msecs: Time in milliseconds to sleep for 1728 * @msecs: Time in milliseconds to sleep for
1742 */ 1729 */
1743 void msleep(unsigned int msecs) 1730 void msleep(unsigned int msecs)
1744 { 1731 {
1745 unsigned long timeout = msecs_to_jiffies(msecs) + 1; 1732 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1746 1733
1747 while (timeout) 1734 while (timeout)
1748 timeout = schedule_timeout_uninterruptible(timeout); 1735 timeout = schedule_timeout_uninterruptible(timeout);
1749 } 1736 }
1750 1737
1751 EXPORT_SYMBOL(msleep); 1738 EXPORT_SYMBOL(msleep);
1752 1739
1753 /** 1740 /**
1754 * msleep_interruptible - sleep waiting for signals 1741 * msleep_interruptible - sleep waiting for signals
1755 * @msecs: Time in milliseconds to sleep for 1742 * @msecs: Time in milliseconds to sleep for
1756 */ 1743 */
1757 unsigned long msleep_interruptible(unsigned int msecs) 1744 unsigned long msleep_interruptible(unsigned int msecs)
1758 { 1745 {
1759 unsigned long timeout = msecs_to_jiffies(msecs) + 1; 1746 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1760 1747
1761 while (timeout && !signal_pending(current)) 1748 while (timeout && !signal_pending(current))
1762 timeout = schedule_timeout_interruptible(timeout); 1749 timeout = schedule_timeout_interruptible(timeout);
1763 return jiffies_to_msecs(timeout); 1750 return jiffies_to_msecs(timeout);
1764 } 1751 }
1765 1752
1766 EXPORT_SYMBOL(msleep_interruptible); 1753 EXPORT_SYMBOL(msleep_interruptible);
1767 1754
1768 static int __sched do_usleep_range(unsigned long min, unsigned long max) 1755 static int __sched do_usleep_range(unsigned long min, unsigned long max)
1769 { 1756 {
1770 ktime_t kmin; 1757 ktime_t kmin;
1771 unsigned long delta; 1758 unsigned long delta;
1772 1759
1773 kmin = ktime_set(0, min * NSEC_PER_USEC); 1760 kmin = ktime_set(0, min * NSEC_PER_USEC);
1774 delta = (max - min) * NSEC_PER_USEC; 1761 delta = (max - min) * NSEC_PER_USEC;
1775 return schedule_hrtimeout_range(&kmin, delta, HRTIMER_MODE_REL); 1762 return schedule_hrtimeout_range(&kmin, delta, HRTIMER_MODE_REL);
1776 } 1763 }
1777 1764
1778 /** 1765 /**
1779 * usleep_range - Drop in replacement for udelay where wakeup is flexible 1766 * usleep_range - Drop in replacement for udelay where wakeup is flexible
1780 * @min: Minimum time in usecs to sleep 1767 * @min: Minimum time in usecs to sleep
1781 * @max: Maximum time in usecs to sleep 1768 * @max: Maximum time in usecs to sleep
1782 */ 1769 */
1783 void usleep_range(unsigned long min, unsigned long max) 1770 void usleep_range(unsigned long min, unsigned long max)
1784 { 1771 {
1785 __set_current_state(TASK_UNINTERRUPTIBLE); 1772 __set_current_state(TASK_UNINTERRUPTIBLE);
1786 do_usleep_range(min, max); 1773 do_usleep_range(min, max);
1787 } 1774 }
1788 EXPORT_SYMBOL(usleep_range); 1775 EXPORT_SYMBOL(usleep_range);
1789 1776