Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit d7240b988017521ebf89edfadd42c0942f166850

Authored by Steven Rostedt
Committed by Ingo Molnar
1 parent 4ab0a9409a
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

generic-ipi: use per cpu data for single cpu ipi calls 

The smp_call_function can be passed a wait parameter telling it to
wait for all the functions running on other CPUs to complete before
returning, or to return without waiting. Unfortunately, this is
currently just a suggestion and not manditory. That is, the
smp_call_function can decide not to return and wait instead.

The reason for this is because it uses kmalloc to allocate storage
to send to the called CPU and that CPU will free it when it is done.
But if we fail to allocate the storage, the stack is used instead.
This means we must wait for the called CPU to finish before
continuing.

Unfortunatly, some callers do no abide by this hint and act as if
the non-wait option is mandatory. The MTRR code for instance will
deadlock if the smp_call_function is set to wait. This is because
the smp_call_function will wait for the other CPUs to finish their
called functions, but those functions are waiting on the caller to
continue.

This patch changes the generic smp_call_function code to use per cpu
variables if the allocation of the data fails for a single CPU call. The
smp_call_function_many will fall back to the smp_call_function_single
if it fails its alloc. The smp_call_function_single is modified
to not force the wait state.

Since we now are using a single data per cpu we must synchronize the
callers to prevent a second caller modifying the data before the
first called IPI functions complete. To do so, I added a flag to
the call_single_data called CSD_FLAG_LOCK. When the single CPU is
called (which can be called when a many call fails an alloc), we
set the LOCK bit on this per cpu data. When the caller finishes
it clears the LOCK bit.

The caller must wait till the LOCK bit is cleared before setting
it. When it is cleared, there is no IPI function using it.

Signed-off-by: Steven Rostedt <srostedt@redhat.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Jens Axboe <jens.axboe@oracle.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

Showing 1 changed file with 33 additions and 3 deletions Inline Diff

1 /* 1 /*
2 * Generic helpers for smp ipi calls 2 * Generic helpers for smp ipi calls
3 * 3 *
4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008 4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
5 * 5 *
6 */ 6 */
7 #include <linux/init.h> 7 #include <linux/init.h>
8 #include <linux/module.h> 8 #include <linux/module.h>
9 #include <linux/percpu.h> 9 #include <linux/percpu.h>
10 #include <linux/rcupdate.h> 10 #include <linux/rcupdate.h>
11 #include <linux/rculist.h> 11 #include <linux/rculist.h>
12 #include <linux/smp.h> 12 #include <linux/smp.h>
13 13
14 static DEFINE_PER_CPU(struct call_single_queue, call_single_queue); 14 static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
15 static LIST_HEAD(call_function_queue); 15 static LIST_HEAD(call_function_queue);
16 __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock); 16 __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);
17 17
18 enum { 18 enum {
19 CSD_FLAG_WAIT = 0x01, 19 CSD_FLAG_WAIT = 0x01,
20 CSD_FLAG_ALLOC = 0x02, 20 CSD_FLAG_ALLOC = 0x02,
21 CSD_FLAG_LOCK = 0x04,
21 }; 22 };
22 23
23 struct call_function_data { 24 struct call_function_data {
24 struct call_single_data csd; 25 struct call_single_data csd;
25 spinlock_t lock; 26 spinlock_t lock;
26 unsigned int refs; 27 unsigned int refs;
27 struct rcu_head rcu_head; 28 struct rcu_head rcu_head;
28 unsigned long cpumask_bits[]; 29 unsigned long cpumask_bits[];
29 }; 30 };
30 31
31 struct call_single_queue { 32 struct call_single_queue {
32 struct list_head list; 33 struct list_head list;
33 spinlock_t lock; 34 spinlock_t lock;
34 }; 35 };
35 36
36 static int __cpuinit init_call_single_data(void) 37 static int __cpuinit init_call_single_data(void)
37 { 38 {
38 int i; 39 int i;
39 40
40 for_each_possible_cpu(i) { 41 for_each_possible_cpu(i) {
41 struct call_single_queue *q = &per_cpu(call_single_queue, i); 42 struct call_single_queue *q = &per_cpu(call_single_queue, i);
42 43
43 spin_lock_init(&q->lock); 44 spin_lock_init(&q->lock);
44 INIT_LIST_HEAD(&q->list); 45 INIT_LIST_HEAD(&q->list);
45 } 46 }
46 return 0; 47 return 0;
47 } 48 }
48 early_initcall(init_call_single_data); 49 early_initcall(init_call_single_data);
49 50
50 static void csd_flag_wait(struct call_single_data *data) 51 static void csd_flag_wait(struct call_single_data *data)
51 { 52 {
52 /* Wait for response */ 53 /* Wait for response */
53 do { 54 do {
54 if (!(data->flags & CSD_FLAG_WAIT)) 55 if (!(data->flags & CSD_FLAG_WAIT))
55 break; 56 break;
56 cpu_relax(); 57 cpu_relax();
57 } while (1); 58 } while (1);
58 } 59 }
59 60
60 /* 61 /*
61 * Insert a previously allocated call_single_data element for execution 62 * Insert a previously allocated call_single_data element for execution
62 * on the given CPU. data must already have ->func, ->info, and ->flags set. 63 * on the given CPU. data must already have ->func, ->info, and ->flags set.
63 */ 64 */
64 static void generic_exec_single(int cpu, struct call_single_data *data) 65 static void generic_exec_single(int cpu, struct call_single_data *data)
65 { 66 {
66 struct call_single_queue *dst = &per_cpu(call_single_queue, cpu); 67 struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
67 int wait = data->flags & CSD_FLAG_WAIT, ipi; 68 int wait = data->flags & CSD_FLAG_WAIT, ipi;
68 unsigned long flags; 69 unsigned long flags;
69 70
70 spin_lock_irqsave(&dst->lock, flags); 71 spin_lock_irqsave(&dst->lock, flags);
71 ipi = list_empty(&dst->list); 72 ipi = list_empty(&dst->list);
72 list_add_tail(&data->list, &dst->list); 73 list_add_tail(&data->list, &dst->list);
73 spin_unlock_irqrestore(&dst->lock, flags); 74 spin_unlock_irqrestore(&dst->lock, flags);
74 75
75 /* 76 /*
76 * Make the list addition visible before sending the ipi. 77 * Make the list addition visible before sending the ipi.
77 */ 78 */
78 smp_mb(); 79 smp_mb();
79 80
80 if (ipi) 81 if (ipi)
81 arch_send_call_function_single_ipi(cpu); 82 arch_send_call_function_single_ipi(cpu);
82 83
83 if (wait) 84 if (wait)
84 csd_flag_wait(data); 85 csd_flag_wait(data);
85 } 86 }
86 87
87 static void rcu_free_call_data(struct rcu_head *head) 88 static void rcu_free_call_data(struct rcu_head *head)
88 { 89 {
89 struct call_function_data *data; 90 struct call_function_data *data;
90 91
91 data = container_of(head, struct call_function_data, rcu_head); 92 data = container_of(head, struct call_function_data, rcu_head);
92 93
93 kfree(data); 94 kfree(data);
94 } 95 }
95 96
96 /* 97 /*
97 * Invoked by arch to handle an IPI for call function. Must be called with 98 * Invoked by arch to handle an IPI for call function. Must be called with
98 * interrupts disabled. 99 * interrupts disabled.
99 */ 100 */
100 void generic_smp_call_function_interrupt(void) 101 void generic_smp_call_function_interrupt(void)
101 { 102 {
102 struct call_function_data *data; 103 struct call_function_data *data;
103 int cpu = get_cpu(); 104 int cpu = get_cpu();
104 105
105 /* 106 /*
106 * It's ok to use list_for_each_rcu() here even though we may delete 107 * It's ok to use list_for_each_rcu() here even though we may delete
107 * 'pos', since list_del_rcu() doesn't clear ->next 108 * 'pos', since list_del_rcu() doesn't clear ->next
108 */ 109 */
109 rcu_read_lock(); 110 rcu_read_lock();
110 list_for_each_entry_rcu(data, &call_function_queue, csd.list) { 111 list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
111 int refs; 112 int refs;
112 113
113 if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits))) 114 if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits)))
114 continue; 115 continue;
115 116
116 data->csd.func(data->csd.info); 117 data->csd.func(data->csd.info);
117 118
118 spin_lock(&data->lock); 119 spin_lock(&data->lock);
119 cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits)); 120 cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));
120 WARN_ON(data->refs == 0); 121 WARN_ON(data->refs == 0);
121 data->refs--; 122 data->refs--;
122 refs = data->refs; 123 refs = data->refs;
123 spin_unlock(&data->lock); 124 spin_unlock(&data->lock);
124 125
125 if (refs) 126 if (refs)
126 continue; 127 continue;
127 128
128 spin_lock(&call_function_lock); 129 spin_lock(&call_function_lock);
129 list_del_rcu(&data->csd.list); 130 list_del_rcu(&data->csd.list);
130 spin_unlock(&call_function_lock); 131 spin_unlock(&call_function_lock);
131 132
132 if (data->csd.flags & CSD_FLAG_WAIT) { 133 if (data->csd.flags & CSD_FLAG_WAIT) {
133 /* 134 /*
134 * serialize stores to data with the flag clear 135 * serialize stores to data with the flag clear
135 * and wakeup 136 * and wakeup
136 */ 137 */
137 smp_wmb(); 138 smp_wmb();
138 data->csd.flags &= ~CSD_FLAG_WAIT; 139 data->csd.flags &= ~CSD_FLAG_WAIT;
139 } 140 }
140 if (data->csd.flags & CSD_FLAG_ALLOC) 141 if (data->csd.flags & CSD_FLAG_ALLOC)
141 call_rcu(&data->rcu_head, rcu_free_call_data); 142 call_rcu(&data->rcu_head, rcu_free_call_data);
142 } 143 }
143 rcu_read_unlock(); 144 rcu_read_unlock();
144 145
145 put_cpu(); 146 put_cpu();
146 } 147 }
147 148
148 /* 149 /*
149 * Invoked by arch to handle an IPI for call function single. Must be called 150 * Invoked by arch to handle an IPI for call function single. Must be called
150 * from the arch with interrupts disabled. 151 * from the arch with interrupts disabled.
151 */ 152 */
152 void generic_smp_call_function_single_interrupt(void) 153 void generic_smp_call_function_single_interrupt(void)
153 { 154 {
154 struct call_single_queue *q = &__get_cpu_var(call_single_queue); 155 struct call_single_queue *q = &__get_cpu_var(call_single_queue);
155 LIST_HEAD(list); 156 LIST_HEAD(list);
156 157
157 /* 158 /*
158 * Need to see other stores to list head for checking whether 159 * Need to see other stores to list head for checking whether
159 * list is empty without holding q->lock 160 * list is empty without holding q->lock
160 */ 161 */
161 smp_read_barrier_depends(); 162 smp_read_barrier_depends();
162 while (!list_empty(&q->list)) { 163 while (!list_empty(&q->list)) {
163 unsigned int data_flags; 164 unsigned int data_flags;
164 165
165 spin_lock(&q->lock); 166 spin_lock(&q->lock);
166 list_replace_init(&q->list, &list); 167 list_replace_init(&q->list, &list);
167 spin_unlock(&q->lock); 168 spin_unlock(&q->lock);
168 169
169 while (!list_empty(&list)) { 170 while (!list_empty(&list)) {
170 struct call_single_data *data; 171 struct call_single_data *data;
171 172
172 data = list_entry(list.next, struct call_single_data, 173 data = list_entry(list.next, struct call_single_data,
173 list); 174 list);
174 list_del(&data->list); 175 list_del(&data->list);
175 176
176 /* 177 /*
177 * 'data' can be invalid after this call if 178 * 'data' can be invalid after this call if
178 * flags == 0 (when called through 179 * flags == 0 (when called through
179 * generic_exec_single(), so save them away before 180 * generic_exec_single(), so save them away before
180 * making the call. 181 * making the call.
181 */ 182 */
182 data_flags = data->flags; 183 data_flags = data->flags;
183 184
184 data->func(data->info); 185 data->func(data->info);
185 186
186 if (data_flags & CSD_FLAG_WAIT) { 187 if (data_flags & CSD_FLAG_WAIT) {
187 smp_wmb(); 188 smp_wmb();
188 data->flags &= ~CSD_FLAG_WAIT; 189 data->flags &= ~CSD_FLAG_WAIT;
190 } else if (data_flags & CSD_FLAG_LOCK) {
191 smp_wmb();
192 data->flags &= ~CSD_FLAG_LOCK;
189 } else if (data_flags & CSD_FLAG_ALLOC) 193 } else if (data_flags & CSD_FLAG_ALLOC)
190 kfree(data); 194 kfree(data);
191 } 195 }
192 /* 196 /*
193 * See comment on outer loop 197 * See comment on outer loop
194 */ 198 */
195 smp_read_barrier_depends(); 199 smp_read_barrier_depends();
196 } 200 }
197 } 201 }
198 202
203 static DEFINE_PER_CPU(struct call_single_data, csd_data);
204
199 /* 205 /*
200 * smp_call_function_single - Run a function on a specific CPU 206 * smp_call_function_single - Run a function on a specific CPU
201 * @func: The function to run. This must be fast and non-blocking. 207 * @func: The function to run. This must be fast and non-blocking.
202 * @info: An arbitrary pointer to pass to the function. 208 * @info: An arbitrary pointer to pass to the function.
203 * @wait: If true, wait until function has completed on other CPUs. 209 * @wait: If true, wait until function has completed on other CPUs.
204 * 210 *
205 * Returns 0 on success, else a negative status code. Note that @wait 211 * Returns 0 on success, else a negative status code. Note that @wait
206 * will be implicitly turned on in case of allocation failures, since 212 * will be implicitly turned on in case of allocation failures, since
207 * we fall back to on-stack allocation. 213 * we fall back to on-stack allocation.
208 */ 214 */
209 int smp_call_function_single(int cpu, void (*func) (void *info), void *info, 215 int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
210 int wait) 216 int wait)
211 { 217 {
212 struct call_single_data d; 218 struct call_single_data d;
213 unsigned long flags; 219 unsigned long flags;
214 /* prevent preemption and reschedule on another processor, 220 /* prevent preemption and reschedule on another processor,
215 as well as CPU removal */ 221 as well as CPU removal */
216 int me = get_cpu(); 222 int me = get_cpu();
217 int err = 0; 223 int err = 0;
218 224
219 /* Can deadlock when called with interrupts disabled */ 225 /* Can deadlock when called with interrupts disabled */
220 WARN_ON(irqs_disabled()); 226 WARN_ON(irqs_disabled());
221 227
222 if (cpu == me) { 228 if (cpu == me) {
223 local_irq_save(flags); 229 local_irq_save(flags);
224 func(info); 230 func(info);
225 local_irq_restore(flags); 231 local_irq_restore(flags);
226 } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { 232 } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
227 struct call_single_data *data = NULL; 233 struct call_single_data *data;
228 234
229 if (!wait) { 235 if (!wait) {
236 /*
237 * We are calling a function on a single CPU
238 * and we are not going to wait for it to finish.
239 * We first try to allocate the data, but if we
240 * fail, we fall back to use a per cpu data to pass
241 * the information to that CPU. Since all callers
242 * of this code will use the same data, we must
243 * synchronize the callers to prevent a new caller
244 * from corrupting the data before the callee
245 * can access it.
246 *
247 * The CSD_FLAG_LOCK is used to let us know when
248 * the IPI handler is done with the data.
249 * The first caller will set it, and the callee
250 * will clear it. The next caller must wait for
251 * it to clear before we set it again. This
252 * will make sure the callee is done with the
253 * data before a new caller will use it.
254 */
230 data = kmalloc(sizeof(*data), GFP_ATOMIC); 255 data = kmalloc(sizeof(*data), GFP_ATOMIC);
231 if (data) 256 if (data)
232 data->flags = CSD_FLAG_ALLOC; 257 data->flags = CSD_FLAG_ALLOC;
233 } 258 else {
234 if (!data) { 259 data = &per_cpu(csd_data, me);
260 while (data->flags & CSD_FLAG_LOCK)
261 cpu_relax();
262 data->flags = CSD_FLAG_LOCK;
263 }
264 } else {
235 data = &d; 265 data = &d;
236 data->flags = CSD_FLAG_WAIT; 266 data->flags = CSD_FLAG_WAIT;
237 } 267 }
238 268
239 data->func = func; 269 data->func = func;
240 data->info = info; 270 data->info = info;
241 generic_exec_single(cpu, data); 271 generic_exec_single(cpu, data);
242 } else { 272 } else {
243 err = -ENXIO; /* CPU not online */ 273 err = -ENXIO; /* CPU not online */
244 } 274 }
245 275
246 put_cpu(); 276 put_cpu();
247 return err; 277 return err;
248 } 278 }
249 EXPORT_SYMBOL(smp_call_function_single); 279 EXPORT_SYMBOL(smp_call_function_single);
250 280
251 /** 281 /**
252 * __smp_call_function_single(): Run a function on another CPU 282 * __smp_call_function_single(): Run a function on another CPU
253 * @cpu: The CPU to run on. 283 * @cpu: The CPU to run on.
254 * @data: Pre-allocated and setup data structure 284 * @data: Pre-allocated and setup data structure
255 * 285 *
256 * Like smp_call_function_single(), but allow caller to pass in a pre-allocated 286 * Like smp_call_function_single(), but allow caller to pass in a pre-allocated
257 * data structure. Useful for embedding @data inside other structures, for 287 * data structure. Useful for embedding @data inside other structures, for
258 * instance. 288 * instance.
259 * 289 *
260 */ 290 */
261 void __smp_call_function_single(int cpu, struct call_single_data *data) 291 void __smp_call_function_single(int cpu, struct call_single_data *data)
262 { 292 {
263 /* Can deadlock when called with interrupts disabled */ 293 /* Can deadlock when called with interrupts disabled */
264 WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled()); 294 WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());
265 295
266 generic_exec_single(cpu, data); 296 generic_exec_single(cpu, data);
267 } 297 }
268 298
269 /* FIXME: Shim for archs using old arch_send_call_function_ipi API. */ 299 /* FIXME: Shim for archs using old arch_send_call_function_ipi API. */
270 #ifndef arch_send_call_function_ipi_mask 300 #ifndef arch_send_call_function_ipi_mask
271 #define arch_send_call_function_ipi_mask(maskp) \ 301 #define arch_send_call_function_ipi_mask(maskp) \
272 arch_send_call_function_ipi(*(maskp)) 302 arch_send_call_function_ipi(*(maskp))
273 #endif 303 #endif
274 304
275 /** 305 /**
276 * smp_call_function_many(): Run a function on a set of other CPUs. 306 * smp_call_function_many(): Run a function on a set of other CPUs.
277 * @mask: The set of cpus to run on (only runs on online subset). 307 * @mask: The set of cpus to run on (only runs on online subset).
278 * @func: The function to run. This must be fast and non-blocking. 308 * @func: The function to run. This must be fast and non-blocking.
279 * @info: An arbitrary pointer to pass to the function. 309 * @info: An arbitrary pointer to pass to the function.
280 * @wait: If true, wait (atomically) until function has completed on other CPUs. 310 * @wait: If true, wait (atomically) until function has completed on other CPUs.
281 * 311 *
282 * If @wait is true, then returns once @func has returned. Note that @wait 312 * If @wait is true, then returns once @func has returned. Note that @wait
283 * will be implicitly turned on in case of allocation failures, since 313 * will be implicitly turned on in case of allocation failures, since
284 * we fall back to on-stack allocation. 314 * we fall back to on-stack allocation.
285 * 315 *
286 * You must not call this function with disabled interrupts or from a 316 * You must not call this function with disabled interrupts or from a
287 * hardware interrupt handler or from a bottom half handler. Preemption 317 * hardware interrupt handler or from a bottom half handler. Preemption
288 * must be disabled when calling this function. 318 * must be disabled when calling this function.
289 */ 319 */
290 void smp_call_function_many(const struct cpumask *mask, 320 void smp_call_function_many(const struct cpumask *mask,
291 void (*func)(void *), void *info, 321 void (*func)(void *), void *info,
292 bool wait) 322 bool wait)
293 { 323 {
294 struct call_function_data *data; 324 struct call_function_data *data;
295 unsigned long flags; 325 unsigned long flags;
296 int cpu, next_cpu; 326 int cpu, next_cpu;
297 327
298 /* Can deadlock when called with interrupts disabled */ 328 /* Can deadlock when called with interrupts disabled */
299 WARN_ON(irqs_disabled()); 329 WARN_ON(irqs_disabled());
300 330
301 /* So, what's a CPU they want? Ignoring this one. */ 331 /* So, what's a CPU they want? Ignoring this one. */
302 cpu = cpumask_first_and(mask, cpu_online_mask); 332 cpu = cpumask_first_and(mask, cpu_online_mask);
303 if (cpu == smp_processor_id()) 333 if (cpu == smp_processor_id())
304 cpu = cpumask_next_and(cpu, mask, cpu_online_mask); 334 cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
305 /* No online cpus? We're done. */ 335 /* No online cpus? We're done. */
306 if (cpu >= nr_cpu_ids) 336 if (cpu >= nr_cpu_ids)
307 return; 337 return;
308 338
309 /* Do we have another CPU which isn't us? */ 339 /* Do we have another CPU which isn't us? */
310 next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask); 340 next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
311 if (next_cpu == smp_processor_id()) 341 if (next_cpu == smp_processor_id())
312 next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask); 342 next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
313 343
314 /* Fastpath: do that cpu by itself. */ 344 /* Fastpath: do that cpu by itself. */
315 if (next_cpu >= nr_cpu_ids) { 345 if (next_cpu >= nr_cpu_ids) {
316 smp_call_function_single(cpu, func, info, wait); 346 smp_call_function_single(cpu, func, info, wait);
317 return; 347 return;
318 } 348 }
319 349
320 data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC); 350 data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC);
321 if (unlikely(!data)) { 351 if (unlikely(!data)) {
322 /* Slow path. */ 352 /* Slow path. */
323 for_each_online_cpu(cpu) { 353 for_each_online_cpu(cpu) {
324 if (cpu == smp_processor_id()) 354 if (cpu == smp_processor_id())
325 continue; 355 continue;
326 if (cpumask_test_cpu(cpu, mask)) 356 if (cpumask_test_cpu(cpu, mask))
327 smp_call_function_single(cpu, func, info, wait); 357 smp_call_function_single(cpu, func, info, wait);
328 } 358 }
329 return; 359 return;
330 } 360 }
331 361
332 spin_lock_init(&data->lock); 362 spin_lock_init(&data->lock);
333 data->csd.flags = CSD_FLAG_ALLOC; 363 data->csd.flags = CSD_FLAG_ALLOC;
334 if (wait) 364 if (wait)
335 data->csd.flags |= CSD_FLAG_WAIT; 365 data->csd.flags |= CSD_FLAG_WAIT;
336 data->csd.func = func; 366 data->csd.func = func;
337 data->csd.info = info; 367 data->csd.info = info;
338 cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask); 368 cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask);
339 cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits)); 369 cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits));
340 data->refs = cpumask_weight(to_cpumask(data->cpumask_bits)); 370 data->refs = cpumask_weight(to_cpumask(data->cpumask_bits));
341 371
342 spin_lock_irqsave(&call_function_lock, flags); 372 spin_lock_irqsave(&call_function_lock, flags);
343 list_add_tail_rcu(&data->csd.list, &call_function_queue); 373 list_add_tail_rcu(&data->csd.list, &call_function_queue);
344 spin_unlock_irqrestore(&call_function_lock, flags); 374 spin_unlock_irqrestore(&call_function_lock, flags);
345 375
346 /* 376 /*
347 * Make the list addition visible before sending the ipi. 377 * Make the list addition visible before sending the ipi.
348 */ 378 */
349 smp_mb(); 379 smp_mb();
350 380
351 /* Send a message to all CPUs in the map */ 381 /* Send a message to all CPUs in the map */
352 arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits)); 382 arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits));
353 383
354 /* optionally wait for the CPUs to complete */ 384 /* optionally wait for the CPUs to complete */
355 if (wait) 385 if (wait)
356 csd_flag_wait(&data->csd); 386 csd_flag_wait(&data->csd);
357 } 387 }
358 EXPORT_SYMBOL(smp_call_function_many); 388 EXPORT_SYMBOL(smp_call_function_many);
359 389
360 /** 390 /**
361 * smp_call_function(): Run a function on all other CPUs. 391 * smp_call_function(): Run a function on all other CPUs.
362 * @func: The function to run. This must be fast and non-blocking. 392 * @func: The function to run. This must be fast and non-blocking.
363 * @info: An arbitrary pointer to pass to the function. 393 * @info: An arbitrary pointer to pass to the function.
364 * @wait: If true, wait (atomically) until function has completed on other CPUs. 394 * @wait: If true, wait (atomically) until function has completed on other CPUs.
365 * 395 *
366 * Returns 0. 396 * Returns 0.
367 * 397 *
368 * If @wait is true, then returns once @func has returned; otherwise 398 * If @wait is true, then returns once @func has returned; otherwise
369 * it returns just before the target cpu calls @func. In case of allocation 399 * it returns just before the target cpu calls @func. In case of allocation
370 * failure, @wait will be implicitly turned on. 400 * failure, @wait will be implicitly turned on.
371 * 401 *
372 * You must not call this function with disabled interrupts or from a 402 * You must not call this function with disabled interrupts or from a
373 * hardware interrupt handler or from a bottom half handler. 403 * hardware interrupt handler or from a bottom half handler.
374 */ 404 */
375 int smp_call_function(void (*func)(void *), void *info, int wait) 405 int smp_call_function(void (*func)(void *), void *info, int wait)
376 { 406 {
377 preempt_disable(); 407 preempt_disable();
378 smp_call_function_many(cpu_online_mask, func, info, wait); 408 smp_call_function_many(cpu_online_mask, func, info, wait);
379 preempt_enable(); 409 preempt_enable();
380 return 0; 410 return 0;
381 } 411 }
382 EXPORT_SYMBOL(smp_call_function); 412 EXPORT_SYMBOL(smp_call_function);
383 413
384 void ipi_call_lock(void) 414 void ipi_call_lock(void)
385 { 415 {
386 spin_lock(&call_function_lock); 416 spin_lock(&call_function_lock);
387 } 417 }
388 418
389 void ipi_call_unlock(void) 419 void ipi_call_unlock(void)
390 { 420 {
391 spin_unlock(&call_function_lock); 421 spin_unlock(&call_function_lock);
392 } 422 }
393 423
394 void ipi_call_lock_irq(void) 424 void ipi_call_lock_irq(void)
395 { 425 {
396 spin_lock_irq(&call_function_lock); 426 spin_lock_irq(&call_function_lock);
397 } 427 }
398 428
399 void ipi_call_unlock_irq(void) 429 void ipi_call_unlock_irq(void)
400 { 430 {
401 spin_unlock_irq(&call_function_lock); 431 spin_unlock_irq(&call_function_lock);
402 } 432 }
403 433