Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 31a6b11fed6ceec07ec4bdfefae56b8252d450cf

Authored by Xiantao Zhang
Committed by Tony Luck
1 parent 96651896b8
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

[IA64] Implement smp_call_function_mask for ia64 

This interface provides more flexible functionality for smp
infrastructure ... e.g. KVM frequently needs to operate on
a subset of cpus.

Signed-off-by: Xiantao Zhang <xiantao.zhang@intel.com>
Signed-off-by: Tony Luck <tony.luck@intel.com>

Showing 2 changed files with 85 additions and 0 deletions Inline Diff

arch/ia64/kernel/smp.c
Diff comments   View file @ 31a6b11
1 /* 1 /*
2 * SMP Support 2 * SMP Support
3 * 3 *
4 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 4 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
5 * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com> 5 * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com>
6 * 6 *
7 * Lots of stuff stolen from arch/alpha/kernel/smp.c 7 * Lots of stuff stolen from arch/alpha/kernel/smp.c
8 * 8 *
9 * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized 9 * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized
10 * the existing code (on the lines of x86 port). 10 * the existing code (on the lines of x86 port).
11 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy 11 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy
12 * calibration on each CPU. 12 * calibration on each CPU.
13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id 13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id
14 * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor 14 * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor
15 * & cpu_online_map now gets done here (instead of setup.c) 15 * & cpu_online_map now gets done here (instead of setup.c)
16 * 99/10/05 davidm Update to bring it in sync with new command-line processing 16 * 99/10/05 davidm Update to bring it in sync with new command-line processing
17 * scheme. 17 * scheme.
18 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and 18 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and
19 * smp_call_function_single to resend IPI on timeouts 19 * smp_call_function_single to resend IPI on timeouts
20 */ 20 */
21 #include <linux/module.h> 21 #include <linux/module.h>
22 #include <linux/kernel.h> 22 #include <linux/kernel.h>
23 #include <linux/sched.h> 23 #include <linux/sched.h>
24 #include <linux/init.h> 24 #include <linux/init.h>
25 #include <linux/interrupt.h> 25 #include <linux/interrupt.h>
26 #include <linux/smp.h> 26 #include <linux/smp.h>
27 #include <linux/kernel_stat.h> 27 #include <linux/kernel_stat.h>
28 #include <linux/mm.h> 28 #include <linux/mm.h>
29 #include <linux/cache.h> 29 #include <linux/cache.h>
30 #include <linux/delay.h> 30 #include <linux/delay.h>
31 #include <linux/efi.h> 31 #include <linux/efi.h>
32 #include <linux/bitops.h> 32 #include <linux/bitops.h>
33 #include <linux/kexec.h> 33 #include <linux/kexec.h>
34 34
35 #include <asm/atomic.h> 35 #include <asm/atomic.h>
36 #include <asm/current.h> 36 #include <asm/current.h>
37 #include <asm/delay.h> 37 #include <asm/delay.h>
38 #include <asm/machvec.h> 38 #include <asm/machvec.h>
39 #include <asm/io.h> 39 #include <asm/io.h>
40 #include <asm/irq.h> 40 #include <asm/irq.h>
41 #include <asm/page.h> 41 #include <asm/page.h>
42 #include <asm/pgalloc.h> 42 #include <asm/pgalloc.h>
43 #include <asm/pgtable.h> 43 #include <asm/pgtable.h>
44 #include <asm/processor.h> 44 #include <asm/processor.h>
45 #include <asm/ptrace.h> 45 #include <asm/ptrace.h>
46 #include <asm/sal.h> 46 #include <asm/sal.h>
47 #include <asm/system.h> 47 #include <asm/system.h>
48 #include <asm/tlbflush.h> 48 #include <asm/tlbflush.h>
49 #include <asm/unistd.h> 49 #include <asm/unistd.h>
50 #include <asm/mca.h> 50 #include <asm/mca.h>
51 51
52 /* 52 /*
53 * Note: alignment of 4 entries/cacheline was empirically determined 53 * Note: alignment of 4 entries/cacheline was empirically determined
54 * to be a good tradeoff between hot cachelines & spreading the array 54 * to be a good tradeoff between hot cachelines & spreading the array
55 * across too many cacheline. 55 * across too many cacheline.
56 */ 56 */
57 static struct local_tlb_flush_counts { 57 static struct local_tlb_flush_counts {
58 unsigned int count; 58 unsigned int count;
59 } __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS]; 59 } __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS];
60 60
61 static DEFINE_PER_CPU(unsigned int, shadow_flush_counts[NR_CPUS]) ____cacheline_aligned; 61 static DEFINE_PER_CPU(unsigned int, shadow_flush_counts[NR_CPUS]) ____cacheline_aligned;
62 62
63 63
64 /* 64 /*
65 * Structure and data for smp_call_function(). This is designed to minimise static memory 65 * Structure and data for smp_call_function(). This is designed to minimise static memory
66 * requirements. It also looks cleaner. 66 * requirements. It also looks cleaner.
67 */ 67 */
68 static __cacheline_aligned DEFINE_SPINLOCK(call_lock); 68 static __cacheline_aligned DEFINE_SPINLOCK(call_lock);
69 69
70 struct call_data_struct { 70 struct call_data_struct {
71 void (*func) (void *info); 71 void (*func) (void *info);
72 void *info; 72 void *info;
73 long wait; 73 long wait;
74 atomic_t started; 74 atomic_t started;
75 atomic_t finished; 75 atomic_t finished;
76 }; 76 };
77 77
78 static volatile struct call_data_struct *call_data; 78 static volatile struct call_data_struct *call_data;
79 79
80 #define IPI_CALL_FUNC 0 80 #define IPI_CALL_FUNC 0
81 #define IPI_CPU_STOP 1 81 #define IPI_CPU_STOP 1
82 #define IPI_KDUMP_CPU_STOP 3 82 #define IPI_KDUMP_CPU_STOP 3
83 83
84 /* This needs to be cacheline aligned because it is written to by *other* CPUs. */ 84 /* This needs to be cacheline aligned because it is written to by *other* CPUs. */
85 static DEFINE_PER_CPU_SHARED_ALIGNED(u64, ipi_operation); 85 static DEFINE_PER_CPU_SHARED_ALIGNED(u64, ipi_operation);
86 86
87 extern void cpu_halt (void); 87 extern void cpu_halt (void);
88 88
89 void 89 void
90 lock_ipi_calllock(void) 90 lock_ipi_calllock(void)
91 { 91 {
92 spin_lock_irq(&call_lock); 92 spin_lock_irq(&call_lock);
93 } 93 }
94 94
95 void 95 void
96 unlock_ipi_calllock(void) 96 unlock_ipi_calllock(void)
97 { 97 {
98 spin_unlock_irq(&call_lock); 98 spin_unlock_irq(&call_lock);
99 } 99 }
100 100
101 static void 101 static void
102 stop_this_cpu (void) 102 stop_this_cpu (void)
103 { 103 {
104 /* 104 /*
105 * Remove this CPU: 105 * Remove this CPU:
106 */ 106 */
107 cpu_clear(smp_processor_id(), cpu_online_map); 107 cpu_clear(smp_processor_id(), cpu_online_map);
108 max_xtp(); 108 max_xtp();
109 local_irq_disable(); 109 local_irq_disable();
110 cpu_halt(); 110 cpu_halt();
111 } 111 }
112 112
113 void 113 void
114 cpu_die(void) 114 cpu_die(void)
115 { 115 {
116 max_xtp(); 116 max_xtp();
117 local_irq_disable(); 117 local_irq_disable();
118 cpu_halt(); 118 cpu_halt();
119 /* Should never be here */ 119 /* Should never be here */
120 BUG(); 120 BUG();
121 for (;;); 121 for (;;);
122 } 122 }
123 123
124 irqreturn_t 124 irqreturn_t
125 handle_IPI (int irq, void *dev_id) 125 handle_IPI (int irq, void *dev_id)
126 { 126 {
127 int this_cpu = get_cpu(); 127 int this_cpu = get_cpu();
128 unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation); 128 unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation);
129 unsigned long ops; 129 unsigned long ops;
130 130
131 mb(); /* Order interrupt and bit testing. */ 131 mb(); /* Order interrupt and bit testing. */
132 while ((ops = xchg(pending_ipis, 0)) != 0) { 132 while ((ops = xchg(pending_ipis, 0)) != 0) {
133 mb(); /* Order bit clearing and data access. */ 133 mb(); /* Order bit clearing and data access. */
134 do { 134 do {
135 unsigned long which; 135 unsigned long which;
136 136
137 which = ffz(~ops); 137 which = ffz(~ops);
138 ops &= ~(1 << which); 138 ops &= ~(1 << which);
139 139
140 switch (which) { 140 switch (which) {
141 case IPI_CALL_FUNC: 141 case IPI_CALL_FUNC:
142 { 142 {
143 struct call_data_struct *data; 143 struct call_data_struct *data;
144 void (*func)(void *info); 144 void (*func)(void *info);
145 void *info; 145 void *info;
146 int wait; 146 int wait;
147 147
148 /* release the 'pointer lock' */ 148 /* release the 'pointer lock' */
149 data = (struct call_data_struct *) call_data; 149 data = (struct call_data_struct *) call_data;
150 func = data->func; 150 func = data->func;
151 info = data->info; 151 info = data->info;
152 wait = data->wait; 152 wait = data->wait;
153 153
154 mb(); 154 mb();
155 atomic_inc(&data->started); 155 atomic_inc(&data->started);
156 /* 156 /*
157 * At this point the structure may be gone unless 157 * At this point the structure may be gone unless
158 * wait is true. 158 * wait is true.
159 */ 159 */
160 (*func)(info); 160 (*func)(info);
161 161
162 /* Notify the sending CPU that the task is done. */ 162 /* Notify the sending CPU that the task is done. */
163 mb(); 163 mb();
164 if (wait) 164 if (wait)
165 atomic_inc(&data->finished); 165 atomic_inc(&data->finished);
166 } 166 }
167 break; 167 break;
168 168
169 case IPI_CPU_STOP: 169 case IPI_CPU_STOP:
170 stop_this_cpu(); 170 stop_this_cpu();
171 break; 171 break;
172 #ifdef CONFIG_KEXEC 172 #ifdef CONFIG_KEXEC
173 case IPI_KDUMP_CPU_STOP: 173 case IPI_KDUMP_CPU_STOP:
174 unw_init_running(kdump_cpu_freeze, NULL); 174 unw_init_running(kdump_cpu_freeze, NULL);
175 break; 175 break;
176 #endif 176 #endif
177 default: 177 default:
178 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which); 178 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which);
179 break; 179 break;
180 } 180 }
181 } while (ops); 181 } while (ops);
182 mb(); /* Order data access and bit testing. */ 182 mb(); /* Order data access and bit testing. */
183 } 183 }
184 put_cpu(); 184 put_cpu();
185 return IRQ_HANDLED; 185 return IRQ_HANDLED;
186 } 186 }
187 187
188 /* 188 /*
189 * Called with preemption disabled. 189 * Called with preemption disabled.
190 */ 190 */
191 static inline void 191 static inline void
192 send_IPI_single (int dest_cpu, int op) 192 send_IPI_single (int dest_cpu, int op)
193 { 193 {
194 set_bit(op, &per_cpu(ipi_operation, dest_cpu)); 194 set_bit(op, &per_cpu(ipi_operation, dest_cpu));
195 platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0); 195 platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0);
196 } 196 }
197 197
198 /* 198 /*
199 * Called with preemption disabled. 199 * Called with preemption disabled.
200 */ 200 */
201 static inline void 201 static inline void
202 send_IPI_allbutself (int op) 202 send_IPI_allbutself (int op)
203 { 203 {
204 unsigned int i; 204 unsigned int i;
205 205
206 for_each_online_cpu(i) { 206 for_each_online_cpu(i) {
207 if (i != smp_processor_id()) 207 if (i != smp_processor_id())
208 send_IPI_single(i, op); 208 send_IPI_single(i, op);
209 } 209 }
210 } 210 }
211 211
212 /* 212 /*
213 * Called with preemption disabled. 213 * Called with preemption disabled.
214 */ 214 */
215 static inline void 215 static inline void
216 send_IPI_mask(cpumask_t mask, int op)
217 {
218 unsigned int cpu;
219
220 for_each_cpu_mask(cpu, mask) {
221 send_IPI_single(cpu, op);
222 }
223 }
224
225 /*
226 * Called with preemption disabled.
227 */
228 static inline void
216 send_IPI_all (int op) 229 send_IPI_all (int op)
217 { 230 {
218 int i; 231 int i;
219 232
220 for_each_online_cpu(i) { 233 for_each_online_cpu(i) {
221 send_IPI_single(i, op); 234 send_IPI_single(i, op);
222 } 235 }
223 } 236 }
224 237
225 /* 238 /*
226 * Called with preemption disabled. 239 * Called with preemption disabled.
227 */ 240 */
228 static inline void 241 static inline void
229 send_IPI_self (int op) 242 send_IPI_self (int op)
230 { 243 {
231 send_IPI_single(smp_processor_id(), op); 244 send_IPI_single(smp_processor_id(), op);
232 } 245 }
233 246
234 #ifdef CONFIG_KEXEC 247 #ifdef CONFIG_KEXEC
235 void 248 void
236 kdump_smp_send_stop(void) 249 kdump_smp_send_stop(void)
237 { 250 {
238 send_IPI_allbutself(IPI_KDUMP_CPU_STOP); 251 send_IPI_allbutself(IPI_KDUMP_CPU_STOP);
239 } 252 }
240 253
241 void 254 void
242 kdump_smp_send_init(void) 255 kdump_smp_send_init(void)
243 { 256 {
244 unsigned int cpu, self_cpu; 257 unsigned int cpu, self_cpu;
245 self_cpu = smp_processor_id(); 258 self_cpu = smp_processor_id();
246 for_each_online_cpu(cpu) { 259 for_each_online_cpu(cpu) {
247 if (cpu != self_cpu) { 260 if (cpu != self_cpu) {
248 if(kdump_status[cpu] == 0) 261 if(kdump_status[cpu] == 0)
249 platform_send_ipi(cpu, 0, IA64_IPI_DM_INIT, 0); 262 platform_send_ipi(cpu, 0, IA64_IPI_DM_INIT, 0);
250 } 263 }
251 } 264 }
252 } 265 }
253 #endif 266 #endif
254 /* 267 /*
255 * Called with preemption disabled. 268 * Called with preemption disabled.
256 */ 269 */
257 void 270 void
258 smp_send_reschedule (int cpu) 271 smp_send_reschedule (int cpu)
259 { 272 {
260 platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0); 273 platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
261 } 274 }
262 275
263 /* 276 /*
264 * Called with preemption disabled. 277 * Called with preemption disabled.
265 */ 278 */
266 static void 279 static void
267 smp_send_local_flush_tlb (int cpu) 280 smp_send_local_flush_tlb (int cpu)
268 { 281 {
269 platform_send_ipi(cpu, IA64_IPI_LOCAL_TLB_FLUSH, IA64_IPI_DM_INT, 0); 282 platform_send_ipi(cpu, IA64_IPI_LOCAL_TLB_FLUSH, IA64_IPI_DM_INT, 0);
270 } 283 }
271 284
272 void 285 void
273 smp_local_flush_tlb(void) 286 smp_local_flush_tlb(void)
274 { 287 {
275 /* 288 /*
276 * Use atomic ops. Otherwise, the load/increment/store sequence from 289 * Use atomic ops. Otherwise, the load/increment/store sequence from
277 * a "++" operation can have the line stolen between the load & store. 290 * a "++" operation can have the line stolen between the load & store.
278 * The overhead of the atomic op in negligible in this case & offers 291 * The overhead of the atomic op in negligible in this case & offers
279 * significant benefit for the brief periods where lots of cpus 292 * significant benefit for the brief periods where lots of cpus
280 * are simultaneously flushing TLBs. 293 * are simultaneously flushing TLBs.
281 */ 294 */
282 ia64_fetchadd(1, &local_tlb_flush_counts[smp_processor_id()].count, acq); 295 ia64_fetchadd(1, &local_tlb_flush_counts[smp_processor_id()].count, acq);
283 local_flush_tlb_all(); 296 local_flush_tlb_all();
284 } 297 }
285 298
286 #define FLUSH_DELAY 5 /* Usec backoff to eliminate excessive cacheline bouncing */ 299 #define FLUSH_DELAY 5 /* Usec backoff to eliminate excessive cacheline bouncing */
287 300
288 void 301 void
289 smp_flush_tlb_cpumask(cpumask_t xcpumask) 302 smp_flush_tlb_cpumask(cpumask_t xcpumask)
290 { 303 {
291 unsigned int *counts = __ia64_per_cpu_var(shadow_flush_counts); 304 unsigned int *counts = __ia64_per_cpu_var(shadow_flush_counts);
292 cpumask_t cpumask = xcpumask; 305 cpumask_t cpumask = xcpumask;
293 int mycpu, cpu, flush_mycpu = 0; 306 int mycpu, cpu, flush_mycpu = 0;
294 307
295 preempt_disable(); 308 preempt_disable();
296 mycpu = smp_processor_id(); 309 mycpu = smp_processor_id();
297 310
298 for_each_cpu_mask(cpu, cpumask) 311 for_each_cpu_mask(cpu, cpumask)
299 counts[cpu] = local_tlb_flush_counts[cpu].count; 312 counts[cpu] = local_tlb_flush_counts[cpu].count;
300 313
301 mb(); 314 mb();
302 for_each_cpu_mask(cpu, cpumask) { 315 for_each_cpu_mask(cpu, cpumask) {
303 if (cpu == mycpu) 316 if (cpu == mycpu)
304 flush_mycpu = 1; 317 flush_mycpu = 1;
305 else 318 else
306 smp_send_local_flush_tlb(cpu); 319 smp_send_local_flush_tlb(cpu);
307 } 320 }
308 321
309 if (flush_mycpu) 322 if (flush_mycpu)
310 smp_local_flush_tlb(); 323 smp_local_flush_tlb();
311 324
312 for_each_cpu_mask(cpu, cpumask) 325 for_each_cpu_mask(cpu, cpumask)
313 while(counts[cpu] == local_tlb_flush_counts[cpu].count) 326 while(counts[cpu] == local_tlb_flush_counts[cpu].count)
314 udelay(FLUSH_DELAY); 327 udelay(FLUSH_DELAY);
315 328
316 preempt_enable(); 329 preempt_enable();
317 } 330 }
318 331
319 void 332 void
320 smp_flush_tlb_all (void) 333 smp_flush_tlb_all (void)
321 { 334 {
322 on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1, 1); 335 on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1, 1);
323 } 336 }
324 337
325 void 338 void
326 smp_flush_tlb_mm (struct mm_struct *mm) 339 smp_flush_tlb_mm (struct mm_struct *mm)
327 { 340 {
328 preempt_disable(); 341 preempt_disable();
329 /* this happens for the common case of a single-threaded fork(): */ 342 /* this happens for the common case of a single-threaded fork(): */
330 if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1)) 343 if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
331 { 344 {
332 local_finish_flush_tlb_mm(mm); 345 local_finish_flush_tlb_mm(mm);
333 preempt_enable(); 346 preempt_enable();
334 return; 347 return;
335 } 348 }
336 349
337 preempt_enable(); 350 preempt_enable();
338 /* 351 /*
339 * We could optimize this further by using mm->cpu_vm_mask to track which CPUs 352 * We could optimize this further by using mm->cpu_vm_mask to track which CPUs
340 * have been running in the address space. It's not clear that this is worth the 353 * have been running in the address space. It's not clear that this is worth the
341 * trouble though: to avoid races, we have to raise the IPI on the target CPU 354 * trouble though: to avoid races, we have to raise the IPI on the target CPU
342 * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is 355 * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is
343 * rather trivial. 356 * rather trivial.
344 */ 357 */
345 on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1); 358 on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1);
346 } 359 }
347 360
348 /* 361 /*
349 * Run a function on a specific CPU 362 * Run a function on a specific CPU
350 * <func> The function to run. This must be fast and non-blocking. 363 * <func> The function to run. This must be fast and non-blocking.
351 * <info> An arbitrary pointer to pass to the function. 364 * <info> An arbitrary pointer to pass to the function.
352 * <nonatomic> Currently unused. 365 * <nonatomic> Currently unused.
353 * <wait> If true, wait until function has completed on other CPUs. 366 * <wait> If true, wait until function has completed on other CPUs.
354 * [RETURNS] 0 on success, else a negative status code. 367 * [RETURNS] 0 on success, else a negative status code.
355 * 368 *
356 * Does not return until the remote CPU is nearly ready to execute <func> 369 * Does not return until the remote CPU is nearly ready to execute <func>
357 * or is or has executed. 370 * or is or has executed.
358 */ 371 */
359 372
360 int 373 int
361 smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int nonatomic, 374 smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int nonatomic,
362 int wait) 375 int wait)
363 { 376 {
364 struct call_data_struct data; 377 struct call_data_struct data;
365 int cpus = 1; 378 int cpus = 1;
366 int me = get_cpu(); /* prevent preemption and reschedule on another processor */ 379 int me = get_cpu(); /* prevent preemption and reschedule on another processor */
367 380
368 if (cpuid == me) { 381 if (cpuid == me) {
369 local_irq_disable(); 382 local_irq_disable();
370 func(info); 383 func(info);
371 local_irq_enable(); 384 local_irq_enable();
372 put_cpu(); 385 put_cpu();
373 return 0; 386 return 0;
374 } 387 }
375 388
376 data.func = func; 389 data.func = func;
377 data.info = info; 390 data.info = info;
378 atomic_set(&data.started, 0); 391 atomic_set(&data.started, 0);
379 data.wait = wait; 392 data.wait = wait;
380 if (wait) 393 if (wait)
381 atomic_set(&data.finished, 0); 394 atomic_set(&data.finished, 0);
382 395
383 spin_lock_bh(&call_lock); 396 spin_lock_bh(&call_lock);
384 397
385 call_data = &data; 398 call_data = &data;
386 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */ 399 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */
387 send_IPI_single(cpuid, IPI_CALL_FUNC); 400 send_IPI_single(cpuid, IPI_CALL_FUNC);
388 401
389 /* Wait for response */ 402 /* Wait for response */
390 while (atomic_read(&data.started) != cpus) 403 while (atomic_read(&data.started) != cpus)
391 cpu_relax(); 404 cpu_relax();
392 405
393 if (wait) 406 if (wait)
394 while (atomic_read(&data.finished) != cpus) 407 while (atomic_read(&data.finished) != cpus)
395 cpu_relax(); 408 cpu_relax();
396 call_data = NULL; 409 call_data = NULL;
397 410
398 spin_unlock_bh(&call_lock); 411 spin_unlock_bh(&call_lock);
399 put_cpu(); 412 put_cpu();
400 return 0; 413 return 0;
401 } 414 }
402 EXPORT_SYMBOL(smp_call_function_single); 415 EXPORT_SYMBOL(smp_call_function_single);
416
417 /**
418 * smp_call_function_mask(): Run a function on a set of other CPUs.
419 * <mask> The set of cpus to run on. Must not include the current cpu.
420 * <func> The function to run. This must be fast and non-blocking.
421 * <info> An arbitrary pointer to pass to the function.
422 * <wait> If true, wait (atomically) until function
423 * has completed on other CPUs.
424 *
425 * Returns 0 on success, else a negative status code.
426 *
427 * If @wait is true, then returns once @func has returned; otherwise
428 * it returns just before the target cpu calls @func.
429 *
430 * You must not call this function with disabled interrupts or from a
431 * hardware interrupt handler or from a bottom half handler.
432 */
433 int smp_call_function_mask(cpumask_t mask,
434 void (*func)(void *), void *info,
435 int wait)
436 {
437 struct call_data_struct data;
438 cpumask_t allbutself;
439 int cpus;
440
441 spin_lock(&call_lock);
442 allbutself = cpu_online_map;
443 cpu_clear(smp_processor_id(), allbutself);
444
445 cpus_and(mask, mask, allbutself);
446 cpus = cpus_weight(mask);
447 if (!cpus) {
448 spin_unlock(&call_lock);
449 return 0;
450 }
451
452 /* Can deadlock when called with interrupts disabled */
453 WARN_ON(irqs_disabled());
454
455 data.func = func;
456 data.info = info;
457 atomic_set(&data.started, 0);
458 data.wait = wait;
459 if (wait)
460 atomic_set(&data.finished, 0);
461
462 call_data = &data;
463 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC*/
464
465 /* Send a message to other CPUs */
466 if (cpus_equal(mask, allbutself))
467 send_IPI_allbutself(IPI_CALL_FUNC);
468 else
469 send_IPI_mask(mask, IPI_CALL_FUNC);
470
471 /* Wait for response */
472 while (atomic_read(&data.started) != cpus)
473 cpu_relax();
474
475 if (wait)
476 while (atomic_read(&data.finished) != cpus)
477 cpu_relax();
478 call_data = NULL;
479
480 spin_unlock(&call_lock);
481 return 0;
482
483 }
484 EXPORT_SYMBOL(smp_call_function_mask);
403 485
404 /* 486 /*
405 * this function sends a 'generic call function' IPI to all other CPUs 487 * this function sends a 'generic call function' IPI to all other CPUs
406 * in the system. 488 * in the system.
407 */ 489 */
408 490
409 /* 491 /*
410 * [SUMMARY] Run a function on all other CPUs. 492 * [SUMMARY] Run a function on all other CPUs.
411 * <func> The function to run. This must be fast and non-blocking. 493 * <func> The function to run. This must be fast and non-blocking.
412 * <info> An arbitrary pointer to pass to the function. 494 * <info> An arbitrary pointer to pass to the function.
413 * <nonatomic> currently unused. 495 * <nonatomic> currently unused.
414 * <wait> If true, wait (atomically) until function has completed on other CPUs. 496 * <wait> If true, wait (atomically) until function has completed on other CPUs.
415 * [RETURNS] 0 on success, else a negative status code. 497 * [RETURNS] 0 on success, else a negative status code.
416 * 498 *
417 * Does not return until remote CPUs are nearly ready to execute <func> or are or have 499 * Does not return until remote CPUs are nearly ready to execute <func> or are or have
418 * executed. 500 * executed.
419 * 501 *
420 * You must not call this function with disabled interrupts or from a 502 * You must not call this function with disabled interrupts or from a
421 * hardware interrupt handler or from a bottom half handler. 503 * hardware interrupt handler or from a bottom half handler.
422 */ 504 */
423 int 505 int
424 smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait) 506 smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait)
425 { 507 {
426 struct call_data_struct data; 508 struct call_data_struct data;
427 int cpus; 509 int cpus;
428 510
429 spin_lock(&call_lock); 511 spin_lock(&call_lock);
430 cpus = num_online_cpus() - 1; 512 cpus = num_online_cpus() - 1;
431 if (!cpus) { 513 if (!cpus) {
432 spin_unlock(&call_lock); 514 spin_unlock(&call_lock);
433 return 0; 515 return 0;
434 } 516 }
435 517
436 /* Can deadlock when called with interrupts disabled */ 518 /* Can deadlock when called with interrupts disabled */
437 WARN_ON(irqs_disabled()); 519 WARN_ON(irqs_disabled());
438 520
439 data.func = func; 521 data.func = func;
440 data.info = info; 522 data.info = info;
441 atomic_set(&data.started, 0); 523 atomic_set(&data.started, 0);
442 data.wait = wait; 524 data.wait = wait;
443 if (wait) 525 if (wait)
444 atomic_set(&data.finished, 0); 526 atomic_set(&data.finished, 0);
445 527
446 call_data = &data; 528 call_data = &data;
447 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */ 529 mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */
448 send_IPI_allbutself(IPI_CALL_FUNC); 530 send_IPI_allbutself(IPI_CALL_FUNC);
449 531
450 /* Wait for response */ 532 /* Wait for response */
451 while (atomic_read(&data.started) != cpus) 533 while (atomic_read(&data.started) != cpus)
452 cpu_relax(); 534 cpu_relax();
453 535
454 if (wait) 536 if (wait)
455 while (atomic_read(&data.finished) != cpus) 537 while (atomic_read(&data.finished) != cpus)
456 cpu_relax(); 538 cpu_relax();
457 call_data = NULL; 539 call_data = NULL;
458 540
459 spin_unlock(&call_lock); 541 spin_unlock(&call_lock);
460 return 0; 542 return 0;
461 } 543 }
462 EXPORT_SYMBOL(smp_call_function); 544 EXPORT_SYMBOL(smp_call_function);
463 545
464 /* 546 /*
465 * this function calls the 'stop' function on all other CPUs in the system. 547 * this function calls the 'stop' function on all other CPUs in the system.
466 */ 548 */
467 void 549 void
468 smp_send_stop (void) 550 smp_send_stop (void)
469 { 551 {
470 send_IPI_allbutself(IPI_CPU_STOP); 552 send_IPI_allbutself(IPI_CPU_STOP);
471 } 553 }
472 554
473 int 555 int
474 setup_profiling_timer (unsigned int multiplier) 556 setup_profiling_timer (unsigned int multiplier)
475 { 557 {
476 return -EINVAL; 558 return -EINVAL;
477 } 559 }
478 560
include/asm-ia64/smp.h
Diff comments   View file @ 31a6b11
1 /* 1 /*
2 * SMP Support 2 * SMP Support
3 * 3 *
4 * Copyright (C) 1999 VA Linux Systems 4 * Copyright (C) 1999 VA Linux Systems
5 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 5 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
6 * (c) Copyright 2001-2003, 2005 Hewlett-Packard Development Company, L.P. 6 * (c) Copyright 2001-2003, 2005 Hewlett-Packard Development Company, L.P.
7 * David Mosberger-Tang <davidm@hpl.hp.com> 7 * David Mosberger-Tang <davidm@hpl.hp.com>
8 * Bjorn Helgaas <bjorn.helgaas@hp.com> 8 * Bjorn Helgaas <bjorn.helgaas@hp.com>
9 */ 9 */
10 #ifndef _ASM_IA64_SMP_H 10 #ifndef _ASM_IA64_SMP_H
11 #define _ASM_IA64_SMP_H 11 #define _ASM_IA64_SMP_H
12 12
13 #include <linux/init.h> 13 #include <linux/init.h>
14 #include <linux/threads.h> 14 #include <linux/threads.h>
15 #include <linux/kernel.h> 15 #include <linux/kernel.h>
16 #include <linux/cpumask.h> 16 #include <linux/cpumask.h>
17 #include <linux/bitops.h> 17 #include <linux/bitops.h>
18 18
19 #include <asm/io.h> 19 #include <asm/io.h>
20 #include <asm/param.h> 20 #include <asm/param.h>
21 #include <asm/processor.h> 21 #include <asm/processor.h>
22 #include <asm/ptrace.h> 22 #include <asm/ptrace.h>
23 23
24 static inline unsigned int 24 static inline unsigned int
25 ia64_get_lid (void) 25 ia64_get_lid (void)
26 { 26 {
27 union { 27 union {
28 struct { 28 struct {
29 unsigned long reserved : 16; 29 unsigned long reserved : 16;
30 unsigned long eid : 8; 30 unsigned long eid : 8;
31 unsigned long id : 8; 31 unsigned long id : 8;
32 unsigned long ignored : 32; 32 unsigned long ignored : 32;
33 } f; 33 } f;
34 unsigned long bits; 34 unsigned long bits;
35 } lid; 35 } lid;
36 36
37 lid.bits = ia64_getreg(_IA64_REG_CR_LID); 37 lid.bits = ia64_getreg(_IA64_REG_CR_LID);
38 return lid.f.id << 8 | lid.f.eid; 38 return lid.f.id << 8 | lid.f.eid;
39 } 39 }
40 40
41 extern int smp_call_function_mask(cpumask_t mask, void (*func)(void *),
42 void *info, int wait);
43
41 #define hard_smp_processor_id() ia64_get_lid() 44 #define hard_smp_processor_id() ia64_get_lid()
42 45
43 #ifdef CONFIG_SMP 46 #ifdef CONFIG_SMP
44 47
45 #define XTP_OFFSET 0x1e0008 48 #define XTP_OFFSET 0x1e0008
46 49
47 #define SMP_IRQ_REDIRECTION (1 << 0) 50 #define SMP_IRQ_REDIRECTION (1 << 0)
48 #define SMP_IPI_REDIRECTION (1 << 1) 51 #define SMP_IPI_REDIRECTION (1 << 1)
49 52
50 #define raw_smp_processor_id() (current_thread_info()->cpu) 53 #define raw_smp_processor_id() (current_thread_info()->cpu)
51 54
52 extern struct smp_boot_data { 55 extern struct smp_boot_data {
53 int cpu_count; 56 int cpu_count;
54 int cpu_phys_id[NR_CPUS]; 57 int cpu_phys_id[NR_CPUS];
55 } smp_boot_data __initdata; 58 } smp_boot_data __initdata;
56 59
57 extern char no_int_routing __devinitdata; 60 extern char no_int_routing __devinitdata;
58 61
59 extern cpumask_t cpu_online_map; 62 extern cpumask_t cpu_online_map;
60 extern cpumask_t cpu_core_map[NR_CPUS]; 63 extern cpumask_t cpu_core_map[NR_CPUS];
61 DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); 64 DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
62 extern int smp_num_siblings; 65 extern int smp_num_siblings;
63 extern void __iomem *ipi_base_addr; 66 extern void __iomem *ipi_base_addr;
64 extern unsigned char smp_int_redirect; 67 extern unsigned char smp_int_redirect;
65 68
66 extern volatile int ia64_cpu_to_sapicid[]; 69 extern volatile int ia64_cpu_to_sapicid[];
67 #define cpu_physical_id(i) ia64_cpu_to_sapicid[i] 70 #define cpu_physical_id(i) ia64_cpu_to_sapicid[i]
68 71
69 extern unsigned long ap_wakeup_vector; 72 extern unsigned long ap_wakeup_vector;
70 73
71 /* 74 /*
72 * Function to map hard smp processor id to logical id. Slow, so don't use this in 75 * Function to map hard smp processor id to logical id. Slow, so don't use this in
73 * performance-critical code. 76 * performance-critical code.
74 */ 77 */
75 static inline int 78 static inline int
76 cpu_logical_id (int cpuid) 79 cpu_logical_id (int cpuid)
77 { 80 {
78 int i; 81 int i;
79 82
80 for (i = 0; i < NR_CPUS; ++i) 83 for (i = 0; i < NR_CPUS; ++i)
81 if (cpu_physical_id(i) == cpuid) 84 if (cpu_physical_id(i) == cpuid)
82 break; 85 break;
83 return i; 86 return i;
84 } 87 }
85 88
86 /* 89 /*
87 * XTP control functions: 90 * XTP control functions:
88 * min_xtp : route all interrupts to this CPU 91 * min_xtp : route all interrupts to this CPU
89 * normal_xtp: nominal XTP value 92 * normal_xtp: nominal XTP value
90 * max_xtp : never deliver interrupts to this CPU. 93 * max_xtp : never deliver interrupts to this CPU.
91 */ 94 */
92 95
93 static inline void 96 static inline void
94 min_xtp (void) 97 min_xtp (void)
95 { 98 {
96 if (smp_int_redirect & SMP_IRQ_REDIRECTION) 99 if (smp_int_redirect & SMP_IRQ_REDIRECTION)
97 writeb(0x00, ipi_base_addr + XTP_OFFSET); /* XTP to min */ 100 writeb(0x00, ipi_base_addr + XTP_OFFSET); /* XTP to min */
98 } 101 }
99 102
100 static inline void 103 static inline void
101 normal_xtp (void) 104 normal_xtp (void)
102 { 105 {
103 if (smp_int_redirect & SMP_IRQ_REDIRECTION) 106 if (smp_int_redirect & SMP_IRQ_REDIRECTION)
104 writeb(0x08, ipi_base_addr + XTP_OFFSET); /* XTP normal */ 107 writeb(0x08, ipi_base_addr + XTP_OFFSET); /* XTP normal */
105 } 108 }
106 109
107 static inline void 110 static inline void
108 max_xtp (void) 111 max_xtp (void)
109 { 112 {
110 if (smp_int_redirect & SMP_IRQ_REDIRECTION) 113 if (smp_int_redirect & SMP_IRQ_REDIRECTION)
111 writeb(0x0f, ipi_base_addr + XTP_OFFSET); /* Set XTP to max */ 114 writeb(0x0f, ipi_base_addr + XTP_OFFSET); /* Set XTP to max */
112 } 115 }
113 116
114 /* Upping and downing of CPUs */ 117 /* Upping and downing of CPUs */
115 extern int __cpu_disable (void); 118 extern int __cpu_disable (void);
116 extern void __cpu_die (unsigned int cpu); 119 extern void __cpu_die (unsigned int cpu);
117 extern void cpu_die (void) __attribute__ ((noreturn)); 120 extern void cpu_die (void) __attribute__ ((noreturn));
118 extern void __init smp_build_cpu_map(void); 121 extern void __init smp_build_cpu_map(void);
119 122
120 extern void __init init_smp_config (void); 123 extern void __init init_smp_config (void);
121 extern void smp_do_timer (struct pt_regs *regs); 124 extern void smp_do_timer (struct pt_regs *regs);
122 125
123 extern void smp_send_reschedule (int cpu); 126 extern void smp_send_reschedule (int cpu);
124 extern void lock_ipi_calllock(void); 127 extern void lock_ipi_calllock(void);
125 extern void unlock_ipi_calllock(void); 128 extern void unlock_ipi_calllock(void);
126 extern void identify_siblings (struct cpuinfo_ia64 *); 129 extern void identify_siblings (struct cpuinfo_ia64 *);
127 extern int is_multithreading_enabled(void); 130 extern int is_multithreading_enabled(void);
128 131
129 #else /* CONFIG_SMP */ 132 #else /* CONFIG_SMP */
130 133
131 #define cpu_logical_id(i) 0 134 #define cpu_logical_id(i) 0
132 #define cpu_physical_id(i) ia64_get_lid() 135 #define cpu_physical_id(i) ia64_get_lid()
133 136
134 #endif /* CONFIG_SMP */ 137 #endif /* CONFIG_SMP */
135 #endif /* _ASM_IA64_SMP_H */ 138 #endif /* _ASM_IA64_SMP_H */
136 139