Blame view
arch/mips/kernel/smp.c
10.2 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 |
/* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) 2000, 2001 Kanoj Sarcar * Copyright (C) 2000, 2001 Ralf Baechle * Copyright (C) 2000, 2001 Silicon Graphics, Inc. * Copyright (C) 2000, 2001, 2003 Broadcom Corporation */ #include <linux/cache.h> #include <linux/delay.h> #include <linux/init.h> #include <linux/interrupt.h> |
631330f58 MIPS: Build fix -... |
25 |
#include <linux/smp.h> |
1da177e4c Linux-2.6.12-rc2 |
26 27 28 29 30 31 32 |
#include <linux/spinlock.h> #include <linux/threads.h> #include <linux/module.h> #include <linux/time.h> #include <linux/timex.h> #include <linux/sched.h> #include <linux/cpumask.h> |
1e35aabab [MIPS] Add topolo... |
33 |
#include <linux/cpu.h> |
4e950f6f0 Remove fs.h from ... |
34 |
#include <linux/err.h> |
8f99a1626 MIPS: Tracing: Ad... |
35 |
#include <linux/ftrace.h> |
1da177e4c Linux-2.6.12-rc2 |
36 |
|
60063497a atomic: use <linu... |
37 |
#include <linux/atomic.h> |
1da177e4c Linux-2.6.12-rc2 |
38 39 |
#include <asm/cpu.h> #include <asm/processor.h> |
39b8d5254 [MIPS] Add suppor... |
40 |
#include <asm/r4k-timer.h> |
1da177e4c Linux-2.6.12-rc2 |
41 42 |
#include <asm/system.h> #include <asm/mmu_context.h> |
7bcf7717b [MIPS] Implement ... |
43 |
#include <asm/time.h> |
1da177e4c Linux-2.6.12-rc2 |
44 |
|
41c594ab6 [MIPS] MT: Improv... |
45 46 47 |
#ifdef CONFIG_MIPS_MT_SMTC #include <asm/mipsmtregs.h> #endif /* CONFIG_MIPS_MT_SMTC */ |
1b2bc75c1 MIPS: Add arch ge... |
48 |
volatile cpumask_t cpu_callin_map; /* Bitmask of started secondaries */ |
2dc2ae344 MIPS: Export __cp... |
49 |
|
1da177e4c Linux-2.6.12-rc2 |
50 |
int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ |
2dc2ae344 MIPS: Export __cp... |
51 |
EXPORT_SYMBOL(__cpu_number_map); |
1da177e4c Linux-2.6.12-rc2 |
52 |
int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ |
2dc2ae344 MIPS: Export __cp... |
53 |
EXPORT_SYMBOL(__cpu_logical_map); |
1da177e4c Linux-2.6.12-rc2 |
54 |
|
0ab7aefc4 [MIPS] MT: Schedu... |
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
/* Number of TCs (or siblings in Intel speak) per CPU core */ int smp_num_siblings = 1; EXPORT_SYMBOL(smp_num_siblings); /* representing the TCs (or siblings in Intel speak) of each logical CPU */ cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; EXPORT_SYMBOL(cpu_sibling_map); /* representing cpus for which sibling maps can be computed */ static cpumask_t cpu_sibling_setup_map; static inline void set_cpu_sibling_map(int cpu) { int i; cpu_set(cpu, cpu_sibling_setup_map); if (smp_num_siblings > 1) { for_each_cpu_mask(i, cpu_sibling_setup_map) { if (cpu_data[cpu].core == cpu_data[i].core) { cpu_set(i, cpu_sibling_map[cpu]); cpu_set(cpu, cpu_sibling_map[i]); } } } else cpu_set(cpu, cpu_sibling_map[cpu]); } |
87353d8ac [MIPS] SMP: Call ... |
82 83 84 85 |
struct plat_smp_ops *mp_ops; __cpuinit void register_smp_ops(struct plat_smp_ops *ops) { |
83738e307 [MIPS] fix warnin... |
86 87 88 |
if (mp_ops) printk(KERN_WARNING "Overriding previously set SMP ops "); |
87353d8ac [MIPS] SMP: Call ... |
89 90 91 |
mp_ops = ops; } |
1da177e4c Linux-2.6.12-rc2 |
92 93 94 95 |
/* * First C code run on the secondary CPUs after being started up by * the master. */ |
4ebd5233f [MIPS] Fix modpos... |
96 |
asmlinkage __cpuinit void start_secondary(void) |
1da177e4c Linux-2.6.12-rc2 |
97 |
{ |
5bfb5d690 [PATCH] sched: di... |
98 |
unsigned int cpu; |
1da177e4c Linux-2.6.12-rc2 |
99 |
|
41c594ab6 [MIPS] MT: Improv... |
100 101 102 103 |
#ifdef CONFIG_MIPS_MT_SMTC /* Only do cpu_probe for first TC of CPU */ if ((read_c0_tcbind() & TCBIND_CURTC) == 0) #endif /* CONFIG_MIPS_MT_SMTC */ |
1da177e4c Linux-2.6.12-rc2 |
104 105 106 |
cpu_probe(); cpu_report(); per_cpu_trap_init(); |
7bcf7717b [MIPS] Implement ... |
107 |
mips_clockevent_init(); |
87353d8ac [MIPS] SMP: Call ... |
108 |
mp_ops->init_secondary(); |
1da177e4c Linux-2.6.12-rc2 |
109 110 111 112 113 114 115 |
/* * XXX parity protection should be folded in here when it's converted * to an option instead of something based on .cputype */ calibrate_delay(); |
5bfb5d690 [PATCH] sched: di... |
116 117 |
preempt_disable(); cpu = smp_processor_id(); |
1da177e4c Linux-2.6.12-rc2 |
118 |
cpu_data[cpu].udelay_val = loops_per_jiffy; |
e545a6140 kernel/cpu.c: cre... |
119 |
notify_cpu_starting(cpu); |
87353d8ac [MIPS] SMP: Call ... |
120 |
mp_ops->smp_finish(); |
0ab7aefc4 [MIPS] MT: Schedu... |
121 |
set_cpu_sibling_map(cpu); |
1da177e4c Linux-2.6.12-rc2 |
122 123 |
cpu_set(cpu, cpu_callin_map); |
39b8d5254 [MIPS] Add suppor... |
124 |
synchronise_count_slave(); |
1da177e4c Linux-2.6.12-rc2 |
125 126 |
cpu_idle(); } |
2f304c0a0 mips: convert to ... |
127 128 129 |
/* * Call into both interrupt handlers, as we share the IPI for them */ |
8f99a1626 MIPS: Tracing: Ad... |
130 |
void __irq_entry smp_call_function_interrupt(void) |
1da177e4c Linux-2.6.12-rc2 |
131 |
{ |
1da177e4c Linux-2.6.12-rc2 |
132 |
irq_enter(); |
2f304c0a0 mips: convert to ... |
133 134 |
generic_smp_call_function_single_interrupt(); generic_smp_call_function_interrupt(); |
1da177e4c Linux-2.6.12-rc2 |
135 |
irq_exit(); |
b4b2917cc [MIPS] Add smp_ca... |
136 |
} |
1da177e4c Linux-2.6.12-rc2 |
137 138 139 140 141 142 |
static void stop_this_cpu(void *dummy) { /* * Remove this CPU: */ cpu_clear(smp_processor_id(), cpu_online_map); |
7920c4d65 MIPS: SMP: Don't ... |
143 144 145 146 |
for (;;) { if (cpu_wait) (*cpu_wait)(); /* Wait if available. */ } |
1da177e4c Linux-2.6.12-rc2 |
147 148 149 150 |
} void smp_send_stop(void) { |
8691e5a8f smp_call_function... |
151 |
smp_call_function(stop_this_cpu, NULL, 0); |
1da177e4c Linux-2.6.12-rc2 |
152 153 154 155 |
} void __init smp_cpus_done(unsigned int max_cpus) { |
87353d8ac [MIPS] SMP: Call ... |
156 |
mp_ops->cpus_done(); |
39b8d5254 [MIPS] Add suppor... |
157 |
synchronise_count_master(); |
1da177e4c Linux-2.6.12-rc2 |
158 159 160 161 162 |
} /* called from main before smp_init() */ void __init smp_prepare_cpus(unsigned int max_cpus) { |
1da177e4c Linux-2.6.12-rc2 |
163 164 |
init_new_context(current, &init_mm); current_thread_info()->cpu = 0; |
87353d8ac [MIPS] SMP: Call ... |
165 |
mp_ops->prepare_cpus(max_cpus); |
0ab7aefc4 [MIPS] MT: Schedu... |
166 |
set_cpu_sibling_map(0); |
320e6aba2 [MIPS] Fix SMP no... |
167 |
#ifndef CONFIG_HOTPLUG_CPU |
4037ac6e2 cpumask: Use acce... |
168 |
init_cpu_present(&cpu_possible_map); |
320e6aba2 [MIPS] Fix SMP no... |
169 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
170 171 172 173 174 |
} /* preload SMP state for boot cpu */ void __devinit smp_prepare_boot_cpu(void) { |
4037ac6e2 cpumask: Use acce... |
175 176 |
set_cpu_possible(0, true); set_cpu_online(0, true); |
1da177e4c Linux-2.6.12-rc2 |
177 178 179 180 |
cpu_set(0, cpu_callin_map); } /* |
b727a6025 Merge do_boot_cpu... |
181 182 183 |
* Called once for each "cpu_possible(cpu)". Needs to spin up the cpu * and keep control until "cpu_online(cpu)" is set. Note: cpu is * physical, not logical. |
1da177e4c Linux-2.6.12-rc2 |
184 |
*/ |
1b2bc75c1 MIPS: Add arch ge... |
185 |
static struct task_struct *cpu_idle_thread[NR_CPUS]; |
6667deb69 MIPS: Move idle t... |
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
struct create_idle { struct work_struct work; struct task_struct *idle; struct completion done; int cpu; }; static void __cpuinit do_fork_idle(struct work_struct *work) { struct create_idle *c_idle = container_of(work, struct create_idle, work); c_idle->idle = fork_idle(c_idle->cpu); complete(&c_idle->done); } |
b282b6f8a [PATCH] Change cp... |
201 |
int __cpuinit __cpu_up(unsigned int cpu) |
1da177e4c Linux-2.6.12-rc2 |
202 203 204 205 |
{ struct task_struct *idle; /* |
b727a6025 Merge do_boot_cpu... |
206 |
* Processor goes to start_secondary(), sets online flag |
1da177e4c Linux-2.6.12-rc2 |
207 208 209 |
* The following code is purely to make sure * Linux can schedule processes on this slave. */ |
1b2bc75c1 MIPS: Add arch ge... |
210 |
if (!cpu_idle_thread[cpu]) { |
6667deb69 MIPS: Move idle t... |
211 212 213 214 215 216 217 218 219 220 221 222 223 |
/* * Schedule work item to avoid forking user task * Ported from arch/x86/kernel/smpboot.c */ struct create_idle c_idle = { .cpu = cpu, .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done), }; INIT_WORK_ONSTACK(&c_idle.work, do_fork_idle); schedule_work(&c_idle.work); wait_for_completion(&c_idle.done); idle = cpu_idle_thread[cpu] = c_idle.idle; |
1b2bc75c1 MIPS: Add arch ge... |
224 225 226 227 228 229 230 |
if (IS_ERR(idle)) panic(KERN_ERR "Fork failed for CPU %d", cpu); } else { idle = cpu_idle_thread[cpu]; init_idle(idle, cpu); } |
1da177e4c Linux-2.6.12-rc2 |
231 |
|
87353d8ac [MIPS] SMP: Call ... |
232 |
mp_ops->boot_secondary(cpu, idle); |
1da177e4c Linux-2.6.12-rc2 |
233 |
|
b727a6025 Merge do_boot_cpu... |
234 235 236 |
/* * Trust is futile. We should really have timeouts ... */ |
1da177e4c Linux-2.6.12-rc2 |
237 238 239 240 241 242 243 |
while (!cpu_isset(cpu, cpu_callin_map)) udelay(100); cpu_set(cpu, cpu_online_map); return 0; } |
1da177e4c Linux-2.6.12-rc2 |
244 245 246 247 248 249 250 251 252 253 254 255 256 |
/* Not really SMP stuff ... */ int setup_profiling_timer(unsigned int multiplier) { return 0; } static void flush_tlb_all_ipi(void *info) { local_flush_tlb_all(); } void flush_tlb_all(void) { |
15c8b6c1a on_each_cpu(): ki... |
257 |
on_each_cpu(flush_tlb_all_ipi, NULL, 1); |
1da177e4c Linux-2.6.12-rc2 |
258 259 260 261 262 263 264 265 |
} static void flush_tlb_mm_ipi(void *mm) { local_flush_tlb_mm((struct mm_struct *)mm); } /* |
25969354a [MIPS] Avoid inte... |
266 267 268 269 270 271 272 273 274 275 276 |
* Special Variant of smp_call_function for use by TLB functions: * * o No return value * o collapses to normal function call on UP kernels * o collapses to normal function call on systems with a single shared * primary cache. * o CONFIG_MIPS_MT_SMTC currently implies there is only one physical core. */ static inline void smp_on_other_tlbs(void (*func) (void *info), void *info) { #ifndef CONFIG_MIPS_MT_SMTC |
8691e5a8f smp_call_function... |
277 |
smp_call_function(func, info, 1); |
25969354a [MIPS] Avoid inte... |
278 279 280 281 282 283 284 285 286 287 288 289 290 291 |
#endif } static inline void smp_on_each_tlb(void (*func) (void *info), void *info) { preempt_disable(); smp_on_other_tlbs(func, info); func(info); preempt_enable(); } /* |
1da177e4c Linux-2.6.12-rc2 |
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 |
* The following tlb flush calls are invoked when old translations are * being torn down, or pte attributes are changing. For single threaded * address spaces, a new context is obtained on the current cpu, and tlb * context on other cpus are invalidated to force a new context allocation * at switch_mm time, should the mm ever be used on other cpus. For * multithreaded address spaces, intercpu interrupts have to be sent. * Another case where intercpu interrupts are required is when the target * mm might be active on another cpu (eg debuggers doing the flushes on * behalf of debugees, kswapd stealing pages from another process etc). * Kanoj 07/00. */ void flush_tlb_mm(struct mm_struct *mm) { preempt_disable(); if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { |
c50cade95 [MIPS] SMP: Kill ... |
309 |
smp_on_other_tlbs(flush_tlb_mm_ipi, mm); |
1da177e4c Linux-2.6.12-rc2 |
310 |
} else { |
b5eb55114 [MIPS] Kill num_o... |
311 312 313 314 |
cpumask_t mask = cpu_online_map; unsigned int cpu; cpu_clear(smp_processor_id(), mask); |
ece8a9e4f [MIPS] SMP: Fix u... |
315 |
for_each_cpu_mask(cpu, mask) |
b5eb55114 [MIPS] Kill num_o... |
316 317 |
if (cpu_context(cpu, mm)) cpu_context(cpu, mm) = 0; |
1da177e4c Linux-2.6.12-rc2 |
318 319 320 321 322 323 324 325 326 327 328 329 330 331 |
} local_flush_tlb_mm(mm); preempt_enable(); } struct flush_tlb_data { struct vm_area_struct *vma; unsigned long addr1; unsigned long addr2; }; static void flush_tlb_range_ipi(void *info) { |
c50cade95 [MIPS] SMP: Kill ... |
332 |
struct flush_tlb_data *fd = info; |
1da177e4c Linux-2.6.12-rc2 |
333 334 335 336 337 338 339 340 341 342 |
local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { struct mm_struct *mm = vma->vm_mm; preempt_disable(); if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { |
89a8a5a6c [MIPS] SMP: Use I... |
343 344 345 346 347 |
struct flush_tlb_data fd = { .vma = vma, .addr1 = start, .addr2 = end, }; |
1da177e4c Linux-2.6.12-rc2 |
348 |
|
c50cade95 [MIPS] SMP: Kill ... |
349 |
smp_on_other_tlbs(flush_tlb_range_ipi, &fd); |
1da177e4c Linux-2.6.12-rc2 |
350 |
} else { |
b5eb55114 [MIPS] Kill num_o... |
351 352 353 354 |
cpumask_t mask = cpu_online_map; unsigned int cpu; cpu_clear(smp_processor_id(), mask); |
ece8a9e4f [MIPS] SMP: Fix u... |
355 |
for_each_cpu_mask(cpu, mask) |
b5eb55114 [MIPS] Kill num_o... |
356 357 |
if (cpu_context(cpu, mm)) cpu_context(cpu, mm) = 0; |
1da177e4c Linux-2.6.12-rc2 |
358 359 360 361 362 363 364 |
} local_flush_tlb_range(vma, start, end); preempt_enable(); } static void flush_tlb_kernel_range_ipi(void *info) { |
c50cade95 [MIPS] SMP: Kill ... |
365 |
struct flush_tlb_data *fd = info; |
1da177e4c Linux-2.6.12-rc2 |
366 367 368 369 370 371 |
local_flush_tlb_kernel_range(fd->addr1, fd->addr2); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) { |
89a8a5a6c [MIPS] SMP: Use I... |
372 373 374 375 |
struct flush_tlb_data fd = { .addr1 = start, .addr2 = end, }; |
1da177e4c Linux-2.6.12-rc2 |
376 |
|
15c8b6c1a on_each_cpu(): ki... |
377 |
on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); |
1da177e4c Linux-2.6.12-rc2 |
378 379 380 381 |
} static void flush_tlb_page_ipi(void *info) { |
c50cade95 [MIPS] SMP: Kill ... |
382 |
struct flush_tlb_data *fd = info; |
1da177e4c Linux-2.6.12-rc2 |
383 384 385 386 387 388 389 390 |
local_flush_tlb_page(fd->vma, fd->addr1); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { preempt_disable(); if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { |
89a8a5a6c [MIPS] SMP: Use I... |
391 392 393 394 |
struct flush_tlb_data fd = { .vma = vma, .addr1 = page, }; |
1da177e4c Linux-2.6.12-rc2 |
395 |
|
c50cade95 [MIPS] SMP: Kill ... |
396 |
smp_on_other_tlbs(flush_tlb_page_ipi, &fd); |
1da177e4c Linux-2.6.12-rc2 |
397 |
} else { |
b5eb55114 [MIPS] Kill num_o... |
398 399 400 401 |
cpumask_t mask = cpu_online_map; unsigned int cpu; cpu_clear(smp_processor_id(), mask); |
ece8a9e4f [MIPS] SMP: Fix u... |
402 |
for_each_cpu_mask(cpu, mask) |
b5eb55114 [MIPS] Kill num_o... |
403 404 |
if (cpu_context(cpu, vma->vm_mm)) cpu_context(cpu, vma->vm_mm) = 0; |
1da177e4c Linux-2.6.12-rc2 |
405 406 407 408 409 410 411 412 413 414 415 416 417 418 |
} local_flush_tlb_page(vma, page); preempt_enable(); } static void flush_tlb_one_ipi(void *info) { unsigned long vaddr = (unsigned long) info; local_flush_tlb_one(vaddr); } void flush_tlb_one(unsigned long vaddr) { |
25969354a [MIPS] Avoid inte... |
419 |
smp_on_each_tlb(flush_tlb_one_ipi, (void *) vaddr); |
1da177e4c Linux-2.6.12-rc2 |
420 421 422 423 |
} EXPORT_SYMBOL(flush_tlb_page); EXPORT_SYMBOL(flush_tlb_one); |