Commit 184748cc50b2dceb8287f9fb657eda48ff8fcfe7
Committed by
Ingo Molnar
1 parent
a4c98f8bbe
Exists in
master
and in
20 other branches
sched: Provide scheduler_ipi() callback in response to smp_send_reschedule()
For future rework of try_to_wake_up() we'd like to push part of that function onto the CPU the task is actually going to run on. In order to do so we need a generic callback from the existing scheduler IPI. This patch introduces such a generic callback: scheduler_ipi() and implements it as a NOP. BenH notes: PowerPC might use this IPI on offline CPUs under rare conditions! Acked-by: Russell King <rmk+kernel@arm.linux.org.uk> Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> Acked-by: Chris Metcalf <cmetcalf@tilera.com> Acked-by: Jesper Nilsson <jesper.nilsson@axis.com> Acked-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> Signed-off-by: Ralf Baechle <ralf@linux-mips.org> Reviewed-by: Frank Rowand <frank.rowand@am.sony.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Nick Piggin <npiggin@kernel.dk> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/20110405152728.744338123@chello.nl
Showing 26 changed files with 63 additions and 50 deletions Side-by-side Diff
- arch/alpha/kernel/smp.c
- arch/arm/kernel/smp.c
- arch/blackfin/mach-common/smp.c
- arch/cris/arch-v32/kernel/smp.c
- arch/ia64/kernel/irq_ia64.c
- arch/ia64/xen/irq_xen.c
- arch/m32r/kernel/smp.c
- arch/mips/cavium-octeon/smp.c
- arch/mips/kernel/smtc.c
- arch/mips/mti-malta/malta-int.c
- arch/mips/pmc-sierra/yosemite/smp.c
- arch/mips/sgi-ip27/ip27-irq.c
- arch/mips/sibyte/bcm1480/smp.c
- arch/mips/sibyte/sb1250/smp.c
- arch/mn10300/kernel/smp.c
- arch/parisc/kernel/smp.c
- arch/powerpc/kernel/smp.c
- arch/s390/kernel/smp.c
- arch/sh/kernel/smp.c
- arch/sparc/kernel/smp_32.c
- arch/sparc/kernel/smp_64.c
- arch/tile/kernel/smp.c
- arch/um/kernel/smp.c
- arch/x86/kernel/smp.c
- arch/x86/xen/smp.c
- include/linux/sched.h
arch/alpha/kernel/smp.c
arch/arm/kernel/smp.c
arch/blackfin/mach-common/smp.c
... | ... | @@ -164,6 +164,9 @@ |
164 | 164 | while (msg_queue->count) { |
165 | 165 | msg = &msg_queue->ipi_message[msg_queue->head]; |
166 | 166 | switch (msg->type) { |
167 | + case BFIN_IPI_RESCHEDULE: | |
168 | + scheduler_ipi(); | |
169 | + break; | |
167 | 170 | case BFIN_IPI_CALL_FUNC: |
168 | 171 | spin_unlock_irqrestore(&msg_queue->lock, flags); |
169 | 172 | ipi_call_function(cpu, msg); |
arch/cris/arch-v32/kernel/smp.c
... | ... | @@ -342,15 +342,18 @@ |
342 | 342 | |
343 | 343 | ipi = REG_RD(intr_vect, irq_regs[smp_processor_id()], rw_ipi); |
344 | 344 | |
345 | + if (ipi.vector & IPI_SCHEDULE) { | |
346 | + scheduler_ipi(); | |
347 | + } | |
345 | 348 | if (ipi.vector & IPI_CALL) { |
346 | - func(info); | |
349 | + func(info); | |
347 | 350 | } |
348 | 351 | if (ipi.vector & IPI_FLUSH_TLB) { |
349 | - if (flush_mm == FLUSH_ALL) | |
350 | - __flush_tlb_all(); | |
351 | - else if (flush_vma == FLUSH_ALL) | |
352 | + if (flush_mm == FLUSH_ALL) | |
353 | + __flush_tlb_all(); | |
354 | + else if (flush_vma == FLUSH_ALL) | |
352 | 355 | __flush_tlb_mm(flush_mm); |
353 | - else | |
356 | + else | |
354 | 357 | __flush_tlb_page(flush_vma, flush_addr); |
355 | 358 | } |
356 | 359 |
arch/ia64/kernel/irq_ia64.c
... | ... | @@ -31,6 +31,7 @@ |
31 | 31 | #include <linux/irq.h> |
32 | 32 | #include <linux/ratelimit.h> |
33 | 33 | #include <linux/acpi.h> |
34 | +#include <linux/sched.h> | |
34 | 35 | |
35 | 36 | #include <asm/delay.h> |
36 | 37 | #include <asm/intrinsics.h> |
... | ... | @@ -496,6 +497,7 @@ |
496 | 497 | smp_local_flush_tlb(); |
497 | 498 | kstat_incr_irqs_this_cpu(irq, desc); |
498 | 499 | } else if (unlikely(IS_RESCHEDULE(vector))) { |
500 | + scheduler_ipi(); | |
499 | 501 | kstat_incr_irqs_this_cpu(irq, desc); |
500 | 502 | } else { |
501 | 503 | ia64_setreg(_IA64_REG_CR_TPR, vector); |
arch/ia64/xen/irq_xen.c
... | ... | @@ -92,6 +92,8 @@ |
92 | 92 | static int xen_slab_ready; |
93 | 93 | |
94 | 94 | #ifdef CONFIG_SMP |
95 | +#include <linux/sched.h> | |
96 | + | |
95 | 97 | /* Dummy stub. Though we may check XEN_RESCHEDULE_VECTOR before __do_IRQ, |
96 | 98 | * it ends up to issue several memory accesses upon percpu data and |
97 | 99 | * thus adds unnecessary traffic to other paths. |
98 | 100 | |
... | ... | @@ -99,7 +101,13 @@ |
99 | 101 | static irqreturn_t |
100 | 102 | xen_dummy_handler(int irq, void *dev_id) |
101 | 103 | { |
104 | + return IRQ_HANDLED; | |
105 | +} | |
102 | 106 | |
107 | +static irqreturn_t | |
108 | +xen_resched_handler(int irq, void *dev_id) | |
109 | +{ | |
110 | + scheduler_ipi(); | |
103 | 111 | return IRQ_HANDLED; |
104 | 112 | } |
105 | 113 | |
... | ... | @@ -110,7 +118,7 @@ |
110 | 118 | }; |
111 | 119 | |
112 | 120 | static struct irqaction xen_resched_irqaction = { |
113 | - .handler = xen_dummy_handler, | |
121 | + .handler = xen_resched_handler, | |
114 | 122 | .flags = IRQF_DISABLED, |
115 | 123 | .name = "resched" |
116 | 124 | }; |
arch/m32r/kernel/smp.c
... | ... | @@ -122,8 +122,6 @@ |
122 | 122 | * |
123 | 123 | * Description: This routine executes on CPU which received |
124 | 124 | * 'RESCHEDULE_IPI'. |
125 | - * Rescheduling is processed at the exit of interrupt | |
126 | - * operation. | |
127 | 125 | * |
128 | 126 | * Born on Date: 2002.02.05 |
129 | 127 | * |
... | ... | @@ -138,7 +136,7 @@ |
138 | 136 | *==========================================================================*/ |
139 | 137 | void smp_reschedule_interrupt(void) |
140 | 138 | { |
141 | - /* nothing to do */ | |
139 | + scheduler_ipi(); | |
142 | 140 | } |
143 | 141 | |
144 | 142 | /*==========================================================================* |
arch/mips/cavium-octeon/smp.c
arch/mips/kernel/smtc.c
arch/mips/mti-malta/malta-int.c
arch/mips/pmc-sierra/yosemite/smp.c
... | ... | @@ -55,6 +55,8 @@ |
55 | 55 | |
56 | 56 | if (status & 0x2) |
57 | 57 | smp_call_function_interrupt(); |
58 | + if (status & 0x4) | |
59 | + scheduler_ipi(); | |
58 | 60 | break; |
59 | 61 | |
60 | 62 | case 1: |
... | ... | @@ -63,6 +65,8 @@ |
63 | 65 | |
64 | 66 | if (status & 0x2) |
65 | 67 | smp_call_function_interrupt(); |
68 | + if (status & 0x4) | |
69 | + scheduler_ipi(); | |
66 | 70 | break; |
67 | 71 | } |
68 | 72 | } |
arch/mips/sgi-ip27/ip27-irq.c
... | ... | @@ -147,8 +147,10 @@ |
147 | 147 | #ifdef CONFIG_SMP |
148 | 148 | if (pend0 & (1UL << CPU_RESCHED_A_IRQ)) { |
149 | 149 | LOCAL_HUB_CLR_INTR(CPU_RESCHED_A_IRQ); |
150 | + scheduler_ipi(); | |
150 | 151 | } else if (pend0 & (1UL << CPU_RESCHED_B_IRQ)) { |
151 | 152 | LOCAL_HUB_CLR_INTR(CPU_RESCHED_B_IRQ); |
153 | + scheduler_ipi(); | |
152 | 154 | } else if (pend0 & (1UL << CPU_CALL_A_IRQ)) { |
153 | 155 | LOCAL_HUB_CLR_INTR(CPU_CALL_A_IRQ); |
154 | 156 | smp_call_function_interrupt(); |
arch/mips/sibyte/bcm1480/smp.c
... | ... | @@ -20,6 +20,7 @@ |
20 | 20 | #include <linux/delay.h> |
21 | 21 | #include <linux/smp.h> |
22 | 22 | #include <linux/kernel_stat.h> |
23 | +#include <linux/sched.h> | |
23 | 24 | |
24 | 25 | #include <asm/mmu_context.h> |
25 | 26 | #include <asm/io.h> |
... | ... | @@ -189,10 +190,8 @@ |
189 | 190 | /* Clear the mailbox to clear the interrupt */ |
190 | 191 | __raw_writeq(((u64)action)<<48, mailbox_0_clear_regs[cpu]); |
191 | 192 | |
192 | - /* | |
193 | - * Nothing to do for SMP_RESCHEDULE_YOURSELF; returning from the | |
194 | - * interrupt will do the reschedule for us | |
195 | - */ | |
193 | + if (action & SMP_RESCHEDULE_YOURSELF) | |
194 | + scheduler_ipi(); | |
196 | 195 | |
197 | 196 | if (action & SMP_CALL_FUNCTION) |
198 | 197 | smp_call_function_interrupt(); |
arch/mips/sibyte/sb1250/smp.c
... | ... | @@ -21,6 +21,7 @@ |
21 | 21 | #include <linux/interrupt.h> |
22 | 22 | #include <linux/smp.h> |
23 | 23 | #include <linux/kernel_stat.h> |
24 | +#include <linux/sched.h> | |
24 | 25 | |
25 | 26 | #include <asm/mmu_context.h> |
26 | 27 | #include <asm/io.h> |
... | ... | @@ -177,10 +178,8 @@ |
177 | 178 | /* Clear the mailbox to clear the interrupt */ |
178 | 179 | ____raw_writeq(((u64)action) << 48, mailbox_clear_regs[cpu]); |
179 | 180 | |
180 | - /* | |
181 | - * Nothing to do for SMP_RESCHEDULE_YOURSELF; returning from the | |
182 | - * interrupt will do the reschedule for us | |
183 | - */ | |
181 | + if (action & SMP_RESCHEDULE_YOURSELF) | |
182 | + scheduler_ipi(); | |
184 | 183 | |
185 | 184 | if (action & SMP_CALL_FUNCTION) |
186 | 185 | smp_call_function_interrupt(); |
arch/mn10300/kernel/smp.c
... | ... | @@ -494,14 +494,11 @@ |
494 | 494 | * @irq: The interrupt number. |
495 | 495 | * @dev_id: The device ID. |
496 | 496 | * |
497 | - * We need do nothing here, since the scheduling will be effected on our way | |
498 | - * back through entry.S. | |
499 | - * | |
500 | 497 | * Returns IRQ_HANDLED to indicate we handled the interrupt successfully. |
501 | 498 | */ |
502 | 499 | static irqreturn_t smp_reschedule_interrupt(int irq, void *dev_id) |
503 | 500 | { |
504 | - /* do nothing */ | |
501 | + scheduler_ipi(); | |
505 | 502 | return IRQ_HANDLED; |
506 | 503 | } |
507 | 504 |
arch/parisc/kernel/smp.c
... | ... | @@ -155,10 +155,7 @@ |
155 | 155 | |
156 | 156 | case IPI_RESCHEDULE: |
157 | 157 | smp_debug(100, KERN_DEBUG "CPU%d IPI_RESCHEDULE\n", this_cpu); |
158 | - /* | |
159 | - * Reschedule callback. Everything to be | |
160 | - * done is done by the interrupt return path. | |
161 | - */ | |
158 | + scheduler_ipi(); | |
162 | 159 | break; |
163 | 160 | |
164 | 161 | case IPI_CALL_FUNC: |
arch/powerpc/kernel/smp.c
... | ... | @@ -116,7 +116,7 @@ |
116 | 116 | generic_smp_call_function_interrupt(); |
117 | 117 | break; |
118 | 118 | case PPC_MSG_RESCHEDULE: |
119 | - /* we notice need_resched on exit */ | |
119 | + scheduler_ipi(); | |
120 | 120 | break; |
121 | 121 | case PPC_MSG_CALL_FUNC_SINGLE: |
122 | 122 | generic_smp_call_function_single_interrupt(); |
... | ... | @@ -146,7 +146,7 @@ |
146 | 146 | |
147 | 147 | static irqreturn_t reschedule_action(int irq, void *data) |
148 | 148 | { |
149 | - /* we just need the return path side effect of checking need_resched */ | |
149 | + scheduler_ipi(); | |
150 | 150 | return IRQ_HANDLED; |
151 | 151 | } |
152 | 152 |
arch/s390/kernel/smp.c
... | ... | @@ -165,11 +165,11 @@ |
165 | 165 | kstat_cpu(smp_processor_id()).irqs[EXTINT_IPI]++; |
166 | 166 | /* |
167 | 167 | * handle bit signal external calls |
168 | - * | |
169 | - * For the ec_schedule signal we have to do nothing. All the work | |
170 | - * is done automatically when we return from the interrupt. | |
171 | 168 | */ |
172 | 169 | bits = xchg(&S390_lowcore.ext_call_fast, 0); |
170 | + | |
171 | + if (test_bit(ec_schedule, &bits)) | |
172 | + scheduler_ipi(); | |
173 | 173 | |
174 | 174 | if (test_bit(ec_call_function, &bits)) |
175 | 175 | generic_smp_call_function_interrupt(); |
arch/sh/kernel/smp.c
... | ... | @@ -20,6 +20,7 @@ |
20 | 20 | #include <linux/module.h> |
21 | 21 | #include <linux/cpu.h> |
22 | 22 | #include <linux/interrupt.h> |
23 | +#include <linux/sched.h> | |
23 | 24 | #include <asm/atomic.h> |
24 | 25 | #include <asm/processor.h> |
25 | 26 | #include <asm/system.h> |
... | ... | @@ -323,6 +324,7 @@ |
323 | 324 | generic_smp_call_function_interrupt(); |
324 | 325 | break; |
325 | 326 | case SMP_MSG_RESCHEDULE: |
327 | + scheduler_ipi(); | |
326 | 328 | break; |
327 | 329 | case SMP_MSG_FUNCTION_SINGLE: |
328 | 330 | generic_smp_call_function_single_interrupt(); |
arch/sparc/kernel/smp_32.c
arch/sparc/kernel/smp_64.c
arch/tile/kernel/smp.c
... | ... | @@ -189,12 +189,8 @@ |
189 | 189 | /* Called when smp_send_reschedule() triggers IRQ_RESCHEDULE. */ |
190 | 190 | static irqreturn_t handle_reschedule_ipi(int irq, void *token) |
191 | 191 | { |
192 | - /* | |
193 | - * Nothing to do here; when we return from interrupt, the | |
194 | - * rescheduling will occur there. But do bump the interrupt | |
195 | - * profiler count in the meantime. | |
196 | - */ | |
197 | 192 | __get_cpu_var(irq_stat).irq_resched_count++; |
193 | + scheduler_ipi(); | |
198 | 194 | |
199 | 195 | return IRQ_HANDLED; |
200 | 196 | } |
arch/um/kernel/smp.c
arch/x86/kernel/smp.c
... | ... | @@ -194,14 +194,13 @@ |
194 | 194 | } |
195 | 195 | |
196 | 196 | /* |
197 | - * Reschedule call back. Nothing to do, | |
198 | - * all the work is done automatically when | |
199 | - * we return from the interrupt. | |
197 | + * Reschedule call back. | |
200 | 198 | */ |
201 | 199 | void smp_reschedule_interrupt(struct pt_regs *regs) |
202 | 200 | { |
203 | 201 | ack_APIC_irq(); |
204 | 202 | inc_irq_stat(irq_resched_count); |
203 | + scheduler_ipi(); | |
205 | 204 | /* |
206 | 205 | * KVM uses this interrupt to force a cpu out of guest mode |
207 | 206 | */ |
arch/x86/xen/smp.c
... | ... | @@ -46,13 +46,12 @@ |
46 | 46 | static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); |
47 | 47 | |
48 | 48 | /* |
49 | - * Reschedule call back. Nothing to do, | |
50 | - * all the work is done automatically when | |
51 | - * we return from the interrupt. | |
49 | + * Reschedule call back. | |
52 | 50 | */ |
53 | 51 | static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) |
54 | 52 | { |
55 | 53 | inc_irq_stat(irq_resched_count); |
54 | + scheduler_ipi(); | |
56 | 55 | |
57 | 56 | return IRQ_HANDLED; |
58 | 57 | } |
include/linux/sched.h
... | ... | @@ -2189,8 +2189,10 @@ |
2189 | 2189 | extern char *get_task_comm(char *to, struct task_struct *tsk); |
2190 | 2190 | |
2191 | 2191 | #ifdef CONFIG_SMP |
2192 | +static inline void scheduler_ipi(void) { } | |
2192 | 2193 | extern unsigned long wait_task_inactive(struct task_struct *, long match_state); |
2193 | 2194 | #else |
2195 | +static inline void scheduler_ipi(void) { } | |
2194 | 2196 | static inline unsigned long wait_task_inactive(struct task_struct *p, |
2195 | 2197 | long match_state) |
2196 | 2198 | { |