x86: convert to generic helpers for IPI function calls

This converts x86, x86-64, and xen to use the new helpers for smp_call_function() and friends, and adds support for smp_call_function_single(). Acked-by: Ingo Molnar <mingo@elte.hu> Acked-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

x86: convert to generic helpers for IPI function calls
This converts x86, x86-64, and xen to use the new helpers for smp_call_function() and friends, and adds support for smp_call_function_single(). Acked-by: Ingo Molnar <mingo@elte.hu> Acked-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
Jens Axboe
1 parent 3d44223327
Showing 20 changed files with 125 additions and 380 deletions Side-by-side Diff
arch/x86/Kconfig
arch/x86/kernel/apic_32.c
arch/x86/kernel/entry_64.S
arch/x86/kernel/i8259_64.c
arch/x86/kernel/smp.c
arch/x86/kernel/smpboot.c
arch/x86/kernel/smpcommon.c
arch/x86/mach-voyager/voyager_smp.c
arch/x86/xen/enlighten.c
arch/x86/xen/mmu.c
arch/x86/xen/smp.c
arch/x86/xen/xen-ops.h
include/asm-x86/hw_irq_32.h
include/asm-x86/hw_irq_64.h
include/asm-x86/mach-default/entry_arch.h
include/asm-x86/mach-default/irq_vectors.h
include/asm-x86/mach-voyager/entry_arch.h
include/asm-x86/mach-voyager/irq_vectors.h
include/asm-x86/smp.h
include/asm-x86/xen/events.h
@@ -168,6 +168,7 @@
 config X86_SMP
 	bool
 	depends on SMP && ((X86_32 && !X86_VOYAGER) || X86_64)
+	select USE_GENERIC_SMP_HELPERS
 	default y
  
 config X86_32_SMP
@@ -1358,6 +1358,10 @@
  
 	/* IPI for generic function call */
 	set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
+
+	/* IPI for single call function */
+	set_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
+				call_function_single_interrupt);
 }
 #endif
  
@@ -711,6 +711,9 @@
 ENTRY(call_function_interrupt)
 	apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
 END(call_function_interrupt)
+ENTRY(call_function_single_interrupt)
+	apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
+END(call_function_single_interrupt)
 ENTRY(irq_move_cleanup_interrupt)
 	apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
 END(irq_move_cleanup_interrupt)
@@ -494,6 +494,10 @@
 	/* IPI for generic function call */
 	set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
  
+	/* IPI for generic single function call */
+	set_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
+				call_function_single_interrupt);
+
 	/* Low priority IPI to cleanup after moving an irq */
 	set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
 #endif
@@ -121,132 +121,23 @@
 	send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
 }
  
-/*
- * Structure and data for smp_call_function(). This is designed to minimise
- * static memory requirements. It also looks cleaner.
- */
-static DEFINE_SPINLOCK(call_lock);
-
-struct call_data_struct {
-	void (*func) (void *info);
-	void *info;
-	atomic_t started;
-	atomic_t finished;
-	int wait;
-};
-
-void lock_ipi_call_lock(void)
+void native_send_call_func_single_ipi(int cpu)
 {
-	spin_lock_irq(&call_lock);
+	send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_SINGLE_VECTOR);
 }
  
-void unlock_ipi_call_lock(void)
+void native_send_call_func_ipi(cpumask_t mask)
 {
-	spin_unlock_irq(&call_lock);
-}
-
-static struct call_data_struct *call_data;
-
-static void __smp_call_function(void (*func) (void *info), void *info,
-				int nonatomic, int wait)
-{
-	struct call_data_struct data;
-	int cpus = num_online_cpus() - 1;
-
-	if (!cpus)
-		return;
-
-	data.func = func;
-	data.info = info;
-	atomic_set(&data.started, 0);
-	data.wait = wait;
-	if (wait)
-		atomic_set(&data.finished, 0);
-
-	call_data = &data;
-	mb();
-
-	/* Send a message to all other CPUs and wait for them to respond */
-	send_IPI_allbutself(CALL_FUNCTION_VECTOR);
-
-	/* Wait for response */
-	while (atomic_read(&data.started) != cpus)
-		cpu_relax();
-
-	if (wait)
-		while (atomic_read(&data.finished) != cpus)
-			cpu_relax();
-}
-
-
-/**
- * smp_call_function_mask(): Run a function on a set of other CPUs.
- * @mask: The set of cpus to run on.  Must not include the current cpu.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait (atomically) until function has completed on other CPUs.
- *
-  * Returns 0 on success, else a negative status code.
- *
- * If @wait is true, then returns once @func has returned; otherwise
- * it returns just before the target cpu calls @func.
- *
- * You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler.
- */
-static int
-native_smp_call_function_mask(cpumask_t mask,
-			      void (*func)(void *), void *info,
-			      int wait)
-{
-	struct call_data_struct data;
 	cpumask_t allbutself;
-	int cpus;
  
-	/* Can deadlock when called with interrupts disabled */
-	WARN_ON(irqs_disabled());
-
-	/* Holding any lock stops cpus from going down. */
-	spin_lock(&call_lock);
-
 	allbutself = cpu_online_map;
 	cpu_clear(smp_processor_id(), allbutself);
  
-	cpus_and(mask, mask, allbutself);
-	cpus = cpus_weight(mask);
-
-	if (!cpus) {
-		spin_unlock(&call_lock);
-		return 0;
-	}
-
-	data.func = func;
-	data.info = info;
-	atomic_set(&data.started, 0);
-	data.wait = wait;
-	if (wait)
-		atomic_set(&data.finished, 0);
-
-	call_data = &data;
-	wmb();
-
-	/* Send a message to other CPUs */
 	if (cpus_equal(mask, allbutself) &&
 	    cpus_equal(cpu_online_map, cpu_callout_map))
 		send_IPI_allbutself(CALL_FUNCTION_VECTOR);
 	else
 		send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
-
-	/* Wait for response */
-	while (atomic_read(&data.started) != cpus)
-		cpu_relax();
-
-	if (wait)
-		while (atomic_read(&data.finished) != cpus)
-			cpu_relax();
-	spin_unlock(&call_lock);
-
-	return 0;
 }
  
 static void stop_this_cpu(void *dummy)
  
  
@@ -268,18 +159,13 @@
  
 static void native_smp_send_stop(void)
 {
-	int nolock;
 	unsigned long flags;
  
 	if (reboot_force)
 		return;
  
-	/* Don't deadlock on the call lock in panic */
-	nolock = !spin_trylock(&call_lock);
+	smp_call_function(stop_this_cpu, NULL, 0, 0);
 	local_irq_save(flags);
-	__smp_call_function(stop_this_cpu, NULL, 0, 0);
-	if (!nolock)
-		spin_unlock(&call_lock);
 	disable_local_APIC();
 	local_irq_restore(flags);
 }
  
  
  
  
@@ -301,33 +187,28 @@
  
 void smp_call_function_interrupt(struct pt_regs *regs)
 {
-	void (*func) (void *info) = call_data->func;
-	void *info = call_data->info;
-	int wait = call_data->wait;
-
 	ack_APIC_irq();
-	/*
-	 * Notify initiating CPU that I've grabbed the data and am
-	 * about to execute the function
-	 */
-	mb();
-	atomic_inc(&call_data->started);
-	/*
-	 * At this point the info structure may be out of scope unless wait==1
-	 */
 	irq_enter();
-	(*func)(info);
+	generic_smp_call_function_interrupt();
 #ifdef CONFIG_X86_32
 	__get_cpu_var(irq_stat).irq_call_count++;
 #else
 	add_pda(irq_call_count, 1);
 #endif
 	irq_exit();
+}
  
-	if (wait) {
-		mb();
-		atomic_inc(&call_data->finished);
-	}
+void smp_call_function_single_interrupt(void)
+{
+	ack_APIC_irq();
+	irq_enter();
+	generic_smp_call_function_single_interrupt();
+#ifdef CONFIG_X86_32
+	__get_cpu_var(irq_stat).irq_call_count++;
+#else
+	add_pda(irq_call_count, 1);
+#endif
+	irq_exit();
 }
  
 struct smp_ops smp_ops = {
@@ -338,7 +219,9 @@
  
 	.smp_send_stop = native_smp_send_stop,
 	.smp_send_reschedule = native_smp_send_reschedule,
-	.smp_call_function_mask = native_smp_call_function_mask,
+
+	.send_call_func_ipi = native_send_call_func_ipi,
+	.send_call_func_single_ipi = native_send_call_func_single_ipi,
 };
 EXPORT_SYMBOL_GPL(smp_ops);
@@ -345,7 +345,7 @@
 	 * lock helps us to not include this cpu in a currently in progress
 	 * smp_call_function().
 	 */
-	lock_ipi_call_lock();
+	ipi_call_lock_irq();
 #ifdef CONFIG_X86_64
 	spin_lock(&vector_lock);
  
@@ -357,7 +357,7 @@
 	spin_unlock(&vector_lock);
 #endif
 	cpu_set(smp_processor_id(), cpu_online_map);
-	unlock_ipi_call_lock();
+	ipi_call_unlock_irq();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
  
 	setup_secondary_clock();
@@ -25,60 +25,4 @@
 	per_cpu(cpu_number, cpu) = cpu;
 }
 #endif
-
-/**
- * smp_call_function(): Run a function on all other CPUs.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: Unused.
- * @wait: If true, wait (atomically) until function has completed on other CPUs.
- *
- * Returns 0 on success, else a negative status code.
- *
- * If @wait is true, then returns once @func has returned; otherwise
- * it returns just before the target cpu calls @func.
- *
- * You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler.
- */
-int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
-		      int wait)
-{
-	return smp_call_function_mask(cpu_online_map, func, info, wait);
-}
-EXPORT_SYMBOL(smp_call_function);
-
-/**
- * smp_call_function_single - Run a function on a specific CPU
- * @cpu: The target CPU.  Cannot be the calling CPU.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @nonatomic: Unused.
- * @wait: If true, wait until function has completed on other CPUs.
- *
- * Returns 0 on success, else a negative status code.
- *
- * If @wait is true, then returns once @func has returned; otherwise
- * it returns just before the target cpu calls @func.
- */
-int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
-			     int nonatomic, int wait)
-{
-	/* prevent preemption and reschedule on another processor */
-	int ret;
-	int me = get_cpu();
-	if (cpu == me) {
-		local_irq_disable();
-		func(info);
-		local_irq_enable();
-		put_cpu();
-		return 0;
-	}
-
-	ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
-
-	put_cpu();
-	return ret;
-}
-EXPORT_SYMBOL(smp_call_function_single);
@@ -955,94 +955,24 @@
 		halt();
 }
  
-static DEFINE_SPINLOCK(call_lock);
-
-struct call_data_struct {
-	void (*func) (void *info);
-	void *info;
-	volatile unsigned long started;
-	volatile unsigned long finished;
-	int wait;
-};
-
-static struct call_data_struct *call_data;
-
 /* execute a thread on a new CPU.  The function to be called must be
  * previously set up.  This is used to schedule a function for
  * execution on all CPUs - set up the function then broadcast a
  * function_interrupt CPI to come here on each CPU */
 static void smp_call_function_interrupt(void)
 {
-	void (*func) (void *info) = call_data->func;
-	void *info = call_data->info;
-	/* must take copy of wait because call_data may be replaced
-	 * unless the function is waiting for us to finish */
-	int wait = call_data->wait;
-	__u8 cpu = smp_processor_id();
-
-	/*
-	 * Notify initiating CPU that I've grabbed the data and am
-	 * about to execute the function
-	 */
-	mb();
-	if (!test_and_clear_bit(cpu, &call_data->started)) {
-		/* If the bit wasn't set, this could be a replay */
-		printk(KERN_WARNING "VOYAGER SMP: CPU %d received call funtion"
-		       " with no call pending\n", cpu);
-		return;
-	}
-	/*
-	 * At this point the info structure may be out of scope unless wait==1
-	 */
 	irq_enter();
-	(*func) (info);
+	generic_smp_call_function_interrupt();
 	__get_cpu_var(irq_stat).irq_call_count++;
 	irq_exit();
-	if (wait) {
-		mb();
-		clear_bit(cpu, &call_data->finished);
-	}
 }
  
-static int
-voyager_smp_call_function_mask(cpumask_t cpumask,
-			       void (*func) (void *info), void *info, int wait)
+static void smp_call_function_single_interrupt(void)
 {
-	struct call_data_struct data;
-	u32 mask = cpus_addr(cpumask)[0];
-
-	mask &= ~(1 << smp_processor_id());
-
-	if (!mask)
-		return 0;
-
-	/* Can deadlock when called with interrupts disabled */
-	WARN_ON(irqs_disabled());
-
-	data.func = func;
-	data.info = info;
-	data.started = mask;
-	data.wait = wait;
-	if (wait)
-		data.finished = mask;
-
-	spin_lock(&call_lock);
-	call_data = &data;
-	wmb();
-	/* Send a message to all other CPUs and wait for them to respond */
-	send_CPI(mask, VIC_CALL_FUNCTION_CPI);
-
-	/* Wait for response */
-	while (data.started)
-		barrier();
-
-	if (wait)
-		while (data.finished)
-			barrier();
-
-	spin_unlock(&call_lock);
-
-	return 0;
+	irq_enter();
+	generic_smp_call_function_single_interrupt();
+	__get_cpu_var(irq_stat).irq_call_count++;
+	irq_exit();
 }
  
 /* Sorry about the name.  In an APIC based system, the APICs
@@ -1099,6 +1029,12 @@
 	smp_call_function_interrupt();
 }
  
+void smp_qic_call_function_single_interrupt(struct pt_regs *regs)
+{
+	ack_QIC_CPI(QIC_CALL_FUNCTION_SINGLE_CPI);
+	smp_call_function_single_interrupt();
+}
+
 void smp_vic_cpi_interrupt(struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
@@ -1119,6 +1055,8 @@
 		smp_enable_irq_interrupt();
 	if (test_and_clear_bit(VIC_CALL_FUNCTION_CPI, &vic_cpi_mailbox[cpu]))
 		smp_call_function_interrupt();
+	if (test_and_clear_bit(VIC_CALL_FUNCTION_SINGLE_CPI, &vic_cpi_mailbox[cpu]))
+		smp_call_function_single_interrupt();
 	set_irq_regs(old_regs);
 }
  
@@ -1862,6 +1800,8 @@
  
 	.smp_send_stop = voyager_smp_send_stop,
 	.smp_send_reschedule = voyager_smp_send_reschedule,
-	.smp_call_function_mask = voyager_smp_call_function_mask,
+
+	.send_call_func_ipi = native_send_call_func_ipi,
+	.send_call_func_single_ipi = native_send_call_func_single_ipi,
 };
@@ -1108,7 +1108,9 @@
  
 	.smp_send_stop = xen_smp_send_stop,
 	.smp_send_reschedule = xen_smp_send_reschedule,
-	.smp_call_function_mask = xen_smp_call_function_mask,
+
+	.send_call_func_ipi = xen_smp_send_call_function_ipi,
+	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
 };
 #endif	/* CONFIG_SMP */
  
@@ -558,7 +558,7 @@
 	}
  
 	if (!cpus_empty(mask))
-		xen_smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
+		smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
 }
 #else
 static void drop_mm_ref(struct mm_struct *mm)
@@ -36,28 +36,15 @@
 #include "mmu.h"
  
 static cpumask_t xen_cpu_initialized_map;
-static DEFINE_PER_CPU(int, resched_irq) = -1;
-static DEFINE_PER_CPU(int, callfunc_irq) = -1;
+
+static DEFINE_PER_CPU(int, resched_irq);
+static DEFINE_PER_CPU(int, callfunc_irq);
+static DEFINE_PER_CPU(int, callfuncsingle_irq);
 static DEFINE_PER_CPU(int, debug_irq) = -1;
  
-/*
- * Structure and data for smp_call_function(). This is designed to minimise
- * static memory requirements. It also looks cleaner.
- */
-static DEFINE_SPINLOCK(call_lock);
-
-struct call_data_struct {
-	void (*func) (void *info);
-	void *info;
-	atomic_t started;
-	atomic_t finished;
-	int wait;
-};
-
 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
+static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
  
-static struct call_data_struct *call_data;
-
 /*
  * Reschedule call back. Nothing to do,
  * all the work is done automatically when
@@ -122,6 +109,17 @@
 		goto fail;
 	per_cpu(debug_irq, cpu) = rc;
  
+	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
+	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
+				    cpu,
+				    xen_call_function_single_interrupt,
+				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
+				    callfunc_name,
+				    NULL);
+	if (rc < 0)
+		goto fail;
+	per_cpu(callfuncsingle_irq, cpu) = rc;
+
 	return 0;
  
  fail:
@@ -131,6 +129,9 @@
 		unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
 	if (per_cpu(debug_irq, cpu) >= 0)
 		unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
+	if (per_cpu(callfuncsingle_irq, cpu) >= 0)
+		unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
+
 	return rc;
 }
  
@@ -338,7 +339,6 @@
 	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
 }
  
-
 static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector)
 {
 	unsigned cpu;
  
  
  
  
  
  
  
@@ -349,84 +349,43 @@
 		xen_send_IPI_one(cpu, vector);
 }
  
-static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
+void xen_smp_send_call_function_ipi(cpumask_t mask)
 {
-	void (*func) (void *info) = call_data->func;
-	void *info = call_data->info;
-	int wait = call_data->wait;
+	int cpu;
  
-	/*
-	 * Notify initiating CPU that I've grabbed the data and am
-	 * about to execute the function
-	 */
-	mb();
-	atomic_inc(&call_data->started);
-	/*
-	 * At this point the info structure may be out of scope unless wait==1
-	 */
+	xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
+
+	/* Make sure other vcpus get a chance to run if they need to. */
+	for_each_cpu_mask(cpu, mask) {
+		if (xen_vcpu_stolen(cpu)) {
+			HYPERVISOR_sched_op(SCHEDOP_yield, 0);
+			break;
+		}
+	}
+}
+
+void xen_smp_send_call_function_single_ipi(int cpu)
+{
+	xen_send_IPI_mask(cpumask_of_cpu(cpu), XEN_CALL_FUNCTION_SINGLE_VECTOR);
+}
+
+static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
+{
 	irq_enter();
-	(*func)(info);
+	generic_smp_call_function_interrupt();
 	__get_cpu_var(irq_stat).irq_call_count++;
 	irq_exit();
  
-	if (wait) {
-		mb();		/* commit everything before setting finished */
-		atomic_inc(&call_data->finished);
-	}
-
 	return IRQ_HANDLED;
 }
  
-int xen_smp_call_function_mask(cpumask_t mask, void (*func)(void *),
-			       void *info, int wait)
+static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
 {
-	struct call_data_struct data;
-	int cpus, cpu;
-	bool yield;
+	irq_enter();
+	generic_smp_call_function_single_interrupt();
+	__get_cpu_var(irq_stat).irq_call_count++;
+	irq_exit();
  
-	/* Holding any lock stops cpus from going down. */
-	spin_lock(&call_lock);
-
-	cpu_clear(smp_processor_id(), mask);
-
-	cpus = cpus_weight(mask);
-	if (!cpus) {
-		spin_unlock(&call_lock);
-		return 0;
-	}
-
-	/* Can deadlock when called with interrupts disabled */
-	WARN_ON(irqs_disabled());
-
-	data.func = func;
-	data.info = info;
-	atomic_set(&data.started, 0);
-	data.wait = wait;
-	if (wait)
-		atomic_set(&data.finished, 0);
-
-	call_data = &data;
-	mb();			/* write everything before IPI */
-
-	/* Send a message to other CPUs and wait for them to respond */
-	xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
-
-	/* Make sure other vcpus get a chance to run if they need to. */
-	yield = false;
-	for_each_cpu_mask(cpu, mask)
-		if (xen_vcpu_stolen(cpu))
-			yield = true;
-
-	if (yield)
-		HYPERVISOR_sched_op(SCHEDOP_yield, 0);
-
-	/* Wait for response */
-	while (atomic_read(&data.started) != cpus ||
-	       (wait && atomic_read(&data.finished) != cpus))
-		cpu_relax();
-
-	spin_unlock(&call_lock);
-
-	return 0;
+	return IRQ_HANDLED;
 }
@@ -46,13 +46,8 @@
  
 void xen_smp_send_stop(void);
 void xen_smp_send_reschedule(int cpu);
-int xen_smp_call_function (void (*func) (void *info), void *info, int nonatomic,
-			   int wait);
-int xen_smp_call_function_single(int cpu, void (*func) (void *info), void *info,
-				 int nonatomic, int wait);
-
-int xen_smp_call_function_mask(cpumask_t mask, void (*func)(void *),
-			       void *info, int wait);
+void xen_smp_send_call_function_ipi(cpumask_t mask);
+void xen_smp_send_call_function_single_ipi(int cpu);
  
  
 /* Declare an asm function, along with symbols needed to make it
@@ -32,6 +32,7 @@
 void reschedule_interrupt(void);
 void invalidate_interrupt(void);
 void call_function_interrupt(void);
+void call_function_single_interrupt(void);
 #endif
  
 #ifdef CONFIG_X86_LOCAL_APIC
@@ -68,6 +68,7 @@
 #define ERROR_APIC_VECTOR	0xfe
 #define RESCHEDULE_VECTOR	0xfd
 #define CALL_FUNCTION_VECTOR	0xfc
+#define	CALL_FUNCTION_SINGLE_VECTOR	0xfb
 /* fb free - please don't readd KDB here because it's useless
    (hint - think what a NMI bit does to a vector) */
 #define THERMAL_APIC_VECTOR	0xfa
@@ -102,6 +103,7 @@
 void error_interrupt(void);
 void reschedule_interrupt(void);
 void call_function_interrupt(void);
+void call_function_single_interrupt(void);
 void irq_move_cleanup_interrupt(void);
 void invalidate_interrupt0(void);
 void invalidate_interrupt1(void);
@@ -13,6 +13,7 @@
 BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR)
 BUILD_INTERRUPT(invalidate_interrupt,INVALIDATE_TLB_VECTOR)
 BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR)
+BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR)
 #endif
  
 /*
@@ -48,6 +48,7 @@
 #define INVALIDATE_TLB_VECTOR	0xfd
 #define RESCHEDULE_VECTOR	0xfc
 #define CALL_FUNCTION_VECTOR	0xfb
+#define CALL_FUNCTION_SINGLE_VECTOR	0xfa
  
 #define THERMAL_APIC_VECTOR	0xf0
 /*
@@ -23,4 +23,5 @@
 BUILD_INTERRUPT(qic_reschedule_interrupt, QIC_RESCHEDULE_CPI);
 BUILD_INTERRUPT(qic_enable_irq_interrupt, QIC_ENABLE_IRQ_CPI);
 BUILD_INTERRUPT(qic_call_function_interrupt, QIC_CALL_FUNCTION_CPI);
+BUILD_INTERRUPT(qic_call_function_single_interrupt, QIC_CALL_FUNCTION_SINGLE_CPI);
@@ -33,6 +33,7 @@
 #define VIC_RESCHEDULE_CPI		4
 #define VIC_ENABLE_IRQ_CPI		5
 #define VIC_CALL_FUNCTION_CPI		6
+#define VIC_CALL_FUNCTION_SINGLE_CPI	7
  
 /* Now the QIC CPIs:  Since we don't need the two initial levels,
  * these are 2 less than the VIC CPIs */
  
@@ -42,9 +43,10 @@
 #define QIC_RESCHEDULE_CPI		(VIC_RESCHEDULE_CPI - QIC_CPI_OFFSET)
 #define QIC_ENABLE_IRQ_CPI		(VIC_ENABLE_IRQ_CPI - QIC_CPI_OFFSET)
 #define QIC_CALL_FUNCTION_CPI		(VIC_CALL_FUNCTION_CPI - QIC_CPI_OFFSET)
+#define QIC_CALL_FUNCTION_SINGLE_CPI	(VIC_CALL_FUNCTION_SINGLE_CPI - QIC_CPI_OFFSET)
  
 #define VIC_START_FAKE_CPI		VIC_TIMER_CPI
-#define VIC_END_FAKE_CPI		VIC_CALL_FUNCTION_CPI
+#define VIC_END_FAKE_CPI		VIC_CALL_FUNCTION_SINGLE_CPI
  
 /* this is the SYS_INT CPI. */
 #define VIC_SYS_INT			8
@@ -59,9 +59,9 @@
  
 	void (*smp_send_stop)(void);
 	void (*smp_send_reschedule)(int cpu);
-	int (*smp_call_function_mask)(cpumask_t mask,
-				      void (*func)(void *info), void *info,
-				      int wait);
+
+	void (*send_call_func_ipi)(cpumask_t mask);
+	void (*send_call_func_single_ipi)(int cpu);
 };
  
 /* Globals due to paravirt */
  
  
  
@@ -103,17 +103,22 @@
 	smp_ops.smp_send_reschedule(cpu);
 }
  
-static inline int smp_call_function_mask(cpumask_t mask,
-					 void (*func) (void *info), void *info,
-					 int wait)
+static inline void arch_send_call_function_single_ipi(int cpu)
 {
-	return smp_ops.smp_call_function_mask(mask, func, info, wait);
+	smp_ops.send_call_func_single_ipi(cpu);
 }
  
+static inline void arch_send_call_function_ipi(cpumask_t mask)
+{
+	smp_ops.send_call_func_ipi(mask);
+}
+
 void native_smp_prepare_boot_cpu(void);
 void native_smp_prepare_cpus(unsigned int max_cpus);
 void native_smp_cpus_done(unsigned int max_cpus);
 int native_cpu_up(unsigned int cpunum);
+void native_send_call_func_ipi(cpumask_t mask);
+void native_send_call_func_single_ipi(int cpu);
  
 extern int __cpu_disable(void);
 extern void __cpu_die(unsigned int cpu);
@@ -202,8 +207,6 @@
 #endif
  
 extern void smp_alloc_memory(void);
-extern void lock_ipi_call_lock(void);
-extern void unlock_ipi_call_lock(void);
 #endif /* __ASSEMBLY__ */
 #endif
@@ -4,6 +4,7 @@
 enum ipi_vector {
 	XEN_RESCHEDULE_VECTOR,
 	XEN_CALL_FUNCTION_VECTOR,
+	XEN_CALL_FUNCTION_SINGLE_VECTOR,
  
 	XEN_NR_IPIS,
 };
...	...	@@ -168,6 +168,7 @@
168	168	config X86_SMP
169	169	bool
170	170	depends on SMP && ((X86_32 && !X86_VOYAGER) \|\| X86_64)
	171	+ select USE_GENERIC_SMP_HELPERS
171	172	default y
172	173
173	174	config X86_32_SMP
...	...	@@ -1358,6 +1358,10 @@
1358	1358
1359	1359	/* IPI for generic function call */
1360	1360	set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
	1361	+
	1362	+ /* IPI for single call function */
	1363	+ set_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
	1364	+ call_function_single_interrupt);
1361	1365	}
1362	1366	#endif
1363	1367
...	...	@@ -711,6 +711,9 @@
711	711	ENTRY(call_function_interrupt)
712	712	apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
713	713	END(call_function_interrupt)
	714	+ENTRY(call_function_single_interrupt)
	715	+ apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
	716	+END(call_function_single_interrupt)
714	717	ENTRY(irq_move_cleanup_interrupt)
715	718	apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
716	719	END(irq_move_cleanup_interrupt)
...	...	@@ -494,6 +494,10 @@
494	494	/* IPI for generic function call */
495	495	set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
496	496
	497	+ /* IPI for generic single function call */
	498	+ set_intr_gate(CALL_FUNCTION_SINGLE_VECTOR,
	499	+ call_function_single_interrupt);
	500	+
497	501	/* Low priority IPI to cleanup after moving an irq */
498	502	set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
499	503	#endif
...	...	@@ -121,132 +121,23 @@
121	121	send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
122	122	}
123	123
124		-/*
125		- * Structure and data for smp_call_function(). This is designed to minimise
126		- * static memory requirements. It also looks cleaner.
127		- */
128		-static DEFINE_SPINLOCK(call_lock);
129		-
130		-struct call_data_struct {
131		- void (func) (void info);
132		- void *info;
133		- atomic_t started;
134		- atomic_t finished;
135		- int wait;
136		-};
137		-
138		-void lock_ipi_call_lock(void)
	124	+void native_send_call_func_single_ipi(int cpu)
139	125	{
140		- spin_lock_irq(&call_lock);
	126	+ send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_SINGLE_VECTOR);
141	127	}
142	128
143		-void unlock_ipi_call_lock(void)
	129	+void native_send_call_func_ipi(cpumask_t mask)
144	130	{
145		- spin_unlock_irq(&call_lock);
146		-}
147		-
148		-static struct call_data_struct *call_data;
149		-
150		-static void __smp_call_function(void (func) (void info), void *info,
151		- int nonatomic, int wait)
152		-{
153		- struct call_data_struct data;
154		- int cpus = num_online_cpus() - 1;
155		-
156		- if (!cpus)
157		- return;
158		-
159		- data.func = func;
160		- data.info = info;
161		- atomic_set(&data.started, 0);
162		- data.wait = wait;
163		- if (wait)
164		- atomic_set(&data.finished, 0);
165		-
166		- call_data = &data;
167		- mb();
168		-
169		- /* Send a message to all other CPUs and wait for them to respond */
170		- send_IPI_allbutself(CALL_FUNCTION_VECTOR);
171		-
172		- /* Wait for response */
173		- while (atomic_read(&data.started) != cpus)
174		- cpu_relax();
175		-
176		- if (wait)
177		- while (atomic_read(&data.finished) != cpus)
178		- cpu_relax();
179		-}
180		-
181		-
182		-/**
183		- * smp_call_function_mask(): Run a function on a set of other CPUs.
184		- * @mask: The set of cpus to run on. Must not include the current cpu.
185		- * @func: The function to run. This must be fast and non-blocking.
186		- * @info: An arbitrary pointer to pass to the function.
187		- * @wait: If true, wait (atomically) until function has completed on other CPUs.
188		- *
189		- * Returns 0 on success, else a negative status code.
190		- *
191		- * If @wait is true, then returns once @func has returned; otherwise
192		- * it returns just before the target cpu calls @func.
193		- *
194		- * You must not call this function with disabled interrupts or from a
195		- * hardware interrupt handler or from a bottom half handler.
196		- */
197		-static int
198		-native_smp_call_function_mask(cpumask_t mask,
199		- void (func)(void ), void *info,
200		- int wait)
201		-{
202		- struct call_data_struct data;
203	131	cpumask_t allbutself;
204		- int cpus;
205	132
206		- /* Can deadlock when called with interrupts disabled */
207		- WARN_ON(irqs_disabled());
208		-
209		- /* Holding any lock stops cpus from going down. */
210		- spin_lock(&call_lock);
211		-
212	133	allbutself = cpu_online_map;
213	134	cpu_clear(smp_processor_id(), allbutself);
214	135
215		- cpus_and(mask, mask, allbutself);
216		- cpus = cpus_weight(mask);
217		-
218		- if (!cpus) {
219		- spin_unlock(&call_lock);
220		- return 0;
221		- }
222		-
223		- data.func = func;
224		- data.info = info;
225		- atomic_set(&data.started, 0);
226		- data.wait = wait;
227		- if (wait)
228		- atomic_set(&data.finished, 0);
229		-
230		- call_data = &data;
231		- wmb();
232		-
233		- /* Send a message to other CPUs */
234	136	if (cpus_equal(mask, allbutself) &&
235	137	cpus_equal(cpu_online_map, cpu_callout_map))
236	138	send_IPI_allbutself(CALL_FUNCTION_VECTOR);
237	139	else
238	140	send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
239		-
240		- /* Wait for response */
241		- while (atomic_read(&data.started) != cpus)
242		- cpu_relax();
243		-
244		- if (wait)
245		- while (atomic_read(&data.finished) != cpus)
246		- cpu_relax();
247		- spin_unlock(&call_lock);
248		-
249		- return 0;
250	141	}
251	142
252	143	static void stop_this_cpu(void *dummy)
253	144
254	145
...	...	@@ -268,18 +159,13 @@
268	159
269	160	static void native_smp_send_stop(void)
270	161	{
271		- int nolock;
272	162	unsigned long flags;
273	163
274	164	if (reboot_force)
275	165	return;
276	166
277		- /* Don't deadlock on the call lock in panic */
278		- nolock = !spin_trylock(&call_lock);
	167	+ smp_call_function(stop_this_cpu, NULL, 0, 0);
279	168	local_irq_save(flags);
280		- __smp_call_function(stop_this_cpu, NULL, 0, 0);
281		- if (!nolock)
282		- spin_unlock(&call_lock);
283	169	disable_local_APIC();
284	170	local_irq_restore(flags);
285	171	}
286	172
287	173
288	174
289	175
...	...	@@ -301,33 +187,28 @@
301	187
302	188	void smp_call_function_interrupt(struct pt_regs *regs)
303	189	{
304		- void (func) (void info) = call_data->func;
305		- void *info = call_data->info;
306		- int wait = call_data->wait;
307		-
308	190	ack_APIC_irq();
309		- /*
310		- * Notify initiating CPU that I've grabbed the data and am
311		- * about to execute the function
312		- */
313		- mb();
314		- atomic_inc(&call_data->started);
315		- /*
316		- * At this point the info structure may be out of scope unless wait==1
317		- */
318	191	irq_enter();
319		- (*func)(info);
	192	+ generic_smp_call_function_interrupt();
320	193	#ifdef CONFIG_X86_32
321	194	__get_cpu_var(irq_stat).irq_call_count++;
322	195	#else
323	196	add_pda(irq_call_count, 1);
324	197	#endif
325	198	irq_exit();
	199	+}
326	200
327		- if (wait) {
328		- mb();
329		- atomic_inc(&call_data->finished);
330		- }
	201	+void smp_call_function_single_interrupt(void)
	202	+{
	203	+ ack_APIC_irq();
	204	+ irq_enter();
	205	+ generic_smp_call_function_single_interrupt();
	206	+#ifdef CONFIG_X86_32
	207	+ __get_cpu_var(irq_stat).irq_call_count++;
	208	+#else
	209	+ add_pda(irq_call_count, 1);
	210	+#endif
	211	+ irq_exit();
331	212	}
332	213
333	214	struct smp_ops smp_ops = {
...	...	@@ -338,7 +219,9 @@
338	219
339	220	.smp_send_stop = native_smp_send_stop,
340	221	.smp_send_reschedule = native_smp_send_reschedule,
341		- .smp_call_function_mask = native_smp_call_function_mask,
	222	+
	223	+ .send_call_func_ipi = native_send_call_func_ipi,
	224	+ .send_call_func_single_ipi = native_send_call_func_single_ipi,
342	225	};
343	226	EXPORT_SYMBOL_GPL(smp_ops);
...	...	@@ -345,7 +345,7 @@
345	345	* lock helps us to not include this cpu in a currently in progress
346	346	* smp_call_function().
347	347	*/
348		- lock_ipi_call_lock();
	348	+ ipi_call_lock_irq();
349	349	#ifdef CONFIG_X86_64
350	350	spin_lock(&vector_lock);
351	351
...	...	@@ -357,7 +357,7 @@
357	357	spin_unlock(&vector_lock);
358	358	#endif
359	359	cpu_set(smp_processor_id(), cpu_online_map);
360		- unlock_ipi_call_lock();
	360	+ ipi_call_unlock_irq();
361	361	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
362	362
363	363	setup_secondary_clock();
...	...	@@ -25,60 +25,4 @@
25	25	per_cpu(cpu_number, cpu) = cpu;
26	26	}
27	27	#endif
28		-
29		-/**
30		- * smp_call_function(): Run a function on all other CPUs.
31		- * @func: The function to run. This must be fast and non-blocking.
32		- * @info: An arbitrary pointer to pass to the function.
33		- * @nonatomic: Unused.
34		- * @wait: If true, wait (atomically) until function has completed on other CPUs.
35		- *
36		- * Returns 0 on success, else a negative status code.
37		- *
38		- * If @wait is true, then returns once @func has returned; otherwise
39		- * it returns just before the target cpu calls @func.
40		- *
41		- * You must not call this function with disabled interrupts or from a
42		- * hardware interrupt handler or from a bottom half handler.
43		- */
44		-int smp_call_function(void (func) (void info), void *info, int nonatomic,
45		- int wait)
46		-{
47		- return smp_call_function_mask(cpu_online_map, func, info, wait);
48		-}
49		-EXPORT_SYMBOL(smp_call_function);
50		-
51		-/**
52		- * smp_call_function_single - Run a function on a specific CPU
53		- * @cpu: The target CPU. Cannot be the calling CPU.
54		- * @func: The function to run. This must be fast and non-blocking.
55		- * @info: An arbitrary pointer to pass to the function.
56		- * @nonatomic: Unused.
57		- * @wait: If true, wait until function has completed on other CPUs.
58		- *
59		- * Returns 0 on success, else a negative status code.
60		- *
61		- * If @wait is true, then returns once @func has returned; otherwise
62		- * it returns just before the target cpu calls @func.
63		- */
64		-int smp_call_function_single(int cpu, void (func) (void info), void *info,
65		- int nonatomic, int wait)
66		-{
67		- /* prevent preemption and reschedule on another processor */
68		- int ret;
69		- int me = get_cpu();
70		- if (cpu == me) {
71		- local_irq_disable();
72		- func(info);
73		- local_irq_enable();
74		- put_cpu();
75		- return 0;
76		- }
77		-
78		- ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
79		-
80		- put_cpu();
81		- return ret;
82		-}
83		-EXPORT_SYMBOL(smp_call_function_single);
...	...	@@ -955,94 +955,24 @@
955	955	halt();
956	956	}
957	957
958		-static DEFINE_SPINLOCK(call_lock);
959		-
960		-struct call_data_struct {
961		- void (func) (void info);
962		- void *info;
963		- volatile unsigned long started;
964		- volatile unsigned long finished;
965		- int wait;
966		-};
967		-
968		-static struct call_data_struct *call_data;
969		-
970	958	/* execute a thread on a new CPU. The function to be called must be
971	959	* previously set up. This is used to schedule a function for
972	960	* execution on all CPUs - set up the function then broadcast a
973	961	* function_interrupt CPI to come here on each CPU */
974	962	static void smp_call_function_interrupt(void)
975	963	{
976		- void (func) (void info) = call_data->func;
977		- void *info = call_data->info;
978		- /* must take copy of wait because call_data may be replaced
979		- * unless the function is waiting for us to finish */
980		- int wait = call_data->wait;
981		- __u8 cpu = smp_processor_id();
982		-
983		- /*
984		- * Notify initiating CPU that I've grabbed the data and am
985		- * about to execute the function
986		- */
987		- mb();
988		- if (!test_and_clear_bit(cpu, &call_data->started)) {
989		- /* If the bit wasn't set, this could be a replay */
990		- printk(KERN_WARNING "VOYAGER SMP: CPU %d received call funtion"
991		- " with no call pending\n", cpu);
992		- return;
993		- }
994		- /*
995		- * At this point the info structure may be out of scope unless wait==1
996		- */
997	964	irq_enter();
998		- (*func) (info);
	965	+ generic_smp_call_function_interrupt();
999	966	__get_cpu_var(irq_stat).irq_call_count++;
1000	967	irq_exit();
1001		- if (wait) {
1002		- mb();
1003		- clear_bit(cpu, &call_data->finished);
1004		- }
1005	968	}
1006	969
1007		-static int
1008		-voyager_smp_call_function_mask(cpumask_t cpumask,
1009		- void (func) (void info), void *info, int wait)
	970	+static void smp_call_function_single_interrupt(void)
1010	971	{
1011		- struct call_data_struct data;
1012		- u32 mask = cpus_addr(cpumask)[0];
1013		-
1014		- mask &= ~(1 << smp_processor_id());
1015		-
1016		- if (!mask)
1017		- return 0;
1018		-
1019		- /* Can deadlock when called with interrupts disabled */
1020		- WARN_ON(irqs_disabled());
1021		-
1022		- data.func = func;
1023		- data.info = info;
1024		- data.started = mask;
1025		- data.wait = wait;
1026		- if (wait)
1027		- data.finished = mask;
1028		-
1029		- spin_lock(&call_lock);
1030		- call_data = &data;
1031		- wmb();
1032		- /* Send a message to all other CPUs and wait for them to respond */
1033		- send_CPI(mask, VIC_CALL_FUNCTION_CPI);
1034		-
1035		- /* Wait for response */
1036		- while (data.started)
1037		- barrier();
1038		-
1039		- if (wait)
1040		- while (data.finished)
1041		- barrier();
1042		-
1043		- spin_unlock(&call_lock);
1044		-
1045		- return 0;
	972	+ irq_enter();
	973	+ generic_smp_call_function_single_interrupt();
	974	+ __get_cpu_var(irq_stat).irq_call_count++;
	975	+ irq_exit();
1046	976	}
1047	977
1048	978	/* Sorry about the name. In an APIC based system, the APICs
...	...	@@ -1099,6 +1029,12 @@
1099	1029	smp_call_function_interrupt();
1100	1030	}
1101	1031
	1032	+void smp_qic_call_function_single_interrupt(struct pt_regs *regs)
	1033	+{
	1034	+ ack_QIC_CPI(QIC_CALL_FUNCTION_SINGLE_CPI);
	1035	+ smp_call_function_single_interrupt();
	1036	+}
	1037	+
1102	1038	void smp_vic_cpi_interrupt(struct pt_regs *regs)
1103	1039	{
1104	1040	struct pt_regs *old_regs = set_irq_regs(regs);
...	...	@@ -1119,6 +1055,8 @@
1119	1055	smp_enable_irq_interrupt();
1120	1056	if (test_and_clear_bit(VIC_CALL_FUNCTION_CPI, &vic_cpi_mailbox[cpu]))
1121	1057	smp_call_function_interrupt();
	1058	+ if (test_and_clear_bit(VIC_CALL_FUNCTION_SINGLE_CPI, &vic_cpi_mailbox[cpu]))
	1059	+ smp_call_function_single_interrupt();
1122	1060	set_irq_regs(old_regs);
1123	1061	}
1124	1062
...	...	@@ -1862,6 +1800,8 @@
1862	1800
1863	1801	.smp_send_stop = voyager_smp_send_stop,
1864	1802	.smp_send_reschedule = voyager_smp_send_reschedule,
1865		- .smp_call_function_mask = voyager_smp_call_function_mask,
	1803	+
	1804	+ .send_call_func_ipi = native_send_call_func_ipi,
	1805	+ .send_call_func_single_ipi = native_send_call_func_single_ipi,
1866	1806	};
...	...	@@ -1108,7 +1108,9 @@
1108	1108
1109	1109	.smp_send_stop = xen_smp_send_stop,
1110	1110	.smp_send_reschedule = xen_smp_send_reschedule,
1111		- .smp_call_function_mask = xen_smp_call_function_mask,
	1111	+
	1112	+ .send_call_func_ipi = xen_smp_send_call_function_ipi,
	1113	+ .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
1112	1114	};
1113	1115	#endif /* CONFIG_SMP */
1114	1116
...	...	@@ -558,7 +558,7 @@
558	558	}
559	559
560	560	if (!cpus_empty(mask))
561		- xen_smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
	561	+ smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
562	562	}
563	563	#else
564	564	static void drop_mm_ref(struct mm_struct *mm)