ARC: [SMP] optimize IPI send and receive

* Don't send an IPI if receiver already has a pending IPI. Atomically piggyback the new msg with pending msg. * IPI receiver looping on xchg() not required References: https://lkml.org/lkml/2013/11/25/232 Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Vineet Gupta <vgupta@synopsys.com>

ARC: [SMP] optimize IPI send and receive
* Don't send an IPI if receiver already has a pending IPI. Atomically piggyback the new msg with pending msg. * IPI receiver looping on xchg() not required References: https://lkml.org/lkml/2013/11/25/232 Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Vineet Gupta <vgupta@synopsys.com>
Vineet Gupta
1 parent f2a4aa5646
Showing 1 changed file with 40 additions and 28 deletions Side-by-side Diff
arch/arc/kernel/smp.c
@@ -215,16 +215,31 @@
 static void ipi_send_msg_one(int cpu, enum ipi_msg_type msg)
 {
 	unsigned long __percpu *ipi_data_ptr = per_cpu_ptr(&ipi_data, cpu);
+	unsigned long old, new;
 	unsigned long flags;
  
 	pr_debug("%d Sending msg [%d] to %d\n", smp_processor_id(), msg, cpu);
  
 	local_irq_save(flags);
  
-	set_bit(msg, ipi_data_ptr);
+	/*
+	 * Atomically write new msg bit (in case others are writing too),
+	 * and read back old value
+	 */
+	do {
+		new = old = *ipi_data_ptr;
+		new |= 1U << msg;
+	} while (cmpxchg(ipi_data_ptr, old, new) != old);
  
-	/* Call the platform specific cross-CPU call function  */
-	if (plat_smp_ops.ipi_send)
+	/*
+	 * Call the platform specific IPI kick function, but avoid if possible:
+	 * Only do so if there's no pending msg from other concurrent sender(s).
+	 * Otherwise, recevier will see this msg as well when it takes the
+	 * IPI corresponding to that msg. This is true, even if it is already in
+	 * IPI handler, because !@old means it has not yet dequeued the msg(s)
+	 * so @new msg can be a free-loader
+	 */
+	if (plat_smp_ops.ipi_send && !old)
 		plat_smp_ops.ipi_send(cpu);
  
 	local_irq_restore(flags);
  
  
  
  
@@ -269,31 +284,23 @@
 	machine_halt();
 }
  
-static inline void __do_IPI(unsigned long pending)
+static inline void __do_IPI(unsigned long msg)
 {
-	while (pending) {
+	switch (msg) {
+	case IPI_RESCHEDULE:
+		scheduler_ipi();
+		break;
  
-		unsigned long msg = __ffs(pending);
+	case IPI_CALL_FUNC:
+		generic_smp_call_function_interrupt();
+		break;
  
-		switch (msg) {
-		case IPI_RESCHEDULE:
-			scheduler_ipi();
-			break;
+	case IPI_CPU_STOP:
+		ipi_cpu_stop();
+		break;
  
-		case IPI_CALL_FUNC:
-			generic_smp_call_function_interrupt();
-			break;
-
-		case IPI_CPU_STOP:
-			ipi_cpu_stop();
-			break;
-
-		default:
-			pr_warn("IPI missing msg\n");
-
-		}
-
-		pending &= ~(1U << msg);
+	default:
+		pr_warn("IPI with unexpected msg %ld\n", msg);
 	}
 }
  
  
@@ -312,11 +319,16 @@
 		plat_smp_ops.ipi_clear(irq);
  
 	/*
-	 * XXX: is this loop really needed
-	 * And do we need to move ipi_clean inside
+	 * "dequeue" the msg corresponding to this IPI (and possibly other
+	 * piggybacked msg from elided IPIs: see ipi_send_msg_one() above)
 	 */
-	while ((pending = xchg(this_cpu_ptr(&ipi_data), 0)) != 0)
-		__do_IPI(pending);
+	pending = xchg(this_cpu_ptr(&ipi_data), 0);
+
+	do {
+		unsigned long msg = __ffs(pending);
+		__do_IPI(msg);
+		pending &= ~(1U << msg);
+	} while (pending);
  
 	return IRQ_HANDLED;
 }
...	...	@@ -215,16 +215,31 @@
215	215	static void ipi_send_msg_one(int cpu, enum ipi_msg_type msg)
216	216	{
217	217	unsigned long __percpu *ipi_data_ptr = per_cpu_ptr(&ipi_data, cpu);
	218	+ unsigned long old, new;
218	219	unsigned long flags;
219	220
220	221	pr_debug("%d Sending msg [%d] to %d\n", smp_processor_id(), msg, cpu);
221	222
222	223	local_irq_save(flags);
223	224
224		- set_bit(msg, ipi_data_ptr);
	225	+ /*
	226	+ * Atomically write new msg bit (in case others are writing too),
	227	+ * and read back old value
	228	+ */
	229	+ do {
	230	+ new = old = *ipi_data_ptr;
	231	+ new \|= 1U << msg;
	232	+ } while (cmpxchg(ipi_data_ptr, old, new) != old);
225	233
226		- /* Call the platform specific cross-CPU call function */
227		- if (plat_smp_ops.ipi_send)
	234	+ /*
	235	+ * Call the platform specific IPI kick function, but avoid if possible:
	236	+ * Only do so if there's no pending msg from other concurrent sender(s).
	237	+ * Otherwise, recevier will see this msg as well when it takes the
	238	+ * IPI corresponding to that msg. This is true, even if it is already in
	239	+ * IPI handler, because !@old means it has not yet dequeued the msg(s)
	240	+ * so @new msg can be a free-loader
	241	+ */
	242	+ if (plat_smp_ops.ipi_send && !old)
228	243	plat_smp_ops.ipi_send(cpu);
229	244
230	245	local_irq_restore(flags);
231	246
232	247
233	248
234	249
...	...	@@ -269,31 +284,23 @@
269	284	machine_halt();
270	285	}
271	286
272		-static inline void __do_IPI(unsigned long pending)
	287	+static inline void __do_IPI(unsigned long msg)
273	288	{
274		- while (pending) {
	289	+ switch (msg) {
	290	+ case IPI_RESCHEDULE:
	291	+ scheduler_ipi();
	292	+ break;
275	293
276		- unsigned long msg = __ffs(pending);
	294	+ case IPI_CALL_FUNC:
	295	+ generic_smp_call_function_interrupt();
	296	+ break;
277	297
278		- switch (msg) {
279		- case IPI_RESCHEDULE:
280		- scheduler_ipi();
281		- break;
	298	+ case IPI_CPU_STOP:
	299	+ ipi_cpu_stop();
	300	+ break;
282	301
283		- case IPI_CALL_FUNC:
284		- generic_smp_call_function_interrupt();
285		- break;
286		-
287		- case IPI_CPU_STOP:
288		- ipi_cpu_stop();
289		- break;
290		-
291		- default:
292		- pr_warn("IPI missing msg\n");
293		-
294		- }
295		-
296		- pending &= ~(1U << msg);
	302	+ default:
	303	+ pr_warn("IPI with unexpected msg %ld\n", msg);
297	304	}
298	305	}
299	306
300	307
...	...	@@ -312,11 +319,16 @@
312	319	plat_smp_ops.ipi_clear(irq);
313	320
314	321	/*
315		- * XXX: is this loop really needed
316		- * And do we need to move ipi_clean inside
	322	+ * "dequeue" the msg corresponding to this IPI (and possibly other
	323	+ * piggybacked msg from elided IPIs: see ipi_send_msg_one() above)
317	324	*/
318		- while ((pending = xchg(this_cpu_ptr(&ipi_data), 0)) != 0)
319		- __do_IPI(pending);
	325	+ pending = xchg(this_cpu_ptr(&ipi_data), 0);
	326	+
	327	+ do {
	328	+ unsigned long msg = __ffs(pending);
	329	+ __do_IPI(msg);
	330	+ pending &= ~(1U << msg);
	331	+ } while (pending);
320	332
321	333	return IRQ_HANDLED;
322	334	}