[MIPS] SMTC: Close tiny holes in the SMTC IPI replay system.

Signed-off-by: Kevin D. Kissell <kevink@paralogos.com> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

[MIPS] SMTC: Close tiny holes in the SMTC IPI replay system.
Signed-off-by: Kevin D. Kissell <kevink@paralogos.com> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
Kevin D. Kissell · Ralf Baechle
1 parent 9cc123631b
Showing 3 changed files with 67 additions and 17 deletions Side-by-side Diff
arch/mips/kernel/entry.S
arch/mips/kernel/smtc.c
include/asm-mips/stackframe.h
@@ -79,11 +79,6 @@
  
 FEXPORT(restore_all)			# restore full frame
 #ifdef CONFIG_MIPS_MT_SMTC
-/* Detect and execute deferred IPI "interrupts" */
-	LONG_L	s0, TI_REGS($28)
-	LONG_S	sp, TI_REGS($28)
-	jal	deferred_smtc_ipi
-	LONG_S	s0, TI_REGS($28)
 #ifdef CONFIG_MIPS_MT_SMTC_IM_BACKSTOP
 /* Re-arm any temporarily masked interrupts not explicitly "acked" */
 	mfc0	v0, CP0_TCSTATUS
@@ -112,6 +107,11 @@
 	xor	t0, t0, t3
 	mtc0	t0, CP0_TCCONTEXT
 #endif /* CONFIG_MIPS_MT_SMTC_IM_BACKSTOP */
+/* Detect and execute deferred IPI "interrupts" */
+	LONG_L	s0, TI_REGS($28)
+	LONG_S	sp, TI_REGS($28)
+	jal	deferred_smtc_ipi
+	LONG_S	s0, TI_REGS($28)
 #endif /* CONFIG_MIPS_MT_SMTC */
 	.set	noat
 	RESTORE_TEMP
@@ -70,7 +70,7 @@
  
 #define IPIBUF_PER_CPU 4
  
-static struct smtc_ipi_q IPIQ[NR_CPUS];
+struct smtc_ipi_q IPIQ[NR_CPUS];
 static struct smtc_ipi_q freeIPIq;
  
  
@@ -297,14 +297,31 @@
 #ifdef CONFIG_MIPS_MT_SMTC
 		.set	mips32r2
 		/*
-		 * This may not really be necessary if ints are already
-		 * inhibited here.
+		 * We need to make sure the read-modify-write
+		 * of Status below isn't perturbed by an interrupt
+		 * or cross-TC access, so we need to do at least a DMT,
+		 * protected by an interrupt-inhibit. But setting IXMT
+		 * also creates a few-cycle window where an IPI could
+		 * be queued and not be detected before potentially
+		 * returning to a WAIT or user-mode loop. It must be
+		 * replayed.
+		 *
+		 * We're in the middle of a context switch, and
+		 * we can't dispatch it directly without trashing
+		 * some registers, so we'll try to detect this unlikely
+		 * case and program a software interrupt in the VPE,
+		 * as would be done for a cross-VPE IPI.  To accomodate
+		 * the handling of that case, we're doing a DVPE instead
+		 * of just a DMT here to protect against other threads.
+		 * This is a lot of cruft to cover a tiny window.
+		 * If you can find a better design, implement it!
+		 *
 		 */
 		mfc0	v0, CP0_TCSTATUS
 		ori	v0, TCSTATUS_IXMT
 		mtc0	v0, CP0_TCSTATUS
 		_ehb
-		DMT	5				# dmt a1
+		DVPE	5				# dvpe a1
 		jal	mips_ihb
 #endif /* CONFIG_MIPS_MT_SMTC */
 		mfc0	a0, CP0_STATUS
  
  
  
@@ -325,17 +342,50 @@
  */
 		LONG_L	v1, PT_TCSTATUS(sp)
 		_ehb
-		mfc0	v0, CP0_TCSTATUS
+		mfc0	a0, CP0_TCSTATUS
 		andi	v1, TCSTATUS_IXMT
-		/* We know that TCStatua.IXMT should be set from above */
-		xori	v0, v0, TCSTATUS_IXMT
-		or	v0, v0, v1
-		mtc0	v0, CP0_TCSTATUS
-		_ehb
-		andi	a1, a1, VPECONTROL_TE
+		bnez	v1, 0f
+
+/*
+ * We'd like to detect any IPIs queued in the tiny window
+ * above and request an software interrupt to service them
+ * when we ERET.
+ *
+ * Computing the offset into the IPIQ array of the executing
+ * TC's IPI queue in-line would be tedious.  We use part of
+ * the TCContext register to hold 16 bits of offset that we
+ * can add in-line to find the queue head.
+ */
+		mfc0	v0, CP0_TCCONTEXT
+		la	a2, IPIQ
+		srl	v0, v0, 16
+		addu	a2, a2, v0
+		LONG_L	v0, 0(a2)
+		beqz	v0, 0f
+/*
+ * If we have a queue, provoke dispatch within the VPE by setting C_SW1
+ */
+		mfc0	v0, CP0_CAUSE
+		ori	v0, v0, C_SW1
+		mtc0	v0, CP0_CAUSE
+0:
+		/*
+		 * This test should really never branch but
+		 * let's be prudent here.  Having atomized
+		 * the shared register modifications, we can
+		 * now EVPE, and must do so before interrupts
+		 * are potentially re-enabled.
+		 */
+		andi	a1, a1, MVPCONTROL_EVP
 		beqz	a1, 1f
-		emt
+		evpe
 1:
+		/* We know that TCStatua.IXMT should be set from above */
+		xori	a0, a0, TCSTATUS_IXMT
+		or	a0, a0, v1
+		mtc0	a0, CP0_TCSTATUS
+		_ehb
+
 		.set	mips0
 #endif /* CONFIG_MIPS_MT_SMTC */
 		LONG_L	v1, PT_EPC(sp)
...	...	@@ -79,11 +79,6 @@
79	79
80	80	FEXPORT(restore_all) # restore full frame
81	81	#ifdef CONFIG_MIPS_MT_SMTC
82		-/* Detect and execute deferred IPI "interrupts" */
83		- LONG_L s0, TI_REGS($28)
84		- LONG_S sp, TI_REGS($28)
85		- jal deferred_smtc_ipi
86		- LONG_S s0, TI_REGS($28)
87	82	#ifdef CONFIG_MIPS_MT_SMTC_IM_BACKSTOP
88	83	/* Re-arm any temporarily masked interrupts not explicitly "acked" */
89	84	mfc0 v0, CP0_TCSTATUS
...	...	@@ -112,6 +107,11 @@
112	107	xor t0, t0, t3
113	108	mtc0 t0, CP0_TCCONTEXT
114	109	#endif /* CONFIG_MIPS_MT_SMTC_IM_BACKSTOP */
	110	+/* Detect and execute deferred IPI "interrupts" */
	111	+ LONG_L s0, TI_REGS($28)
	112	+ LONG_S sp, TI_REGS($28)
	113	+ jal deferred_smtc_ipi
	114	+ LONG_S s0, TI_REGS($28)
115	115	#endif /* CONFIG_MIPS_MT_SMTC */
116	116	.set noat
117	117	RESTORE_TEMP
...	...	@@ -70,7 +70,7 @@
70	70
71	71	#define IPIBUF_PER_CPU 4
72	72
73		-static struct smtc_ipi_q IPIQ[NR_CPUS];
	73	+struct smtc_ipi_q IPIQ[NR_CPUS];
74	74	static struct smtc_ipi_q freeIPIq;
75	75
76	76
...	...	@@ -297,14 +297,31 @@
297	297	#ifdef CONFIG_MIPS_MT_SMTC
298	298	.set mips32r2
299	299	/*
300		- * This may not really be necessary if ints are already
301		- * inhibited here.
	300	+ * We need to make sure the read-modify-write
	301	+ * of Status below isn't perturbed by an interrupt
	302	+ * or cross-TC access, so we need to do at least a DMT,
	303	+ * protected by an interrupt-inhibit. But setting IXMT
	304	+ * also creates a few-cycle window where an IPI could
	305	+ * be queued and not be detected before potentially
	306	+ * returning to a WAIT or user-mode loop. It must be
	307	+ * replayed.
	308	+ *
	309	+ * We're in the middle of a context switch, and
	310	+ * we can't dispatch it directly without trashing
	311	+ * some registers, so we'll try to detect this unlikely
	312	+ * case and program a software interrupt in the VPE,
	313	+ * as would be done for a cross-VPE IPI. To accomodate
	314	+ * the handling of that case, we're doing a DVPE instead
	315	+ * of just a DMT here to protect against other threads.
	316	+ * This is a lot of cruft to cover a tiny window.
	317	+ * If you can find a better design, implement it!
	318	+ *
302	319	*/
303	320	mfc0 v0, CP0_TCSTATUS
304	321	ori v0, TCSTATUS_IXMT
305	322	mtc0 v0, CP0_TCSTATUS
306	323	_ehb
307		- DMT 5 # dmt a1
	324	+ DVPE 5 # dvpe a1
308	325	jal mips_ihb
309	326	#endif /* CONFIG_MIPS_MT_SMTC */
310	327	mfc0 a0, CP0_STATUS
311	328
312	329
313	330
...	...	@@ -325,17 +342,50 @@
325	342	*/
326	343	LONG_L v1, PT_TCSTATUS(sp)
327	344	_ehb
328		- mfc0 v0, CP0_TCSTATUS
	345	+ mfc0 a0, CP0_TCSTATUS
329	346	andi v1, TCSTATUS_IXMT
330		- /* We know that TCStatua.IXMT should be set from above */
331		- xori v0, v0, TCSTATUS_IXMT
332		- or v0, v0, v1
333		- mtc0 v0, CP0_TCSTATUS
334		- _ehb
335		- andi a1, a1, VPECONTROL_TE
	347	+ bnez v1, 0f
	348	+
	349	+/*
	350	+ * We'd like to detect any IPIs queued in the tiny window
	351	+ * above and request an software interrupt to service them
	352	+ * when we ERET.
	353	+ *
	354	+ * Computing the offset into the IPIQ array of the executing
	355	+ * TC's IPI queue in-line would be tedious. We use part of
	356	+ * the TCContext register to hold 16 bits of offset that we
	357	+ * can add in-line to find the queue head.
	358	+ */
	359	+ mfc0 v0, CP0_TCCONTEXT
	360	+ la a2, IPIQ
	361	+ srl v0, v0, 16
	362	+ addu a2, a2, v0
	363	+ LONG_L v0, 0(a2)
	364	+ beqz v0, 0f
	365	+/*
	366	+ * If we have a queue, provoke dispatch within the VPE by setting C_SW1
	367	+ */
	368	+ mfc0 v0, CP0_CAUSE
	369	+ ori v0, v0, C_SW1
	370	+ mtc0 v0, CP0_CAUSE
	371	+0:
	372	+ /*
	373	+ * This test should really never branch but
	374	+ * let's be prudent here. Having atomized
	375	+ * the shared register modifications, we can
	376	+ * now EVPE, and must do so before interrupts
	377	+ * are potentially re-enabled.
	378	+ */
	379	+ andi a1, a1, MVPCONTROL_EVP
336	380	beqz a1, 1f
337		- emt
	381	+ evpe
338	382	1:
	383	+ /* We know that TCStatua.IXMT should be set from above */
	384	+ xori a0, a0, TCSTATUS_IXMT
	385	+ or a0, a0, v1
	386	+ mtc0 a0, CP0_TCSTATUS
	387	+ _ehb
	388	+
339	389	.set mips0
340	390	#endif /* CONFIG_MIPS_MT_SMTC */
341	391	LONG_L v1, PT_EPC(sp)