MIPS Kprobes: Refactor branch emulation

This patch refactors MIPS branch emulation code so as to allow skipping delay slot instruction in case of branch likely instructions when branch is not taken. This is useful for keeping the code common for use cases like kprobes where one would like to handle the branch instructions keeping the delay slot instuction also in picture for branch likely instructions. Also allow emulation when instruction to be decoded is not at pt_regs->cp0_epc as in case of kprobes where pt_regs->cp0_epc points to the breakpoint instruction. The patch also exports the function for modules. Signed-off-by: Maneesh Soni <manesoni@cisco.com> Signed-off-by: Victor Kamensky <kamensky@cisco.com> Cc: David Daney <david.daney@cavium.com> Cc: ananth@in.ibm.com Cc: linux-kernel@vger.kernel.org Cc: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2913/ Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

MIPS Kprobes: Refactor branch emulation
This patch refactors MIPS branch emulation code so as to allow skipping delay slot instruction in case of branch likely instructions when branch is not taken. This is useful for keeping the code common for use cases like kprobes where one would like to handle the branch instructions keeping the delay slot instuction also in picture for branch likely instructions. Also allow emulation when instruction to be decoded is not at pt_regs->cp0_epc as in case of kprobes where pt_regs->cp0_epc points to the breakpoint instruction. The patch also exports the function for modules. Signed-off-by: Maneesh Soni <manesoni@cisco.com> Signed-off-by: Victor Kamensky <kamensky@cisco.com> Cc: David Daney <david.daney@cavium.com> Cc: ananth@in.ibm.com Cc: linux-kernel@vger.kernel.org Cc: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/2913/ Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
Maneesh Soni · Ralf Baechle
1 parent 9233c1ee71
Showing 3 changed files with 90 additions and 45 deletions Side-by-side Diff
arch/mips/include/asm/branch.h
arch/mips/kernel/branch.c
arch/mips/math-emu/cp1emu.c
@@ -9,6 +9,7 @@
 #define _ASM_BRANCH_H
  
 #include <asm/ptrace.h>
+#include <asm/inst.h>
  
 static inline int delay_slot(struct pt_regs *regs)
 {
  
@@ -23,7 +24,11 @@
 	return regs->cp0_epc + 4;
 }
  
+#define BRANCH_LIKELY_TAKEN 0x0001
+
 extern int __compute_return_epc(struct pt_regs *regs);
+extern int __compute_return_epc_for_insn(struct pt_regs *regs,
+					 union mips_instruction insn);
  
 static inline int compute_return_epc(struct pt_regs *regs)
 {
@@ -9,6 +9,7 @@
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/signal.h>
+#include <linux/module.h>
 #include <asm/branch.h>
 #include <asm/cpu.h>
 #include <asm/cpu-features.h>
  
  
  
  
@@ -17,29 +18,23 @@
 #include <asm/ptrace.h>
 #include <asm/uaccess.h>
  
-/*
- * Compute the return address and do emulate branch simulation, if required.
+/**
+ * __compute_return_epc_for_insn - Computes the return address and do emulate
+ *				    branch simulation, if required.
+ *
+ * @regs:	Pointer to pt_regs
+ * @insn:	branch instruction to decode
+ * @returns:	-EFAULT on error and forces SIGBUS, and on success
+ *		returns 0 or BRANCH_LIKELY_TAKEN as appropriate after
+ *		evaluating the branch.
  */
-int __compute_return_epc(struct pt_regs *regs)
+int __compute_return_epc_for_insn(struct pt_regs *regs,
+				   union mips_instruction insn)
 {
-	unsigned int __user *addr;
 	unsigned int bit, fcr31, dspcontrol;
-	long epc;
-	union mips_instruction insn;
+	long epc = regs->cp0_epc;
+	int ret = 0;
  
-	epc = regs->cp0_epc;
-	if (epc & 3)
-		goto unaligned;
-
-	/*
-	 * Read the instruction
-	 */
-	addr = (unsigned int __user *) epc;
-	if (__get_user(insn.word, addr)) {
-		force_sig(SIGSEGV, current);
-		return -EFAULT;
-	}
-
 	switch (insn.i_format.opcode) {
 	/*
 	 * jr and jalr are in r_format format.
  
  
  
@@ -64,18 +59,22 @@
 		switch (insn.i_format.rt) {
 	 	case bltz_op:
 		case bltzl_op:
-			if ((long)regs->regs[insn.i_format.rs] < 0)
+			if ((long)regs->regs[insn.i_format.rs] < 0) {
 				epc = epc + 4 + (insn.i_format.simmediate << 2);
-			else
+				if (insn.i_format.rt == bltzl_op)
+					ret = BRANCH_LIKELY_TAKEN;
+			} else
 				epc += 8;
 			regs->cp0_epc = epc;
 			break;
  
 		case bgez_op:
 		case bgezl_op:
-			if ((long)regs->regs[insn.i_format.rs] >= 0)
+			if ((long)regs->regs[insn.i_format.rs] >= 0) {
 				epc = epc + 4 + (insn.i_format.simmediate << 2);
-			else
+				if (insn.i_format.rt == bgezl_op)
+					ret = BRANCH_LIKELY_TAKEN;
+			} else
 				epc += 8;
 			regs->cp0_epc = epc;
 			break;
  
@@ -83,9 +82,11 @@
 		case bltzal_op:
 		case bltzall_op:
 			regs->regs[31] = epc + 8;
-			if ((long)regs->regs[insn.i_format.rs] < 0)
+			if ((long)regs->regs[insn.i_format.rs] < 0) {
 				epc = epc + 4 + (insn.i_format.simmediate << 2);
-			else
+				if (insn.i_format.rt == bltzall_op)
+					ret = BRANCH_LIKELY_TAKEN;
+			} else
 				epc += 8;
 			regs->cp0_epc = epc;
 			break;
  
  
@@ -93,12 +94,15 @@
 		case bgezal_op:
 		case bgezall_op:
 			regs->regs[31] = epc + 8;
-			if ((long)regs->regs[insn.i_format.rs] >= 0)
+			if ((long)regs->regs[insn.i_format.rs] >= 0) {
 				epc = epc + 4 + (insn.i_format.simmediate << 2);
-			else
+				if (insn.i_format.rt == bgezall_op)
+					ret = BRANCH_LIKELY_TAKEN;
+			} else
 				epc += 8;
 			regs->cp0_epc = epc;
 			break;
+
 		case bposge32_op:
 			if (!cpu_has_dsp)
 				goto sigill;
  
@@ -133,9 +137,11 @@
 	case beq_op:
 	case beql_op:
 		if (regs->regs[insn.i_format.rs] ==
-		    regs->regs[insn.i_format.rt])
+		    regs->regs[insn.i_format.rt]) {
 			epc = epc + 4 + (insn.i_format.simmediate << 2);
-		else
+			if (insn.i_format.rt == beql_op)
+				ret = BRANCH_LIKELY_TAKEN;
+		} else
 			epc += 8;
 		regs->cp0_epc = epc;
 		break;
  
@@ -143,9 +149,11 @@
 	case bne_op:
 	case bnel_op:
 		if (regs->regs[insn.i_format.rs] !=
-		    regs->regs[insn.i_format.rt])
+		    regs->regs[insn.i_format.rt]) {
 			epc = epc + 4 + (insn.i_format.simmediate << 2);
-		else
+			if (insn.i_format.rt == bnel_op)
+				ret = BRANCH_LIKELY_TAKEN;
+		} else
 			epc += 8;
 		regs->cp0_epc = epc;
 		break;
  
@@ -153,9 +161,11 @@
 	case blez_op: /* not really i_format */
 	case blezl_op:
 		/* rt field assumed to be zero */
-		if ((long)regs->regs[insn.i_format.rs] <= 0)
+		if ((long)regs->regs[insn.i_format.rs] <= 0) {
 			epc = epc + 4 + (insn.i_format.simmediate << 2);
-		else
+			if (insn.i_format.rt == bnel_op)
+				ret = BRANCH_LIKELY_TAKEN;
+		} else
 			epc += 8;
 		regs->cp0_epc = epc;
 		break;
  
@@ -163,9 +173,11 @@
 	case bgtz_op:
 	case bgtzl_op:
 		/* rt field assumed to be zero */
-		if ((long)regs->regs[insn.i_format.rs] > 0)
+		if ((long)regs->regs[insn.i_format.rs] > 0) {
 			epc = epc + 4 + (insn.i_format.simmediate << 2);
-		else
+			if (insn.i_format.rt == bnel_op)
+				ret = BRANCH_LIKELY_TAKEN;
+		} else
 			epc += 8;
 		regs->cp0_epc = epc;
 		break;
  
  
  
@@ -187,18 +199,22 @@
 		switch (insn.i_format.rt & 3) {
 		case 0:	/* bc1f */
 		case 2:	/* bc1fl */
-			if (~fcr31 & (1 << bit))
+			if (~fcr31 & (1 << bit)) {
 				epc = epc + 4 + (insn.i_format.simmediate << 2);
-			else
+				if (insn.i_format.rt == 2)
+					ret = BRANCH_LIKELY_TAKEN;
+			} else
 				epc += 8;
 			regs->cp0_epc = epc;
 			break;
  
 		case 1:	/* bc1t */
 		case 3:	/* bc1tl */
-			if (fcr31 & (1 << bit))
+			if (fcr31 & (1 << bit)) {
 				epc = epc + 4 + (insn.i_format.simmediate << 2);
-			else
+				if (insn.i_format.rt == 3)
+					ret = BRANCH_LIKELY_TAKEN;
+			} else
 				epc += 8;
 			regs->cp0_epc = epc;
 			break;
  
  
  
  
@@ -239,16 +255,40 @@
 #endif
 	}
  
-	return 0;
+	return ret;
  
-unaligned:
-	printk("%s: unaligned epc - sending SIGBUS.\n", current->comm);
+sigill:
+	printk("%s: DSP branch but not DSP ASE - sending SIGBUS.\n", current->comm);
 	force_sig(SIGBUS, current);
 	return -EFAULT;
+}
+EXPORT_SYMBOL_GPL(__compute_return_epc_for_insn);
  
-sigill:
-	printk("%s: DSP branch but not DSP ASE - sending SIGBUS.\n", current->comm);
+int __compute_return_epc(struct pt_regs *regs)
+{
+	unsigned int __user *addr;
+	long epc;
+	union mips_instruction insn;
+
+	epc = regs->cp0_epc;
+	if (epc & 3)
+		goto unaligned;
+
+	/*
+	 * Read the instruction
+	 */
+	addr = (unsigned int __user *) epc;
+	if (__get_user(insn.word, addr)) {
+		force_sig(SIGSEGV, current);
+		return -EFAULT;
+	}
+
+	return __compute_return_epc_for_insn(regs, insn);
+
+unaligned:
+	printk("%s: unaligned epc - sending SIGBUS.\n", current->comm);
 	force_sig(SIGBUS, current);
 	return -EFAULT;
+
 }
@@ -245,7 +245,7 @@
 		 */
 		emulpc = xcp->cp0_epc + 4;	/* Snapshot emulation target */
  
-		if (__compute_return_epc(xcp)) {
+		if (__compute_return_epc(xcp) < 0) {
 #ifdef CP1DBG
 			printk("failed to emulate branch at %p\n",
 				(void *) (xcp->cp0_epc));
...	...	@@ -9,6 +9,7 @@
9	9	#define _ASM_BRANCH_H
10	10
11	11	#include <asm/ptrace.h>
	12	+#include <asm/inst.h>
12	13
13	14	static inline int delay_slot(struct pt_regs *regs)
14	15	{
15	16
...	...	@@ -23,7 +24,11 @@
23	24	return regs->cp0_epc + 4;
24	25	}
25	26
	27	+#define BRANCH_LIKELY_TAKEN 0x0001
	28	+
26	29	extern int __compute_return_epc(struct pt_regs *regs);
	30	+extern int __compute_return_epc_for_insn(struct pt_regs *regs,
	31	+ union mips_instruction insn);
27	32
28	33	static inline int compute_return_epc(struct pt_regs *regs)
29	34	{
...	...	@@ -9,6 +9,7 @@
9	9	#include <linux/kernel.h>
10	10	#include <linux/sched.h>
11	11	#include <linux/signal.h>
	12	+#include <linux/module.h>
12	13	#include <asm/branch.h>
13	14	#include <asm/cpu.h>
14	15	#include <asm/cpu-features.h>
15	16
16	17
17	18
18	19
...	...	@@ -17,29 +18,23 @@
17	18	#include <asm/ptrace.h>
18	19	#include <asm/uaccess.h>
19	20
20		-/*
21		- * Compute the return address and do emulate branch simulation, if required.
	21	+/**
	22	+ * __compute_return_epc_for_insn - Computes the return address and do emulate
	23	+ * branch simulation, if required.
	24	+ *
	25	+ * @regs: Pointer to pt_regs
	26	+ * @insn: branch instruction to decode
	27	+ * @returns: -EFAULT on error and forces SIGBUS, and on success
	28	+ * returns 0 or BRANCH_LIKELY_TAKEN as appropriate after
	29	+ * evaluating the branch.
22	30	*/
23		-int __compute_return_epc(struct pt_regs *regs)
	31	+int __compute_return_epc_for_insn(struct pt_regs *regs,
	32	+ union mips_instruction insn)
24	33	{
25		- unsigned int __user *addr;
26	34	unsigned int bit, fcr31, dspcontrol;
27		- long epc;
28		- union mips_instruction insn;
	35	+ long epc = regs->cp0_epc;
	36	+ int ret = 0;
29	37
30		- epc = regs->cp0_epc;
31		- if (epc & 3)
32		- goto unaligned;
33		-
34		- /*
35		- * Read the instruction
36		- */
37		- addr = (unsigned int __user *) epc;
38		- if (__get_user(insn.word, addr)) {
39		- force_sig(SIGSEGV, current);
40		- return -EFAULT;
41		- }
42		-
43	38	switch (insn.i_format.opcode) {
44	39	/*
45	40	* jr and jalr are in r_format format.
46	41
47	42
48	43
...	...	@@ -64,18 +59,22 @@
64	59	switch (insn.i_format.rt) {
65	60	case bltz_op:
66	61	case bltzl_op:
67		- if ((long)regs->regs[insn.i_format.rs] < 0)
	62	+ if ((long)regs->regs[insn.i_format.rs] < 0) {
68	63	epc = epc + 4 + (insn.i_format.simmediate << 2);
69		- else
	64	+ if (insn.i_format.rt == bltzl_op)
	65	+ ret = BRANCH_LIKELY_TAKEN;
	66	+ } else
70	67	epc += 8;
71	68	regs->cp0_epc = epc;
72	69	break;
73	70
74	71	case bgez_op:
75	72	case bgezl_op:
76		- if ((long)regs->regs[insn.i_format.rs] >= 0)
	73	+ if ((long)regs->regs[insn.i_format.rs] >= 0) {
77	74	epc = epc + 4 + (insn.i_format.simmediate << 2);
78		- else
	75	+ if (insn.i_format.rt == bgezl_op)
	76	+ ret = BRANCH_LIKELY_TAKEN;
	77	+ } else
79	78	epc += 8;
80	79	regs->cp0_epc = epc;
81	80	break;
82	81
...	...	@@ -83,9 +82,11 @@
83	82	case bltzal_op:
84	83	case bltzall_op:
85	84	regs->regs[31] = epc + 8;
86		- if ((long)regs->regs[insn.i_format.rs] < 0)
	85	+ if ((long)regs->regs[insn.i_format.rs] < 0) {
87	86	epc = epc + 4 + (insn.i_format.simmediate << 2);
88		- else
	87	+ if (insn.i_format.rt == bltzall_op)
	88	+ ret = BRANCH_LIKELY_TAKEN;
	89	+ } else
89	90	epc += 8;
90	91	regs->cp0_epc = epc;
91	92	break;
92	93
93	94
...	...	@@ -93,12 +94,15 @@
93	94	case bgezal_op:
94	95	case bgezall_op:
95	96	regs->regs[31] = epc + 8;
96		- if ((long)regs->regs[insn.i_format.rs] >= 0)
	97	+ if ((long)regs->regs[insn.i_format.rs] >= 0) {
97	98	epc = epc + 4 + (insn.i_format.simmediate << 2);
98		- else
	99	+ if (insn.i_format.rt == bgezall_op)
	100	+ ret = BRANCH_LIKELY_TAKEN;
	101	+ } else
99	102	epc += 8;
100	103	regs->cp0_epc = epc;
101	104	break;
	105	+
102	106	case bposge32_op:
103	107	if (!cpu_has_dsp)
104	108	goto sigill;
105	109
...	...	@@ -133,9 +137,11 @@
133	137	case beq_op:
134	138	case beql_op:
135	139	if (regs->regs[insn.i_format.rs] ==
136		- regs->regs[insn.i_format.rt])
	140	+ regs->regs[insn.i_format.rt]) {
137	141	epc = epc + 4 + (insn.i_format.simmediate << 2);
138		- else
	142	+ if (insn.i_format.rt == beql_op)
	143	+ ret = BRANCH_LIKELY_TAKEN;
	144	+ } else
139	145	epc += 8;
140	146	regs->cp0_epc = epc;
141	147	break;
142	148
...	...	@@ -143,9 +149,11 @@
143	149	case bne_op:
144	150	case bnel_op:
145	151	if (regs->regs[insn.i_format.rs] !=
146		- regs->regs[insn.i_format.rt])
	152	+ regs->regs[insn.i_format.rt]) {
147	153	epc = epc + 4 + (insn.i_format.simmediate << 2);
148		- else
	154	+ if (insn.i_format.rt == bnel_op)
	155	+ ret = BRANCH_LIKELY_TAKEN;
	156	+ } else
149	157	epc += 8;
150	158	regs->cp0_epc = epc;
151	159	break;
152	160
...	...	@@ -153,9 +161,11 @@
153	161	case blez_op: /* not really i_format */
154	162	case blezl_op:
155	163	/* rt field assumed to be zero */
156		- if ((long)regs->regs[insn.i_format.rs] <= 0)
	164	+ if ((long)regs->regs[insn.i_format.rs] <= 0) {
157	165	epc = epc + 4 + (insn.i_format.simmediate << 2);
158		- else
	166	+ if (insn.i_format.rt == bnel_op)
	167	+ ret = BRANCH_LIKELY_TAKEN;
	168	+ } else
159	169	epc += 8;
160	170	regs->cp0_epc = epc;
161	171	break;
162	172
...	...	@@ -163,9 +173,11 @@
163	173	case bgtz_op:
164	174	case bgtzl_op:
165	175	/* rt field assumed to be zero */
166		- if ((long)regs->regs[insn.i_format.rs] > 0)
	176	+ if ((long)regs->regs[insn.i_format.rs] > 0) {
167	177	epc = epc + 4 + (insn.i_format.simmediate << 2);
168		- else
	178	+ if (insn.i_format.rt == bnel_op)
	179	+ ret = BRANCH_LIKELY_TAKEN;
	180	+ } else
169	181	epc += 8;
170	182	regs->cp0_epc = epc;
171	183	break;
172	184
173	185
174	186
...	...	@@ -187,18 +199,22 @@
187	199	switch (insn.i_format.rt & 3) {
188	200	case 0: /* bc1f */
189	201	case 2: /* bc1fl */
190		- if (~fcr31 & (1 << bit))
	202	+ if (~fcr31 & (1 << bit)) {
191	203	epc = epc + 4 + (insn.i_format.simmediate << 2);
192		- else
	204	+ if (insn.i_format.rt == 2)
	205	+ ret = BRANCH_LIKELY_TAKEN;
	206	+ } else
193	207	epc += 8;
194	208	regs->cp0_epc = epc;
195	209	break;
196	210
197	211	case 1: /* bc1t */
198	212	case 3: /* bc1tl */
199		- if (fcr31 & (1 << bit))
	213	+ if (fcr31 & (1 << bit)) {
200	214	epc = epc + 4 + (insn.i_format.simmediate << 2);
201		- else
	215	+ if (insn.i_format.rt == 3)
	216	+ ret = BRANCH_LIKELY_TAKEN;
	217	+ } else
202	218	epc += 8;
203	219	regs->cp0_epc = epc;
204	220	break;
205	221
206	222
207	223
208	224
...	...	@@ -239,16 +255,40 @@
239	255	#endif
240	256	}
241	257
242		- return 0;
	258	+ return ret;
243	259
244		-unaligned:
245		- printk("%s: unaligned epc - sending SIGBUS.\n", current->comm);
	260	+sigill:
	261	+ printk("%s: DSP branch but not DSP ASE - sending SIGBUS.\n", current->comm);
246	262	force_sig(SIGBUS, current);
247	263	return -EFAULT;
	264	+}
	265	+EXPORT_SYMBOL_GPL(__compute_return_epc_for_insn);
248	266
249		-sigill:
250		- printk("%s: DSP branch but not DSP ASE - sending SIGBUS.\n", current->comm);
	267	+int __compute_return_epc(struct pt_regs *regs)
	268	+{
	269	+ unsigned int __user *addr;
	270	+ long epc;
	271	+ union mips_instruction insn;
	272	+
	273	+ epc = regs->cp0_epc;
	274	+ if (epc & 3)
	275	+ goto unaligned;
	276	+
	277	+ /*
	278	+ * Read the instruction
	279	+ */
	280	+ addr = (unsigned int __user *) epc;
	281	+ if (__get_user(insn.word, addr)) {
	282	+ force_sig(SIGSEGV, current);
	283	+ return -EFAULT;
	284	+ }
	285	+
	286	+ return __compute_return_epc_for_insn(regs, insn);
	287	+
	288	+unaligned:
	289	+ printk("%s: unaligned epc - sending SIGBUS.\n", current->comm);
251	290	force_sig(SIGBUS, current);
252	291	return -EFAULT;
	292	+
253	293	}
...	...	@@ -245,7 +245,7 @@
245	245	*/
246	246	emulpc = xcp->cp0_epc + 4; /* Snapshot emulation target */
247	247
248		- if (__compute_return_epc(xcp)) {
	248	+ if (__compute_return_epc(xcp) < 0) {
249	249	#ifdef CP1DBG
250	250	printk("failed to emulate branch at %p\n",
251	251	(void *) (xcp->cp0_epc));