x86, extable: Switch to relative exception table entries

Switch to using relative exception table entries on x86. On i386, this has the advantage that the exception table entries don't need to be relocated; on x86-64 this means the exception table entries take up only half the space. In either case, a 32-bit delta is sufficient, as the range of kernel code addresses is limited. Since part of the goal is to avoid needing to adjust the entries when the kernel is relocated, the old trick of using addresses in the NULL pointer range to indicate uaccess_err no longer works (and unlike RISC architectures we can't use a flag bit); instead use an delta just below +2G to indicate these special entries. The reach is still limited to a single instruction. Signed-off-by: H. Peter Anvin <hpa@zytor.com> Cc: David Daney <david.daney@cavium.com> Link: http://lkml.kernel.org/r/CA%2B55aFyijf43qSu3N9nWHEBwaGbb7T2Oq9A=9EyR=Jtyqfq_cQ@mail.gmail.com

x86, extable: Switch to relative exception table entries
Switch to using relative exception table entries on x86. On i386, this has the advantage that the exception table entries don't need to be relocated; on x86-64 this means the exception table entries take up only half the space. In either case, a 32-bit delta is sufficient, as the range of kernel code addresses is limited. Since part of the goal is to avoid needing to adjust the entries when the kernel is relocated, the old trick of using addresses in the NULL pointer range to indicate uaccess_err no longer works (and unlike RISC architectures we can't use a flag bit); instead use an delta just below +2G to indicate these special entries. The reach is still limited to a single instruction. Signed-off-by: H. Peter Anvin <hpa@zytor.com> Cc: David Daney <david.daney@cavium.com> Link: http://lkml.kernel.org/r/CA%2B55aFyijf43qSu3N9nWHEBwaGbb7T2Oq9A=9EyR=Jtyqfq_cQ@mail.gmail.com
H. Peter Anvin
1 parent fa574a48a1
Showing 3 changed files with 146 additions and 22 deletions Side-by-side Diff
arch/x86/include/asm/asm.h
arch/x86/include/asm/uaccess.h
arch/x86/mm/extable.c
@@ -42,26 +42,30 @@
 #ifdef __ASSEMBLY__
 # define _ASM_EXTABLE(from,to)					\
 	.pushsection "__ex_table","a" ;				\
-	_ASM_ALIGN ;						\
-	_ASM_PTR from , to ;					\
+	.balign 8 ;						\
+	.long (from) - . ;					\
+	.long (to) - . ;					\
 	.popsection
  
 # define _ASM_EXTABLE_EX(from,to)				\
 	.pushsection "__ex_table","a" ;				\
-	_ASM_ALIGN ;						\
-	_ASM_PTR from , (to) - (from) ;				\
+	.balign 8 ;						\
+	.long (from) - . ;					\
+	.long (to) - . + 0x7ffffff0 ;				\
 	.popsection
 #else
 # define _ASM_EXTABLE(from,to)					\
 	" .pushsection \"__ex_table\",\"a\"\n"			\
-	_ASM_ALIGN "\n" 					\
-	_ASM_PTR #from "," #to "\n" 				\
+	" .balign 8\n"						\
+	" .long (" #from ") - .\n"				\
+	" .long (" #to ") - .\n"				\
 	" .popsection\n"
  
 # define _ASM_EXTABLE_EX(from,to)				\
 	" .pushsection \"__ex_table\",\"a\"\n"			\
-	_ASM_ALIGN "\n" 					\
-	_ASM_PTR #from ",(" #to ")-(" #from ")\n" 		\
+	" .balign 8\n"						\
+	" .long (" #from ") - .\n"				\
+	" .long (" #to ") - . + 0x7ffffff0\n"			\
 	" .popsection\n"
 #endif
  
@@ -79,11 +79,12 @@
 #define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
  
 /*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue.  No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
+ * The exception table consists of pairs of addresses relative to the
+ * exception table enty itself: the first is the address of an
+ * instruction that is allowed to fault, and the second is the address
+ * at which the program should continue.  No registers are modified,
+ * so it is entirely up to the continuation code to figure out what to
+ * do.
  *
  * All the routines below use bits of fixup code that are out of line
  * with the main instruction path.  This means when everything is well,
  
  
@@ -92,10 +93,14 @@
  */
  
 struct exception_table_entry {
-	unsigned long insn, fixup;
+	int insn, fixup;
 };
+/* This is not the generic standard exception_table_entry format */
+#define ARCH_HAS_SORT_EXTABLE
+#define ARCH_HAS_SEARCH_EXTABLE
  
 extern int fixup_exception(struct pt_regs *regs);
+extern int early_fixup_exception(unsigned long *ip);
  
 /*
  * These are the main single-value transfer routines.  They automatically
 #include <linux/module.h>
 #include <linux/spinlock.h>
+#include <linux/sort.h>
 #include <asm/uaccess.h>
  
+static inline unsigned long
+ex_insn_addr(const struct exception_table_entry *x)
+{
+	return (unsigned long)&x->insn + x->insn;
+}
+static inline unsigned long
+ex_fixup_addr(const struct exception_table_entry *x)
+{
+	return (unsigned long)&x->fixup + x->fixup;
+}
  
 int fixup_exception(struct pt_regs *regs)
 {
 	const struct exception_table_entry *fixup;
+	unsigned long new_ip;
  
 #ifdef CONFIG_PNPBIOS
 	if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) {
  
  
@@ -23,13 +35,14 @@
  
 	fixup = search_exception_tables(regs->ip);
 	if (fixup) {
-		/* If fixup is less than 16, it means uaccess error */
-		if (fixup->fixup < 16) {
+		new_ip = ex_fixup_addr(fixup);
+
+		if (fixup->fixup - fixup->insn >= 0x7ffffff0 - 4) {
+			/* Special hack for uaccess_err */
 			current_thread_info()->uaccess_err = 1;
-			regs->ip += fixup->fixup;
-			return 1;
+			new_ip -= 0x7ffffff0;
 		}
-		regs->ip = fixup->fixup;
+		regs->ip = new_ip;
 		return 1;
 	}
  
  
  
  
@@ -40,16 +53,118 @@
 int __init early_fixup_exception(unsigned long *ip)
 {
 	const struct exception_table_entry *fixup;
+	unsigned long new_ip;
  
 	fixup = search_exception_tables(*ip);
 	if (fixup) {
-		if (fixup->fixup < 16)
-			return 0; /* Not supported during early boot */
+		new_ip = ex_fixup_addr(fixup);
  
-		*ip = fixup->fixup;
+		if (fixup->fixup - fixup->insn >= 0x7ffffff0 - 4) {
+			/* uaccess handling not supported during early boot */
+			return 0;
+		}
+
+		*ip = new_ip;
 		return 1;
 	}
  
 	return 0;
 }
+
+/*
+ * Search one exception table for an entry corresponding to the
+ * given instruction address, and return the address of the entry,
+ * or NULL if none is found.
+ * We use a binary search, and thus we assume that the table is
+ * already sorted.
+ */
+const struct exception_table_entry *
+search_extable(const struct exception_table_entry *first,
+	       const struct exception_table_entry *last,
+	       unsigned long value)
+{
+	while (first <= last) {
+		const struct exception_table_entry *mid;
+		unsigned long addr;
+
+		mid = ((last - first) >> 1) + first;
+		addr = ex_insn_addr(mid);
+		if (addr < value)
+			first = mid + 1;
+		else if (addr > value)
+			last = mid - 1;
+		else
+			return mid;
+        }
+        return NULL;
+}
+
+/*
+ * The exception table needs to be sorted so that the binary
+ * search that we use to find entries in it works properly.
+ * This is used both for the kernel exception table and for
+ * the exception tables of modules that get loaded.
+ *
+ */
+static int cmp_ex(const void *a, const void *b)
+{
+	const struct exception_table_entry *x = a, *y = b;
+
+	/*
+	 * This value will always end up fittin in an int, because on
+	 * both i386 and x86-64 the kernel symbol-reachable address
+	 * space is < 2 GiB.
+	 *
+	 * This compare is only valid after normalization.
+	 */
+	return x->insn - y->insn;
+}
+
+void sort_extable(struct exception_table_entry *start,
+		  struct exception_table_entry *finish)
+{
+	struct exception_table_entry *p;
+	int i;
+
+	/* Convert all entries to being relative to the start of the section */
+	i = 0;
+	for (p = start; p < finish; p++) {
+		p->insn += i;
+		i += 4;
+		p->fixup += i;
+		i += 4;
+	}
+
+	sort(start, finish - start, sizeof(struct exception_table_entry),
+	     cmp_ex, NULL);
+
+	/* Denormalize all entries */
+	i = 0;
+	for (p = start; p < finish; p++) {
+		p->insn -= i;
+		i += 4;
+		p->fixup -= i;
+		i += 4;
+	}
+}
+
+#ifdef CONFIG_MODULES
+/*
+ * If the exception table is sorted, any referring to the module init
+ * will be at the beginning or the end.
+ */
+void trim_init_extable(struct module *m)
+{
+	/*trim the beginning*/
+	while (m->num_exentries &&
+	       within_module_init(ex_insn_addr(&m->extable[0]), m)) {
+		m->extable++;
+		m->num_exentries--;
+	}
+	/*trim the end*/
+	while (m->num_exentries &&
+	       within_module_init(ex_insn_addr(&m->extable[m->num_exentries-1]), m))
+		m->num_exentries--;
+}
+#endif /* CONFIG_MODULES */
...	...	@@ -42,26 +42,30 @@
42	42	#ifdef __ASSEMBLY__
43	43	# define _ASM_EXTABLE(from,to) \
44	44	.pushsection "__ex_table","a" ; \
45		- _ASM_ALIGN ; \
46		- _ASM_PTR from , to ; \
	45	+ .balign 8 ; \
	46	+ .long (from) - . ; \
	47	+ .long (to) - . ; \
47	48	.popsection
48	49
49	50	# define _ASM_EXTABLE_EX(from,to) \
50	51	.pushsection "__ex_table","a" ; \
51		- _ASM_ALIGN ; \
52		- _ASM_PTR from , (to) - (from) ; \
	52	+ .balign 8 ; \
	53	+ .long (from) - . ; \
	54	+ .long (to) - . + 0x7ffffff0 ; \
53	55	.popsection
54	56	#else
55	57	# define _ASM_EXTABLE(from,to) \
56	58	" .pushsection \"__ex_table\",\"a\"\n" \
57		- _ASM_ALIGN "\n" \
58		- _ASM_PTR #from "," #to "\n" \
	59	+ " .balign 8\n" \
	60	+ " .long (" #from ") - .\n" \
	61	+ " .long (" #to ") - .\n" \
59	62	" .popsection\n"
60	63
61	64	# define _ASM_EXTABLE_EX(from,to) \
62	65	" .pushsection \"__ex_table\",\"a\"\n" \
63		- _ASM_ALIGN "\n" \
64		- _ASM_PTR #from ",(" #to ")-(" #from ")\n" \
	66	+ " .balign 8\n" \
	67	+ " .long (" #from ") - .\n" \
	68	+ " .long (" #to ") - . + 0x7ffffff0\n" \
65	69	" .popsection\n"
66	70	#endif
67	71
...	...	@@ -79,11 +79,12 @@
79	79	#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
80	80
81	81	/*
82		- * The exception table consists of pairs of addresses: the first is the
83		- * address of an instruction that is allowed to fault, and the second is
84		- * the address at which the program should continue. No registers are
85		- * modified, so it is entirely up to the continuation code to figure out
86		- * what to do.
	82	+ * The exception table consists of pairs of addresses relative to the
	83	+ * exception table enty itself: the first is the address of an
	84	+ * instruction that is allowed to fault, and the second is the address
	85	+ * at which the program should continue. No registers are modified,
	86	+ * so it is entirely up to the continuation code to figure out what to
	87	+ * do.
87	88	*
88	89	* All the routines below use bits of fixup code that are out of line
89	90	* with the main instruction path. This means when everything is well,
90	91
91	92
...	...	@@ -92,10 +93,14 @@
92	93	*/
93	94
94	95	struct exception_table_entry {
95		- unsigned long insn, fixup;
	96	+ int insn, fixup;
96	97	};
	98	+/* This is not the generic standard exception_table_entry format */
	99	+#define ARCH_HAS_SORT_EXTABLE
	100	+#define ARCH_HAS_SEARCH_EXTABLE
97	101
98	102	extern int fixup_exception(struct pt_regs *regs);
	103	+extern int early_fixup_exception(unsigned long *ip);
99	104
100	105	/*
101	106	* These are the main single-value transfer routines. They automatically
1	1	#include <linux/module.h>
2	2	#include <linux/spinlock.h>
	3	+#include <linux/sort.h>
3	4	#include <asm/uaccess.h>
4	5
	6	+static inline unsigned long
	7	+ex_insn_addr(const struct exception_table_entry *x)
	8	+{
	9	+ return (unsigned long)&x->insn + x->insn;
	10	+}
	11	+static inline unsigned long
	12	+ex_fixup_addr(const struct exception_table_entry *x)
	13	+{
	14	+ return (unsigned long)&x->fixup + x->fixup;
	15	+}
5	16
6	17	int fixup_exception(struct pt_regs *regs)
7	18	{
8	19	const struct exception_table_entry *fixup;
	20	+ unsigned long new_ip;
9	21
10	22	#ifdef CONFIG_PNPBIOS
11	23	if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) {
12	24
13	25
...	...	@@ -23,13 +35,14 @@
23	35
24	36	fixup = search_exception_tables(regs->ip);
25	37	if (fixup) {
26		- /* If fixup is less than 16, it means uaccess error */
27		- if (fixup->fixup < 16) {
	38	+ new_ip = ex_fixup_addr(fixup);
	39	+
	40	+ if (fixup->fixup - fixup->insn >= 0x7ffffff0 - 4) {
	41	+ /* Special hack for uaccess_err */
28	42	current_thread_info()->uaccess_err = 1;
29		- regs->ip += fixup->fixup;
30		- return 1;
	43	+ new_ip -= 0x7ffffff0;
31	44	}
32		- regs->ip = fixup->fixup;
	45	+ regs->ip = new_ip;
33	46	return 1;
34	47	}
35	48
36	49
37	50
38	51
...	...	@@ -40,16 +53,118 @@
40	53	int __init early_fixup_exception(unsigned long *ip)
41	54	{
42	55	const struct exception_table_entry *fixup;
	56	+ unsigned long new_ip;
43	57
44	58	fixup = search_exception_tables(*ip);
45	59	if (fixup) {
46		- if (fixup->fixup < 16)
47		- return 0; /* Not supported during early boot */
	60	+ new_ip = ex_fixup_addr(fixup);
48	61
49		- *ip = fixup->fixup;
	62	+ if (fixup->fixup - fixup->insn >= 0x7ffffff0 - 4) {
	63	+ /* uaccess handling not supported during early boot */
	64	+ return 0;
	65	+ }
	66	+
	67	+ *ip = new_ip;
50	68	return 1;
51	69	}
52	70
53	71	return 0;
54	72	}
	73	+
	74	+/*
	75	+ * Search one exception table for an entry corresponding to the
	76	+ * given instruction address, and return the address of the entry,
	77	+ * or NULL if none is found.
	78	+ * We use a binary search, and thus we assume that the table is
	79	+ * already sorted.
	80	+ */
	81	+const struct exception_table_entry *
	82	+search_extable(const struct exception_table_entry *first,
	83	+ const struct exception_table_entry *last,
	84	+ unsigned long value)
	85	+{
	86	+ while (first <= last) {
	87	+ const struct exception_table_entry *mid;
	88	+ unsigned long addr;
	89	+
	90	+ mid = ((last - first) >> 1) + first;
	91	+ addr = ex_insn_addr(mid);
	92	+ if (addr < value)
	93	+ first = mid + 1;
	94	+ else if (addr > value)
	95	+ last = mid - 1;
	96	+ else
	97	+ return mid;
	98	+ }
	99	+ return NULL;
	100	+}
	101	+
	102	+/*
	103	+ * The exception table needs to be sorted so that the binary
	104	+ * search that we use to find entries in it works properly.
	105	+ * This is used both for the kernel exception table and for
	106	+ * the exception tables of modules that get loaded.
	107	+ *
	108	+ */
	109	+static int cmp_ex(const void a, const void b)
	110	+{
	111	+ const struct exception_table_entry x = a, y = b;
	112	+
	113	+ /*
	114	+ * This value will always end up fittin in an int, because on
	115	+ * both i386 and x86-64 the kernel symbol-reachable address
	116	+ * space is < 2 GiB.
	117	+ *
	118	+ * This compare is only valid after normalization.
	119	+ */
	120	+ return x->insn - y->insn;
	121	+}
	122	+
	123	+void sort_extable(struct exception_table_entry *start,
	124	+ struct exception_table_entry *finish)
	125	+{
	126	+ struct exception_table_entry *p;
	127	+ int i;
	128	+
	129	+ /* Convert all entries to being relative to the start of the section */
	130	+ i = 0;
	131	+ for (p = start; p < finish; p++) {
	132	+ p->insn += i;
	133	+ i += 4;
	134	+ p->fixup += i;
	135	+ i += 4;
	136	+ }
	137	+
	138	+ sort(start, finish - start, sizeof(struct exception_table_entry),
	139	+ cmp_ex, NULL);
	140	+
	141	+ /* Denormalize all entries */
	142	+ i = 0;
	143	+ for (p = start; p < finish; p++) {
	144	+ p->insn -= i;
	145	+ i += 4;
	146	+ p->fixup -= i;
	147	+ i += 4;
	148	+ }
	149	+}
	150	+
	151	+#ifdef CONFIG_MODULES
	152	+/*
	153	+ * If the exception table is sorted, any referring to the module init
	154	+ * will be at the beginning or the end.
	155	+ */
	156	+void trim_init_extable(struct module *m)
	157	+{
	158	+ /trim the beginning/
	159	+ while (m->num_exentries &&
	160	+ within_module_init(ex_insn_addr(&m->extable[0]), m)) {
	161	+ m->extable++;
	162	+ m->num_exentries--;
	163	+ }
	164	+ /trim the end/
	165	+ while (m->num_exentries &&
	166	+ within_module_init(ex_insn_addr(&m->extable[m->num_exentries-1]), m))
	167	+ m->num_exentries--;
	168	+}
	169	+#endif /* CONFIG_MODULES */