s390/exceptions: switch to relative exception table entries

This is the s390 port of 70627654 "x86, extable: Switch to relative exception table entries". Reduces the size of our exception tables by 50% on 64 bit builds. Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com> Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

s390/exceptions: switch to relative exception table entries
This is the s390 port of 70627654 "x86, extable: Switch to relative exception table entries". Reduces the size of our exception tables by 50% on 64 bit builds. Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com> Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Heiko Carstens · Martin Schwidefsky
1 parent 24996edce5
Showing 10 changed files with 132 additions and 33 deletions Side-by-side Diff
arch/s390/include/asm/processor.h
arch/s390/include/asm/uaccess.h
arch/s390/kernel/early.c
arch/s390/kernel/entry64.S
arch/s390/kernel/kprobes.c
arch/s390/kernel/traps.c
arch/s390/mm/Makefile
arch/s390/mm/extable.c
arch/s390/mm/fault.c
scripts/sortextable.c
@@ -11,6 +11,8 @@
 #ifndef __ASM_S390_PROCESSOR_H
 #define __ASM_S390_PROCESSOR_H
  
+#ifndef __ASSEMBLY__
+
 #include <linux/linkage.h>
 #include <linux/irqflags.h>
 #include <asm/cpu.h>
@@ -348,23 +350,6 @@
  
 #define ARCH_LOW_ADDRESS_LIMIT	0x7fffffffUL
  
-/*
- * Helper macro for exception table entries
- */
-#ifndef CONFIG_64BIT
-#define EX_TABLE(_fault,_target)			\
-	".section __ex_table,\"a\"\n"			\
-	"	.align 4\n"				\
-	"	.long  " #_fault "," #_target "\n"	\
-	".previous\n"
-#else
-#define EX_TABLE(_fault,_target)			\
-	".section __ex_table,\"a\"\n"			\
-	"	.align 8\n"				\
-	"	.quad  " #_fault "," #_target "\n"	\
-	".previous\n"
-#endif
-
 extern int memcpy_real(void *, void *, size_t);
 extern void memcpy_absolute(void *, void *, size_t);
  
@@ -375,5 +360,26 @@
 	memcpy_absolute(&(dest), &__tmp, sizeof(__tmp));	\
 }
  
-#endif                                 /* __ASM_S390_PROCESSOR_H           */
+/*
+ * Helper macro for exception table entries
+ */
+#define EX_TABLE(_fault, _target)	\
+	".section __ex_table,\"a\"\n"	\
+	".align	4\n"			\
+	".long	(" #_fault ") - .\n"	\
+	".long	(" #_target ") - .\n"	\
+	".previous\n"
+
+#else /* __ASSEMBLY__ */
+
+#define EX_TABLE(_fault, _target)	\
+	.section __ex_table,"a"	;	\
+	.align	4 ;			\
+	.long	(_fault) - . ;		\
+	.long	(_target) - . ;		\
+	.previous
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* __ASM_S390_PROCESSOR_H */
@@ -76,8 +76,21 @@
  
 struct exception_table_entry
 {
-        unsigned long insn, fixup;
+	int insn, fixup;
 };
+
+static inline unsigned long extable_insn(const struct exception_table_entry *x)
+{
+	return (unsigned long)&x->insn + x->insn;
+}
+
+static inline unsigned long extable_fixup(const struct exception_table_entry *x)
+{
+	return (unsigned long)&x->fixup + x->fixup;
+}
+
+#define ARCH_HAS_SORT_EXTABLE
+#define ARCH_HAS_SEARCH_EXTABLE
  
 struct uaccess_ops {
 	size_t (*copy_from_user)(size_t, const void __user *, void *);
@@ -255,14 +255,14 @@
  
 static void early_pgm_check_handler(void)
 {
-	unsigned long addr;
 	const struct exception_table_entry *fixup;
+	unsigned long addr;
  
 	addr = S390_lowcore.program_old_psw.addr;
 	fixup = search_exception_tables(addr & PSW_ADDR_INSN);
 	if (!fixup)
 		disabled_wait(0);
-	S390_lowcore.program_old_psw.addr = fixup->fixup | PSW_ADDR_AMODE;
+	S390_lowcore.program_old_psw.addr = extable_fixup(fixup)|PSW_ADDR_AMODE;
 }
  
 static noinline __init void setup_lowcore_early(void)
@@ -10,6 +10,7 @@
  
 #include <linux/init.h>
 #include <linux/linkage.h>
+#include <asm/processor.h>
 #include <asm/cache.h>
 #include <asm/errno.h>
 #include <asm/ptrace.h>
@@ -1008,9 +1009,7 @@
 .Lhost_id:
 	.quad	0
  
-	.section __ex_table,"a"
-	.quad	sie_loop,sie_fault
-	.previous
+	EX_TABLE(sie_loop,sie_fault)
 #endif
  
 		.section .rodata, "a"
@@ -547,7 +547,7 @@
 		 */
 		entry = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
 		if (entry) {
-			regs->psw.addr = entry->fixup | PSW_ADDR_AMODE;
+			regs->psw.addr = extable_fixup(entry) | PSW_ADDR_AMODE;
 			return 1;
 		}
  
@@ -322,7 +322,7 @@
                 const struct exception_table_entry *fixup;
                 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
                 if (fixup)
-                        regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
+			regs->psw.addr = extable_fixup(fixup) | PSW_ADDR_AMODE;
 		else {
 			enum bug_trap_type btt;
  
@@ -3,7 +3,7 @@
 #
  
 obj-y	 := init.o fault.o extmem.o mmap.o vmem.o pgtable.o maccess.o \
-	    page-states.o gup.o
+	    page-states.o gup.o extable.o
 obj-$(CONFIG_CMM) += cmm.o
 obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
 obj-$(CONFIG_DEBUG_SET_MODULE_RONX) += pageattr.o
+#include <linux/module.h>
+#include <linux/sort.h>
+#include <asm/uaccess.h>
+
+/*
+ * Search one exception table for an entry corresponding to the
+ * given instruction address, and return the address of the entry,
+ * or NULL if none is found.
+ * We use a binary search, and thus we assume that the table is
+ * already sorted.
+ */
+const struct exception_table_entry *
+search_extable(const struct exception_table_entry *first,
+	       const struct exception_table_entry *last,
+	       unsigned long value)
+{
+	const struct exception_table_entry *mid;
+	unsigned long addr;
+
+	while (first <= last) {
+		mid = ((last - first) >> 1) + first;
+		addr = extable_insn(mid);
+		if (addr < value)
+			first = mid + 1;
+		else if (addr > value)
+			last = mid - 1;
+		else
+			return mid;
+	}
+	return NULL;
+}
+
+/*
+ * The exception table needs to be sorted so that the binary
+ * search that we use to find entries in it works properly.
+ * This is used both for the kernel exception table and for
+ * the exception tables of modules that get loaded.
+ *
+ */
+static int cmp_ex(const void *a, const void *b)
+{
+	const struct exception_table_entry *x = a, *y = b;
+
+	/* This compare is only valid after normalization. */
+	return x->insn - y->insn;
+}
+
+void sort_extable(struct exception_table_entry *start,
+		  struct exception_table_entry *finish)
+{
+	struct exception_table_entry *p;
+	int i;
+
+	/* Normalize entries to being relative to the start of the section */
+	for (p = start, i = 0; p < finish; p++, i += 8)
+		p->insn += i;
+	sort(start, finish - start, sizeof(*start), cmp_ex, NULL);
+	/* Denormalize all entries */
+	for (p = start, i = 0; p < finish; p++, i += 8)
+		p->insn -= i;
+}
+
+#ifdef CONFIG_MODULES
+/*
+ * If the exception table is sorted, any referring to the module init
+ * will be at the beginning or the end.
+ */
+void trim_init_extable(struct module *m)
+{
+	/* Trim the beginning */
+	while (m->num_exentries &&
+	       within_module_init(extable_insn(&m->extable[0]), m)) {
+		m->extable++;
+		m->num_exentries--;
+	}
+	/* Trim the end */
+	while (m->num_exentries &&
+	       within_module_init(extable_insn(&m->extable[m->num_exentries-1]), m))
+		m->num_exentries--;
+}
+#endif /* CONFIG_MODULES */
@@ -163,7 +163,7 @@
 	/* Are we prepared to handle this kernel fault?  */
 	fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
 	if (fixup) {
-		regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
+		regs->psw.addr = extable_fixup(fixup) | PSW_ADDR_AMODE;
 		return;
 	}
  
@@ -161,7 +161,7 @@
 #define SORTEXTABLE_64
 #include "sortextable.h"
  
-static int compare_x86_table(const void *a, const void *b)
+static int compare_relative_table(const void *a, const void *b)
 {
 	int32_t av = (int32_t)r(a);
 	int32_t bv = (int32_t)r(b);
@@ -173,7 +173,7 @@
 	return 0;
 }
  
-static void sort_x86_table(char *extab_image, int image_size)
+static void sort_relative_table(char *extab_image, int image_size)
 {
 	int i;
  
@@ -188,7 +188,7 @@
 		i += 4;
 	}
  
-	qsort(extab_image, image_size / 8, 8, compare_x86_table);
+	qsort(extab_image, image_size / 8, 8, compare_relative_table);
  
 	/* Now denormalize. */
 	i = 0;
  
@@ -245,9 +245,9 @@
 		break;
 	case EM_386:
 	case EM_X86_64:
-		custom_sort = sort_x86_table;
-		break;
 	case EM_S390:
+		custom_sort = sort_relative_table;
+		break;
 	case EM_MIPS:
 		break;
 	}  /* end switch */
...	...	@@ -11,6 +11,8 @@
11	11	#ifndef __ASM_S390_PROCESSOR_H
12	12	#define __ASM_S390_PROCESSOR_H
13	13
	14	+#ifndef __ASSEMBLY__
	15	+
14	16	#include <linux/linkage.h>
15	17	#include <linux/irqflags.h>
16	18	#include <asm/cpu.h>
...	...	@@ -348,23 +350,6 @@
348	350
349	351	#define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL
350	352
351		-/*
352		- * Helper macro for exception table entries
353		- */
354		-#ifndef CONFIG_64BIT
355		-#define EX_TABLE(_fault,_target) \
356		- ".section __ex_table,\"a\"\n" \
357		- " .align 4\n" \
358		- " .long " #_fault "," #_target "\n" \
359		- ".previous\n"
360		-#else
361		-#define EX_TABLE(_fault,_target) \
362		- ".section __ex_table,\"a\"\n" \
363		- " .align 8\n" \
364		- " .quad " #_fault "," #_target "\n" \
365		- ".previous\n"
366		-#endif
367		-
368	353	extern int memcpy_real(void , void , size_t);
369	354	extern void memcpy_absolute(void , void , size_t);
370	355
...	...	@@ -375,5 +360,26 @@
375	360	memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \
376	361	}
377	362
378		-#endif /* __ASM_S390_PROCESSOR_H */
	363	+/*
	364	+ * Helper macro for exception table entries
	365	+ */
	366	+#define EX_TABLE(_fault, _target) \
	367	+ ".section __ex_table,\"a\"\n" \
	368	+ ".align 4\n" \
	369	+ ".long (" #_fault ") - .\n" \
	370	+ ".long (" #_target ") - .\n" \
	371	+ ".previous\n"
	372	+
	373	+#else /* __ASSEMBLY__ */
	374	+
	375	+#define EX_TABLE(_fault, _target) \
	376	+ .section __ex_table,"a" ; \
	377	+ .align 4 ; \
	378	+ .long (_fault) - . ; \
	379	+ .long (_target) - . ; \
	380	+ .previous
	381	+
	382	+#endif /* __ASSEMBLY__ */
	383	+
	384	+#endif /* __ASM_S390_PROCESSOR_H */
...	...	@@ -76,8 +76,21 @@
76	76
77	77	struct exception_table_entry
78	78	{
79		- unsigned long insn, fixup;
	79	+ int insn, fixup;
80	80	};
	81	+
	82	+static inline unsigned long extable_insn(const struct exception_table_entry *x)
	83	+{
	84	+ return (unsigned long)&x->insn + x->insn;
	85	+}
	86	+
	87	+static inline unsigned long extable_fixup(const struct exception_table_entry *x)
	88	+{
	89	+ return (unsigned long)&x->fixup + x->fixup;
	90	+}
	91	+
	92	+#define ARCH_HAS_SORT_EXTABLE
	93	+#define ARCH_HAS_SEARCH_EXTABLE
81	94
82	95	struct uaccess_ops {
83	96	size_t (copy_from_user)(size_t, const void __user , void *);
...	...	@@ -255,14 +255,14 @@
255	255
256	256	static void early_pgm_check_handler(void)
257	257	{
258		- unsigned long addr;
259	258	const struct exception_table_entry *fixup;
	259	+ unsigned long addr;
260	260
261	261	addr = S390_lowcore.program_old_psw.addr;
262	262	fixup = search_exception_tables(addr & PSW_ADDR_INSN);
263	263	if (!fixup)
264	264	disabled_wait(0);
265		- S390_lowcore.program_old_psw.addr = fixup->fixup \| PSW_ADDR_AMODE;
	265	+ S390_lowcore.program_old_psw.addr = extable_fixup(fixup)\|PSW_ADDR_AMODE;
266	266	}
267	267
268	268	static noinline __init void setup_lowcore_early(void)
...	...	@@ -10,6 +10,7 @@
10	10
11	11	#include <linux/init.h>
12	12	#include <linux/linkage.h>
	13	+#include <asm/processor.h>
13	14	#include <asm/cache.h>
14	15	#include <asm/errno.h>
15	16	#include <asm/ptrace.h>
...	...	@@ -1008,9 +1009,7 @@
1008	1009	.Lhost_id:
1009	1010	.quad 0
1010	1011
1011		- .section __ex_table,"a"
1012		- .quad sie_loop,sie_fault
1013		- .previous
	1012	+ EX_TABLE(sie_loop,sie_fault)
1014	1013	#endif
1015	1014
1016	1015	.section .rodata, "a"
...	...	@@ -547,7 +547,7 @@
547	547	*/
548	548	entry = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
549	549	if (entry) {
550		- regs->psw.addr = entry->fixup \| PSW_ADDR_AMODE;
	550	+ regs->psw.addr = extable_fixup(entry) \| PSW_ADDR_AMODE;
551	551	return 1;
552	552	}
553	553
...	...	@@ -322,7 +322,7 @@
322	322	const struct exception_table_entry *fixup;
323	323	fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
324	324	if (fixup)
325		- regs->psw.addr = fixup->fixup \| PSW_ADDR_AMODE;
	325	+ regs->psw.addr = extable_fixup(fixup) \| PSW_ADDR_AMODE;
326	326	else {
327	327	enum bug_trap_type btt;
328	328
...	...	@@ -3,7 +3,7 @@
3	3	#
4	4
5	5	obj-y := init.o fault.o extmem.o mmap.o vmem.o pgtable.o maccess.o \
6		- page-states.o gup.o
	6	+ page-states.o gup.o extable.o
7	7	obj-$(CONFIG_CMM) += cmm.o
8	8	obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
9	9	obj-$(CONFIG_DEBUG_SET_MODULE_RONX) += pageattr.o
	1	+#include <linux/module.h>
	2	+#include <linux/sort.h>
	3	+#include <asm/uaccess.h>
	4	+
	5	+/*
	6	+ * Search one exception table for an entry corresponding to the
	7	+ * given instruction address, and return the address of the entry,
	8	+ * or NULL if none is found.
	9	+ * We use a binary search, and thus we assume that the table is
	10	+ * already sorted.
	11	+ */
	12	+const struct exception_table_entry *
	13	+search_extable(const struct exception_table_entry *first,
	14	+ const struct exception_table_entry *last,
	15	+ unsigned long value)
	16	+{
	17	+ const struct exception_table_entry *mid;
	18	+ unsigned long addr;
	19	+
	20	+ while (first <= last) {
	21	+ mid = ((last - first) >> 1) + first;
	22	+ addr = extable_insn(mid);
	23	+ if (addr < value)
	24	+ first = mid + 1;
	25	+ else if (addr > value)
	26	+ last = mid - 1;
	27	+ else
	28	+ return mid;
	29	+ }
	30	+ return NULL;
	31	+}
	32	+
	33	+/*
	34	+ * The exception table needs to be sorted so that the binary
	35	+ * search that we use to find entries in it works properly.
	36	+ * This is used both for the kernel exception table and for
	37	+ * the exception tables of modules that get loaded.
	38	+ *
	39	+ */
	40	+static int cmp_ex(const void a, const void b)
	41	+{
	42	+ const struct exception_table_entry x = a, y = b;
	43	+
	44	+ /* This compare is only valid after normalization. */
	45	+ return x->insn - y->insn;
	46	+}
	47	+
	48	+void sort_extable(struct exception_table_entry *start,
	49	+ struct exception_table_entry *finish)
	50	+{
	51	+ struct exception_table_entry *p;
	52	+ int i;
	53	+
	54	+ /* Normalize entries to being relative to the start of the section */
	55	+ for (p = start, i = 0; p < finish; p++, i += 8)
	56	+ p->insn += i;
	57	+ sort(start, finish - start, sizeof(*start), cmp_ex, NULL);
	58	+ /* Denormalize all entries */
	59	+ for (p = start, i = 0; p < finish; p++, i += 8)
	60	+ p->insn -= i;
	61	+}
	62	+
	63	+#ifdef CONFIG_MODULES
	64	+/*
	65	+ * If the exception table is sorted, any referring to the module init
	66	+ * will be at the beginning or the end.
	67	+ */
	68	+void trim_init_extable(struct module *m)
	69	+{
	70	+ /* Trim the beginning */
	71	+ while (m->num_exentries &&
	72	+ within_module_init(extable_insn(&m->extable[0]), m)) {
	73	+ m->extable++;
	74	+ m->num_exentries--;
	75	+ }
	76	+ /* Trim the end */
	77	+ while (m->num_exentries &&
	78	+ within_module_init(extable_insn(&m->extable[m->num_exentries-1]), m))
	79	+ m->num_exentries--;
	80	+}
	81	+#endif /* CONFIG_MODULES */
...	...	@@ -163,7 +163,7 @@
163	163	/* Are we prepared to handle this kernel fault? */
164	164	fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
165	165	if (fixup) {
166		- regs->psw.addr = fixup->fixup \| PSW_ADDR_AMODE;
	166	+ regs->psw.addr = extable_fixup(fixup) \| PSW_ADDR_AMODE;
167	167	return;
168	168	}
169	169
...	...	@@ -161,7 +161,7 @@
161	161	#define SORTEXTABLE_64
162	162	#include "sortextable.h"
163	163
164		-static int compare_x86_table(const void a, const void b)
	164	+static int compare_relative_table(const void a, const void b)
165	165	{
166	166	int32_t av = (int32_t)r(a);
167	167	int32_t bv = (int32_t)r(b);
...	...	@@ -173,7 +173,7 @@
173	173	return 0;
174	174	}
175	175
176		-static void sort_x86_table(char *extab_image, int image_size)
	176	+static void sort_relative_table(char *extab_image, int image_size)
177	177	{
178	178	int i;
179	179
...	...	@@ -188,7 +188,7 @@
188	188	i += 4;
189	189	}
190	190
191		- qsort(extab_image, image_size / 8, 8, compare_x86_table);
	191	+ qsort(extab_image, image_size / 8, 8, compare_relative_table);
192	192
193	193	/* Now denormalize. */
194	194	i = 0;
195	195
...	...	@@ -245,9 +245,9 @@
245	245	break;
246	246	case EM_386:
247	247	case EM_X86_64:
248		- custom_sort = sort_x86_table;
249		- break;
250	248	case EM_S390:
	249	+ custom_sort = sort_relative_table;
	250	+ break;
251	251	case EM_MIPS:
252	252	break;
253	253	} /* end switch */