x32: Add x32 VDSO support

Add support for the x32 VDSO. The x32 VDSO takes advantage of the similarity between the x86-64 and the x32 ABIs to contain the same content, only the container is different, as the x32 VDSO obviously is an x32 shared object. Signed-off-by: H. Peter Anvin <hpa@zytor.com>

x32: Add x32 VDSO support
Add support for the x32 VDSO. The x32 VDSO takes advantage of the similarity between the x86-64 and the x32 ABIs to contain the same content, only the container is different, as the x32 VDSO obviously is an x32 shared object. Signed-off-by: H. Peter Anvin <hpa@zytor.com>
H. J. Lu · H. Peter Anvin
1 parent 5fd92e65a6
Showing 6 changed files with 177 additions and 9 deletions Side-by-side Diff
arch/x86/vdso/.gitignore
arch/x86/vdso/Makefile
arch/x86/vdso/vdso32-setup.c
arch/x86/vdso/vdsox32.S
arch/x86/vdso/vdsox32.lds.S
arch/x86/vdso/vma.c
 vdso.lds
 vdso-syms.lds
+vdsox32.lds
+vdsox32-syms.lds
 vdso32-syms.lds
 vdso32-syscall-syms.lds
 vdso32-sysenter-syms.lds
@@ -3,21 +3,29 @@
 #
  
 VDSO64-$(CONFIG_X86_64)		:= y
+VDSOX32-$(CONFIG_X86_X32_ABI)	:= y
 VDSO32-$(CONFIG_X86_32)		:= y
 VDSO32-$(CONFIG_COMPAT)		:= y
  
 vdso-install-$(VDSO64-y)	+= vdso.so
+vdso-install-$(VDSOX32-y)	+= vdsox32.so
 vdso-install-$(VDSO32-y)	+= $(vdso32-images)
  
  
 # files to link into the vdso
 vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o
  
+vobjs-$(VDSOX32-y) += $(vobjx32s-compat)
+
+# Filter out x32 objects.
+vobj64s := $(filter-out $(vobjx32s-compat),$(vobjs-y))
+
 # files to link into kernel
 obj-$(VDSO64-y)			+= vma.o vdso.o
+obj-$(VDSOX32-y)		+= vdsox32.o
 obj-$(VDSO32-y)			+= vdso32.o vdso32-setup.o
  
-vobjs := $(foreach F,$(vobjs-y),$(obj)/$F)
+vobjs := $(foreach F,$(vobj64s),$(obj)/$F)
  
 $(obj)/vdso.o: $(obj)/vdso.so
  
@@ -71,6 +79,42 @@
  
 $(obj)/%-syms.lds: $(obj)/%.so.dbg FORCE
 	$(call if_changed,vdsosym)
+
+#
+# X32 processes use x32 vDSO to access 64bit kernel data.
+#
+# Build x32 vDSO image:
+# 1. Compile x32 vDSO as 64bit.
+# 2. Convert object files to x32.
+# 3. Build x32 VDSO image with x32 objects, which contains 64bit codes
+# so that it can reach 64bit address space with 64bit pointers.
+#
+
+targets += vdsox32-syms.lds
+obj-$(VDSOX32-y)		+= vdsox32-syms.lds
+
+CPPFLAGS_vdsox32.lds = $(CPPFLAGS_vdso.lds)
+VDSO_LDFLAGS_vdsox32.lds = -Wl,-m,elf32_x86_64 \
+			   -Wl,-soname=linux-vdso.so.1 \
+			   -Wl,-z,max-page-size=4096 \
+			   -Wl,-z,common-page-size=4096
+
+vobjx32s-y := $(vobj64s:.o=-x32.o)
+vobjx32s := $(foreach F,$(vobjx32s-y),$(obj)/$F)
+
+# Convert 64bit object file to x32 for x32 vDSO.
+quiet_cmd_x32 = X32     $@
+      cmd_x32 = $(OBJCOPY) -O elf32-x86-64 $< $@
+
+$(obj)/%-x32.o: $(obj)/%.o FORCE
+	$(call if_changed,x32)
+
+targets += vdsox32.so vdsox32.so.dbg vdsox32.lds $(vobjx32s-y)
+
+$(obj)/vdsox32.o: $(src)/vdsox32.S $(obj)/vdsox32.so
+
+$(obj)/vdsox32.so.dbg: $(src)/vdsox32.lds $(vobjx32s) FORCE
+	$(call if_changed,vdso)
  
 #
 # Build multiple 32-bit vDSO images to choose from at boot time.
@@ -317,6 +317,12 @@
 	int ret = 0;
 	bool compat;
  
+#ifdef CONFIG_X86_X32_ABI
+	extern int x32_setup_additional_pages(struct linux_binprm *, int);
+	if (test_thread_flag(TIF_X32))
+		return x32_setup_additional_pages (bprm, uses_interp);
+#endif
+
 	if (vdso_enabled == VDSO_DISABLED)
 		return 0;
  
+#include <asm/page_types.h>
+#include <linux/linkage.h>
+#include <linux/init.h>
+
+__PAGE_ALIGNED_DATA
+
+	.globl vdsox32_start, vdsox32_end
+	.align PAGE_SIZE
+vdsox32_start:
+	.incbin "arch/x86/vdso/vdsox32.so"
+vdsox32_end:
+	.align PAGE_SIZE /* extra data here leaks to userspace. */
+
+.previous
+
+	.globl vdsox32_pages
+	.bss
+	.align 8
+	.type vdsox32_pages, @object
+vdsox32_pages:
+	.zero (vdsox32_end - vdsox32_start + PAGE_SIZE - 1) / PAGE_SIZE * 8
+	.size vdsox32_pages, .-vdsox32_pages
+/*
+ * Linker script for x32 vDSO.
+ * We #include the file to define the layout details.
+ * Here we only choose the prelinked virtual address.
+ *
+ * This file defines the version script giving the user-exported symbols in
+ * the DSO.  We can define local symbols here called VDSO* to make their
+ * values visible using the asm-x86/vdso.h macros from the kernel proper.
+ */
+
+#define VDSO_PRELINK 0
+#include "vdso-layout.lds.S"
+
+/*
+ * This controls what userland symbols we export from the vDSO.
+ */
+VERSION {
+	LINUX_2.6 {
+	global:
+		clock_gettime;
+		__vdso_clock_gettime;
+		gettimeofday;
+		__vdso_gettimeofday;
+		getcpu;
+		__vdso_getcpu;
+		time;
+		__vdso_time;
+	local: *;
+	};
+}
+
+VDSOX32_PRELINK = VDSO_PRELINK;
@@ -24,8 +24,45 @@
 extern struct page *vdso_pages[];
 static unsigned vdso_size;
  
-static void __init patch_vdso(void *vdso, size_t len)
+#ifdef CONFIG_X86_X32_ABI
+extern char vdsox32_start[], vdsox32_end[];
+extern struct page *vdsox32_pages[];
+static unsigned vdsox32_size;
+
+static void __init patch_vdsox32(void *vdso, size_t len)
 {
+	Elf32_Ehdr *hdr = vdso;
+	Elf32_Shdr *sechdrs, *alt_sec = 0;
+	char *secstrings;
+	void *alt_data;
+	int i;
+
+	BUG_ON(len < sizeof(Elf32_Ehdr));
+	BUG_ON(memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0);
+
+	sechdrs = (void *)hdr + hdr->e_shoff;
+	secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
+
+	for (i = 1; i < hdr->e_shnum; i++) {
+		Elf32_Shdr *shdr = &sechdrs[i];
+		if (!strcmp(secstrings + shdr->sh_name, ".altinstructions")) {
+			alt_sec = shdr;
+			goto found;
+		}
+	}
+
+	/* If we get here, it's probably a bug. */
+	pr_warning("patch_vdsox32: .altinstructions not found\n");
+	return;  /* nothing to patch */
+
+found:
+	alt_data = (void *)hdr + alt_sec->sh_offset;
+	apply_alternatives(alt_data, alt_data + alt_sec->sh_size);
+}
+#endif
+
+static void __init patch_vdso64(void *vdso, size_t len)
+{
 	Elf64_Ehdr *hdr = vdso;
 	Elf64_Shdr *sechdrs, *alt_sec = 0;
 	char *secstrings;
@@ -47,7 +84,7 @@
 	}
  
 	/* If we get here, it's probably a bug. */
-	pr_warning("patch_vdso: .altinstructions not found\n");
+	pr_warning("patch_vdso64: .altinstructions not found\n");
 	return;  /* nothing to patch */
  
 found:
  
@@ -60,12 +97,20 @@
 	int npages = (vdso_end - vdso_start + PAGE_SIZE - 1) / PAGE_SIZE;
 	int i;
  
-	patch_vdso(vdso_start, vdso_end - vdso_start);
+	patch_vdso64(vdso_start, vdso_end - vdso_start);
  
 	vdso_size = npages << PAGE_SHIFT;
 	for (i = 0; i < npages; i++)
 		vdso_pages[i] = virt_to_page(vdso_start + i*PAGE_SIZE);
  
+#ifdef CONFIG_X86_X32_ABI
+	patch_vdsox32(vdsox32_start, vdsox32_end - vdsox32_start);
+	npages = (vdsox32_end - vdsox32_start + PAGE_SIZE - 1) / PAGE_SIZE;
+	vdsox32_size = npages << PAGE_SHIFT;
+	for (i = 0; i < npages; i++)
+		vdsox32_pages[i] = virt_to_page(vdsox32_start + i*PAGE_SIZE);
+#endif
+
 	return 0;
 }
 subsys_initcall(init_vdso);
@@ -103,7 +148,10 @@
  
 /* Setup a VMA at program startup for the vsyscall page.
    Not called for compat tasks */
-int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
+static int setup_additional_pages(struct linux_binprm *bprm,
+				  int uses_interp,
+				  struct page **pages,
+				  unsigned size)
 {
 	struct mm_struct *mm = current->mm;
 	unsigned long addr;
@@ -113,8 +161,8 @@
 		return 0;
  
 	down_write(&mm->mmap_sem);
-	addr = vdso_addr(mm->start_stack, vdso_size);
-	addr = get_unmapped_area(NULL, addr, vdso_size, 0, 0);
+	addr = vdso_addr(mm->start_stack, size);
+	addr = get_unmapped_area(NULL, addr, size, 0, 0);
 	if (IS_ERR_VALUE(addr)) {
 		ret = addr;
 		goto up_fail;
  
@@ -122,11 +170,11 @@
  
 	current->mm->context.vdso = (void *)addr;
  
-	ret = install_special_mapping(mm, addr, vdso_size,
+	ret = install_special_mapping(mm, addr, size,
 				      VM_READ|VM_EXEC|
 				      VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC|
 				      VM_ALWAYSDUMP,
-				      vdso_pages);
+				      pages);
 	if (ret) {
 		current->mm->context.vdso = NULL;
 		goto up_fail;
@@ -136,6 +184,20 @@
 	up_write(&mm->mmap_sem);
 	return ret;
 }
+
+int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
+{
+	return setup_additional_pages (bprm, uses_interp, vdso_pages,
+				       vdso_size);
+}
+
+#ifdef CONFIG_X86_X32_ABI
+int x32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
+{
+	return setup_additional_pages (bprm, uses_interp, vdsox32_pages,
+				       vdsox32_size);
+}
+#endif
  
 static __init int vdso_setup(char *s)
 {
1	1	vdso.lds
2	2	vdso-syms.lds
	3	+vdsox32.lds
	4	+vdsox32-syms.lds
3	5	vdso32-syms.lds
4	6	vdso32-syscall-syms.lds
5	7	vdso32-sysenter-syms.lds
...	...	@@ -3,21 +3,29 @@
3	3	#
4	4
5	5	VDSO64-$(CONFIG_X86_64) := y
	6	+VDSOX32-$(CONFIG_X86_X32_ABI) := y
6	7	VDSO32-$(CONFIG_X86_32) := y
7	8	VDSO32-$(CONFIG_COMPAT) := y
8	9
9	10	vdso-install-$(VDSO64-y) += vdso.so
	11	+vdso-install-$(VDSOX32-y) += vdsox32.so
10	12	vdso-install-$(VDSO32-y) += $(vdso32-images)
11	13
12	14
13	15	# files to link into the vdso
14	16	vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o
15	17
	18	+vobjs-$(VDSOX32-y) += $(vobjx32s-compat)
	19	+
	20	+# Filter out x32 objects.
	21	+vobj64s := $(filter-out $(vobjx32s-compat),$(vobjs-y))
	22	+
16	23	# files to link into kernel
17	24	obj-$(VDSO64-y) += vma.o vdso.o
	25	+obj-$(VDSOX32-y) += vdsox32.o
18	26	obj-$(VDSO32-y) += vdso32.o vdso32-setup.o
19	27
20		-vobjs := $(foreach F,$(vobjs-y),$(obj)/$F)
	28	+vobjs := $(foreach F,$(vobj64s),$(obj)/$F)
21	29
22	30	$(obj)/vdso.o: $(obj)/vdso.so
23	31
...	...	@@ -71,6 +79,42 @@
71	79
72	80	$(obj)/%-syms.lds: $(obj)/%.so.dbg FORCE
73	81	$(call if_changed,vdsosym)
	82	+
	83	+#
	84	+# X32 processes use x32 vDSO to access 64bit kernel data.
	85	+#
	86	+# Build x32 vDSO image:
	87	+# 1. Compile x32 vDSO as 64bit.
	88	+# 2. Convert object files to x32.
	89	+# 3. Build x32 VDSO image with x32 objects, which contains 64bit codes
	90	+# so that it can reach 64bit address space with 64bit pointers.
	91	+#
	92	+
	93	+targets += vdsox32-syms.lds
	94	+obj-$(VDSOX32-y) += vdsox32-syms.lds
	95	+
	96	+CPPFLAGS_vdsox32.lds = $(CPPFLAGS_vdso.lds)
	97	+VDSO_LDFLAGS_vdsox32.lds = -Wl,-m,elf32_x86_64 \
	98	+ -Wl,-soname=linux-vdso.so.1 \
	99	+ -Wl,-z,max-page-size=4096 \
	100	+ -Wl,-z,common-page-size=4096
	101	+
	102	+vobjx32s-y := $(vobj64s:.o=-x32.o)
	103	+vobjx32s := $(foreach F,$(vobjx32s-y),$(obj)/$F)
	104	+
	105	+# Convert 64bit object file to x32 for x32 vDSO.
	106	+quiet_cmd_x32 = X32 $@
	107	+ cmd_x32 = $(OBJCOPY) -O elf32-x86-64 $< $@
	108	+
	109	+$(obj)/%-x32.o: $(obj)/%.o FORCE
	110	+ $(call if_changed,x32)
	111	+
	112	+targets += vdsox32.so vdsox32.so.dbg vdsox32.lds $(vobjx32s-y)
	113	+
	114	+$(obj)/vdsox32.o: $(src)/vdsox32.S $(obj)/vdsox32.so
	115	+
	116	+$(obj)/vdsox32.so.dbg: $(src)/vdsox32.lds $(vobjx32s) FORCE
	117	+ $(call if_changed,vdso)
74	118
75	119	#
76	120	# Build multiple 32-bit vDSO images to choose from at boot time.
...	...	@@ -317,6 +317,12 @@
317	317	int ret = 0;
318	318	bool compat;
319	319
	320	+#ifdef CONFIG_X86_X32_ABI
	321	+ extern int x32_setup_additional_pages(struct linux_binprm *, int);
	322	+ if (test_thread_flag(TIF_X32))
	323	+ return x32_setup_additional_pages (bprm, uses_interp);
	324	+#endif
	325	+
320	326	if (vdso_enabled == VDSO_DISABLED)
321	327	return 0;
322	328
	1	+#include <asm/page_types.h>
	2	+#include <linux/linkage.h>
	3	+#include <linux/init.h>
	4	+
	5	+__PAGE_ALIGNED_DATA
	6	+
	7	+ .globl vdsox32_start, vdsox32_end
	8	+ .align PAGE_SIZE
	9	+vdsox32_start:
	10	+ .incbin "arch/x86/vdso/vdsox32.so"
	11	+vdsox32_end:
	12	+ .align PAGE_SIZE /* extra data here leaks to userspace. */
	13	+
	14	+.previous
	15	+
	16	+ .globl vdsox32_pages
	17	+ .bss
	18	+ .align 8
	19	+ .type vdsox32_pages, @object
	20	+vdsox32_pages:
	21	+ .zero (vdsox32_end - vdsox32_start + PAGE_SIZE - 1) / PAGE_SIZE * 8
	22	+ .size vdsox32_pages, .-vdsox32_pages
	1	+/*
	2	+ * Linker script for x32 vDSO.
	3	+ * We #include the file to define the layout details.
	4	+ * Here we only choose the prelinked virtual address.
	5	+ *
	6	+ * This file defines the version script giving the user-exported symbols in
	7	+ * the DSO. We can define local symbols here called VDSO* to make their
	8	+ * values visible using the asm-x86/vdso.h macros from the kernel proper.
	9	+ */
	10	+
	11	+#define VDSO_PRELINK 0
	12	+#include "vdso-layout.lds.S"
	13	+
	14	+/*
	15	+ * This controls what userland symbols we export from the vDSO.
	16	+ */
	17	+VERSION {
	18	+ LINUX_2.6 {
	19	+ global:
	20	+ clock_gettime;
	21	+ __vdso_clock_gettime;
	22	+ gettimeofday;
	23	+ __vdso_gettimeofday;
	24	+ getcpu;
	25	+ __vdso_getcpu;
	26	+ time;
	27	+ __vdso_time;
	28	+ local: *;
	29	+ };
	30	+}
	31	+
	32	+VDSOX32_PRELINK = VDSO_PRELINK;
...	...	@@ -24,8 +24,45 @@
24	24	extern struct page *vdso_pages[];
25	25	static unsigned vdso_size;
26	26
27		-static void __init patch_vdso(void *vdso, size_t len)
	27	+#ifdef CONFIG_X86_X32_ABI
	28	+extern char vdsox32_start[], vdsox32_end[];
	29	+extern struct page *vdsox32_pages[];
	30	+static unsigned vdsox32_size;
	31	+
	32	+static void __init patch_vdsox32(void *vdso, size_t len)
28	33	{
	34	+ Elf32_Ehdr *hdr = vdso;
	35	+ Elf32_Shdr sechdrs, alt_sec = 0;
	36	+ char *secstrings;
	37	+ void *alt_data;
	38	+ int i;
	39	+
	40	+ BUG_ON(len < sizeof(Elf32_Ehdr));
	41	+ BUG_ON(memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0);
	42	+
	43	+ sechdrs = (void *)hdr + hdr->e_shoff;
	44	+ secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
	45	+
	46	+ for (i = 1; i < hdr->e_shnum; i++) {
	47	+ Elf32_Shdr *shdr = &sechdrs[i];
	48	+ if (!strcmp(secstrings + shdr->sh_name, ".altinstructions")) {
	49	+ alt_sec = shdr;
	50	+ goto found;
	51	+ }
	52	+ }
	53	+
	54	+ /* If we get here, it's probably a bug. */
	55	+ pr_warning("patch_vdsox32: .altinstructions not found\n");
	56	+ return; /* nothing to patch */
	57	+
	58	+found:
	59	+ alt_data = (void *)hdr + alt_sec->sh_offset;
	60	+ apply_alternatives(alt_data, alt_data + alt_sec->sh_size);
	61	+}
	62	+#endif
	63	+
	64	+static void __init patch_vdso64(void *vdso, size_t len)
	65	+{
29	66	Elf64_Ehdr *hdr = vdso;
30	67	Elf64_Shdr sechdrs, alt_sec = 0;
31	68	char *secstrings;
...	...	@@ -47,7 +84,7 @@
47	84	}
48	85
49	86	/* If we get here, it's probably a bug. */
50		- pr_warning("patch_vdso: .altinstructions not found\n");
	87	+ pr_warning("patch_vdso64: .altinstructions not found\n");
51	88	return; /* nothing to patch */
52	89
53	90	found:
54	91
...	...	@@ -60,12 +97,20 @@
60	97	int npages = (vdso_end - vdso_start + PAGE_SIZE - 1) / PAGE_SIZE;
61	98	int i;
62	99
63		- patch_vdso(vdso_start, vdso_end - vdso_start);
	100	+ patch_vdso64(vdso_start, vdso_end - vdso_start);
64	101
65	102	vdso_size = npages << PAGE_SHIFT;
66	103	for (i = 0; i < npages; i++)
67	104	vdso_pages[i] = virt_to_page(vdso_start + i*PAGE_SIZE);
68	105
	106	+#ifdef CONFIG_X86_X32_ABI
	107	+ patch_vdsox32(vdsox32_start, vdsox32_end - vdsox32_start);
	108	+ npages = (vdsox32_end - vdsox32_start + PAGE_SIZE - 1) / PAGE_SIZE;
	109	+ vdsox32_size = npages << PAGE_SHIFT;
	110	+ for (i = 0; i < npages; i++)
	111	+ vdsox32_pages[i] = virt_to_page(vdsox32_start + i*PAGE_SIZE);
	112	+#endif
	113	+
69	114	return 0;
70	115	}
71	116	subsys_initcall(init_vdso);
...	...	@@ -103,7 +148,10 @@
103	148
104	149	/* Setup a VMA at program startup for the vsyscall page.
105	150	Not called for compat tasks */
106		-int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
	151	+static int setup_additional_pages(struct linux_binprm *bprm,
	152	+ int uses_interp,
	153	+ struct page **pages,
	154	+ unsigned size)
107	155	{
108	156	struct mm_struct *mm = current->mm;
109	157	unsigned long addr;
...	...	@@ -113,8 +161,8 @@
113	161	return 0;
114	162
115	163	down_write(&mm->mmap_sem);
116		- addr = vdso_addr(mm->start_stack, vdso_size);
117		- addr = get_unmapped_area(NULL, addr, vdso_size, 0, 0);
	164	+ addr = vdso_addr(mm->start_stack, size);
	165	+ addr = get_unmapped_area(NULL, addr, size, 0, 0);
118	166	if (IS_ERR_VALUE(addr)) {
119	167	ret = addr;
120	168	goto up_fail;
121	169
...	...	@@ -122,11 +170,11 @@
122	170
123	171	current->mm->context.vdso = (void *)addr;
124	172
125		- ret = install_special_mapping(mm, addr, vdso_size,
	173	+ ret = install_special_mapping(mm, addr, size,
126	174	VM_READ\|VM_EXEC\|
127	175	VM_MAYREAD\|VM_MAYWRITE\|VM_MAYEXEC\|
128	176	VM_ALWAYSDUMP,
129		- vdso_pages);
	177	+ pages);
130	178	if (ret) {
131	179	current->mm->context.vdso = NULL;
132	180	goto up_fail;
...	...	@@ -136,6 +184,20 @@
136	184	up_write(&mm->mmap_sem);
137	185	return ret;
138	186	}
	187	+
	188	+int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
	189	+{
	190	+ return setup_additional_pages (bprm, uses_interp, vdso_pages,
	191	+ vdso_size);
	192	+}
	193	+
	194	+#ifdef CONFIG_X86_X32_ABI
	195	+int x32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
	196	+{
	197	+ return setup_additional_pages (bprm, uses_interp, vdsox32_pages,
	198	+ vdsox32_size);
	199	+}
	200	+#endif
139	201
140	202	static __init int vdso_setup(char *s)
141	203	{