asmlinkage_protect replaces prevent_tail_call

The prevent_tail_call() macro works around the problem of the compiler clobbering argument words on the stack, which for asmlinkage functions is the caller's (user's) struct pt_regs. The tail/sibling-call optimization is not the only way that the compiler can decide to use stack argument words as scratch space, which we have to prevent. Other optimizations can do it too. Until we have new compiler support to make "asmlinkage" binding on the compiler's own use of the stack argument frame, we have work around all the manifestations of this issue that crop up. More cases seem to be prevented by also keeping the incoming argument variables live at the end of the function. This makes their original stack slots attractive places to leave those variables, so the compiler tends not clobber them for something else. It's still no guarantee, but it handles some observed cases that prevent_tail_call() did not. Signed-off-by: Roland McGrath <roland@redhat.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

asmlinkage_protect replaces prevent_tail_call
The prevent_tail_call() macro works around the problem of the compiler clobbering argument words on the stack, which for asmlinkage functions is the caller's (user's) struct pt_regs. The tail/sibling-call optimization is not the only way that the compiler can decide to use stack argument words as scratch space, which we have to prevent. Other optimizations can do it too. Until we have new compiler support to make "asmlinkage" binding on the compiler's own use of the stack argument frame, we have work around all the manifestations of this issue that crop up. More cases seem to be prevented by also keeping the incoming argument variables live at the end of the function. This makes their original stack slots attractive places to leave those variables, so the compiler tends not clobber them for something else. It's still no guarantee, but it handles some observed cases that prevent_tail_call() did not. Signed-off-by: Roland McGrath <roland@redhat.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Roland McGrath · Linus Torvalds
1 parent 783e391b7b
Showing 6 changed files with 44 additions and 22 deletions Side-by-side Diff
arch/x86/kernel/tls.c
fs/open.c
include/asm-x86/linkage.h
include/linux/linkage.h
kernel/exit.c
kernel/uid16.c
@@ -92,7 +92,7 @@
 asmlinkage int sys_set_thread_area(struct user_desc __user *u_info)
 {
 	int ret = do_set_thread_area(current, -1, u_info, 1);
-	prevent_tail_call(ret);
+	asmlinkage_protect(1, ret, u_info);
 	return ret;
 }
  
@@ -142,7 +142,7 @@
 asmlinkage int sys_get_thread_area(struct user_desc __user *u_info)
 {
 	int ret = do_get_thread_area(current, -1, u_info);
-	prevent_tail_call(ret);
+	asmlinkage_protect(1, ret, u_info);
 	return ret;
 }
  
@@ -335,7 +335,7 @@
 {
 	long ret = do_sys_ftruncate(fd, length, 1);
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(2, ret, fd, length);
 	return ret;
 }
  
@@ -350,7 +350,7 @@
 {
 	long ret = do_sys_ftruncate(fd, length, 0);
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(2, ret, fd, length);
 	return ret;
 }
 #endif
@@ -1067,7 +1067,7 @@
  
 	ret = do_sys_open(AT_FDCWD, filename, flags, mode);
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(3, ret, filename, flags, mode);
 	return ret;
 }
  
@@ -1081,7 +1081,7 @@
  
 	ret = do_sys_open(dfd, filename, flags, mode);
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(4, ret, dfd, filename, flags, mode);
 	return ret;
 }
  
@@ -8,12 +8,34 @@
  
 #ifdef CONFIG_X86_32
 #define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0)))
-#define prevent_tail_call(ret) __asm__ ("" : "=r" (ret) : "0" (ret))
 /*
  * For 32-bit UML - mark functions implemented in assembly that use
  * regparm input parameters:
  */
 #define asmregparm __attribute__((regparm(3)))
+
+#define asmlinkage_protect(n, ret, args...) \
+	__asmlinkage_protect##n(ret, ##args)
+#define __asmlinkage_protect_n(ret, args...) \
+	__asm__ __volatile__ ("" : "=r" (ret) : "0" (ret), ##args)
+#define __asmlinkage_protect0(ret) \
+	__asmlinkage_protect_n(ret)
+#define __asmlinkage_protect1(ret, arg1) \
+	__asmlinkage_protect_n(ret, "g" (arg1))
+#define __asmlinkage_protect2(ret, arg1, arg2) \
+	__asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2))
+#define __asmlinkage_protect3(ret, arg1, arg2, arg3) \
+	__asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3))
+#define __asmlinkage_protect4(ret, arg1, arg2, arg3, arg4) \
+	__asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
+			      "g" (arg4))
+#define __asmlinkage_protect5(ret, arg1, arg2, arg3, arg4, arg5) \
+	__asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
+			      "g" (arg4), "g" (arg5))
+#define __asmlinkage_protect6(ret, arg1, arg2, arg3, arg4, arg5, arg6) \
+	__asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
+			      "g" (arg4), "g" (arg5), "g" (arg6))
+
 #endif
  
 #ifdef CONFIG_X86_ALIGNMENT_16
@@ -17,8 +17,8 @@
 # define asmregparm
 #endif
  
-#ifndef prevent_tail_call
-# define prevent_tail_call(ret) do { } while (0)
+#ifndef asmlinkage_protect
+# define asmlinkage_protect(n, ret, args...)	do { } while (0)
 #endif
  
 #ifndef __ALIGN
@@ -1608,7 +1608,7 @@
 	put_pid(pid);
  
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(5, ret, which, upid, infop, options, ru);
 	return ret;
 }
  
@@ -1640,7 +1640,7 @@
 	put_pid(pid);
  
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(4, ret, upid, stat_addr, options, ru);
 	return ret;
 }
  
@@ -21,7 +21,7 @@
 {
 	long ret = sys_chown(filename, low2highuid(user), low2highgid(group));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(3, ret, filename, user, group);
 	return ret;
 }
  
@@ -29,7 +29,7 @@
 {
 	long ret = sys_lchown(filename, low2highuid(user), low2highgid(group));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(3, ret, filename, user, group);
 	return ret;
 }
  
@@ -37,7 +37,7 @@
 {
 	long ret = sys_fchown(fd, low2highuid(user), low2highgid(group));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(3, ret, fd, user, group);
 	return ret;
 }
  
@@ -45,7 +45,7 @@
 {
 	long ret = sys_setregid(low2highgid(rgid), low2highgid(egid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(2, ret, rgid, egid);
 	return ret;
 }
  
@@ -53,7 +53,7 @@
 {
 	long ret = sys_setgid(low2highgid(gid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(1, ret, gid);
 	return ret;
 }
  
@@ -61,7 +61,7 @@
 {
 	long ret = sys_setreuid(low2highuid(ruid), low2highuid(euid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(2, ret, ruid, euid);
 	return ret;
 }
  
@@ -69,7 +69,7 @@
 {
 	long ret = sys_setuid(low2highuid(uid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(1, ret, uid);
 	return ret;
 }
  
@@ -78,7 +78,7 @@
 	long ret = sys_setresuid(low2highuid(ruid), low2highuid(euid),
 				 low2highuid(suid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(3, ret, ruid, euid, suid);
 	return ret;
 }
  
@@ -98,7 +98,7 @@
 	long ret = sys_setresgid(low2highgid(rgid), low2highgid(egid),
 				 low2highgid(sgid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(3, ret, rgid, egid, sgid);
 	return ret;
 }
  
@@ -117,7 +117,7 @@
 {
 	long ret = sys_setfsuid(low2highuid(uid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(1, ret, uid);
 	return ret;
 }
  
@@ -125,7 +125,7 @@
 {
 	long ret = sys_setfsgid(low2highgid(gid));
 	/* avoid REGPARM breakage on x86: */
-	prevent_tail_call(ret);
+	asmlinkage_protect(1, ret, gid);
 	return ret;
 }
...	...	@@ -92,7 +92,7 @@
92	92	asmlinkage int sys_set_thread_area(struct user_desc __user *u_info)
93	93	{
94	94	int ret = do_set_thread_area(current, -1, u_info, 1);
95		- prevent_tail_call(ret);
	95	+ asmlinkage_protect(1, ret, u_info);
96	96	return ret;
97	97	}
98	98
...	...	@@ -142,7 +142,7 @@
142	142	asmlinkage int sys_get_thread_area(struct user_desc __user *u_info)
143	143	{
144	144	int ret = do_get_thread_area(current, -1, u_info);
145		- prevent_tail_call(ret);
	145	+ asmlinkage_protect(1, ret, u_info);
146	146	return ret;
147	147	}
148	148
...	...	@@ -335,7 +335,7 @@
335	335	{
336	336	long ret = do_sys_ftruncate(fd, length, 1);
337	337	/* avoid REGPARM breakage on x86: */
338		- prevent_tail_call(ret);
	338	+ asmlinkage_protect(2, ret, fd, length);
339	339	return ret;
340	340	}
341	341
...	...	@@ -350,7 +350,7 @@
350	350	{
351	351	long ret = do_sys_ftruncate(fd, length, 0);
352	352	/* avoid REGPARM breakage on x86: */
353		- prevent_tail_call(ret);
	353	+ asmlinkage_protect(2, ret, fd, length);
354	354	return ret;
355	355	}
356	356	#endif
...	...	@@ -1067,7 +1067,7 @@
1067	1067
1068	1068	ret = do_sys_open(AT_FDCWD, filename, flags, mode);
1069	1069	/* avoid REGPARM breakage on x86: */
1070		- prevent_tail_call(ret);
	1070	+ asmlinkage_protect(3, ret, filename, flags, mode);
1071	1071	return ret;
1072	1072	}
1073	1073
...	...	@@ -1081,7 +1081,7 @@
1081	1081
1082	1082	ret = do_sys_open(dfd, filename, flags, mode);
1083	1083	/* avoid REGPARM breakage on x86: */
1084		- prevent_tail_call(ret);
	1084	+ asmlinkage_protect(4, ret, dfd, filename, flags, mode);
1085	1085	return ret;
1086	1086	}
1087	1087
...	...	@@ -8,12 +8,34 @@
8	8
9	9	#ifdef CONFIG_X86_32
10	10	#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0)))
11		-#define prevent_tail_call(ret) __asm__ ("" : "=r" (ret) : "0" (ret))
12	11	/*
13	12	* For 32-bit UML - mark functions implemented in assembly that use
14	13	* regparm input parameters:
15	14	*/
16	15	#define asmregparm __attribute__((regparm(3)))
	16	+
	17	+#define asmlinkage_protect(n, ret, args...) \
	18	+ __asmlinkage_protect##n(ret, ##args)
	19	+#define __asmlinkage_protect_n(ret, args...) \
	20	+ __asm__ __volatile__ ("" : "=r" (ret) : "0" (ret), ##args)
	21	+#define __asmlinkage_protect0(ret) \
	22	+ __asmlinkage_protect_n(ret)
	23	+#define __asmlinkage_protect1(ret, arg1) \
	24	+ __asmlinkage_protect_n(ret, "g" (arg1))
	25	+#define __asmlinkage_protect2(ret, arg1, arg2) \
	26	+ __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2))
	27	+#define __asmlinkage_protect3(ret, arg1, arg2, arg3) \
	28	+ __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3))
	29	+#define __asmlinkage_protect4(ret, arg1, arg2, arg3, arg4) \
	30	+ __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
	31	+ "g" (arg4))
	32	+#define __asmlinkage_protect5(ret, arg1, arg2, arg3, arg4, arg5) \
	33	+ __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
	34	+ "g" (arg4), "g" (arg5))
	35	+#define __asmlinkage_protect6(ret, arg1, arg2, arg3, arg4, arg5, arg6) \
	36	+ __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
	37	+ "g" (arg4), "g" (arg5), "g" (arg6))
	38	+
17	39	#endif
18	40
19	41	#ifdef CONFIG_X86_ALIGNMENT_16
...	...	@@ -17,8 +17,8 @@
17	17	# define asmregparm
18	18	#endif
19	19
20		-#ifndef prevent_tail_call
21		-# define prevent_tail_call(ret) do { } while (0)
	20	+#ifndef asmlinkage_protect
	21	+# define asmlinkage_protect(n, ret, args...) do { } while (0)
22	22	#endif
23	23
24	24	#ifndef __ALIGN
...	...	@@ -1608,7 +1608,7 @@
1608	1608	put_pid(pid);
1609	1609
1610	1610	/* avoid REGPARM breakage on x86: */
1611		- prevent_tail_call(ret);
	1611	+ asmlinkage_protect(5, ret, which, upid, infop, options, ru);
1612	1612	return ret;
1613	1613	}
1614	1614
...	...	@@ -1640,7 +1640,7 @@
1640	1640	put_pid(pid);
1641	1641
1642	1642	/* avoid REGPARM breakage on x86: */
1643		- prevent_tail_call(ret);
	1643	+ asmlinkage_protect(4, ret, upid, stat_addr, options, ru);
1644	1644	return ret;
1645	1645	}
1646	1646
...	...	@@ -21,7 +21,7 @@
21	21	{
22	22	long ret = sys_chown(filename, low2highuid(user), low2highgid(group));
23	23	/* avoid REGPARM breakage on x86: */
24		- prevent_tail_call(ret);
	24	+ asmlinkage_protect(3, ret, filename, user, group);
25	25	return ret;
26	26	}
27	27
...	...	@@ -29,7 +29,7 @@
29	29	{
30	30	long ret = sys_lchown(filename, low2highuid(user), low2highgid(group));
31	31	/* avoid REGPARM breakage on x86: */
32		- prevent_tail_call(ret);
	32	+ asmlinkage_protect(3, ret, filename, user, group);
33	33	return ret;
34	34	}
35	35
...	...	@@ -37,7 +37,7 @@
37	37	{
38	38	long ret = sys_fchown(fd, low2highuid(user), low2highgid(group));
39	39	/* avoid REGPARM breakage on x86: */
40		- prevent_tail_call(ret);
	40	+ asmlinkage_protect(3, ret, fd, user, group);
41	41	return ret;
42	42	}
43	43
...	...	@@ -45,7 +45,7 @@
45	45	{
46	46	long ret = sys_setregid(low2highgid(rgid), low2highgid(egid));
47	47	/* avoid REGPARM breakage on x86: */
48		- prevent_tail_call(ret);
	48	+ asmlinkage_protect(2, ret, rgid, egid);
49	49	return ret;
50	50	}
51	51
...	...	@@ -53,7 +53,7 @@
53	53	{
54	54	long ret = sys_setgid(low2highgid(gid));
55	55	/* avoid REGPARM breakage on x86: */
56		- prevent_tail_call(ret);
	56	+ asmlinkage_protect(1, ret, gid);
57	57	return ret;
58	58	}
59	59
...	...	@@ -61,7 +61,7 @@
61	61	{
62	62	long ret = sys_setreuid(low2highuid(ruid), low2highuid(euid));
63	63	/* avoid REGPARM breakage on x86: */
64		- prevent_tail_call(ret);
	64	+ asmlinkage_protect(2, ret, ruid, euid);
65	65	return ret;
66	66	}
67	67
...	...	@@ -69,7 +69,7 @@
69	69	{
70	70	long ret = sys_setuid(low2highuid(uid));
71	71	/* avoid REGPARM breakage on x86: */
72		- prevent_tail_call(ret);
	72	+ asmlinkage_protect(1, ret, uid);
73	73	return ret;
74	74	}
75	75
...	...	@@ -78,7 +78,7 @@
78	78	long ret = sys_setresuid(low2highuid(ruid), low2highuid(euid),
79	79	low2highuid(suid));
80	80	/* avoid REGPARM breakage on x86: */
81		- prevent_tail_call(ret);
	81	+ asmlinkage_protect(3, ret, ruid, euid, suid);
82	82	return ret;
83	83	}
84	84
...	...	@@ -98,7 +98,7 @@
98	98	long ret = sys_setresgid(low2highgid(rgid), low2highgid(egid),
99	99	low2highgid(sgid));
100	100	/* avoid REGPARM breakage on x86: */
101		- prevent_tail_call(ret);
	101	+ asmlinkage_protect(3, ret, rgid, egid, sgid);
102	102	return ret;
103	103	}
104	104
...	...	@@ -117,7 +117,7 @@
117	117	{
118	118	long ret = sys_setfsuid(low2highuid(uid));
119	119	/* avoid REGPARM breakage on x86: */
120		- prevent_tail_call(ret);
	120	+ asmlinkage_protect(1, ret, uid);
121	121	return ret;
122	122	}
123	123
...	...	@@ -125,7 +125,7 @@
125	125	{
126	126	long ret = sys_setfsgid(low2highgid(gid));
127	127	/* avoid REGPARM breakage on x86: */
128		- prevent_tail_call(ret);
	128	+ asmlinkage_protect(1, ret, gid);
129	129	return ret;
130	130	}
131	131