[PATCH] x86: cmpxchg improvements

This adjusts i386's cmpxchg patterns so that - for word and long cmpxchg-es the compiler can utilize all possible registers - cmpxchg8b gets disabled when the minimum specified hardware architectur doesn't support it (like was already happening for the byte, word, and long ones). Signed-off-by: Jan Beulich <jbeulich@novell.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] x86: cmpxchg improvements
This adjusts i386's cmpxchg patterns so that - for word and long cmpxchg-es the compiler can utilize all possible registers - cmpxchg8b gets disabled when the minimum specified hardware architectur doesn't support it (like was already happening for the byte, word, and long ones). Signed-off-by: Jan Beulich <jbeulich@novell.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Jan Beulich · Linus Torvalds
1 parent dacb16b1a0
Showing 2 changed files with 35 additions and 3 deletions Side-by-side Diff
arch/i386/Kconfig
include/asm-i386/system.h
@@ -424,6 +424,11 @@
 	depends on !M386
 	default y
  
+config X86_CMPXCHG64
+	bool
+	depends on !M386 && !M486
+	default y
+
 config X86_ALIGNMENT_16
 	bool
 	depends on MWINCHIP3D || MWINCHIP2 || MWINCHIPC6 || MCYRIXIII || X86_ELAN || MK6 || M586MMX || M586TSC || M586 || M486 || MVIAC3_2 || MGEODEGX1
@@ -167,6 +167,8 @@
 #define __xg(x) ((struct __xchg_dummy *)(x))
  
  
+#ifdef CONFIG_X86_CMPXCHG64
+
 /*
  * The semantics of XCHGCMP8B are a bit strange, this is why
  * there is a loop and the loading of %%eax and %%edx has to
@@ -221,6 +223,8 @@
  __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \
  __set_64bit(ptr, ll_low(value), ll_high(value)) )
  
+#endif
+
 /*
  * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
  * Note 2: xchg has side effect, so that attribute volatile is necessary,
@@ -259,7 +263,6 @@
  
 #ifdef CONFIG_X86_CMPXCHG
 #define __HAVE_ARCH_CMPXCHG 1
-#endif
  
 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 				      unsigned long new, int size)
  
@@ -275,13 +278,13 @@
 	case 2:
 		__asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
 				     : "=a"(prev)
-				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
+				     : "r"(new), "m"(*__xg(ptr)), "0"(old)
 				     : "memory");
 		return prev;
 	case 4:
 		__asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
 				     : "=a"(prev)
-				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
+				     : "r"(new), "m"(*__xg(ptr)), "0"(old)
 				     : "memory");
 		return prev;
 	}
@@ -291,6 +294,30 @@
 #define cmpxchg(ptr,o,n)\
 	((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
 					(unsigned long)(n),sizeof(*(ptr))))
+
+#endif
+
+#ifdef CONFIG_X86_CMPXCHG64
+
+static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old,
+				      unsigned long long new)
+{
+	unsigned long long prev;
+	__asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
+			     : "=A"(prev)
+			     : "b"((unsigned long)new),
+			       "c"((unsigned long)(new >> 32)),
+			       "m"(*__xg(ptr)),
+			       "0"(old)
+			     : "memory");
+	return prev;
+}
+
+#define cmpxchg64(ptr,o,n)\
+	((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
+					(unsigned long long)(n)))
+
+#endif
  
 #ifdef __KERNEL__
 struct alt_instr {
...	...	@@ -424,6 +424,11 @@
424	424	depends on !M386
425	425	default y
426	426
	427	+config X86_CMPXCHG64
	428	+ bool
	429	+ depends on !M386 && !M486
	430	+ default y
	431	+
427	432	config X86_ALIGNMENT_16
428	433	bool
429	434	depends on MWINCHIP3D \|\| MWINCHIP2 \|\| MWINCHIPC6 \|\| MCYRIXIII \|\| X86_ELAN \|\| MK6 \|\| M586MMX \|\| M586TSC \|\| M586 \|\| M486 \|\| MVIAC3_2 \|\| MGEODEGX1
...	...	@@ -167,6 +167,8 @@
167	167	#define __xg(x) ((struct __xchg_dummy *)(x))
168	168
169	169
	170	+#ifdef CONFIG_X86_CMPXCHG64
	171	+
170	172	/*
171	173	* The semantics of XCHGCMP8B are a bit strange, this is why
172	174	* there is a loop and the loading of %%eax and %%edx has to
...	...	@@ -221,6 +223,8 @@
221	223	__set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \
222	224	__set_64bit(ptr, ll_low(value), ll_high(value)) )
223	225
	226	+#endif
	227	+
224	228	/*
225	229	* Note: no "lock" prefix even on SMP: xchg always implies lock anyway
226	230	* Note 2: xchg has side effect, so that attribute volatile is necessary,
...	...	@@ -259,7 +263,6 @@
259	263
260	264	#ifdef CONFIG_X86_CMPXCHG
261	265	#define __HAVE_ARCH_CMPXCHG 1
262		-#endif
263	266
264	267	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
265	268	unsigned long new, int size)
266	269
...	...	@@ -275,13 +278,13 @@
275	278	case 2:
276	279	__asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
277	280	: "=a"(prev)
278		- : "q"(new), "m"(*__xg(ptr)), "0"(old)
	281	+ : "r"(new), "m"(*__xg(ptr)), "0"(old)
279	282	: "memory");
280	283	return prev;
281	284	case 4:
282	285	__asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
283	286	: "=a"(prev)
284		- : "q"(new), "m"(*__xg(ptr)), "0"(old)
	287	+ : "r"(new), "m"(*__xg(ptr)), "0"(old)
285	288	: "memory");
286	289	return prev;
287	290	}
...	...	@@ -291,6 +294,30 @@
291	294	#define cmpxchg(ptr,o,n)\
292	295	((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
293	296	(unsigned long)(n),sizeof(*(ptr))))
	297	+
	298	+#endif
	299	+
	300	+#ifdef CONFIG_X86_CMPXCHG64
	301	+
	302	+static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old,
	303	+ unsigned long long new)
	304	+{
	305	+ unsigned long long prev;
	306	+ __asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
	307	+ : "=A"(prev)
	308	+ : "b"((unsigned long)new),
	309	+ "c"((unsigned long)(new >> 32)),
	310	+ "m"(*__xg(ptr)),
	311	+ "0"(old)
	312	+ : "memory");
	313	+ return prev;
	314	+}
	315	+
	316	+#define cmpxchg64(ptr,o,n)\
	317	+ ((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
	318	+ (unsigned long long)(n)))
	319	+
	320	+#endif
294	321
295	322	#ifdef __KERNEL__
296	323	struct alt_instr {