Merge branch 'sched-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kerne…

…l/git/tip/linux-2.6-tip * 'sched-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: sched_clock: Fix atomicity/continuity bug by using cmpxchg64() x86: Provide an alternative() based cmpxchg64()

Merge branch 'sched-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kerne…
…l/git/tip/linux-2.6-tip * 'sched-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: sched_clock: Fix atomicity/continuity bug by using cmpxchg64() x86: Provide an alternative() based cmpxchg64()
Linus Torvalds
2 parents e399835c34 152f9d0710
Showing 5 changed files Side-by-side Diff
arch/x86/include/asm/cmpxchg_32.h
arch/x86/kernel/i386_ksyms_32.c
arch/x86/lib/Makefile
arch/x86/lib/cmpxchg8b_emu.S
kernel/sched_clock.c
@@ -312,19 +312,23 @@
  
 extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
  
-#define cmpxchg64(ptr, o, n)						\
-({									\
-	__typeof__(*(ptr)) __ret;					\
-	if (likely(boot_cpu_data.x86 > 4))				\
-		__ret = (__typeof__(*(ptr)))__cmpxchg64((ptr),		\
-				(unsigned long long)(o),		\
-				(unsigned long long)(n));		\
-	else								\
-		__ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr),	\
-				(unsigned long long)(o),		\
-				(unsigned long long)(n));		\
-	__ret;								\
-})
+#define cmpxchg64(ptr, o, n)					\
+({								\
+	__typeof__(*(ptr)) __ret;				\
+	__typeof__(*(ptr)) __old = (o);				\
+	__typeof__(*(ptr)) __new = (n);				\
+	alternative_io("call cmpxchg8b_emu",			\
+			"lock; cmpxchg8b (%%esi)" ,		\
+		       X86_FEATURE_CX8,				\
+		       "=A" (__ret),				\
+		       "S" ((ptr)), "0" (__old),		\
+		       "b" ((unsigned int)__new),		\
+		       "c" ((unsigned int)(__new>>32))		\
+		       : "memory");				\
+	__ret; })
+
+
+
 #define cmpxchg64_local(ptr, o, n)					\
 ({									\
 	__typeof__(*(ptr)) __ret;					\
@@ -10,6 +10,14 @@
 EXPORT_SYMBOL(mcount);
 #endif
  
+/*
+ * Note, this is a prototype to get at the symbol for
+ * the export, but dont use it from C code, it is used
+ * by assembly code and is not using C calling convention!
+ */
+extern void cmpxchg8b_emu(void);
+EXPORT_SYMBOL(cmpxchg8b_emu);
+
 /* Networking helper routines. */
 EXPORT_SYMBOL(csum_partial_copy_generic);
  
@@ -15,7 +15,7 @@
         obj-y += atomic64_32.o
         lib-y += checksum_32.o
         lib-y += strstr_32.o
-        lib-y += semaphore_32.o string_32.o
+        lib-y += semaphore_32.o string_32.o cmpxchg8b_emu.o
  
         lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o
 else
+/*
+ *	This program is free software; you can redistribute it and/or
+ *	modify it under the terms of the GNU General Public License
+ *	as published by the Free Software Foundation; version 2
+ *	of the License.
+ *
+ */
+
+#include <linux/linkage.h>
+#include <asm/alternative-asm.h>
+#include <asm/frame.h>
+#include <asm/dwarf2.h>
+
+
+.text
+
+/*
+ * Inputs:
+ * %esi : memory location to compare
+ * %eax : low 32 bits of old value
+ * %edx : high 32 bits of old value
+ * %ebx : low 32 bits of new value
+ * %ecx : high 32 bits of new value
+ */
+ENTRY(cmpxchg8b_emu)
+CFI_STARTPROC
+
+#
+# Emulate 'cmpxchg8b (%esi)' on UP except we don't
+# set the whole ZF thing (caller will just compare
+# eax:edx with the expected value)
+#
+cmpxchg8b_emu:
+	pushfl
+	cli
+
+	cmpl  (%esi), %eax
+	jne not_same
+	cmpl 4(%esi), %edx
+	jne half_same
+
+	movl %ebx,  (%esi)
+	movl %ecx, 4(%esi)
+
+	popfl
+	ret
+
+ not_same:
+	movl  (%esi), %eax
+ half_same:
+	movl 4(%esi), %edx
+
+	popfl
+	ret
+
+CFI_ENDPROC
+ENDPROC(cmpxchg8b_emu)
@@ -127,7 +127,7 @@
 	clock = wrap_max(clock, min_clock);
 	clock = wrap_min(clock, max_clock);
  
-	if (cmpxchg(&scd->clock, old_clock, clock) != old_clock)
+	if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock)
 		goto again;
  
 	return clock;
@@ -163,7 +163,7 @@
 		val = remote_clock;
 	}
  
-	if (cmpxchg(ptr, old_val, val) != old_val)
+	if (cmpxchg64(ptr, old_val, val) != old_val)
 		goto again;
  
 	return val;
...	...	@@ -312,19 +312,23 @@
312	312
313	313	extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
314	314
315		-#define cmpxchg64(ptr, o, n) \
316		-({ \
317		- __typeof__(*(ptr)) __ret; \
318		- if (likely(boot_cpu_data.x86 > 4)) \
319		- __ret = (__typeof__(*(ptr)))__cmpxchg64((ptr), \
320		- (unsigned long long)(o), \
321		- (unsigned long long)(n)); \
322		- else \
323		- __ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr), \
324		- (unsigned long long)(o), \
325		- (unsigned long long)(n)); \
326		- __ret; \
327		-})
	315	+#define cmpxchg64(ptr, o, n) \
	316	+({ \
	317	+ __typeof__(*(ptr)) __ret; \
	318	+ __typeof__(*(ptr)) __old = (o); \
	319	+ __typeof__(*(ptr)) __new = (n); \
	320	+ alternative_io("call cmpxchg8b_emu", \
	321	+ "lock; cmpxchg8b (%%esi)" , \
	322	+ X86_FEATURE_CX8, \
	323	+ "=A" (__ret), \
	324	+ "S" ((ptr)), "0" (__old), \
	325	+ "b" ((unsigned int)__new), \
	326	+ "c" ((unsigned int)(__new>>32)) \
	327	+ : "memory"); \
	328	+ __ret; })
	329	+
	330	+
	331	+
328	332	#define cmpxchg64_local(ptr, o, n) \
329	333	({ \
330	334	__typeof__(*(ptr)) __ret; \
...	...	@@ -10,6 +10,14 @@
10	10	EXPORT_SYMBOL(mcount);
11	11	#endif
12	12
	13	+/*
	14	+ * Note, this is a prototype to get at the symbol for
	15	+ * the export, but dont use it from C code, it is used
	16	+ * by assembly code and is not using C calling convention!
	17	+ */
	18	+extern void cmpxchg8b_emu(void);
	19	+EXPORT_SYMBOL(cmpxchg8b_emu);
	20	+
13	21	/* Networking helper routines. */
14	22	EXPORT_SYMBOL(csum_partial_copy_generic);
15	23
...	...	@@ -15,7 +15,7 @@
15	15	obj-y += atomic64_32.o
16	16	lib-y += checksum_32.o
17	17	lib-y += strstr_32.o
18		- lib-y += semaphore_32.o string_32.o
	18	+ lib-y += semaphore_32.o string_32.o cmpxchg8b_emu.o
19	19
20	20	lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o
21	21	else
	1	+/*
	2	+ * This program is free software; you can redistribute it and/or
	3	+ * modify it under the terms of the GNU General Public License
	4	+ * as published by the Free Software Foundation; version 2
	5	+ * of the License.
	6	+ *
	7	+ */
	8	+
	9	+#include <linux/linkage.h>
	10	+#include <asm/alternative-asm.h>
	11	+#include <asm/frame.h>
	12	+#include <asm/dwarf2.h>
	13	+
	14	+
	15	+.text
	16	+
	17	+/*
	18	+ * Inputs:
	19	+ * %esi : memory location to compare
	20	+ * %eax : low 32 bits of old value
	21	+ * %edx : high 32 bits of old value
	22	+ * %ebx : low 32 bits of new value
	23	+ * %ecx : high 32 bits of new value
	24	+ */
	25	+ENTRY(cmpxchg8b_emu)
	26	+CFI_STARTPROC
	27	+
	28	+#
	29	+# Emulate 'cmpxchg8b (%esi)' on UP except we don't
	30	+# set the whole ZF thing (caller will just compare
	31	+# eax:edx with the expected value)
	32	+#
	33	+cmpxchg8b_emu:
	34	+ pushfl
	35	+ cli
	36	+
	37	+ cmpl (%esi), %eax
	38	+ jne not_same
	39	+ cmpl 4(%esi), %edx
	40	+ jne half_same
	41	+
	42	+ movl %ebx, (%esi)
	43	+ movl %ecx, 4(%esi)
	44	+
	45	+ popfl
	46	+ ret
	47	+
	48	+ not_same:
	49	+ movl (%esi), %eax
	50	+ half_same:
	51	+ movl 4(%esi), %edx
	52	+
	53	+ popfl
	54	+ ret
	55	+
	56	+CFI_ENDPROC
	57	+ENDPROC(cmpxchg8b_emu)
...	...	@@ -127,7 +127,7 @@
127	127	clock = wrap_max(clock, min_clock);
128	128	clock = wrap_min(clock, max_clock);
129	129
130		- if (cmpxchg(&scd->clock, old_clock, clock) != old_clock)
	130	+ if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock)
131	131	goto again;
132	132
133	133	return clock;
...	...	@@ -163,7 +163,7 @@
163	163	val = remote_clock;
164	164	}
165	165
166		- if (cmpxchg(ptr, old_val, val) != old_val)
	166	+ if (cmpxchg64(ptr, old_val, val) != old_val)
167	167	goto again;
168	168
169	169	return val;