Merge branch 'x86-idle-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'x86-idle-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: x86, hotplug: In the MWAIT case of play_dead, CLFLUSH the cache line x86, hotplug: Move WBINVD back outside the play_dead loop x86, hotplug: Use mwait to offline a processor, fix the legacy case x86, mwait: Move mwait constants to a common header file

Merge branch 'x86-idle-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'x86-idle-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: x86, hotplug: In the MWAIT case of play_dead, CLFLUSH the cache line x86, hotplug: Move WBINVD back outside the play_dead loop x86, hotplug: Use mwait to offline a processor, fix the legacy case x86, mwait: Move mwait constants to a common header file
Linus Torvalds
2 parents b6f7e38dbb ce5f68246b
Showing 6 changed files Side-by-side Diff
arch/x86/include/asm/mwait.h
arch/x86/include/asm/processor.h
arch/x86/kernel/acpi/cstate.c
arch/x86/kernel/smpboot.c
drivers/acpi/acpi_pad.c
drivers/idle/intel_idle.c
+#ifndef _ASM_X86_MWAIT_H
+#define _ASM_X86_MWAIT_H
+
+#define MWAIT_SUBSTATE_MASK		0xf
+#define MWAIT_CSTATE_MASK		0xf
+#define MWAIT_SUBSTATE_SIZE		4
+#define MWAIT_MAX_NUM_CSTATES		8
+
+#define CPUID_MWAIT_LEAF		5
+#define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1
+#define CPUID5_ECX_INTERRUPT_BREAK	0x2
+
+#define MWAIT_ECX_INTERRUPT_BREAK	0x1
+
+#endif /* _ASM_X86_MWAIT_H */
@@ -766,29 +766,6 @@
 extern unsigned long		idle_nomwait;
 extern bool			c1e_detected;
  
-/*
- * on systems with caches, caches must be flashed as the absolute
- * last instruction before going into a suspended halt.  Otherwise,
- * dirty data can linger in the cache and become stale on resume,
- * leading to strange errors.
- *
- * perform a variety of operations to guarantee that the compiler
- * will not reorder instructions.  wbinvd itself is serializing
- * so the processor will not reorder.
- *
- * Systems without cache can just go into halt.
- */
-static inline void wbinvd_halt(void)
-{
-	mb();
-	/* check for clflush to determine if wbinvd is legal */
-	if (cpu_has_clflush)
-		asm volatile("cli; wbinvd; 1: hlt; jmp 1b" : : : "memory");
-	else
-		while (1)
-			halt();
-}
-
 extern void enable_sep_cpu(void);
 extern int sysenter_setup(void);
  
@@ -13,6 +13,7 @@
  
 #include <acpi/processor.h>
 #include <asm/acpi.h>
+#include <asm/mwait.h>
  
 /*
  * Initialize bm_flags based on the CPU cache properties
@@ -64,16 +65,6 @@
 static struct cstate_entry __percpu *cpu_cstate_entry;	/* per CPU ptr */
  
 static short mwait_supported[ACPI_PROCESSOR_MAX_POWER];
-
-#define MWAIT_SUBSTATE_MASK	(0xf)
-#define MWAIT_CSTATE_MASK	(0xf)
-#define MWAIT_SUBSTATE_SIZE	(4)
-
-#define CPUID_MWAIT_LEAF (5)
-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
-#define CPUID5_ECX_INTERRUPT_BREAK	(0x2)
-
-#define MWAIT_ECX_INTERRUPT_BREAK	(0x1)
  
 #define NATIVE_CSTATE_BEYOND_HALT	(2)
  
@@ -62,6 +62,7 @@
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <asm/mtrr.h>
+#include <asm/mwait.h>
 #include <asm/vmi.h>
 #include <asm/apic.h>
 #include <asm/setup.h>
  
@@ -1395,11 +1396,88 @@
 	local_irq_disable();
 }
  
+/*
+ * We need to flush the caches before going to sleep, lest we have
+ * dirty data in our caches when we come back up.
+ */
+static inline void mwait_play_dead(void)
+{
+	unsigned int eax, ebx, ecx, edx;
+	unsigned int highest_cstate = 0;
+	unsigned int highest_subcstate = 0;
+	int i;
+	void *mwait_ptr;
+
+	if (!cpu_has(&current_cpu_data, X86_FEATURE_MWAIT))
+		return;
+	if (!cpu_has(&current_cpu_data, X86_FEATURE_CLFLSH))
+		return;
+	if (current_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
+		return;
+
+	eax = CPUID_MWAIT_LEAF;
+	ecx = 0;
+	native_cpuid(&eax, &ebx, &ecx, &edx);
+
+	/*
+	 * eax will be 0 if EDX enumeration is not valid.
+	 * Initialized below to cstate, sub_cstate value when EDX is valid.
+	 */
+	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
+		eax = 0;
+	} else {
+		edx >>= MWAIT_SUBSTATE_SIZE;
+		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
+			if (edx & MWAIT_SUBSTATE_MASK) {
+				highest_cstate = i;
+				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
+			}
+		}
+		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
+			(highest_subcstate - 1);
+	}
+
+	/*
+	 * This should be a memory location in a cache line which is
+	 * unlikely to be touched by other processors.  The actual
+	 * content is immaterial as it is not actually modified in any way.
+	 */
+	mwait_ptr = &current_thread_info()->flags;
+
+	wbinvd();
+
+	while (1) {
+		/*
+		 * The CLFLUSH is a workaround for erratum AAI65 for
+		 * the Xeon 7400 series.  It's not clear it is actually
+		 * needed, but it should be harmless in either case.
+		 * The WBINVD is insufficient due to the spurious-wakeup
+		 * case where we return around the loop.
+		 */
+		clflush(mwait_ptr);
+		__monitor(mwait_ptr, 0, 0);
+		mb();
+		__mwait(eax, 0);
+	}
+}
+
+static inline void hlt_play_dead(void)
+{
+	if (current_cpu_data.x86 >= 4)
+		wbinvd();
+
+	while (1) {
+		native_halt();
+	}
+}
+
 void native_play_dead(void)
 {
 	play_dead_common();
 	tboot_shutdown(TB_SHUTDOWN_WFS);
-	wbinvd_halt();
+
+	mwait_play_dead();	/* Only returns on failure */
+	hlt_play_dead();
 }
  
 #else /* ... !CONFIG_HOTPLUG_CPU */
@@ -30,18 +30,13 @@
 #include <linux/slab.h>
 #include <acpi/acpi_bus.h>
 #include <acpi/acpi_drivers.h>
+#include <asm/mwait.h>
  
 #define ACPI_PROCESSOR_AGGREGATOR_CLASS	"acpi_pad"
 #define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
 #define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
 static DEFINE_MUTEX(isolated_cpus_lock);
  
-#define MWAIT_SUBSTATE_MASK	(0xf)
-#define MWAIT_CSTATE_MASK	(0xf)
-#define MWAIT_SUBSTATE_SIZE	(4)
-#define CPUID_MWAIT_LEAF (5)
-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
-#define CPUID5_ECX_INTERRUPT_BREAK	(0x2)
 static unsigned long power_saving_mwait_eax;
  
 static unsigned char tsc_detected_unstable;
@@ -59,17 +59,10 @@
 #include <linux/hrtimer.h>	/* ktime_get_real() */
 #include <trace/events/power.h>
 #include <linux/sched.h>
+#include <asm/mwait.h>
  
 #define INTEL_IDLE_VERSION "0.4"
 #define PREFIX "intel_idle: "
-
-#define MWAIT_SUBSTATE_MASK	(0xf)
-#define MWAIT_CSTATE_MASK	(0xf)
-#define MWAIT_SUBSTATE_SIZE	(4)
-#define MWAIT_MAX_NUM_CSTATES	8
-#define CPUID_MWAIT_LEAF (5)
-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
-#define CPUID5_ECX_INTERRUPT_BREAK	(0x2)
  
 static struct cpuidle_driver intel_idle_driver = {
 	.name = "intel_idle",
	1	+#ifndef _ASM_X86_MWAIT_H
	2	+#define _ASM_X86_MWAIT_H
	3	+
	4	+#define MWAIT_SUBSTATE_MASK 0xf
	5	+#define MWAIT_CSTATE_MASK 0xf
	6	+#define MWAIT_SUBSTATE_SIZE 4
	7	+#define MWAIT_MAX_NUM_CSTATES 8
	8	+
	9	+#define CPUID_MWAIT_LEAF 5
	10	+#define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1
	11	+#define CPUID5_ECX_INTERRUPT_BREAK 0x2
	12	+
	13	+#define MWAIT_ECX_INTERRUPT_BREAK 0x1
	14	+
	15	+#endif /* _ASM_X86_MWAIT_H */
...	...	@@ -766,29 +766,6 @@
766	766	extern unsigned long idle_nomwait;
767	767	extern bool c1e_detected;
768	768
769		-/*
770		- * on systems with caches, caches must be flashed as the absolute
771		- * last instruction before going into a suspended halt. Otherwise,
772		- * dirty data can linger in the cache and become stale on resume,
773		- * leading to strange errors.
774		- *
775		- * perform a variety of operations to guarantee that the compiler
776		- * will not reorder instructions. wbinvd itself is serializing
777		- * so the processor will not reorder.
778		- *
779		- * Systems without cache can just go into halt.
780		- */
781		-static inline void wbinvd_halt(void)
782		-{
783		- mb();
784		- /* check for clflush to determine if wbinvd is legal */
785		- if (cpu_has_clflush)
786		- asm volatile("cli; wbinvd; 1: hlt; jmp 1b" : : : "memory");
787		- else
788		- while (1)
789		- halt();
790		-}
791		-
792	769	extern void enable_sep_cpu(void);
793	770	extern int sysenter_setup(void);
794	771
...	...	@@ -13,6 +13,7 @@
13	13
14	14	#include <acpi/processor.h>
15	15	#include <asm/acpi.h>
	16	+#include <asm/mwait.h>
16	17
17	18	/*
18	19	* Initialize bm_flags based on the CPU cache properties
...	...	@@ -64,16 +65,6 @@
64	65	static struct cstate_entry __percpu cpu_cstate_entry; / per CPU ptr */
65	66
66	67	static short mwait_supported[ACPI_PROCESSOR_MAX_POWER];
67		-
68		-#define MWAIT_SUBSTATE_MASK (0xf)
69		-#define MWAIT_CSTATE_MASK (0xf)
70		-#define MWAIT_SUBSTATE_SIZE (4)
71		-
72		-#define CPUID_MWAIT_LEAF (5)
73		-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
74		-#define CPUID5_ECX_INTERRUPT_BREAK (0x2)
75		-
76		-#define MWAIT_ECX_INTERRUPT_BREAK (0x1)
77	68
78	69	#define NATIVE_CSTATE_BEYOND_HALT (2)
79	70
...	...	@@ -62,6 +62,7 @@
62	62	#include <asm/pgtable.h>
63	63	#include <asm/tlbflush.h>
64	64	#include <asm/mtrr.h>
	65	+#include <asm/mwait.h>
65	66	#include <asm/vmi.h>
66	67	#include <asm/apic.h>
67	68	#include <asm/setup.h>
68	69
...	...	@@ -1395,11 +1396,88 @@
1395	1396	local_irq_disable();
1396	1397	}
1397	1398
	1399	+/*
	1400	+ * We need to flush the caches before going to sleep, lest we have
	1401	+ * dirty data in our caches when we come back up.
	1402	+ */
	1403	+static inline void mwait_play_dead(void)
	1404	+{
	1405	+ unsigned int eax, ebx, ecx, edx;
	1406	+ unsigned int highest_cstate = 0;
	1407	+ unsigned int highest_subcstate = 0;
	1408	+ int i;
	1409	+ void *mwait_ptr;
	1410	+
	1411	+ if (!cpu_has(&current_cpu_data, X86_FEATURE_MWAIT))
	1412	+ return;
	1413	+ if (!cpu_has(&current_cpu_data, X86_FEATURE_CLFLSH))
	1414	+ return;
	1415	+ if (current_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
	1416	+ return;
	1417	+
	1418	+ eax = CPUID_MWAIT_LEAF;
	1419	+ ecx = 0;
	1420	+ native_cpuid(&eax, &ebx, &ecx, &edx);
	1421	+
	1422	+ /*
	1423	+ * eax will be 0 if EDX enumeration is not valid.
	1424	+ * Initialized below to cstate, sub_cstate value when EDX is valid.
	1425	+ */
	1426	+ if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
	1427	+ eax = 0;
	1428	+ } else {
	1429	+ edx >>= MWAIT_SUBSTATE_SIZE;
	1430	+ for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
	1431	+ if (edx & MWAIT_SUBSTATE_MASK) {
	1432	+ highest_cstate = i;
	1433	+ highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
	1434	+ }
	1435	+ }
	1436	+ eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) \|
	1437	+ (highest_subcstate - 1);
	1438	+ }
	1439	+
	1440	+ /*
	1441	+ * This should be a memory location in a cache line which is
	1442	+ * unlikely to be touched by other processors. The actual
	1443	+ * content is immaterial as it is not actually modified in any way.
	1444	+ */
	1445	+ mwait_ptr = &current_thread_info()->flags;
	1446	+
	1447	+ wbinvd();
	1448	+
	1449	+ while (1) {
	1450	+ /*
	1451	+ * The CLFLUSH is a workaround for erratum AAI65 for
	1452	+ * the Xeon 7400 series. It's not clear it is actually
	1453	+ * needed, but it should be harmless in either case.
	1454	+ * The WBINVD is insufficient due to the spurious-wakeup
	1455	+ * case where we return around the loop.
	1456	+ */
	1457	+ clflush(mwait_ptr);
	1458	+ __monitor(mwait_ptr, 0, 0);
	1459	+ mb();
	1460	+ __mwait(eax, 0);
	1461	+ }
	1462	+}
	1463	+
	1464	+static inline void hlt_play_dead(void)
	1465	+{
	1466	+ if (current_cpu_data.x86 >= 4)
	1467	+ wbinvd();
	1468	+
	1469	+ while (1) {
	1470	+ native_halt();
	1471	+ }
	1472	+}
	1473	+
1398	1474	void native_play_dead(void)
1399	1475	{
1400	1476	play_dead_common();
1401	1477	tboot_shutdown(TB_SHUTDOWN_WFS);
1402		- wbinvd_halt();
	1478	+
	1479	+ mwait_play_dead(); /* Only returns on failure */
	1480	+ hlt_play_dead();
1403	1481	}
1404	1482
1405	1483	#else /* ... !CONFIG_HOTPLUG_CPU */
...	...	@@ -30,18 +30,13 @@
30	30	#include <linux/slab.h>
31	31	#include <acpi/acpi_bus.h>
32	32	#include <acpi/acpi_drivers.h>
	33	+#include <asm/mwait.h>
33	34
34	35	#define ACPI_PROCESSOR_AGGREGATOR_CLASS "acpi_pad"
35	36	#define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
36	37	#define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
37	38	static DEFINE_MUTEX(isolated_cpus_lock);
38	39
39		-#define MWAIT_SUBSTATE_MASK (0xf)
40		-#define MWAIT_CSTATE_MASK (0xf)
41		-#define MWAIT_SUBSTATE_SIZE (4)
42		-#define CPUID_MWAIT_LEAF (5)
43		-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
44		-#define CPUID5_ECX_INTERRUPT_BREAK (0x2)
45	40	static unsigned long power_saving_mwait_eax;
46	41
47	42	static unsigned char tsc_detected_unstable;
...	...	@@ -59,17 +59,10 @@
59	59	#include <linux/hrtimer.h> /* ktime_get_real() */
60	60	#include <trace/events/power.h>
61	61	#include <linux/sched.h>
	62	+#include <asm/mwait.h>
62	63
63	64	#define INTEL_IDLE_VERSION "0.4"
64	65	#define PREFIX "intel_idle: "
65		-
66		-#define MWAIT_SUBSTATE_MASK (0xf)
67		-#define MWAIT_CSTATE_MASK (0xf)
68		-#define MWAIT_SUBSTATE_SIZE (4)
69		-#define MWAIT_MAX_NUM_CSTATES 8
70		-#define CPUID_MWAIT_LEAF (5)
71		-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
72		-#define CPUID5_ECX_INTERRUPT_BREAK (0x2)
73	66
74	67	static struct cpuidle_driver intel_idle_driver = {
75	68	.name = "intel_idle",