intel_idle: disable NHM/WSM HW C-state auto-demotion

Hardware C-state auto-demotion is a mechanism where the HW overrides the OS C-state request, instead demoting to a shallower state, which is less expensive, but saves less power. Modern Linux should generally get exactly the states it requests. In particular, when a CPU is taken off-line, it must not be demoted, else it can prevent the entire package from reaching deep C-states. https://bugzilla.kernel.org/show_bug.cgi?id=25252 Signed-off-by: Len Brown <len.brown@intel.com>

intel_idle: disable NHM/WSM HW C-state auto-demotion
Hardware C-state auto-demotion is a mechanism where the HW overrides the OS C-state request, instead demoting to a shallower state, which is less expensive, but saves less power. Modern Linux should generally get exactly the states it requests. In particular, when a CPU is taken off-line, it must not be demoted, else it can prevent the entire package from reaching deep C-states. https://bugzilla.kernel.org/show_bug.cgi?id=25252 Signed-off-by: Len Brown <len.brown@intel.com>
Len Brown
1 parent 100b33c8bd
Showing 2 changed files with 24 additions and 0 deletions Side-by-side Diff
arch/x86/include/asm/msr-index.h
drivers/idle/intel_idle.c
@@ -36,6 +36,10 @@
 #define MSR_IA32_PERFCTR1		0x000000c2
 #define MSR_FSB_FREQ			0x000000cd
  
+#define MSR_NHM_SNB_PKG_CST_CFG_CTL	0x000000e2
+#define NHM_C3_AUTO_DEMOTE		(1UL << 25)
+#define NHM_C1_AUTO_DEMOTE		(1UL << 26)
+
 #define MSR_MTRRcap			0x000000fe
 #define MSR_IA32_BBL_CR_CTL		0x00000119
  
@@ -62,6 +62,7 @@
 #include <linux/notifier.h>
 #include <linux/cpu.h>
 #include <asm/mwait.h>
+#include <asm/msr.h>
  
 #define INTEL_IDLE_VERSION "0.4"
 #define PREFIX "intel_idle: "
@@ -85,6 +86,12 @@
 static struct cpuidle_state *cpuidle_state_table;
  
 /*
+ * Hardware C-state auto-demotion may not always be optimal.
+ * Indicate which enable bits to clear here.
+ */
+static unsigned long long auto_demotion_disable_flags;
+
+/*
  * Set this flag for states where the HW flushes the TLB for us
  * and so we don't need cross-calls to keep it consistent.
  * If this flag is set, SW flushes the TLB, so even if the
@@ -281,6 +288,15 @@
 	.notifier_call = setup_broadcast_cpuhp_notify,
 };
  
+static void auto_demotion_disable(void *dummy)
+{
+	unsigned long long msr_bits;
+
+	rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
+	msr_bits &= ~auto_demotion_disable_flags;
+	wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
+}
+
 /*
  * intel_idle_probe()
  */
@@ -324,6 +340,8 @@
 	case 0x25:	/* Westmere */
 	case 0x2C:	/* Westmere */
 		cpuidle_state_table = nehalem_cstates;
+		auto_demotion_disable_flags =
+			(NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE);
 		break;
  
 	case 0x1C:	/* 28 - Atom Processor */
@@ -436,6 +454,8 @@
 			return -EIO;
 		}
 	}
+	if (auto_demotion_disable_flags)
+		smp_call_function(auto_demotion_disable, NULL, 1);
  
 	return 0;
 }
...	...	@@ -36,6 +36,10 @@
36	36	#define MSR_IA32_PERFCTR1 0x000000c2
37	37	#define MSR_FSB_FREQ 0x000000cd
38	38
	39	+#define MSR_NHM_SNB_PKG_CST_CFG_CTL 0x000000e2
	40	+#define NHM_C3_AUTO_DEMOTE (1UL << 25)
	41	+#define NHM_C1_AUTO_DEMOTE (1UL << 26)
	42	+
39	43	#define MSR_MTRRcap 0x000000fe
40	44	#define MSR_IA32_BBL_CR_CTL 0x00000119
41	45
...	...	@@ -62,6 +62,7 @@
62	62	#include <linux/notifier.h>
63	63	#include <linux/cpu.h>
64	64	#include <asm/mwait.h>
	65	+#include <asm/msr.h>
65	66
66	67	#define INTEL_IDLE_VERSION "0.4"
67	68	#define PREFIX "intel_idle: "
...	...	@@ -85,6 +86,12 @@
85	86	static struct cpuidle_state *cpuidle_state_table;
86	87
87	88	/*
	89	+ * Hardware C-state auto-demotion may not always be optimal.
	90	+ * Indicate which enable bits to clear here.
	91	+ */
	92	+static unsigned long long auto_demotion_disable_flags;
	93	+
	94	+/*
88	95	* Set this flag for states where the HW flushes the TLB for us
89	96	* and so we don't need cross-calls to keep it consistent.
90	97	* If this flag is set, SW flushes the TLB, so even if the
...	...	@@ -281,6 +288,15 @@
281	288	.notifier_call = setup_broadcast_cpuhp_notify,
282	289	};
283	290
	291	+static void auto_demotion_disable(void *dummy)
	292	+{
	293	+ unsigned long long msr_bits;
	294	+
	295	+ rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
	296	+ msr_bits &= ~auto_demotion_disable_flags;
	297	+ wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
	298	+}
	299	+
284	300	/*
285	301	* intel_idle_probe()
286	302	*/
...	...	@@ -324,6 +340,8 @@
324	340	case 0x25: /* Westmere */
325	341	case 0x2C: /* Westmere */
326	342	cpuidle_state_table = nehalem_cstates;
	343	+ auto_demotion_disable_flags =
	344	+ (NHM_C1_AUTO_DEMOTE \| NHM_C3_AUTO_DEMOTE);
327	345	break;
328	346
329	347	case 0x1C: /* 28 - Atom Processor */
...	...	@@ -436,6 +454,8 @@
436	454	return -EIO;
437	455	}
438	456	}
	457	+ if (auto_demotion_disable_flags)
	458	+ smp_call_function(auto_demotion_disable, NULL, 1);
439	459
440	460	return 0;
441	461	}