sched: reintroduce cache-hot affinity

reintroduce a simplified version of cache-hot/cold scheduling affinity. This improves performance with certain SMP workloads, such as sysbench. Signed-off-by: Ingo Molnar <mingo@elte.hu>

sched: reintroduce cache-hot affinity
reintroduce a simplified version of cache-hot/cold scheduling affinity. This improves performance with certain SMP workloads, such as sysbench. Signed-off-by: Ingo Molnar <mingo@elte.hu>
Ingo Molnar
1 parent e5f32a3856
Showing 4 changed files with 38 additions and 0 deletions Side-by-side Diff
include/linux/sched.h
kernel/sched.c
kernel/sched_fair.c
kernel/sysctl.c
@@ -1415,6 +1415,7 @@
 extern unsigned int sysctl_sched_batch_wakeup_granularity;
 extern unsigned int sysctl_sched_child_runs_first;
 extern unsigned int sysctl_sched_features;
+extern unsigned int sysctl_sched_migration_cost;
 #endif
  
 extern unsigned int sysctl_sched_compat_yield;
@@ -2119,6 +2119,17 @@
 }
  
 /*
+ * Is this task likely cache-hot:
+ */
+static inline int
+task_hot(struct task_struct *p, unsigned long long now, struct sched_domain *sd)
+{
+	s64 delta = now - p->se.exec_start;
+
+	return delta < (long long)sysctl_sched_migration_cost;
+}
+
+/*
  * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
  */
 static
@@ -2139,6 +2150,22 @@
 	if (task_running(rq, p))
 		return 0;
  
+	/*
+	 * Aggressive migration if:
+	 * 1) task is cache cold, or
+	 * 2) too many balance attempts have failed.
+	 */
+
+	if (sd->nr_balance_failed > sd->cache_nice_tries) {
+#ifdef CONFIG_SCHEDSTATS
+		if (task_hot(p, rq->clock, sd))
+			schedstat_inc(sd, lb_hot_gained[idle]);
+#endif
+		return 1;
+	}
+
+	if (task_hot(p, rq->clock, sd))
+		return 0;
 	return 1;
 }
  
@@ -74,6 +74,8 @@
  */
 const_debug unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
  
+const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
+
 /**************************************************************
  * CFS operations on generic schedulable entities:
  */
@@ -277,6 +277,14 @@
 		.mode		= 0644,
 		.proc_handler	= &proc_dointvec,
 	},
+	{
+		.ctl_name	= CTL_UNNUMBERED,
+		.procname	= "sched_migration_cost",
+		.data		= &sysctl_sched_migration_cost,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec,
+	},
 #endif
 	{
 		.ctl_name	= CTL_UNNUMBERED,
...	...	@@ -1415,6 +1415,7 @@
1415	1415	extern unsigned int sysctl_sched_batch_wakeup_granularity;
1416	1416	extern unsigned int sysctl_sched_child_runs_first;
1417	1417	extern unsigned int sysctl_sched_features;
	1418	+extern unsigned int sysctl_sched_migration_cost;
1418	1419	#endif
1419	1420
1420	1421	extern unsigned int sysctl_sched_compat_yield;
...	...	@@ -2119,6 +2119,17 @@
2119	2119	}
2120	2120
2121	2121	/*
	2122	+ * Is this task likely cache-hot:
	2123	+ */
	2124	+static inline int
	2125	+task_hot(struct task_struct p, unsigned long long now, struct sched_domain sd)
	2126	+{
	2127	+ s64 delta = now - p->se.exec_start;
	2128	+
	2129	+ return delta < (long long)sysctl_sched_migration_cost;
	2130	+}
	2131	+
	2132	+/*
2122	2133	* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
2123	2134	*/
2124	2135	static
...	...	@@ -2139,6 +2150,22 @@
2139	2150	if (task_running(rq, p))
2140	2151	return 0;
2141	2152
	2153	+ /*
	2154	+ * Aggressive migration if:
	2155	+ * 1) task is cache cold, or
	2156	+ * 2) too many balance attempts have failed.
	2157	+ */
	2158	+
	2159	+ if (sd->nr_balance_failed > sd->cache_nice_tries) {
	2160	+#ifdef CONFIG_SCHEDSTATS
	2161	+ if (task_hot(p, rq->clock, sd))
	2162	+ schedstat_inc(sd, lb_hot_gained[idle]);
	2163	+#endif
	2164	+ return 1;
	2165	+ }
	2166	+
	2167	+ if (task_hot(p, rq->clock, sd))
	2168	+ return 0;
2142	2169	return 1;
2143	2170	}
2144	2171
...	...	@@ -74,6 +74,8 @@
74	74	*/
75	75	const_debug unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
76	76
	77	+const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
	78	+
77	79	/**************************************************************
78	80	* CFS operations on generic schedulable entities:
79	81	*/
...	...	@@ -277,6 +277,14 @@
277	277	.mode = 0644,
278	278	.proc_handler = &proc_dointvec,
279	279	},
	280	+ {
	281	+ .ctl_name = CTL_UNNUMBERED,
	282	+ .procname = "sched_migration_cost",
	283	+ .data = &sysctl_sched_migration_cost,
	284	+ .maxlen = sizeof(unsigned int),
	285	+ .mode = 0644,
	286	+ .proc_handler = &proc_dointvec,
	287	+ },
280	288	#endif
281	289	{
282	290	.ctl_name = CTL_UNNUMBERED,