Merge git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched

* git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched: sched: fix new task startup crash sched: fix !SYSFS build breakage sched: fix improper load balance across sched domain sched: more robust sd-sysctl entry freeing

Merge git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched
* git://git.kernel.org/pub/scm/linux/kernel/git/mingo/linux-2.6-sched: sched: fix new task startup crash sched: fix !SYSFS build breakage sched: fix improper load balance across sched domain sched: more robust sd-sysctl entry freeing
Linus Torvalds
2 parents b6257a9036 b9dca1e0fc
Showing 4 changed files Side-by-side Diff
include/linux/sched.h
kernel/sched.c
kernel/sched_fair.c
kernel/user.c
@@ -535,9 +535,11 @@
  
 #ifdef CONFIG_FAIR_USER_SCHED
 	struct task_group *tg;
+#ifdef CONFIG_SYSFS
 	struct kset kset;
 	struct subsys_attribute user_attr;
 	struct work_struct work;
+#endif
 #endif
 };
  
@@ -1712,7 +1712,7 @@
  
 	p->prio = effective_prio(p);
  
-	if (!p->sched_class->task_new || !current->se.on_rq || !rq->cfs.curr) {
+	if (!p->sched_class->task_new || !current->se.on_rq) {
 		activate_task(rq, p, 0);
 	} else {
 		/*
@@ -2336,7 +2336,7 @@
 	unsigned long max_pull;
 	unsigned long busiest_load_per_task, busiest_nr_running;
 	unsigned long this_load_per_task, this_nr_running;
-	int load_idx;
+	int load_idx, group_imb = 0;
 #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
 	int power_savings_balance = 1;
 	unsigned long leader_nr_running = 0, min_load_per_task = 0;
  
@@ -2355,9 +2355,10 @@
 		load_idx = sd->idle_idx;
  
 	do {
-		unsigned long load, group_capacity;
+		unsigned long load, group_capacity, max_cpu_load, min_cpu_load;
 		int local_group;
 		int i;
+		int __group_imb = 0;
 		unsigned int balance_cpu = -1, first_idle_cpu = 0;
 		unsigned long sum_nr_running, sum_weighted_load;
  
@@ -2368,6 +2369,8 @@
  
 		/* Tally up the load of all CPUs in the group */
 		sum_weighted_load = sum_nr_running = avg_load = 0;
+		max_cpu_load = 0;
+		min_cpu_load = ~0UL;
  
 		for_each_cpu_mask(i, group->cpumask) {
 			struct rq *rq;
  
@@ -2388,8 +2391,13 @@
 				}
  
 				load = target_load(i, load_idx);
-			} else
+			} else {
 				load = source_load(i, load_idx);
+				if (load > max_cpu_load)
+					max_cpu_load = load;
+				if (min_cpu_load > load)
+					min_cpu_load = load;
+			}
  
 			avg_load += load;
 			sum_nr_running += rq->nr_running;
@@ -2415,6 +2423,9 @@
 		avg_load = sg_div_cpu_power(group,
 				avg_load * SCHED_LOAD_SCALE);
  
+		if ((max_cpu_load - min_cpu_load) > SCHED_LOAD_SCALE)
+			__group_imb = 1;
+
 		group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
  
 		if (local_group) {
  
@@ -2423,11 +2434,12 @@
 			this_nr_running = sum_nr_running;
 			this_load_per_task = sum_weighted_load;
 		} else if (avg_load > max_load &&
-			   sum_nr_running > group_capacity) {
+			   (sum_nr_running > group_capacity || __group_imb)) {
 			max_load = avg_load;
 			busiest = group;
 			busiest_nr_running = sum_nr_running;
 			busiest_load_per_task = sum_weighted_load;
+			group_imb = __group_imb;
 		}
  
 #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
@@ -2499,6 +2511,9 @@
 		goto out_balanced;
  
 	busiest_load_per_task /= busiest_nr_running;
+	if (group_imb)
+		busiest_load_per_task = min(busiest_load_per_task, avg_load);
+
 	/*
 	 * We're trying to get all the cpus to the average_load, so we don't
 	 * want to push ourselves above the average load, nor do we wish to
  
  
@@ -5282,11 +5297,20 @@
  
 static void sd_free_ctl_entry(struct ctl_table **tablep)
 {
-	struct ctl_table *entry = *tablep;
+	struct ctl_table *entry;
  
-	for (entry = *tablep; entry->procname; entry++)
+	/*
+	 * In the intermediate directories, both the child directory and
+	 * procname are dynamically allocated and could fail but the mode
+	 * will always be set.  In the lowest directory the names are
+	 * static strings and all have proc handlers.
+	 */
+	for (entry = *tablep; entry->mode; entry++) {
 		if (entry->child)
 			sd_free_ctl_entry(&entry->child);
+		if (entry->proc_handler == NULL)
+			kfree(entry->procname);
+	}
  
 	kfree(*tablep);
 	*tablep = NULL;
@@ -1031,12 +1031,8 @@
 		swap(curr->vruntime, se->vruntime);
 	}
  
-	update_stats_enqueue(cfs_rq, se);
-	check_spread(cfs_rq, se);
-	check_spread(cfs_rq, curr);
-	__enqueue_entity(cfs_rq, se);
-	account_entity_enqueue(cfs_rq, se);
 	se->peer_preempt = 0;
+	enqueue_task_fair(rq, p, 0);
 	resched_task(rq->curr);
 }
  
@@ -84,9 +84,6 @@
  
 #ifdef CONFIG_FAIR_USER_SCHED
  
-static struct kobject uids_kobject; /* represents /sys/kernel/uids directory */
-static DEFINE_MUTEX(uids_mutex);
-
 static void sched_destroy_user(struct user_struct *up)
 {
 	sched_destroy_group(up->tg);
@@ -108,6 +105,19 @@
 	sched_move_task(p);
 }
  
+#else	/* CONFIG_FAIR_USER_SCHED */
+
+static void sched_destroy_user(struct user_struct *up) { }
+static int sched_create_user(struct user_struct *up) { return 0; }
+static void sched_switch_user(struct task_struct *p) { }
+
+#endif	/* CONFIG_FAIR_USER_SCHED */
+
+#if defined(CONFIG_FAIR_USER_SCHED) && defined(CONFIG_SYSFS)
+
+static struct kobject uids_kobject; /* represents /sys/kernel/uids directory */
+static DEFINE_MUTEX(uids_mutex);
+
 static inline void uids_mutex_lock(void)
 {
 	mutex_lock(&uids_mutex);
  
@@ -254,11 +264,8 @@
 	schedule_work(&up->work);
 }
  
-#else	/* CONFIG_FAIR_USER_SCHED */
+#else	/* CONFIG_FAIR_USER_SCHED && CONFIG_SYSFS */
  
-static void sched_destroy_user(struct user_struct *up) { }
-static int sched_create_user(struct user_struct *up) { return 0; }
-static void sched_switch_user(struct task_struct *p) { }
 static inline int user_kobject_create(struct user_struct *up) { return 0; }
 static inline void uids_mutex_lock(void) { }
 static inline void uids_mutex_unlock(void) { }
@@ -277,7 +284,7 @@
 	kmem_cache_free(uid_cachep, up);
 }
  
-#endif	/* CONFIG_FAIR_USER_SCHED */
+#endif
  
 /*
  * Locate the user_struct for the passed UID.  If found, take a ref on it.  The
...	...	@@ -535,9 +535,11 @@
535	535
536	536	#ifdef CONFIG_FAIR_USER_SCHED
537	537	struct task_group *tg;
	538	+#ifdef CONFIG_SYSFS
538	539	struct kset kset;
539	540	struct subsys_attribute user_attr;
540	541	struct work_struct work;
	542	+#endif
541	543	#endif
542	544	};
543	545
...	...	@@ -1712,7 +1712,7 @@
1712	1712
1713	1713	p->prio = effective_prio(p);
1714	1714
1715		- if (!p->sched_class->task_new \|\| !current->se.on_rq \|\| !rq->cfs.curr) {
	1715	+ if (!p->sched_class->task_new \|\| !current->se.on_rq) {
1716	1716	activate_task(rq, p, 0);
1717	1717	} else {
1718	1718	/*
...	...	@@ -2336,7 +2336,7 @@
2336	2336	unsigned long max_pull;
2337	2337	unsigned long busiest_load_per_task, busiest_nr_running;
2338	2338	unsigned long this_load_per_task, this_nr_running;
2339		- int load_idx;
	2339	+ int load_idx, group_imb = 0;
2340	2340	#if defined(CONFIG_SCHED_MC) \|\| defined(CONFIG_SCHED_SMT)
2341	2341	int power_savings_balance = 1;
2342	2342	unsigned long leader_nr_running = 0, min_load_per_task = 0;
2343	2343
...	...	@@ -2355,9 +2355,10 @@
2355	2355	load_idx = sd->idle_idx;
2356	2356
2357	2357	do {
2358		- unsigned long load, group_capacity;
	2358	+ unsigned long load, group_capacity, max_cpu_load, min_cpu_load;
2359	2359	int local_group;
2360	2360	int i;
	2361	+ int __group_imb = 0;
2361	2362	unsigned int balance_cpu = -1, first_idle_cpu = 0;
2362	2363	unsigned long sum_nr_running, sum_weighted_load;
2363	2364
...	...	@@ -2368,6 +2369,8 @@
2368	2369
2369	2370	/* Tally up the load of all CPUs in the group */
2370	2371	sum_weighted_load = sum_nr_running = avg_load = 0;
	2372	+ max_cpu_load = 0;
	2373	+ min_cpu_load = ~0UL;
2371	2374
2372	2375	for_each_cpu_mask(i, group->cpumask) {
2373	2376	struct rq *rq;
2374	2377
...	...	@@ -2388,8 +2391,13 @@
2388	2391	}
2389	2392
2390	2393	load = target_load(i, load_idx);
2391		- } else
	2394	+ } else {
2392	2395	load = source_load(i, load_idx);
	2396	+ if (load > max_cpu_load)
	2397	+ max_cpu_load = load;
	2398	+ if (min_cpu_load > load)
	2399	+ min_cpu_load = load;
	2400	+ }
2393	2401
2394	2402	avg_load += load;
2395	2403	sum_nr_running += rq->nr_running;
...	...	@@ -2415,6 +2423,9 @@
2415	2423	avg_load = sg_div_cpu_power(group,
2416	2424	avg_load * SCHED_LOAD_SCALE);
2417	2425
	2426	+ if ((max_cpu_load - min_cpu_load) > SCHED_LOAD_SCALE)
	2427	+ __group_imb = 1;
	2428	+
2418	2429	group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
2419	2430
2420	2431	if (local_group) {
2421	2432
...	...	@@ -2423,11 +2434,12 @@
2423	2434	this_nr_running = sum_nr_running;
2424	2435	this_load_per_task = sum_weighted_load;
2425	2436	} else if (avg_load > max_load &&
2426		- sum_nr_running > group_capacity) {
	2437	+ (sum_nr_running > group_capacity \|\| __group_imb)) {
2427	2438	max_load = avg_load;
2428	2439	busiest = group;
2429	2440	busiest_nr_running = sum_nr_running;
2430	2441	busiest_load_per_task = sum_weighted_load;
	2442	+ group_imb = __group_imb;
2431	2443	}
2432	2444
2433	2445	#if defined(CONFIG_SCHED_MC) \|\| defined(CONFIG_SCHED_SMT)
...	...	@@ -2499,6 +2511,9 @@
2499	2511	goto out_balanced;
2500	2512
2501	2513	busiest_load_per_task /= busiest_nr_running;
	2514	+ if (group_imb)
	2515	+ busiest_load_per_task = min(busiest_load_per_task, avg_load);
	2516	+
2502	2517	/*
2503	2518	* We're trying to get all the cpus to the average_load, so we don't
2504	2519	* want to push ourselves above the average load, nor do we wish to
2505	2520
2506	2521
...	...	@@ -5282,11 +5297,20 @@
5282	5297
5283	5298	static void sd_free_ctl_entry(struct ctl_table **tablep)
5284	5299	{
5285		- struct ctl_table entry = tablep;
	5300	+ struct ctl_table *entry;
5286	5301
5287		- for (entry = *tablep; entry->procname; entry++)
	5302	+ /*
	5303	+ * In the intermediate directories, both the child directory and
	5304	+ * procname are dynamically allocated and could fail but the mode
	5305	+ * will always be set. In the lowest directory the names are
	5306	+ * static strings and all have proc handlers.
	5307	+ */
	5308	+ for (entry = *tablep; entry->mode; entry++) {
5288	5309	if (entry->child)
5289	5310	sd_free_ctl_entry(&entry->child);
	5311	+ if (entry->proc_handler == NULL)
	5312	+ kfree(entry->procname);
	5313	+ }
5290	5314
5291	5315	kfree(*tablep);
5292	5316	*tablep = NULL;
...	...	@@ -1031,12 +1031,8 @@
1031	1031	swap(curr->vruntime, se->vruntime);
1032	1032	}
1033	1033
1034		- update_stats_enqueue(cfs_rq, se);
1035		- check_spread(cfs_rq, se);
1036		- check_spread(cfs_rq, curr);
1037		- __enqueue_entity(cfs_rq, se);
1038		- account_entity_enqueue(cfs_rq, se);
1039	1034	se->peer_preempt = 0;
	1035	+ enqueue_task_fair(rq, p, 0);
1040	1036	resched_task(rq->curr);
1041	1037	}
1042	1038
...	...	@@ -84,9 +84,6 @@
84	84
85	85	#ifdef CONFIG_FAIR_USER_SCHED
86	86
87		-static struct kobject uids_kobject; /* represents /sys/kernel/uids directory */
88		-static DEFINE_MUTEX(uids_mutex);
89		-
90	87	static void sched_destroy_user(struct user_struct *up)
91	88	{
92	89	sched_destroy_group(up->tg);
...	...	@@ -108,6 +105,19 @@
108	105	sched_move_task(p);
109	106	}
110	107
	108	+#else /* CONFIG_FAIR_USER_SCHED */
	109	+
	110	+static void sched_destroy_user(struct user_struct *up) { }
	111	+static int sched_create_user(struct user_struct *up) { return 0; }
	112	+static void sched_switch_user(struct task_struct *p) { }
	113	+
	114	+#endif /* CONFIG_FAIR_USER_SCHED */
	115	+
	116	+#if defined(CONFIG_FAIR_USER_SCHED) && defined(CONFIG_SYSFS)
	117	+
	118	+static struct kobject uids_kobject; /* represents /sys/kernel/uids directory */
	119	+static DEFINE_MUTEX(uids_mutex);
	120	+
111	121	static inline void uids_mutex_lock(void)
112	122	{
113	123	mutex_lock(&uids_mutex);
114	124
...	...	@@ -254,11 +264,8 @@
254	264	schedule_work(&up->work);
255	265	}
256	266
257		-#else /* CONFIG_FAIR_USER_SCHED */
	267	+#else /* CONFIG_FAIR_USER_SCHED && CONFIG_SYSFS */
258	268
259		-static void sched_destroy_user(struct user_struct *up) { }
260		-static int sched_create_user(struct user_struct *up) { return 0; }
261		-static void sched_switch_user(struct task_struct *p) { }
262	269	static inline int user_kobject_create(struct user_struct *up) { return 0; }
263	270	static inline void uids_mutex_lock(void) { }
264	271	static inline void uids_mutex_unlock(void) { }
...	...	@@ -277,7 +284,7 @@
277	284	kmem_cache_free(uid_cachep, up);
278	285	}
279	286
280		-#endif /* CONFIG_FAIR_USER_SCHED */
	287	+#endif
281	288
282	289	/*
283	290	* Locate the user_struct for the passed UID. If found, take a ref on it. The