Eric Lee / linux-smarc-t335x-v3.2

Commit a1bc5a4eee990a1f290735c8694d0aebdad095fa

Authored by David Rientjes 2009-04-03 07:57:54 +0800

Committed by Linus Torvalds 2009-04-03 10:04:57 +0800

Exists in master and in 4 other branches

cpusets: replace zone allowed functions with node allowed

The cpuset_zone_allowed() variants are actually only a function of the
zone's node.

Cc: Paul Menage <menage@google.com>
Acked-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Randy Dunlap <randy.dunlap@oracle.com>
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 2 changed files with 52 additions and 40 deletions Inline Diff

include/linux/cpuset.h
kernel/cpuset.c

include/linux/cpuset.h

Diff comments View file @ a1bc5a4

 #ifndef _LINUX_CPUSET_H
 #define _LINUX_CPUSET_H
 /*
  *  cpuset interface
  *
  *  Copyright (C) 2003 BULL SA
  *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
  *
  */
 #include <linux/sched.h>
 #include <linux/cpumask.h>
 #include <linux/nodemask.h>
 #include <linux/cgroup.h>
+#include <linux/mm.h>
 #ifdef CONFIG_CPUSETS
 extern int number_of_cpusets;	/* How many cpusets are defined in system? */
 extern int cpuset_init_early(void);
 extern int cpuset_init(void);
 extern void cpuset_init_smp(void);
 extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
 extern void cpuset_cpus_allowed_locked(struct task_struct *p,
 				       struct cpumask *mask);
 extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
 #define cpuset_current_mems_allowed (current->mems_allowed)
 void cpuset_init_current_mems_allowed(void);
 void cpuset_update_task_memory_state(void);
 int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask);
-extern int __cpuset_zone_allowed_softwall(struct zone *z, gfp_t gfp_mask);
+extern int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask);
-extern int __cpuset_zone_allowed_hardwall(struct zone *z, gfp_t gfp_mask);
+extern int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask);
-static int inline cpuset_zone_allowed_softwall(struct zone *z, gfp_t gfp_mask)
+static inline int cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
 {
 	return number_of_cpusets <= 1 ||
-		__cpuset_zone_allowed_softwall(z, gfp_mask);
+		__cpuset_node_allowed_softwall(node, gfp_mask);
 }
-static int inline cpuset_zone_allowed_hardwall(struct zone *z, gfp_t gfp_mask)
+static inline int cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
 {
 	return number_of_cpusets <= 1 ||
-		__cpuset_zone_allowed_hardwall(z, gfp_mask);
+		__cpuset_node_allowed_hardwall(node, gfp_mask);
 }
+static inline int cpuset_zone_allowed_softwall(struct zone *z, gfp_t gfp_mask)
+{
+	return cpuset_node_allowed_softwall(zone_to_nid(z), gfp_mask);
+}
+static inline int cpuset_zone_allowed_hardwall(struct zone *z, gfp_t gfp_mask)
+{
+	return cpuset_node_allowed_hardwall(zone_to_nid(z), gfp_mask);
+}
 extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
 					  const struct task_struct *tsk2);
 #define cpuset_memory_pressure_bump() 				\
 	do {							\
 		if (cpuset_memory_pressure_enabled)		\
 			__cpuset_memory_pressure_bump();	\
 	} while (0)
 extern int cpuset_memory_pressure_enabled;
 extern void __cpuset_memory_pressure_bump(void);
 extern const struct file_operations proc_cpuset_operations;
 struct seq_file;
 extern void cpuset_task_status_allowed(struct seq_file *m,
 					struct task_struct *task);
 extern void cpuset_lock(void);
 extern void cpuset_unlock(void);
 extern int cpuset_mem_spread_node(void);
 static inline int cpuset_do_page_mem_spread(void)
 {
 	return current->flags & PF_SPREAD_PAGE;
 }
 static inline int cpuset_do_slab_mem_spread(void)
 {
 	return current->flags & PF_SPREAD_SLAB;
 }
 extern int current_cpuset_is_being_rebound(void);
 extern void rebuild_sched_domains(void);
 extern void cpuset_print_task_mems_allowed(struct task_struct *p);
 #else /* !CONFIG_CPUSETS */
 static inline int cpuset_init_early(void) { return 0; }
 static inline int cpuset_init(void) { return 0; }
 static inline void cpuset_init_smp(void) {}
 static inline void cpuset_cpus_allowed(struct task_struct *p,
 				       struct cpumask *mask)
 {
 	cpumask_copy(mask, cpu_possible_mask);
 }
 static inline void cpuset_cpus_allowed_locked(struct task_struct *p,
 					      struct cpumask *mask)
 {
 	cpumask_copy(mask, cpu_possible_mask);
 }
 static inline nodemask_t cpuset_mems_allowed(struct task_struct *p)
 {
 	return node_possible_map;
 }
 #define cpuset_current_mems_allowed (node_states[N_HIGH_MEMORY])
 static inline void cpuset_init_current_mems_allowed(void) {}
 static inline void cpuset_update_task_memory_state(void) {}
 static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
+{
+	return 1;
+}
+static inline int cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
+{
+	return 1;
+}
+static inline int cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
 {
 	return 1;
 }
 static inline int cpuset_zone_allowed_softwall(struct zone *z, gfp_t gfp_mask)
 {
 	return 1;
 }
 static inline int cpuset_zone_allowed_hardwall(struct zone *z, gfp_t gfp_mask)
 {
 	return 1;
 }
 static inline int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
 						 const struct task_struct *tsk2)
 {
 	return 1;
 }
 static inline void cpuset_memory_pressure_bump(void) {}
 static inline void cpuset_task_status_allowed(struct seq_file *m,
 						struct task_struct *task)
 {
 }
 static inline void cpuset_lock(void) {}
 static inline void cpuset_unlock(void) {}
 static inline int cpuset_mem_spread_node(void)
 {
 	return 0;
 }
 static inline int cpuset_do_page_mem_spread(void)
 {
 	return 0;
 }
 static inline int cpuset_do_slab_mem_spread(void)
 {
 	return 0;
 }
 static inline int current_cpuset_is_being_rebound(void)
 {
 	return 0;
 }
 static inline void rebuild_sched_domains(void)
 {
 	partition_sched_domains(1, NULL, NULL);
 }
 static inline void cpuset_print_task_mems_allowed(struct task_struct *p)
 {
 }
 #endif /* !CONFIG_CPUSETS */
 #endif /* _LINUX_CPUSET_H */

kernel/cpuset.c

Diff comments View file @ a1bc5a4

1	/*	1	/*
2	* kernel/cpuset.c	2	* kernel/cpuset.c
3	*	3	*
4	* Processor and Memory placement constraints for sets of tasks.	4	* Processor and Memory placement constraints for sets of tasks.
5	*	5	*
6	* Copyright (C) 2003 BULL SA.	6	* Copyright (C) 2003 BULL SA.
7	* Copyright (C) 2004-2007 Silicon Graphics, Inc.	7	* Copyright (C) 2004-2007 Silicon Graphics, Inc.
8	* Copyright (C) 2006 Google, Inc	8	* Copyright (C) 2006 Google, Inc
9	*	9	*
10	* Portions derived from Patrick Mochel's sysfs code.	10	* Portions derived from Patrick Mochel's sysfs code.
11	* sysfs is Copyright (c) 2001-3 Patrick Mochel	11	* sysfs is Copyright (c) 2001-3 Patrick Mochel
12	*	12	*
13	* 2003-10-10 Written by Simon Derr.	13	* 2003-10-10 Written by Simon Derr.
14	* 2003-10-22 Updates by Stephen Hemminger.	14	* 2003-10-22 Updates by Stephen Hemminger.
15	* 2004 May-July Rework by Paul Jackson.	15	* 2004 May-July Rework by Paul Jackson.
16	* 2006 Rework by Paul Menage to use generic cgroups	16	* 2006 Rework by Paul Menage to use generic cgroups
17	* 2008 Rework of the scheduler domains and CPU hotplug handling	17	* 2008 Rework of the scheduler domains and CPU hotplug handling
18	* by Max Krasnyansky	18	* by Max Krasnyansky
19	*	19	*
20	* This file is subject to the terms and conditions of the GNU General Public	20	* This file is subject to the terms and conditions of the GNU General Public
21	* License. See the file COPYING in the main directory of the Linux	21	* License. See the file COPYING in the main directory of the Linux
22	* distribution for more details.	22	* distribution for more details.
23	*/	23	*/
24		24
25	#include <linux/cpu.h>	25	#include <linux/cpu.h>
26	#include <linux/cpumask.h>	26	#include <linux/cpumask.h>
27	#include <linux/cpuset.h>	27	#include <linux/cpuset.h>
28	#include <linux/err.h>	28	#include <linux/err.h>
29	#include <linux/errno.h>	29	#include <linux/errno.h>
30	#include <linux/file.h>	30	#include <linux/file.h>
31	#include <linux/fs.h>	31	#include <linux/fs.h>
32	#include <linux/init.h>	32	#include <linux/init.h>
33	#include <linux/interrupt.h>	33	#include <linux/interrupt.h>
34	#include <linux/kernel.h>	34	#include <linux/kernel.h>
35	#include <linux/kmod.h>	35	#include <linux/kmod.h>
36	#include <linux/list.h>	36	#include <linux/list.h>
37	#include <linux/mempolicy.h>	37	#include <linux/mempolicy.h>
38	#include <linux/mm.h>	38	#include <linux/mm.h>
39	#include <linux/memory.h>	39	#include <linux/memory.h>
40	#include <linux/module.h>	40	#include <linux/module.h>
41	#include <linux/mount.h>	41	#include <linux/mount.h>
42	#include <linux/namei.h>	42	#include <linux/namei.h>
43	#include <linux/pagemap.h>	43	#include <linux/pagemap.h>
44	#include <linux/proc_fs.h>	44	#include <linux/proc_fs.h>
45	#include <linux/rcupdate.h>	45	#include <linux/rcupdate.h>
46	#include <linux/sched.h>	46	#include <linux/sched.h>
47	#include <linux/seq_file.h>	47	#include <linux/seq_file.h>
48	#include <linux/security.h>	48	#include <linux/security.h>
49	#include <linux/slab.h>	49	#include <linux/slab.h>
50	#include <linux/spinlock.h>	50	#include <linux/spinlock.h>
51	#include <linux/stat.h>	51	#include <linux/stat.h>
52	#include <linux/string.h>	52	#include <linux/string.h>
53	#include <linux/time.h>	53	#include <linux/time.h>
54	#include <linux/backing-dev.h>	54	#include <linux/backing-dev.h>
55	#include <linux/sort.h>	55	#include <linux/sort.h>
56		56
57	#include <asm/uaccess.h>	57	#include <asm/uaccess.h>
58	#include <asm/atomic.h>	58	#include <asm/atomic.h>
59	#include <linux/mutex.h>	59	#include <linux/mutex.h>
60	#include <linux/workqueue.h>	60	#include <linux/workqueue.h>
61	#include <linux/cgroup.h>	61	#include <linux/cgroup.h>
62		62
63	/*	63	/*
64	* Workqueue for cpuset related tasks.	64	* Workqueue for cpuset related tasks.
65	*	65	*
66	* Using kevent workqueue may cause deadlock when memory_migrate	66	* Using kevent workqueue may cause deadlock when memory_migrate
67	* is set. So we create a separate workqueue thread for cpuset.	67	* is set. So we create a separate workqueue thread for cpuset.
68	*/	68	*/
69	static struct workqueue_struct *cpuset_wq;	69	static struct workqueue_struct *cpuset_wq;
70		70
71	/*	71	/*
72	* Tracks how many cpusets are currently defined in system.	72	* Tracks how many cpusets are currently defined in system.
73	* When there is only one cpuset (the root cpuset) we can	73	* When there is only one cpuset (the root cpuset) we can
74	* short circuit some hooks.	74	* short circuit some hooks.
75	*/	75	*/
76	int number_of_cpusets __read_mostly;	76	int number_of_cpusets __read_mostly;
77		77
78	/* Forward declare cgroup structures */	78	/* Forward declare cgroup structures */
79	struct cgroup_subsys cpuset_subsys;	79	struct cgroup_subsys cpuset_subsys;
80	struct cpuset;	80	struct cpuset;
81		81
82	/* See "Frequency meter" comments, below. */	82	/* See "Frequency meter" comments, below. */
83		83
84	struct fmeter {	84	struct fmeter {
85	int cnt; /* unprocessed events count */	85	int cnt; /* unprocessed events count */
86	int val; /* most recent output value */	86	int val; /* most recent output value */
87	time_t time; /* clock (secs) when val computed */	87	time_t time; /* clock (secs) when val computed */
88	spinlock_t lock; /* guards read or write of above */	88	spinlock_t lock; /* guards read or write of above */
89	};	89	};
90		90
91	struct cpuset {	91	struct cpuset {
92	struct cgroup_subsys_state css;	92	struct cgroup_subsys_state css;
93		93
94	unsigned long flags; /* "unsigned long" so bitops work */	94	unsigned long flags; /* "unsigned long" so bitops work */
95	cpumask_var_t cpus_allowed; /* CPUs allowed to tasks in cpuset */	95	cpumask_var_t cpus_allowed; /* CPUs allowed to tasks in cpuset */
96	nodemask_t mems_allowed; /* Memory Nodes allowed to tasks */	96	nodemask_t mems_allowed; /* Memory Nodes allowed to tasks */
97		97
98	struct cpuset parent; / my parent */	98	struct cpuset parent; / my parent */
99		99
100	/*	100	/*
101	* Copy of global cpuset_mems_generation as of the most	101	* Copy of global cpuset_mems_generation as of the most
102	* recent time this cpuset changed its mems_allowed.	102	* recent time this cpuset changed its mems_allowed.
103	*/	103	*/
104	int mems_generation;	104	int mems_generation;
105		105
106	struct fmeter fmeter; /* memory_pressure filter */	106	struct fmeter fmeter; /* memory_pressure filter */
107		107
108	/* partition number for rebuild_sched_domains() */	108	/* partition number for rebuild_sched_domains() */
109	int pn;	109	int pn;
110		110
111	/* for custom sched domain */	111	/* for custom sched domain */
112	int relax_domain_level;	112	int relax_domain_level;
113		113
114	/* used for walking a cpuset heirarchy */	114	/* used for walking a cpuset heirarchy */
115	struct list_head stack_list;	115	struct list_head stack_list;
116	};	116	};
117		117
118	/* Retrieve the cpuset for a cgroup */	118	/* Retrieve the cpuset for a cgroup */
119	static inline struct cpuset cgroup_cs(struct cgroup cont)	119	static inline struct cpuset cgroup_cs(struct cgroup cont)
120	{	120	{
121	return container_of(cgroup_subsys_state(cont, cpuset_subsys_id),	121	return container_of(cgroup_subsys_state(cont, cpuset_subsys_id),
122	struct cpuset, css);	122	struct cpuset, css);
123	}	123	}
124		124
125	/* Retrieve the cpuset for a task */	125	/* Retrieve the cpuset for a task */
126	static inline struct cpuset task_cs(struct task_struct task)	126	static inline struct cpuset task_cs(struct task_struct task)
127	{	127	{
128	return container_of(task_subsys_state(task, cpuset_subsys_id),	128	return container_of(task_subsys_state(task, cpuset_subsys_id),
129	struct cpuset, css);	129	struct cpuset, css);
130	}	130	}
131		131
132	/* bits in struct cpuset flags field */	132	/* bits in struct cpuset flags field */
133	typedef enum {	133	typedef enum {
134	CS_CPU_EXCLUSIVE,	134	CS_CPU_EXCLUSIVE,
135	CS_MEM_EXCLUSIVE,	135	CS_MEM_EXCLUSIVE,
136	CS_MEM_HARDWALL,	136	CS_MEM_HARDWALL,
137	CS_MEMORY_MIGRATE,	137	CS_MEMORY_MIGRATE,
138	CS_SCHED_LOAD_BALANCE,	138	CS_SCHED_LOAD_BALANCE,
139	CS_SPREAD_PAGE,	139	CS_SPREAD_PAGE,
140	CS_SPREAD_SLAB,	140	CS_SPREAD_SLAB,
141	} cpuset_flagbits_t;	141	} cpuset_flagbits_t;
142		142
143	/* convenient tests for these bits */	143	/* convenient tests for these bits */
144	static inline int is_cpu_exclusive(const struct cpuset *cs)	144	static inline int is_cpu_exclusive(const struct cpuset *cs)
145	{	145	{
146	return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);	146	return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
147	}	147	}
148		148
149	static inline int is_mem_exclusive(const struct cpuset *cs)	149	static inline int is_mem_exclusive(const struct cpuset *cs)
150	{	150	{
151	return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);	151	return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
152	}	152	}
153		153
154	static inline int is_mem_hardwall(const struct cpuset *cs)	154	static inline int is_mem_hardwall(const struct cpuset *cs)
155	{	155	{
156	return test_bit(CS_MEM_HARDWALL, &cs->flags);	156	return test_bit(CS_MEM_HARDWALL, &cs->flags);
157	}	157	}
158		158
159	static inline int is_sched_load_balance(const struct cpuset *cs)	159	static inline int is_sched_load_balance(const struct cpuset *cs)
160	{	160	{
161	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);	161	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
162	}	162	}
163		163
164	static inline int is_memory_migrate(const struct cpuset *cs)	164	static inline int is_memory_migrate(const struct cpuset *cs)
165	{	165	{
166	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);	166	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
167	}	167	}
168		168
169	static inline int is_spread_page(const struct cpuset *cs)	169	static inline int is_spread_page(const struct cpuset *cs)
170	{	170	{
171	return test_bit(CS_SPREAD_PAGE, &cs->flags);	171	return test_bit(CS_SPREAD_PAGE, &cs->flags);
172	}	172	}
173		173
174	static inline int is_spread_slab(const struct cpuset *cs)	174	static inline int is_spread_slab(const struct cpuset *cs)
175	{	175	{
176	return test_bit(CS_SPREAD_SLAB, &cs->flags);	176	return test_bit(CS_SPREAD_SLAB, &cs->flags);
177	}	177	}
178		178
179	/*	179	/*
180	* Increment this integer everytime any cpuset changes its	180	* Increment this integer everytime any cpuset changes its
181	* mems_allowed value. Users of cpusets can track this generation	181	* mems_allowed value. Users of cpusets can track this generation
182	* number, and avoid having to lock and reload mems_allowed unless	182	* number, and avoid having to lock and reload mems_allowed unless
183	* the cpuset they're using changes generation.	183	* the cpuset they're using changes generation.
184	*	184	*
185	* A single, global generation is needed because cpuset_attach_task() could	185	* A single, global generation is needed because cpuset_attach_task() could
186	* reattach a task to a different cpuset, which must not have its	186	* reattach a task to a different cpuset, which must not have its
187	* generation numbers aliased with those of that tasks previous cpuset.	187	* generation numbers aliased with those of that tasks previous cpuset.
188	*	188	*
189	* Generations are needed for mems_allowed because one task cannot	189	* Generations are needed for mems_allowed because one task cannot
190	* modify another's memory placement. So we must enable every task,	190	* modify another's memory placement. So we must enable every task,
191	* on every visit to __alloc_pages(), to efficiently check whether	191	* on every visit to __alloc_pages(), to efficiently check whether
192	* its current->cpuset->mems_allowed has changed, requiring an update	192	* its current->cpuset->mems_allowed has changed, requiring an update
193	* of its current->mems_allowed.	193	* of its current->mems_allowed.
194	*	194	*
195	* Since writes to cpuset_mems_generation are guarded by the cgroup lock	195	* Since writes to cpuset_mems_generation are guarded by the cgroup lock
196	* there is no need to mark it atomic.	196	* there is no need to mark it atomic.
197	*/	197	*/
198	static int cpuset_mems_generation;	198	static int cpuset_mems_generation;
199		199
200	static struct cpuset top_cpuset = {	200	static struct cpuset top_cpuset = {
201	.flags = ((1 << CS_CPU_EXCLUSIVE) \| (1 << CS_MEM_EXCLUSIVE)),	201	.flags = ((1 << CS_CPU_EXCLUSIVE) \| (1 << CS_MEM_EXCLUSIVE)),
202	};	202	};
203		203
204	/*	204	/*
205	* There are two global mutexes guarding cpuset structures. The first	205	* There are two global mutexes guarding cpuset structures. The first
206	* is the main control groups cgroup_mutex, accessed via	206	* is the main control groups cgroup_mutex, accessed via
207	* cgroup_lock()/cgroup_unlock(). The second is the cpuset-specific	207	* cgroup_lock()/cgroup_unlock(). The second is the cpuset-specific
208	* callback_mutex, below. They can nest. It is ok to first take	208	* callback_mutex, below. They can nest. It is ok to first take
209	* cgroup_mutex, then nest callback_mutex. We also require taking	209	* cgroup_mutex, then nest callback_mutex. We also require taking
210	* task_lock() when dereferencing a task's cpuset pointer. See "The	210	* task_lock() when dereferencing a task's cpuset pointer. See "The
211	* task_lock() exception", at the end of this comment.	211	* task_lock() exception", at the end of this comment.
212	*	212	*
213	* A task must hold both mutexes to modify cpusets. If a task	213	* A task must hold both mutexes to modify cpusets. If a task
214	* holds cgroup_mutex, then it blocks others wanting that mutex,	214	* holds cgroup_mutex, then it blocks others wanting that mutex,
215	* ensuring that it is the only task able to also acquire callback_mutex	215	* ensuring that it is the only task able to also acquire callback_mutex
216	* and be able to modify cpusets. It can perform various checks on	216	* and be able to modify cpusets. It can perform various checks on
217	* the cpuset structure first, knowing nothing will change. It can	217	* the cpuset structure first, knowing nothing will change. It can
218	* also allocate memory while just holding cgroup_mutex. While it is	218	* also allocate memory while just holding cgroup_mutex. While it is
219	* performing these checks, various callback routines can briefly	219	* performing these checks, various callback routines can briefly
220	* acquire callback_mutex to query cpusets. Once it is ready to make	220	* acquire callback_mutex to query cpusets. Once it is ready to make
221	* the changes, it takes callback_mutex, blocking everyone else.	221	* the changes, it takes callback_mutex, blocking everyone else.
222	*	222	*
223	* Calls to the kernel memory allocator can not be made while holding	223	* Calls to the kernel memory allocator can not be made while holding
224	* callback_mutex, as that would risk double tripping on callback_mutex	224	* callback_mutex, as that would risk double tripping on callback_mutex
225	* from one of the callbacks into the cpuset code from within	225	* from one of the callbacks into the cpuset code from within
226	* __alloc_pages().	226	* __alloc_pages().
227	*	227	*
228	* If a task is only holding callback_mutex, then it has read-only	228	* If a task is only holding callback_mutex, then it has read-only
229	* access to cpusets.	229	* access to cpusets.
230	*	230	*
231	* The task_struct fields mems_allowed and mems_generation may only	231	* The task_struct fields mems_allowed and mems_generation may only
232	* be accessed in the context of that task, so require no locks.	232	* be accessed in the context of that task, so require no locks.
233	*	233	*
234	* The cpuset_common_file_read() handlers only hold callback_mutex across	234	* The cpuset_common_file_read() handlers only hold callback_mutex across
235	* small pieces of code, such as when reading out possibly multi-word	235	* small pieces of code, such as when reading out possibly multi-word
236	* cpumasks and nodemasks.	236	* cpumasks and nodemasks.
237	*	237	*
238	* Accessing a task's cpuset should be done in accordance with the	238	* Accessing a task's cpuset should be done in accordance with the
239	* guidelines for accessing subsystem state in kernel/cgroup.c	239	* guidelines for accessing subsystem state in kernel/cgroup.c
240	*/	240	*/
241		241
242	static DEFINE_MUTEX(callback_mutex);	242	static DEFINE_MUTEX(callback_mutex);
243		243
244	/*	244	/*
245	* cpuset_buffer_lock protects both the cpuset_name and cpuset_nodelist	245	* cpuset_buffer_lock protects both the cpuset_name and cpuset_nodelist
246	* buffers. They are statically allocated to prevent using excess stack	246	* buffers. They are statically allocated to prevent using excess stack
247	* when calling cpuset_print_task_mems_allowed().	247	* when calling cpuset_print_task_mems_allowed().
248	*/	248	*/
249	#define CPUSET_NAME_LEN (128)	249	#define CPUSET_NAME_LEN (128)
250	#define CPUSET_NODELIST_LEN (256)	250	#define CPUSET_NODELIST_LEN (256)
251	static char cpuset_name[CPUSET_NAME_LEN];	251	static char cpuset_name[CPUSET_NAME_LEN];
252	static char cpuset_nodelist[CPUSET_NODELIST_LEN];	252	static char cpuset_nodelist[CPUSET_NODELIST_LEN];
253	static DEFINE_SPINLOCK(cpuset_buffer_lock);	253	static DEFINE_SPINLOCK(cpuset_buffer_lock);
254		254
255	/*	255	/*
256	* This is ugly, but preserves the userspace API for existing cpuset	256	* This is ugly, but preserves the userspace API for existing cpuset
257	* users. If someone tries to mount the "cpuset" filesystem, we	257	* users. If someone tries to mount the "cpuset" filesystem, we
258	* silently switch it to mount "cgroup" instead	258	* silently switch it to mount "cgroup" instead
259	*/	259	*/
260	static int cpuset_get_sb(struct file_system_type *fs_type,	260	static int cpuset_get_sb(struct file_system_type *fs_type,
261	int flags, const char *unused_dev_name,	261	int flags, const char *unused_dev_name,
262	void data, struct vfsmount mnt)	262	void data, struct vfsmount mnt)
263	{	263	{
264	struct file_system_type *cgroup_fs = get_fs_type("cgroup");	264	struct file_system_type *cgroup_fs = get_fs_type("cgroup");
265	int ret = -ENODEV;	265	int ret = -ENODEV;
266	if (cgroup_fs) {	266	if (cgroup_fs) {
267	char mountopts[] =	267	char mountopts[] =
268	"cpuset,noprefix,"	268	"cpuset,noprefix,"
269	"release_agent=/sbin/cpuset_release_agent";	269	"release_agent=/sbin/cpuset_release_agent";
270	ret = cgroup_fs->get_sb(cgroup_fs, flags,	270	ret = cgroup_fs->get_sb(cgroup_fs, flags,
271	unused_dev_name, mountopts, mnt);	271	unused_dev_name, mountopts, mnt);
272	put_filesystem(cgroup_fs);	272	put_filesystem(cgroup_fs);
273	}	273	}
274	return ret;	274	return ret;
275	}	275	}
276		276
277	static struct file_system_type cpuset_fs_type = {	277	static struct file_system_type cpuset_fs_type = {
278	.name = "cpuset",	278	.name = "cpuset",
279	.get_sb = cpuset_get_sb,	279	.get_sb = cpuset_get_sb,
280	};	280	};
281		281
282	/*	282	/*
283	* Return in pmask the portion of a cpusets's cpus_allowed that	283	* Return in pmask the portion of a cpusets's cpus_allowed that
284	* are online. If none are online, walk up the cpuset hierarchy	284	* are online. If none are online, walk up the cpuset hierarchy
285	* until we find one that does have some online cpus. If we get	285	* until we find one that does have some online cpus. If we get
286	* all the way to the top and still haven't found any online cpus,	286	* all the way to the top and still haven't found any online cpus,
287	* return cpu_online_map. Or if passed a NULL cs from an exit'ing	287	* return cpu_online_map. Or if passed a NULL cs from an exit'ing
288	* task, return cpu_online_map.	288	* task, return cpu_online_map.
289	*	289	*
290	* One way or another, we guarantee to return some non-empty subset	290	* One way or another, we guarantee to return some non-empty subset
291	* of cpu_online_map.	291	* of cpu_online_map.
292	*	292	*
293	* Call with callback_mutex held.	293	* Call with callback_mutex held.
294	*/	294	*/
295		295
296	static void guarantee_online_cpus(const struct cpuset *cs,	296	static void guarantee_online_cpus(const struct cpuset *cs,
297	struct cpumask *pmask)	297	struct cpumask *pmask)
298	{	298	{
299	while (cs && !cpumask_intersects(cs->cpus_allowed, cpu_online_mask))	299	while (cs && !cpumask_intersects(cs->cpus_allowed, cpu_online_mask))
300	cs = cs->parent;	300	cs = cs->parent;
301	if (cs)	301	if (cs)
302	cpumask_and(pmask, cs->cpus_allowed, cpu_online_mask);	302	cpumask_and(pmask, cs->cpus_allowed, cpu_online_mask);
303	else	303	else
304	cpumask_copy(pmask, cpu_online_mask);	304	cpumask_copy(pmask, cpu_online_mask);
305	BUG_ON(!cpumask_intersects(pmask, cpu_online_mask));	305	BUG_ON(!cpumask_intersects(pmask, cpu_online_mask));
306	}	306	}
307		307
308	/*	308	/*
309	* Return in *pmask the portion of a cpusets's mems_allowed that	309	* Return in *pmask the portion of a cpusets's mems_allowed that
310	* are online, with memory. If none are online with memory, walk	310	* are online, with memory. If none are online with memory, walk
311	* up the cpuset hierarchy until we find one that does have some	311	* up the cpuset hierarchy until we find one that does have some
312	* online mems. If we get all the way to the top and still haven't	312	* online mems. If we get all the way to the top and still haven't
313	* found any online mems, return node_states[N_HIGH_MEMORY].	313	* found any online mems, return node_states[N_HIGH_MEMORY].
314	*	314	*
315	* One way or another, we guarantee to return some non-empty subset	315	* One way or another, we guarantee to return some non-empty subset
316	* of node_states[N_HIGH_MEMORY].	316	* of node_states[N_HIGH_MEMORY].
317	*	317	*
318	* Call with callback_mutex held.	318	* Call with callback_mutex held.
319	*/	319	*/
320		320
321	static void guarantee_online_mems(const struct cpuset cs, nodemask_t pmask)	321	static void guarantee_online_mems(const struct cpuset cs, nodemask_t pmask)
322	{	322	{
323	while (cs && !nodes_intersects(cs->mems_allowed,	323	while (cs && !nodes_intersects(cs->mems_allowed,
324	node_states[N_HIGH_MEMORY]))	324	node_states[N_HIGH_MEMORY]))
325	cs = cs->parent;	325	cs = cs->parent;
326	if (cs)	326	if (cs)
327	nodes_and(*pmask, cs->mems_allowed,	327	nodes_and(*pmask, cs->mems_allowed,
328	node_states[N_HIGH_MEMORY]);	328	node_states[N_HIGH_MEMORY]);
329	else	329	else
330	*pmask = node_states[N_HIGH_MEMORY];	330	*pmask = node_states[N_HIGH_MEMORY];
331	BUG_ON(!nodes_intersects(*pmask, node_states[N_HIGH_MEMORY]));	331	BUG_ON(!nodes_intersects(*pmask, node_states[N_HIGH_MEMORY]));
332	}	332	}
333		333
334	/**	334	/**
335	* cpuset_update_task_memory_state - update task memory placement	335	* cpuset_update_task_memory_state - update task memory placement
336	*	336	*
337	* If the current tasks cpusets mems_allowed changed behind our	337	* If the current tasks cpusets mems_allowed changed behind our
338	* backs, update current->mems_allowed, mems_generation and task NUMA	338	* backs, update current->mems_allowed, mems_generation and task NUMA
339	* mempolicy to the new value.	339	* mempolicy to the new value.
340	*	340	*
341	* Task mempolicy is updated by rebinding it relative to the	341	* Task mempolicy is updated by rebinding it relative to the
342	* current->cpuset if a task has its memory placement changed.	342	* current->cpuset if a task has its memory placement changed.
343	* Do not call this routine if in_interrupt().	343	* Do not call this routine if in_interrupt().
344	*	344	*
345	* Call without callback_mutex or task_lock() held. May be	345	* Call without callback_mutex or task_lock() held. May be
346	* called with or without cgroup_mutex held. Thanks in part to	346	* called with or without cgroup_mutex held. Thanks in part to
347	* 'the_top_cpuset_hack', the task's cpuset pointer will never	347	* 'the_top_cpuset_hack', the task's cpuset pointer will never
348	* be NULL. This routine also might acquire callback_mutex during	348	* be NULL. This routine also might acquire callback_mutex during
349	* call.	349	* call.
350	*	350	*
351	* Reading current->cpuset->mems_generation doesn't need task_lock	351	* Reading current->cpuset->mems_generation doesn't need task_lock
352	* to guard the current->cpuset derefence, because it is guarded	352	* to guard the current->cpuset derefence, because it is guarded
353	* from concurrent freeing of current->cpuset using RCU.	353	* from concurrent freeing of current->cpuset using RCU.
354	*	354	*
355	* The rcu_dereference() is technically probably not needed,	355	* The rcu_dereference() is technically probably not needed,
356	* as I don't actually mind if I see a new cpuset pointer but	356	* as I don't actually mind if I see a new cpuset pointer but
357	* an old value of mems_generation. However this really only	357	* an old value of mems_generation. However this really only
358	* matters on alpha systems using cpusets heavily. If I dropped	358	* matters on alpha systems using cpusets heavily. If I dropped
359	* that rcu_dereference(), it would save them a memory barrier.	359	* that rcu_dereference(), it would save them a memory barrier.
360	* For all other arch's, rcu_dereference is a no-op anyway, and for	360	* For all other arch's, rcu_dereference is a no-op anyway, and for
361	* alpha systems not using cpusets, another planned optimization,	361	* alpha systems not using cpusets, another planned optimization,
362	* avoiding the rcu critical section for tasks in the root cpuset	362	* avoiding the rcu critical section for tasks in the root cpuset
363	* which is statically allocated, so can't vanish, will make this	363	* which is statically allocated, so can't vanish, will make this
364	* irrelevant. Better to use RCU as intended, than to engage in	364	* irrelevant. Better to use RCU as intended, than to engage in
365	* some cute trick to save a memory barrier that is impossible to	365	* some cute trick to save a memory barrier that is impossible to
366	* test, for alpha systems using cpusets heavily, which might not	366	* test, for alpha systems using cpusets heavily, which might not
367	* even exist.	367	* even exist.
368	*	368	*
369	* This routine is needed to update the per-task mems_allowed data,	369	* This routine is needed to update the per-task mems_allowed data,
370	* within the tasks context, when it is trying to allocate memory	370	* within the tasks context, when it is trying to allocate memory
371	* (in various mm/mempolicy.c routines) and notices that some other	371	* (in various mm/mempolicy.c routines) and notices that some other
372	* task has been modifying its cpuset.	372	* task has been modifying its cpuset.
373	*/	373	*/
374		374
375	void cpuset_update_task_memory_state(void)	375	void cpuset_update_task_memory_state(void)
376	{	376	{
377	int my_cpusets_mem_gen;	377	int my_cpusets_mem_gen;
378	struct task_struct *tsk = current;	378	struct task_struct *tsk = current;
379	struct cpuset *cs;	379	struct cpuset *cs;
380		380
381	rcu_read_lock();	381	rcu_read_lock();
382	my_cpusets_mem_gen = task_cs(tsk)->mems_generation;	382	my_cpusets_mem_gen = task_cs(tsk)->mems_generation;
383	rcu_read_unlock();	383	rcu_read_unlock();
384		384
385	if (my_cpusets_mem_gen != tsk->cpuset_mems_generation) {	385	if (my_cpusets_mem_gen != tsk->cpuset_mems_generation) {
386	mutex_lock(&callback_mutex);	386	mutex_lock(&callback_mutex);
387	task_lock(tsk);	387	task_lock(tsk);
388	cs = task_cs(tsk); /* Maybe changed when task not locked */	388	cs = task_cs(tsk); /* Maybe changed when task not locked */
389	guarantee_online_mems(cs, &tsk->mems_allowed);	389	guarantee_online_mems(cs, &tsk->mems_allowed);
390	tsk->cpuset_mems_generation = cs->mems_generation;	390	tsk->cpuset_mems_generation = cs->mems_generation;
391	if (is_spread_page(cs))	391	if (is_spread_page(cs))
392	tsk->flags \|= PF_SPREAD_PAGE;	392	tsk->flags \|= PF_SPREAD_PAGE;
393	else	393	else
394	tsk->flags &= ~PF_SPREAD_PAGE;	394	tsk->flags &= ~PF_SPREAD_PAGE;
395	if (is_spread_slab(cs))	395	if (is_spread_slab(cs))
396	tsk->flags \|= PF_SPREAD_SLAB;	396	tsk->flags \|= PF_SPREAD_SLAB;
397	else	397	else
398	tsk->flags &= ~PF_SPREAD_SLAB;	398	tsk->flags &= ~PF_SPREAD_SLAB;
399	task_unlock(tsk);	399	task_unlock(tsk);
400	mutex_unlock(&callback_mutex);	400	mutex_unlock(&callback_mutex);
401	mpol_rebind_task(tsk, &tsk->mems_allowed);	401	mpol_rebind_task(tsk, &tsk->mems_allowed);
402	}	402	}
403	}	403	}
404		404
405	/*	405	/*
406	* is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?	406	* is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
407	*	407	*
408	* One cpuset is a subset of another if all its allowed CPUs and	408	* One cpuset is a subset of another if all its allowed CPUs and
409	* Memory Nodes are a subset of the other, and its exclusive flags	409	* Memory Nodes are a subset of the other, and its exclusive flags
410	* are only set if the other's are set. Call holding cgroup_mutex.	410	* are only set if the other's are set. Call holding cgroup_mutex.
411	*/	411	*/
412		412
413	static int is_cpuset_subset(const struct cpuset p, const struct cpuset q)	413	static int is_cpuset_subset(const struct cpuset p, const struct cpuset q)
414	{	414	{
415	return cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&	415	return cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&
416	nodes_subset(p->mems_allowed, q->mems_allowed) &&	416	nodes_subset(p->mems_allowed, q->mems_allowed) &&
417	is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&	417	is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
418	is_mem_exclusive(p) <= is_mem_exclusive(q);	418	is_mem_exclusive(p) <= is_mem_exclusive(q);
419	}	419	}
420		420
421	/**	421	/**
422	* alloc_trial_cpuset - allocate a trial cpuset	422	* alloc_trial_cpuset - allocate a trial cpuset
423	* @cs: the cpuset that the trial cpuset duplicates	423	* @cs: the cpuset that the trial cpuset duplicates
424	*/	424	*/
425	static struct cpuset alloc_trial_cpuset(const struct cpuset cs)	425	static struct cpuset alloc_trial_cpuset(const struct cpuset cs)
426	{	426	{
427	struct cpuset *trial;	427	struct cpuset *trial;
428		428
429	trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);	429	trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
430	if (!trial)	430	if (!trial)
431	return NULL;	431	return NULL;
432		432
433	if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) {	433	if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) {
434	kfree(trial);	434	kfree(trial);
435	return NULL;	435	return NULL;
436	}	436	}
437	cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);	437	cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
438		438
439	return trial;	439	return trial;
440	}	440	}
441		441
442	/**	442	/**
443	* free_trial_cpuset - free the trial cpuset	443	* free_trial_cpuset - free the trial cpuset
444	* @trial: the trial cpuset to be freed	444	* @trial: the trial cpuset to be freed
445	*/	445	*/
446	static void free_trial_cpuset(struct cpuset *trial)	446	static void free_trial_cpuset(struct cpuset *trial)
447	{	447	{
448	free_cpumask_var(trial->cpus_allowed);	448	free_cpumask_var(trial->cpus_allowed);
449	kfree(trial);	449	kfree(trial);
450	}	450	}
451		451
452	/*	452	/*
453	* validate_change() - Used to validate that any proposed cpuset change	453	* validate_change() - Used to validate that any proposed cpuset change
454	* follows the structural rules for cpusets.	454	* follows the structural rules for cpusets.
455	*	455	*
456	* If we replaced the flag and mask values of the current cpuset	456	* If we replaced the flag and mask values of the current cpuset
457	* (cur) with those values in the trial cpuset (trial), would	457	* (cur) with those values in the trial cpuset (trial), would
458	* our various subset and exclusive rules still be valid? Presumes	458	* our various subset and exclusive rules still be valid? Presumes
459	* cgroup_mutex held.	459	* cgroup_mutex held.
460	*	460	*
461	* 'cur' is the address of an actual, in-use cpuset. Operations	461	* 'cur' is the address of an actual, in-use cpuset. Operations
462	* such as list traversal that depend on the actual address of the	462	* such as list traversal that depend on the actual address of the
463	* cpuset in the list must use cur below, not trial.	463	* cpuset in the list must use cur below, not trial.
464	*	464	*
465	* 'trial' is the address of bulk structure copy of cur, with	465	* 'trial' is the address of bulk structure copy of cur, with
466	* perhaps one or more of the fields cpus_allowed, mems_allowed,	466	* perhaps one or more of the fields cpus_allowed, mems_allowed,
467	* or flags changed to new, trial values.	467	* or flags changed to new, trial values.
468	*	468	*
469	* Return 0 if valid, -errno if not.	469	* Return 0 if valid, -errno if not.
470	*/	470	*/
471		471
472	static int validate_change(const struct cpuset cur, const struct cpuset trial)	472	static int validate_change(const struct cpuset cur, const struct cpuset trial)
473	{	473	{
474	struct cgroup *cont;	474	struct cgroup *cont;
475	struct cpuset c, par;	475	struct cpuset c, par;
476		476
477	/* Each of our child cpusets must be a subset of us */	477	/* Each of our child cpusets must be a subset of us */
478	list_for_each_entry(cont, &cur->css.cgroup->children, sibling) {	478	list_for_each_entry(cont, &cur->css.cgroup->children, sibling) {
479	if (!is_cpuset_subset(cgroup_cs(cont), trial))	479	if (!is_cpuset_subset(cgroup_cs(cont), trial))
480	return -EBUSY;	480	return -EBUSY;
481	}	481	}
482		482
483	/* Remaining checks don't apply to root cpuset */	483	/* Remaining checks don't apply to root cpuset */
484	if (cur == &top_cpuset)	484	if (cur == &top_cpuset)
485	return 0;	485	return 0;
486		486
487	par = cur->parent;	487	par = cur->parent;
488		488
489	/* We must be a subset of our parent cpuset */	489	/* We must be a subset of our parent cpuset */
490	if (!is_cpuset_subset(trial, par))	490	if (!is_cpuset_subset(trial, par))
491	return -EACCES;	491	return -EACCES;
492		492
493	/*	493	/*
494	* If either I or some sibling (!= me) is exclusive, we can't	494	* If either I or some sibling (!= me) is exclusive, we can't
495	* overlap	495	* overlap
496	*/	496	*/
497	list_for_each_entry(cont, &par->css.cgroup->children, sibling) {	497	list_for_each_entry(cont, &par->css.cgroup->children, sibling) {
498	c = cgroup_cs(cont);	498	c = cgroup_cs(cont);
499	if ((is_cpu_exclusive(trial) \|\| is_cpu_exclusive(c)) &&	499	if ((is_cpu_exclusive(trial) \|\| is_cpu_exclusive(c)) &&
500	c != cur &&	500	c != cur &&
501	cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))	501	cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
502	return -EINVAL;	502	return -EINVAL;
503	if ((is_mem_exclusive(trial) \|\| is_mem_exclusive(c)) &&	503	if ((is_mem_exclusive(trial) \|\| is_mem_exclusive(c)) &&
504	c != cur &&	504	c != cur &&
505	nodes_intersects(trial->mems_allowed, c->mems_allowed))	505	nodes_intersects(trial->mems_allowed, c->mems_allowed))
506	return -EINVAL;	506	return -EINVAL;
507	}	507	}
508		508
509	/* Cpusets with tasks can't have empty cpus_allowed or mems_allowed */	509	/* Cpusets with tasks can't have empty cpus_allowed or mems_allowed */
510	if (cgroup_task_count(cur->css.cgroup)) {	510	if (cgroup_task_count(cur->css.cgroup)) {
511	if (cpumask_empty(trial->cpus_allowed) \|\|	511	if (cpumask_empty(trial->cpus_allowed) \|\|
512	nodes_empty(trial->mems_allowed)) {	512	nodes_empty(trial->mems_allowed)) {
513	return -ENOSPC;	513	return -ENOSPC;
514	}	514	}
515	}	515	}
516		516
517	return 0;	517	return 0;
518	}	518	}
519		519
520	/*	520	/*
521	* Helper routine for generate_sched_domains().	521	* Helper routine for generate_sched_domains().
522	* Do cpusets a, b have overlapping cpus_allowed masks?	522	* Do cpusets a, b have overlapping cpus_allowed masks?
523	*/	523	*/
524	static int cpusets_overlap(struct cpuset a, struct cpuset b)	524	static int cpusets_overlap(struct cpuset a, struct cpuset b)
525	{	525	{
526	return cpumask_intersects(a->cpus_allowed, b->cpus_allowed);	526	return cpumask_intersects(a->cpus_allowed, b->cpus_allowed);
527	}	527	}
528		528
529	static void	529	static void
530	update_domain_attr(struct sched_domain_attr dattr, struct cpuset c)	530	update_domain_attr(struct sched_domain_attr dattr, struct cpuset c)
531	{	531	{
532	if (dattr->relax_domain_level < c->relax_domain_level)	532	if (dattr->relax_domain_level < c->relax_domain_level)
533	dattr->relax_domain_level = c->relax_domain_level;	533	dattr->relax_domain_level = c->relax_domain_level;
534	return;	534	return;
535	}	535	}
536		536
537	static void	537	static void
538	update_domain_attr_tree(struct sched_domain_attr dattr, struct cpuset c)	538	update_domain_attr_tree(struct sched_domain_attr dattr, struct cpuset c)
539	{	539	{
540	LIST_HEAD(q);	540	LIST_HEAD(q);
541		541
542	list_add(&c->stack_list, &q);	542	list_add(&c->stack_list, &q);
543	while (!list_empty(&q)) {	543	while (!list_empty(&q)) {
544	struct cpuset *cp;	544	struct cpuset *cp;
545	struct cgroup *cont;	545	struct cgroup *cont;
546	struct cpuset *child;	546	struct cpuset *child;
547		547
548	cp = list_first_entry(&q, struct cpuset, stack_list);	548	cp = list_first_entry(&q, struct cpuset, stack_list);
549	list_del(q.next);	549	list_del(q.next);
550		550
551	if (cpumask_empty(cp->cpus_allowed))	551	if (cpumask_empty(cp->cpus_allowed))
552	continue;	552	continue;
553		553
554	if (is_sched_load_balance(cp))	554	if (is_sched_load_balance(cp))
555	update_domain_attr(dattr, cp);	555	update_domain_attr(dattr, cp);
556		556
557	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {	557	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {
558	child = cgroup_cs(cont);	558	child = cgroup_cs(cont);
559	list_add_tail(&child->stack_list, &q);	559	list_add_tail(&child->stack_list, &q);
560	}	560	}
561	}	561	}
562	}	562	}
563		563
564	/*	564	/*
565	* generate_sched_domains()	565	* generate_sched_domains()
566	*	566	*
567	* This function builds a partial partition of the systems CPUs	567	* This function builds a partial partition of the systems CPUs
568	* A 'partial partition' is a set of non-overlapping subsets whose	568	* A 'partial partition' is a set of non-overlapping subsets whose
569	* union is a subset of that set.	569	* union is a subset of that set.
570	* The output of this function needs to be passed to kernel/sched.c	570	* The output of this function needs to be passed to kernel/sched.c
571	* partition_sched_domains() routine, which will rebuild the scheduler's	571	* partition_sched_domains() routine, which will rebuild the scheduler's
572	* load balancing domains (sched domains) as specified by that partial	572	* load balancing domains (sched domains) as specified by that partial
573	* partition.	573	* partition.
574	*	574	*
575	* See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt	575	* See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt
576	* for a background explanation of this.	576	* for a background explanation of this.
577	*	577	*
578	* Does not return errors, on the theory that the callers of this	578	* Does not return errors, on the theory that the callers of this
579	* routine would rather not worry about failures to rebuild sched	579	* routine would rather not worry about failures to rebuild sched
580	* domains when operating in the severe memory shortage situations	580	* domains when operating in the severe memory shortage situations
581	* that could cause allocation failures below.	581	* that could cause allocation failures below.
582	*	582	*
583	* Must be called with cgroup_lock held.	583	* Must be called with cgroup_lock held.
584	*	584	*
585	* The three key local variables below are:	585	* The three key local variables below are:
586	* q - a linked-list queue of cpuset pointers, used to implement a	586	* q - a linked-list queue of cpuset pointers, used to implement a
587	* top-down scan of all cpusets. This scan loads a pointer	587	* top-down scan of all cpusets. This scan loads a pointer
588	* to each cpuset marked is_sched_load_balance into the	588	* to each cpuset marked is_sched_load_balance into the
589	* array 'csa'. For our purposes, rebuilding the schedulers	589	* array 'csa'. For our purposes, rebuilding the schedulers
590	* sched domains, we can ignore !is_sched_load_balance cpusets.	590	* sched domains, we can ignore !is_sched_load_balance cpusets.
591	* csa - (for CpuSet Array) Array of pointers to all the cpusets	591	* csa - (for CpuSet Array) Array of pointers to all the cpusets
592	* that need to be load balanced, for convenient iterative	592	* that need to be load balanced, for convenient iterative
593	* access by the subsequent code that finds the best partition,	593	* access by the subsequent code that finds the best partition,
594	* i.e the set of domains (subsets) of CPUs such that the	594	* i.e the set of domains (subsets) of CPUs such that the
595	* cpus_allowed of every cpuset marked is_sched_load_balance	595	* cpus_allowed of every cpuset marked is_sched_load_balance
596	* is a subset of one of these domains, while there are as	596	* is a subset of one of these domains, while there are as
597	* many such domains as possible, each as small as possible.	597	* many such domains as possible, each as small as possible.
598	* doms - Conversion of 'csa' to an array of cpumasks, for passing to	598	* doms - Conversion of 'csa' to an array of cpumasks, for passing to
599	* the kernel/sched.c routine partition_sched_domains() in a	599	* the kernel/sched.c routine partition_sched_domains() in a
600	* convenient format, that can be easily compared to the prior	600	* convenient format, that can be easily compared to the prior
601	* value to determine what partition elements (sched domains)	601	* value to determine what partition elements (sched domains)
602	* were changed (added or removed.)	602	* were changed (added or removed.)
603	*	603	*
604	* Finding the best partition (set of domains):	604	* Finding the best partition (set of domains):
605	* The triple nested loops below over i, j, k scan over the	605	* The triple nested loops below over i, j, k scan over the
606	* load balanced cpusets (using the array of cpuset pointers in	606	* load balanced cpusets (using the array of cpuset pointers in
607	* csa[]) looking for pairs of cpusets that have overlapping	607	* csa[]) looking for pairs of cpusets that have overlapping
608	* cpus_allowed, but which don't have the same 'pn' partition	608	* cpus_allowed, but which don't have the same 'pn' partition
609	* number and gives them in the same partition number. It keeps	609	* number and gives them in the same partition number. It keeps
610	* looping on the 'restart' label until it can no longer find	610	* looping on the 'restart' label until it can no longer find
611	* any such pairs.	611	* any such pairs.
612	*	612	*
613	* The union of the cpus_allowed masks from the set of	613	* The union of the cpus_allowed masks from the set of
614	* all cpusets having the same 'pn' value then form the one	614	* all cpusets having the same 'pn' value then form the one
615	* element of the partition (one sched domain) to be passed to	615	* element of the partition (one sched domain) to be passed to
616	* partition_sched_domains().	616	* partition_sched_domains().
617	*/	617	*/
618	/* FIXME: see the FIXME in partition_sched_domains() */	618	/* FIXME: see the FIXME in partition_sched_domains() */
619	static int generate_sched_domains(struct cpumask **domains,	619	static int generate_sched_domains(struct cpumask **domains,
620	struct sched_domain_attr **attributes)	620	struct sched_domain_attr **attributes)
621	{	621	{
622	LIST_HEAD(q); /* queue of cpusets to be scanned */	622	LIST_HEAD(q); /* queue of cpusets to be scanned */
623	struct cpuset cp; / scans q */	623	struct cpuset cp; / scans q */
624	struct cpuset *csa; / array of all cpuset ptrs */	624	struct cpuset *csa; / array of all cpuset ptrs */
625	int csn; /* how many cpuset ptrs in csa so far */	625	int csn; /* how many cpuset ptrs in csa so far */
626	int i, j, k; /* indices for partition finding loops */	626	int i, j, k; /* indices for partition finding loops */
627	struct cpumask doms; / resulting partition; i.e. sched domains */	627	struct cpumask doms; / resulting partition; i.e. sched domains */
628	struct sched_domain_attr dattr; / attributes for custom domains */	628	struct sched_domain_attr dattr; / attributes for custom domains */
629	int ndoms = 0; /* number of sched domains in result */	629	int ndoms = 0; /* number of sched domains in result */
630	int nslot; /* next empty doms[] struct cpumask slot */	630	int nslot; /* next empty doms[] struct cpumask slot */
631		631
632	doms = NULL;	632	doms = NULL;
633	dattr = NULL;	633	dattr = NULL;
634	csa = NULL;	634	csa = NULL;
635		635
636	/* Special case for the 99% of systems with one, full, sched domain */	636	/* Special case for the 99% of systems with one, full, sched domain */
637	if (is_sched_load_balance(&top_cpuset)) {	637	if (is_sched_load_balance(&top_cpuset)) {
638	doms = kmalloc(cpumask_size(), GFP_KERNEL);	638	doms = kmalloc(cpumask_size(), GFP_KERNEL);
639	if (!doms)	639	if (!doms)
640	goto done;	640	goto done;
641		641
642	dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);	642	dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
643	if (dattr) {	643	if (dattr) {
644	*dattr = SD_ATTR_INIT;	644	*dattr = SD_ATTR_INIT;
645	update_domain_attr_tree(dattr, &top_cpuset);	645	update_domain_attr_tree(dattr, &top_cpuset);
646	}	646	}
647	cpumask_copy(doms, top_cpuset.cpus_allowed);	647	cpumask_copy(doms, top_cpuset.cpus_allowed);
648		648
649	ndoms = 1;	649	ndoms = 1;
650	goto done;	650	goto done;
651	}	651	}
652		652
653	csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);	653	csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);
654	if (!csa)	654	if (!csa)
655	goto done;	655	goto done;
656	csn = 0;	656	csn = 0;
657		657
658	list_add(&top_cpuset.stack_list, &q);	658	list_add(&top_cpuset.stack_list, &q);
659	while (!list_empty(&q)) {	659	while (!list_empty(&q)) {
660	struct cgroup *cont;	660	struct cgroup *cont;
661	struct cpuset child; / scans child cpusets of cp */	661	struct cpuset child; / scans child cpusets of cp */
662		662
663	cp = list_first_entry(&q, struct cpuset, stack_list);	663	cp = list_first_entry(&q, struct cpuset, stack_list);
664	list_del(q.next);	664	list_del(q.next);
665		665
666	if (cpumask_empty(cp->cpus_allowed))	666	if (cpumask_empty(cp->cpus_allowed))
667	continue;	667	continue;
668		668
669	/*	669	/*
670	* All child cpusets contain a subset of the parent's cpus, so	670	* All child cpusets contain a subset of the parent's cpus, so
671	* just skip them, and then we call update_domain_attr_tree()	671	* just skip them, and then we call update_domain_attr_tree()
672	* to calc relax_domain_level of the corresponding sched	672	* to calc relax_domain_level of the corresponding sched
673	* domain.	673	* domain.
674	*/	674	*/
675	if (is_sched_load_balance(cp)) {	675	if (is_sched_load_balance(cp)) {
676	csa[csn++] = cp;	676	csa[csn++] = cp;
677	continue;	677	continue;
678	}	678	}
679		679
680	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {	680	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {
681	child = cgroup_cs(cont);	681	child = cgroup_cs(cont);
682	list_add_tail(&child->stack_list, &q);	682	list_add_tail(&child->stack_list, &q);
683	}	683	}
684	}	684	}
685		685
686	for (i = 0; i < csn; i++)	686	for (i = 0; i < csn; i++)
687	csa[i]->pn = i;	687	csa[i]->pn = i;
688	ndoms = csn;	688	ndoms = csn;
689		689
690	restart:	690	restart:
691	/* Find the best partition (set of sched domains) */	691	/* Find the best partition (set of sched domains) */
692	for (i = 0; i < csn; i++) {	692	for (i = 0; i < csn; i++) {
693	struct cpuset *a = csa[i];	693	struct cpuset *a = csa[i];
694	int apn = a->pn;	694	int apn = a->pn;
695		695
696	for (j = 0; j < csn; j++) {	696	for (j = 0; j < csn; j++) {
697	struct cpuset *b = csa[j];	697	struct cpuset *b = csa[j];
698	int bpn = b->pn;	698	int bpn = b->pn;
699		699
700	if (apn != bpn && cpusets_overlap(a, b)) {	700	if (apn != bpn && cpusets_overlap(a, b)) {
701	for (k = 0; k < csn; k++) {	701	for (k = 0; k < csn; k++) {
702	struct cpuset *c = csa[k];	702	struct cpuset *c = csa[k];
703		703
704	if (c->pn == bpn)	704	if (c->pn == bpn)
705	c->pn = apn;	705	c->pn = apn;
706	}	706	}
707	ndoms--; /* one less element */	707	ndoms--; /* one less element */
708	goto restart;	708	goto restart;
709	}	709	}
710	}	710	}
711	}	711	}
712		712
713	/*	713	/*
714	* Now we know how many domains to create.	714	* Now we know how many domains to create.
715	* Convert <csn, csa> to <ndoms, doms> and populate cpu masks.	715	* Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
716	*/	716	*/
717	doms = kmalloc(ndoms * cpumask_size(), GFP_KERNEL);	717	doms = kmalloc(ndoms * cpumask_size(), GFP_KERNEL);
718	if (!doms)	718	if (!doms)
719	goto done;	719	goto done;
720		720
721	/*	721	/*
722	* The rest of the code, including the scheduler, can deal with	722	* The rest of the code, including the scheduler, can deal with
723	* dattr==NULL case. No need to abort if alloc fails.	723	* dattr==NULL case. No need to abort if alloc fails.
724	*/	724	*/
725	dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);	725	dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);
726		726
727	for (nslot = 0, i = 0; i < csn; i++) {	727	for (nslot = 0, i = 0; i < csn; i++) {
728	struct cpuset *a = csa[i];	728	struct cpuset *a = csa[i];
729	struct cpumask *dp;	729	struct cpumask *dp;
730	int apn = a->pn;	730	int apn = a->pn;
731		731
732	if (apn < 0) {	732	if (apn < 0) {
733	/* Skip completed partitions */	733	/* Skip completed partitions */
734	continue;	734	continue;
735	}	735	}
736		736
737	dp = doms + nslot;	737	dp = doms + nslot;
738		738
739	if (nslot == ndoms) {	739	if (nslot == ndoms) {
740	static int warnings = 10;	740	static int warnings = 10;
741	if (warnings) {	741	if (warnings) {
742	printk(KERN_WARNING	742	printk(KERN_WARNING
743	"rebuild_sched_domains confused:"	743	"rebuild_sched_domains confused:"
744	" nslot %d, ndoms %d, csn %d, i %d,"	744	" nslot %d, ndoms %d, csn %d, i %d,"
745	" apn %d\n",	745	" apn %d\n",
746	nslot, ndoms, csn, i, apn);	746	nslot, ndoms, csn, i, apn);
747	warnings--;	747	warnings--;
748	}	748	}
749	continue;	749	continue;
750	}	750	}
751		751
752	cpumask_clear(dp);	752	cpumask_clear(dp);
753	if (dattr)	753	if (dattr)
754	*(dattr + nslot) = SD_ATTR_INIT;	754	*(dattr + nslot) = SD_ATTR_INIT;
755	for (j = i; j < csn; j++) {	755	for (j = i; j < csn; j++) {
756	struct cpuset *b = csa[j];	756	struct cpuset *b = csa[j];
757		757
758	if (apn == b->pn) {	758	if (apn == b->pn) {
759	cpumask_or(dp, dp, b->cpus_allowed);	759	cpumask_or(dp, dp, b->cpus_allowed);
760	if (dattr)	760	if (dattr)
761	update_domain_attr_tree(dattr + nslot, b);	761	update_domain_attr_tree(dattr + nslot, b);
762		762
763	/* Done with this partition */	763	/* Done with this partition */
764	b->pn = -1;	764	b->pn = -1;
765	}	765	}
766	}	766	}
767	nslot++;	767	nslot++;
768	}	768	}
769	BUG_ON(nslot != ndoms);	769	BUG_ON(nslot != ndoms);
770		770
771	done:	771	done:
772	kfree(csa);	772	kfree(csa);
773		773
774	/*	774	/*
775	* Fallback to the default domain if kmalloc() failed.	775	* Fallback to the default domain if kmalloc() failed.
776	* See comments in partition_sched_domains().	776	* See comments in partition_sched_domains().
777	*/	777	*/
778	if (doms == NULL)	778	if (doms == NULL)
779	ndoms = 1;	779	ndoms = 1;
780		780
781	*domains = doms;	781	*domains = doms;
782	*attributes = dattr;	782	*attributes = dattr;
783	return ndoms;	783	return ndoms;
784	}	784	}
785		785
786	/*	786	/*
787	* Rebuild scheduler domains.	787	* Rebuild scheduler domains.
788	*	788	*
789	* Call with neither cgroup_mutex held nor within get_online_cpus().	789	* Call with neither cgroup_mutex held nor within get_online_cpus().
790	* Takes both cgroup_mutex and get_online_cpus().	790	* Takes both cgroup_mutex and get_online_cpus().
791	*	791	*
792	* Cannot be directly called from cpuset code handling changes	792	* Cannot be directly called from cpuset code handling changes
793	* to the cpuset pseudo-filesystem, because it cannot be called	793	* to the cpuset pseudo-filesystem, because it cannot be called
794	* from code that already holds cgroup_mutex.	794	* from code that already holds cgroup_mutex.
795	*/	795	*/
796	static void do_rebuild_sched_domains(struct work_struct *unused)	796	static void do_rebuild_sched_domains(struct work_struct *unused)
797	{	797	{
798	struct sched_domain_attr *attr;	798	struct sched_domain_attr *attr;
799	struct cpumask *doms;	799	struct cpumask *doms;
800	int ndoms;	800	int ndoms;
801		801
802	get_online_cpus();	802	get_online_cpus();
803		803
804	/* Generate domain masks and attrs */	804	/* Generate domain masks and attrs */
805	cgroup_lock();	805	cgroup_lock();
806	ndoms = generate_sched_domains(&doms, &attr);	806	ndoms = generate_sched_domains(&doms, &attr);
807	cgroup_unlock();	807	cgroup_unlock();
808		808
809	/* Have scheduler rebuild the domains */	809	/* Have scheduler rebuild the domains */
810	partition_sched_domains(ndoms, doms, attr);	810	partition_sched_domains(ndoms, doms, attr);
811		811
812	put_online_cpus();	812	put_online_cpus();
813	}	813	}
814		814
815	static DECLARE_WORK(rebuild_sched_domains_work, do_rebuild_sched_domains);	815	static DECLARE_WORK(rebuild_sched_domains_work, do_rebuild_sched_domains);
816		816
817	/*	817	/*
818	* Rebuild scheduler domains, asynchronously via workqueue.	818	* Rebuild scheduler domains, asynchronously via workqueue.
819	*	819	*
820	* If the flag 'sched_load_balance' of any cpuset with non-empty	820	* If the flag 'sched_load_balance' of any cpuset with non-empty
821	* 'cpus' changes, or if the 'cpus' allowed changes in any cpuset	821	* 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
822	* which has that flag enabled, or if any cpuset with a non-empty	822	* which has that flag enabled, or if any cpuset with a non-empty
823	* 'cpus' is removed, then call this routine to rebuild the	823	* 'cpus' is removed, then call this routine to rebuild the
824	* scheduler's dynamic sched domains.	824	* scheduler's dynamic sched domains.
825	*	825	*
826	* The rebuild_sched_domains() and partition_sched_domains()	826	* The rebuild_sched_domains() and partition_sched_domains()
827	* routines must nest cgroup_lock() inside get_online_cpus(),	827	* routines must nest cgroup_lock() inside get_online_cpus(),
828	* but such cpuset changes as these must nest that locking the	828	* but such cpuset changes as these must nest that locking the
829	* other way, holding cgroup_lock() for much of the code.	829	* other way, holding cgroup_lock() for much of the code.
830	*	830	*
831	* So in order to avoid an ABBA deadlock, the cpuset code handling	831	* So in order to avoid an ABBA deadlock, the cpuset code handling
832	* these user changes delegates the actual sched domain rebuilding	832	* these user changes delegates the actual sched domain rebuilding
833	* to a separate workqueue thread, which ends up processing the	833	* to a separate workqueue thread, which ends up processing the
834	* above do_rebuild_sched_domains() function.	834	* above do_rebuild_sched_domains() function.
835	*/	835	*/
836	static void async_rebuild_sched_domains(void)	836	static void async_rebuild_sched_domains(void)
837	{	837	{
838	queue_work(cpuset_wq, &rebuild_sched_domains_work);	838	queue_work(cpuset_wq, &rebuild_sched_domains_work);
839	}	839	}
840		840
841	/*	841	/*
842	* Accomplishes the same scheduler domain rebuild as the above	842	* Accomplishes the same scheduler domain rebuild as the above
843	* async_rebuild_sched_domains(), however it directly calls the	843	* async_rebuild_sched_domains(), however it directly calls the
844	* rebuild routine synchronously rather than calling it via an	844	* rebuild routine synchronously rather than calling it via an
845	* asynchronous work thread.	845	* asynchronous work thread.
846	*	846	*
847	* This can only be called from code that is not holding	847	* This can only be called from code that is not holding
848	* cgroup_mutex (not nested in a cgroup_lock() call.)	848	* cgroup_mutex (not nested in a cgroup_lock() call.)
849	*/	849	*/
850	void rebuild_sched_domains(void)	850	void rebuild_sched_domains(void)
851	{	851	{
852	do_rebuild_sched_domains(NULL);	852	do_rebuild_sched_domains(NULL);
853	}	853	}
854		854
855	/**	855	/**
856	* cpuset_test_cpumask - test a task's cpus_allowed versus its cpuset's	856	* cpuset_test_cpumask - test a task's cpus_allowed versus its cpuset's
857	* @tsk: task to test	857	* @tsk: task to test
858	* @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner	858	* @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner
859	*	859	*
860	* Call with cgroup_mutex held. May take callback_mutex during call.	860	* Call with cgroup_mutex held. May take callback_mutex during call.
861	* Called for each task in a cgroup by cgroup_scan_tasks().	861	* Called for each task in a cgroup by cgroup_scan_tasks().
862	* Return nonzero if this tasks's cpus_allowed mask should be changed (in other	862	* Return nonzero if this tasks's cpus_allowed mask should be changed (in other
863	* words, if its mask is not equal to its cpuset's mask).	863	* words, if its mask is not equal to its cpuset's mask).
864	*/	864	*/
865	static int cpuset_test_cpumask(struct task_struct *tsk,	865	static int cpuset_test_cpumask(struct task_struct *tsk,
866	struct cgroup_scanner *scan)	866	struct cgroup_scanner *scan)
867	{	867	{
868	return !cpumask_equal(&tsk->cpus_allowed,	868	return !cpumask_equal(&tsk->cpus_allowed,
869	(cgroup_cs(scan->cg))->cpus_allowed);	869	(cgroup_cs(scan->cg))->cpus_allowed);
870	}	870	}
871		871
872	/**	872	/**
873	* cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's	873	* cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's
874	* @tsk: task to test	874	* @tsk: task to test
875	* @scan: struct cgroup_scanner containing the cgroup of the task	875	* @scan: struct cgroup_scanner containing the cgroup of the task
876	*	876	*
877	* Called by cgroup_scan_tasks() for each task in a cgroup whose	877	* Called by cgroup_scan_tasks() for each task in a cgroup whose
878	* cpus_allowed mask needs to be changed.	878	* cpus_allowed mask needs to be changed.
879	*	879	*
880	* We don't need to re-check for the cgroup/cpuset membership, since we're	880	* We don't need to re-check for the cgroup/cpuset membership, since we're
881	* holding cgroup_lock() at this point.	881	* holding cgroup_lock() at this point.
882	*/	882	*/
883	static void cpuset_change_cpumask(struct task_struct *tsk,	883	static void cpuset_change_cpumask(struct task_struct *tsk,
884	struct cgroup_scanner *scan)	884	struct cgroup_scanner *scan)
885	{	885	{
886	set_cpus_allowed_ptr(tsk, ((cgroup_cs(scan->cg))->cpus_allowed));	886	set_cpus_allowed_ptr(tsk, ((cgroup_cs(scan->cg))->cpus_allowed));
887	}	887	}
888		888
889	/**	889	/**
890	* update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.	890	* update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
891	* @cs: the cpuset in which each task's cpus_allowed mask needs to be changed	891	* @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
892	* @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()	892	* @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
893	*	893	*
894	* Called with cgroup_mutex held	894	* Called with cgroup_mutex held
895	*	895	*
896	* The cgroup_scan_tasks() function will scan all the tasks in a cgroup,	896	* The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
897	* calling callback functions for each.	897	* calling callback functions for each.
898	*	898	*
899	* No return value. It's guaranteed that cgroup_scan_tasks() always returns 0	899	* No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
900	* if @heap != NULL.	900	* if @heap != NULL.
901	*/	901	*/
902	static void update_tasks_cpumask(struct cpuset cs, struct ptr_heap heap)	902	static void update_tasks_cpumask(struct cpuset cs, struct ptr_heap heap)
903	{	903	{
904	struct cgroup_scanner scan;	904	struct cgroup_scanner scan;
905		905
906	scan.cg = cs->css.cgroup;	906	scan.cg = cs->css.cgroup;
907	scan.test_task = cpuset_test_cpumask;	907	scan.test_task = cpuset_test_cpumask;
908	scan.process_task = cpuset_change_cpumask;	908	scan.process_task = cpuset_change_cpumask;
909	scan.heap = heap;	909	scan.heap = heap;
910	cgroup_scan_tasks(&scan);	910	cgroup_scan_tasks(&scan);
911	}	911	}
912		912
913	/**	913	/**
914	* update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it	914	* update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
915	* @cs: the cpuset to consider	915	* @cs: the cpuset to consider
916	* @buf: buffer of cpu numbers written to this cpuset	916	* @buf: buffer of cpu numbers written to this cpuset
917	*/	917	*/
918	static int update_cpumask(struct cpuset cs, struct cpuset trialcs,	918	static int update_cpumask(struct cpuset cs, struct cpuset trialcs,
919	const char *buf)	919	const char *buf)
920	{	920	{
921	struct ptr_heap heap;	921	struct ptr_heap heap;
922	int retval;	922	int retval;
923	int is_load_balanced;	923	int is_load_balanced;
924		924
925	/* top_cpuset.cpus_allowed tracks cpu_online_map; it's read-only */	925	/* top_cpuset.cpus_allowed tracks cpu_online_map; it's read-only */
926	if (cs == &top_cpuset)	926	if (cs == &top_cpuset)
927	return -EACCES;	927	return -EACCES;
928		928
929	/*	929	/*
930	* An empty cpus_allowed is ok only if the cpuset has no tasks.	930	* An empty cpus_allowed is ok only if the cpuset has no tasks.
931	* Since cpulist_parse() fails on an empty mask, we special case	931	* Since cpulist_parse() fails on an empty mask, we special case
932	* that parsing. The validate_change() call ensures that cpusets	932	* that parsing. The validate_change() call ensures that cpusets
933	* with tasks have cpus.	933	* with tasks have cpus.
934	*/	934	*/
935	if (!*buf) {	935	if (!*buf) {
936	cpumask_clear(trialcs->cpus_allowed);	936	cpumask_clear(trialcs->cpus_allowed);
937	} else {	937	} else {
938	retval = cpulist_parse(buf, trialcs->cpus_allowed);	938	retval = cpulist_parse(buf, trialcs->cpus_allowed);
939	if (retval < 0)	939	if (retval < 0)
940	return retval;	940	return retval;
941		941
942	if (!cpumask_subset(trialcs->cpus_allowed, cpu_online_mask))	942	if (!cpumask_subset(trialcs->cpus_allowed, cpu_online_mask))
943	return -EINVAL;	943	return -EINVAL;
944	}	944	}
945	retval = validate_change(cs, trialcs);	945	retval = validate_change(cs, trialcs);
946	if (retval < 0)	946	if (retval < 0)
947	return retval;	947	return retval;
948		948
949	/* Nothing to do if the cpus didn't change */	949	/* Nothing to do if the cpus didn't change */
950	if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))	950	if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
951	return 0;	951	return 0;
952		952
953	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);	953	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
954	if (retval)	954	if (retval)
955	return retval;	955	return retval;
956		956
957	is_load_balanced = is_sched_load_balance(trialcs);	957	is_load_balanced = is_sched_load_balance(trialcs);
958		958
959	mutex_lock(&callback_mutex);	959	mutex_lock(&callback_mutex);
960	cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);	960	cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
961	mutex_unlock(&callback_mutex);	961	mutex_unlock(&callback_mutex);
962		962
963	/*	963	/*
964	* Scan tasks in the cpuset, and update the cpumasks of any	964	* Scan tasks in the cpuset, and update the cpumasks of any
965	* that need an update.	965	* that need an update.
966	*/	966	*/
967	update_tasks_cpumask(cs, &heap);	967	update_tasks_cpumask(cs, &heap);
968		968
969	heap_free(&heap);	969	heap_free(&heap);
970		970
971	if (is_load_balanced)	971	if (is_load_balanced)
972	async_rebuild_sched_domains();	972	async_rebuild_sched_domains();
973	return 0;	973	return 0;
974	}	974	}
975		975
976	/*	976	/*
977	* cpuset_migrate_mm	977	* cpuset_migrate_mm
978	*	978	*
979	* Migrate memory region from one set of nodes to another.	979	* Migrate memory region from one set of nodes to another.
980	*	980	*
981	* Temporarilly set tasks mems_allowed to target nodes of migration,	981	* Temporarilly set tasks mems_allowed to target nodes of migration,
982	* so that the migration code can allocate pages on these nodes.	982	* so that the migration code can allocate pages on these nodes.
983	*	983	*
984	* Call holding cgroup_mutex, so current's cpuset won't change	984	* Call holding cgroup_mutex, so current's cpuset won't change
985	* during this call, as manage_mutex holds off any cpuset_attach()	985	* during this call, as manage_mutex holds off any cpuset_attach()
986	* calls. Therefore we don't need to take task_lock around the	986	* calls. Therefore we don't need to take task_lock around the
987	* call to guarantee_online_mems(), as we know no one is changing	987	* call to guarantee_online_mems(), as we know no one is changing
988	* our task's cpuset.	988	* our task's cpuset.
989	*	989	*
990	* Hold callback_mutex around the two modifications of our tasks	990	* Hold callback_mutex around the two modifications of our tasks
991	* mems_allowed to synchronize with cpuset_mems_allowed().	991	* mems_allowed to synchronize with cpuset_mems_allowed().
992	*	992	*
993	* While the mm_struct we are migrating is typically from some	993	* While the mm_struct we are migrating is typically from some
994	* other task, the task_struct mems_allowed that we are hacking	994	* other task, the task_struct mems_allowed that we are hacking
995	* is for our current task, which must allocate new pages for that	995	* is for our current task, which must allocate new pages for that
996	* migrating memory region.	996	* migrating memory region.
997	*	997	*
998	* We call cpuset_update_task_memory_state() before hacking	998	* We call cpuset_update_task_memory_state() before hacking
999	* our tasks mems_allowed, so that we are assured of being in	999	* our tasks mems_allowed, so that we are assured of being in
1000	* sync with our tasks cpuset, and in particular, callbacks to	1000	* sync with our tasks cpuset, and in particular, callbacks to
1001	* cpuset_update_task_memory_state() from nested page allocations	1001	* cpuset_update_task_memory_state() from nested page allocations
1002	* won't see any mismatch of our cpuset and task mems_generation	1002	* won't see any mismatch of our cpuset and task mems_generation
1003	* values, so won't overwrite our hacked tasks mems_allowed	1003	* values, so won't overwrite our hacked tasks mems_allowed
1004	* nodemask.	1004	* nodemask.
1005	*/	1005	*/
1006		1006
1007	static void cpuset_migrate_mm(struct mm_struct mm, const nodemask_t from,	1007	static void cpuset_migrate_mm(struct mm_struct mm, const nodemask_t from,
1008	const nodemask_t *to)	1008	const nodemask_t *to)
1009	{	1009	{
1010	struct task_struct *tsk = current;	1010	struct task_struct *tsk = current;
1011		1011
1012	cpuset_update_task_memory_state();	1012	cpuset_update_task_memory_state();
1013		1013
1014	mutex_lock(&callback_mutex);	1014	mutex_lock(&callback_mutex);
1015	tsk->mems_allowed = *to;	1015	tsk->mems_allowed = *to;
1016	mutex_unlock(&callback_mutex);	1016	mutex_unlock(&callback_mutex);
1017		1017
1018	do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);	1018	do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);
1019		1019
1020	mutex_lock(&callback_mutex);	1020	mutex_lock(&callback_mutex);
1021	guarantee_online_mems(task_cs(tsk),&tsk->mems_allowed);	1021	guarantee_online_mems(task_cs(tsk),&tsk->mems_allowed);
1022	mutex_unlock(&callback_mutex);	1022	mutex_unlock(&callback_mutex);
1023	}	1023	}
1024		1024
1025	/*	1025	/*
1026	* Rebind task's vmas to cpuset's new mems_allowed, and migrate pages to new	1026	* Rebind task's vmas to cpuset's new mems_allowed, and migrate pages to new
1027	* nodes if memory_migrate flag is set. Called with cgroup_mutex held.	1027	* nodes if memory_migrate flag is set. Called with cgroup_mutex held.
1028	*/	1028	*/
1029	static void cpuset_change_nodemask(struct task_struct *p,	1029	static void cpuset_change_nodemask(struct task_struct *p,
1030	struct cgroup_scanner *scan)	1030	struct cgroup_scanner *scan)
1031	{	1031	{
1032	struct mm_struct *mm;	1032	struct mm_struct *mm;
1033	struct cpuset *cs;	1033	struct cpuset *cs;
1034	int migrate;	1034	int migrate;
1035	const nodemask_t *oldmem = scan->data;	1035	const nodemask_t *oldmem = scan->data;
1036		1036
1037	mm = get_task_mm(p);	1037	mm = get_task_mm(p);
1038	if (!mm)	1038	if (!mm)
1039	return;	1039	return;
1040		1040
1041	cs = cgroup_cs(scan->cg);	1041	cs = cgroup_cs(scan->cg);
1042	migrate = is_memory_migrate(cs);	1042	migrate = is_memory_migrate(cs);
1043		1043
1044	mpol_rebind_mm(mm, &cs->mems_allowed);	1044	mpol_rebind_mm(mm, &cs->mems_allowed);
1045	if (migrate)	1045	if (migrate)
1046	cpuset_migrate_mm(mm, oldmem, &cs->mems_allowed);	1046	cpuset_migrate_mm(mm, oldmem, &cs->mems_allowed);
1047	mmput(mm);	1047	mmput(mm);
1048	}	1048	}
1049		1049
1050	static void *cpuset_being_rebound;	1050	static void *cpuset_being_rebound;
1051		1051
1052	/**	1052	/**
1053	* update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.	1053	* update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
1054	* @cs: the cpuset in which each task's mems_allowed mask needs to be changed	1054	* @cs: the cpuset in which each task's mems_allowed mask needs to be changed
1055	* @oldmem: old mems_allowed of cpuset cs	1055	* @oldmem: old mems_allowed of cpuset cs
1056	* @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()	1056	* @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
1057	*	1057	*
1058	* Called with cgroup_mutex held	1058	* Called with cgroup_mutex held
1059	* No return value. It's guaranteed that cgroup_scan_tasks() always returns 0	1059	* No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
1060	* if @heap != NULL.	1060	* if @heap != NULL.
1061	*/	1061	*/
1062	static void update_tasks_nodemask(struct cpuset cs, const nodemask_t oldmem,	1062	static void update_tasks_nodemask(struct cpuset cs, const nodemask_t oldmem,
1063	struct ptr_heap *heap)	1063	struct ptr_heap *heap)
1064	{	1064	{
1065	struct cgroup_scanner scan;	1065	struct cgroup_scanner scan;
1066		1066
1067	cpuset_being_rebound = cs; /* causes mpol_dup() rebind */	1067	cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
1068		1068
1069	scan.cg = cs->css.cgroup;	1069	scan.cg = cs->css.cgroup;
1070	scan.test_task = NULL;	1070	scan.test_task = NULL;
1071	scan.process_task = cpuset_change_nodemask;	1071	scan.process_task = cpuset_change_nodemask;
1072	scan.heap = heap;	1072	scan.heap = heap;
1073	scan.data = (nodemask_t *)oldmem;	1073	scan.data = (nodemask_t *)oldmem;
1074		1074
1075	/*	1075	/*
1076	* The mpol_rebind_mm() call takes mmap_sem, which we couldn't	1076	* The mpol_rebind_mm() call takes mmap_sem, which we couldn't
1077	* take while holding tasklist_lock. Forks can happen - the	1077	* take while holding tasklist_lock. Forks can happen - the
1078	* mpol_dup() cpuset_being_rebound check will catch such forks,	1078	* mpol_dup() cpuset_being_rebound check will catch such forks,
1079	* and rebind their vma mempolicies too. Because we still hold	1079	* and rebind their vma mempolicies too. Because we still hold
1080	* the global cgroup_mutex, we know that no other rebind effort	1080	* the global cgroup_mutex, we know that no other rebind effort
1081	* will be contending for the global variable cpuset_being_rebound.	1081	* will be contending for the global variable cpuset_being_rebound.
1082	* It's ok if we rebind the same mm twice; mpol_rebind_mm()	1082	* It's ok if we rebind the same mm twice; mpol_rebind_mm()
1083	* is idempotent. Also migrate pages in each mm to new nodes.	1083	* is idempotent. Also migrate pages in each mm to new nodes.
1084	*/	1084	*/
1085	cgroup_scan_tasks(&scan);	1085	cgroup_scan_tasks(&scan);
1086		1086
1087	/* We're done rebinding vmas to this cpuset's new mems_allowed. */	1087	/* We're done rebinding vmas to this cpuset's new mems_allowed. */
1088	cpuset_being_rebound = NULL;	1088	cpuset_being_rebound = NULL;
1089	}	1089	}
1090		1090
1091	/*	1091	/*
1092	* Handle user request to change the 'mems' memory placement	1092	* Handle user request to change the 'mems' memory placement
1093	* of a cpuset. Needs to validate the request, update the	1093	* of a cpuset. Needs to validate the request, update the
1094	* cpusets mems_allowed and mems_generation, and for each	1094	* cpusets mems_allowed and mems_generation, and for each
1095	* task in the cpuset, rebind any vma mempolicies and if	1095	* task in the cpuset, rebind any vma mempolicies and if
1096	* the cpuset is marked 'memory_migrate', migrate the tasks	1096	* the cpuset is marked 'memory_migrate', migrate the tasks
1097	* pages to the new memory.	1097	* pages to the new memory.
1098	*	1098	*
1099	* Call with cgroup_mutex held. May take callback_mutex during call.	1099	* Call with cgroup_mutex held. May take callback_mutex during call.
1100	* Will take tasklist_lock, scan tasklist for tasks in cpuset cs,	1100	* Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
1101	* lock each such tasks mm->mmap_sem, scan its vma's and rebind	1101	* lock each such tasks mm->mmap_sem, scan its vma's and rebind
1102	* their mempolicies to the cpusets new mems_allowed.	1102	* their mempolicies to the cpusets new mems_allowed.
1103	*/	1103	*/
1104	static int update_nodemask(struct cpuset cs, struct cpuset trialcs,	1104	static int update_nodemask(struct cpuset cs, struct cpuset trialcs,
1105	const char *buf)	1105	const char *buf)
1106	{	1106	{
1107	nodemask_t oldmem;	1107	nodemask_t oldmem;
1108	int retval;	1108	int retval;
1109	struct ptr_heap heap;	1109	struct ptr_heap heap;
1110		1110
1111	/*	1111	/*
1112	* top_cpuset.mems_allowed tracks node_stats[N_HIGH_MEMORY];	1112	* top_cpuset.mems_allowed tracks node_stats[N_HIGH_MEMORY];
1113	* it's read-only	1113	* it's read-only
1114	*/	1114	*/
1115	if (cs == &top_cpuset)	1115	if (cs == &top_cpuset)
1116	return -EACCES;	1116	return -EACCES;
1117		1117
1118	/*	1118	/*
1119	* An empty mems_allowed is ok iff there are no tasks in the cpuset.	1119	* An empty mems_allowed is ok iff there are no tasks in the cpuset.
1120	* Since nodelist_parse() fails on an empty mask, we special case	1120	* Since nodelist_parse() fails on an empty mask, we special case
1121	* that parsing. The validate_change() call ensures that cpusets	1121	* that parsing. The validate_change() call ensures that cpusets
1122	* with tasks have memory.	1122	* with tasks have memory.
1123	*/	1123	*/
1124	if (!*buf) {	1124	if (!*buf) {
1125	nodes_clear(trialcs->mems_allowed);	1125	nodes_clear(trialcs->mems_allowed);
1126	} else {	1126	} else {
1127	retval = nodelist_parse(buf, trialcs->mems_allowed);	1127	retval = nodelist_parse(buf, trialcs->mems_allowed);
1128	if (retval < 0)	1128	if (retval < 0)
1129	goto done;	1129	goto done;
1130		1130
1131	if (!nodes_subset(trialcs->mems_allowed,	1131	if (!nodes_subset(trialcs->mems_allowed,
1132	node_states[N_HIGH_MEMORY]))	1132	node_states[N_HIGH_MEMORY]))
1133	return -EINVAL;	1133	return -EINVAL;
1134	}	1134	}
1135	oldmem = cs->mems_allowed;	1135	oldmem = cs->mems_allowed;
1136	if (nodes_equal(oldmem, trialcs->mems_allowed)) {	1136	if (nodes_equal(oldmem, trialcs->mems_allowed)) {
1137	retval = 0; /* Too easy - nothing to do */	1137	retval = 0; /* Too easy - nothing to do */
1138	goto done;	1138	goto done;
1139	}	1139	}
1140	retval = validate_change(cs, trialcs);	1140	retval = validate_change(cs, trialcs);
1141	if (retval < 0)	1141	if (retval < 0)
1142	goto done;	1142	goto done;
1143		1143
1144	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);	1144	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
1145	if (retval < 0)	1145	if (retval < 0)
1146	goto done;	1146	goto done;
1147		1147
1148	mutex_lock(&callback_mutex);	1148	mutex_lock(&callback_mutex);
1149	cs->mems_allowed = trialcs->mems_allowed;	1149	cs->mems_allowed = trialcs->mems_allowed;
1150	cs->mems_generation = cpuset_mems_generation++;	1150	cs->mems_generation = cpuset_mems_generation++;
1151	mutex_unlock(&callback_mutex);	1151	mutex_unlock(&callback_mutex);
1152		1152
1153	update_tasks_nodemask(cs, &oldmem, &heap);	1153	update_tasks_nodemask(cs, &oldmem, &heap);
1154		1154
1155	heap_free(&heap);	1155	heap_free(&heap);
1156	done:	1156	done:
1157	return retval;	1157	return retval;
1158	}	1158	}
1159		1159
1160	int current_cpuset_is_being_rebound(void)	1160	int current_cpuset_is_being_rebound(void)
1161	{	1161	{
1162	return task_cs(current) == cpuset_being_rebound;	1162	return task_cs(current) == cpuset_being_rebound;
1163	}	1163	}
1164		1164
1165	static int update_relax_domain_level(struct cpuset *cs, s64 val)	1165	static int update_relax_domain_level(struct cpuset *cs, s64 val)
1166	{	1166	{
1167	if (val < -1 \|\| val >= SD_LV_MAX)	1167	if (val < -1 \|\| val >= SD_LV_MAX)
1168	return -EINVAL;	1168	return -EINVAL;
1169		1169
1170	if (val != cs->relax_domain_level) {	1170	if (val != cs->relax_domain_level) {
1171	cs->relax_domain_level = val;	1171	cs->relax_domain_level = val;
1172	if (!cpumask_empty(cs->cpus_allowed) &&	1172	if (!cpumask_empty(cs->cpus_allowed) &&
1173	is_sched_load_balance(cs))	1173	is_sched_load_balance(cs))
1174	async_rebuild_sched_domains();	1174	async_rebuild_sched_domains();
1175	}	1175	}
1176		1176
1177	return 0;	1177	return 0;
1178	}	1178	}
1179		1179
1180	/*	1180	/*
1181	* update_flag - read a 0 or a 1 in a file and update associated flag	1181	* update_flag - read a 0 or a 1 in a file and update associated flag
1182	* bit: the bit to update (see cpuset_flagbits_t)	1182	* bit: the bit to update (see cpuset_flagbits_t)
1183	* cs: the cpuset to update	1183	* cs: the cpuset to update
1184	* turning_on: whether the flag is being set or cleared	1184	* turning_on: whether the flag is being set or cleared
1185	*	1185	*
1186	* Call with cgroup_mutex held.	1186	* Call with cgroup_mutex held.
1187	*/	1187	*/
1188		1188
1189	static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,	1189	static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1190	int turning_on)	1190	int turning_on)
1191	{	1191	{
1192	struct cpuset *trialcs;	1192	struct cpuset *trialcs;
1193	int err;	1193	int err;
1194	int balance_flag_changed;	1194	int balance_flag_changed;
1195		1195
1196	trialcs = alloc_trial_cpuset(cs);	1196	trialcs = alloc_trial_cpuset(cs);
1197	if (!trialcs)	1197	if (!trialcs)
1198	return -ENOMEM;	1198	return -ENOMEM;
1199		1199
1200	if (turning_on)	1200	if (turning_on)
1201	set_bit(bit, &trialcs->flags);	1201	set_bit(bit, &trialcs->flags);
1202	else	1202	else
1203	clear_bit(bit, &trialcs->flags);	1203	clear_bit(bit, &trialcs->flags);
1204		1204
1205	err = validate_change(cs, trialcs);	1205	err = validate_change(cs, trialcs);
1206	if (err < 0)	1206	if (err < 0)
1207	goto out;	1207	goto out;
1208		1208
1209	balance_flag_changed = (is_sched_load_balance(cs) !=	1209	balance_flag_changed = (is_sched_load_balance(cs) !=
1210	is_sched_load_balance(trialcs));	1210	is_sched_load_balance(trialcs));
1211		1211
1212	mutex_lock(&callback_mutex);	1212	mutex_lock(&callback_mutex);
1213	cs->flags = trialcs->flags;	1213	cs->flags = trialcs->flags;
1214	mutex_unlock(&callback_mutex);	1214	mutex_unlock(&callback_mutex);
1215		1215
1216	if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)	1216	if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
1217	async_rebuild_sched_domains();	1217	async_rebuild_sched_domains();
1218		1218
1219	out:	1219	out:
1220	free_trial_cpuset(trialcs);	1220	free_trial_cpuset(trialcs);
1221	return err;	1221	return err;
1222	}	1222	}
1223		1223
1224	/*	1224	/*
1225	* Frequency meter - How fast is some event occurring?	1225	* Frequency meter - How fast is some event occurring?
1226	*	1226	*
1227	* These routines manage a digitally filtered, constant time based,	1227	* These routines manage a digitally filtered, constant time based,
1228	* event frequency meter. There are four routines:	1228	* event frequency meter. There are four routines:
1229	* fmeter_init() - initialize a frequency meter.	1229	* fmeter_init() - initialize a frequency meter.
1230	* fmeter_markevent() - called each time the event happens.	1230	* fmeter_markevent() - called each time the event happens.
1231	* fmeter_getrate() - returns the recent rate of such events.	1231	* fmeter_getrate() - returns the recent rate of such events.
1232	* fmeter_update() - internal routine used to update fmeter.	1232	* fmeter_update() - internal routine used to update fmeter.
1233	*	1233	*
1234	* A common data structure is passed to each of these routines,	1234	* A common data structure is passed to each of these routines,
1235	* which is used to keep track of the state required to manage the	1235	* which is used to keep track of the state required to manage the
1236	* frequency meter and its digital filter.	1236	* frequency meter and its digital filter.
1237	*	1237	*
1238	* The filter works on the number of events marked per unit time.	1238	* The filter works on the number of events marked per unit time.
1239	* The filter is single-pole low-pass recursive (IIR). The time unit	1239	* The filter is single-pole low-pass recursive (IIR). The time unit
1240	* is 1 second. Arithmetic is done using 32-bit integers scaled to	1240	* is 1 second. Arithmetic is done using 32-bit integers scaled to
1241	* simulate 3 decimal digits of precision (multiplied by 1000).	1241	* simulate 3 decimal digits of precision (multiplied by 1000).
1242	*	1242	*
1243	* With an FM_COEF of 933, and a time base of 1 second, the filter	1243	* With an FM_COEF of 933, and a time base of 1 second, the filter
1244	* has a half-life of 10 seconds, meaning that if the events quit	1244	* has a half-life of 10 seconds, meaning that if the events quit
1245	* happening, then the rate returned from the fmeter_getrate()	1245	* happening, then the rate returned from the fmeter_getrate()
1246	* will be cut in half each 10 seconds, until it converges to zero.	1246	* will be cut in half each 10 seconds, until it converges to zero.
1247	*	1247	*
1248	* It is not worth doing a real infinitely recursive filter. If more	1248	* It is not worth doing a real infinitely recursive filter. If more
1249	* than FM_MAXTICKS ticks have elapsed since the last filter event,	1249	* than FM_MAXTICKS ticks have elapsed since the last filter event,
1250	* just compute FM_MAXTICKS ticks worth, by which point the level	1250	* just compute FM_MAXTICKS ticks worth, by which point the level
1251	* will be stable.	1251	* will be stable.
1252	*	1252	*
1253	* Limit the count of unprocessed events to FM_MAXCNT, so as to avoid	1253	* Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
1254	* arithmetic overflow in the fmeter_update() routine.	1254	* arithmetic overflow in the fmeter_update() routine.
1255	*	1255	*
1256	* Given the simple 32 bit integer arithmetic used, this meter works	1256	* Given the simple 32 bit integer arithmetic used, this meter works
1257	* best for reporting rates between one per millisecond (msec) and	1257	* best for reporting rates between one per millisecond (msec) and
1258	* one per 32 (approx) seconds. At constant rates faster than one	1258	* one per 32 (approx) seconds. At constant rates faster than one
1259	* per msec it maxes out at values just under 1,000,000. At constant	1259	* per msec it maxes out at values just under 1,000,000. At constant
1260	* rates between one per msec, and one per second it will stabilize	1260	* rates between one per msec, and one per second it will stabilize
1261	* to a value N*1000, where N is the rate of events per second.	1261	* to a value N*1000, where N is the rate of events per second.
1262	* At constant rates between one per second and one per 32 seconds,	1262	* At constant rates between one per second and one per 32 seconds,
1263	* it will be choppy, moving up on the seconds that have an event,	1263	* it will be choppy, moving up on the seconds that have an event,
1264	* and then decaying until the next event. At rates slower than	1264	* and then decaying until the next event. At rates slower than
1265	* about one in 32 seconds, it decays all the way back to zero between	1265	* about one in 32 seconds, it decays all the way back to zero between
1266	* each event.	1266	* each event.
1267	*/	1267	*/
1268		1268
1269	#define FM_COEF 933 /* coefficient for half-life of 10 secs */	1269	#define FM_COEF 933 /* coefficient for half-life of 10 secs */
1270	#define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */	1270	#define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */
1271	#define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */	1271	#define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */
1272	#define FM_SCALE 1000 /* faux fixed point scale */	1272	#define FM_SCALE 1000 /* faux fixed point scale */
1273		1273
1274	/* Initialize a frequency meter */	1274	/* Initialize a frequency meter */
1275	static void fmeter_init(struct fmeter *fmp)	1275	static void fmeter_init(struct fmeter *fmp)
1276	{	1276	{
1277	fmp->cnt = 0;	1277	fmp->cnt = 0;
1278	fmp->val = 0;	1278	fmp->val = 0;
1279	fmp->time = 0;	1279	fmp->time = 0;
1280	spin_lock_init(&fmp->lock);	1280	spin_lock_init(&fmp->lock);
1281	}	1281	}
1282		1282
1283	/* Internal meter update - process cnt events and update value */	1283	/* Internal meter update - process cnt events and update value */
1284	static void fmeter_update(struct fmeter *fmp)	1284	static void fmeter_update(struct fmeter *fmp)
1285	{	1285	{
1286	time_t now = get_seconds();	1286	time_t now = get_seconds();
1287	time_t ticks = now - fmp->time;	1287	time_t ticks = now - fmp->time;
1288		1288
1289	if (ticks == 0)	1289	if (ticks == 0)
1290	return;	1290	return;
1291		1291
1292	ticks = min(FM_MAXTICKS, ticks);	1292	ticks = min(FM_MAXTICKS, ticks);
1293	while (ticks-- > 0)	1293	while (ticks-- > 0)
1294	fmp->val = (FM_COEF * fmp->val) / FM_SCALE;	1294	fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
1295	fmp->time = now;	1295	fmp->time = now;
1296		1296
1297	fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;	1297	fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
1298	fmp->cnt = 0;	1298	fmp->cnt = 0;
1299	}	1299	}
1300		1300
1301	/* Process any previous ticks, then bump cnt by one (times scale). */	1301	/* Process any previous ticks, then bump cnt by one (times scale). */
1302	static void fmeter_markevent(struct fmeter *fmp)	1302	static void fmeter_markevent(struct fmeter *fmp)
1303	{	1303	{
1304	spin_lock(&fmp->lock);	1304	spin_lock(&fmp->lock);
1305	fmeter_update(fmp);	1305	fmeter_update(fmp);
1306	fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);	1306	fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
1307	spin_unlock(&fmp->lock);	1307	spin_unlock(&fmp->lock);
1308	}	1308	}
1309		1309
1310	/* Process any previous ticks, then return current value. */	1310	/* Process any previous ticks, then return current value. */
1311	static int fmeter_getrate(struct fmeter *fmp)	1311	static int fmeter_getrate(struct fmeter *fmp)
1312	{	1312	{
1313	int val;	1313	int val;
1314		1314
1315	spin_lock(&fmp->lock);	1315	spin_lock(&fmp->lock);
1316	fmeter_update(fmp);	1316	fmeter_update(fmp);
1317	val = fmp->val;	1317	val = fmp->val;
1318	spin_unlock(&fmp->lock);	1318	spin_unlock(&fmp->lock);
1319	return val;	1319	return val;
1320	}	1320	}
1321		1321
1322	/* Protected by cgroup_lock */	1322	/* Protected by cgroup_lock */
1323	static cpumask_var_t cpus_attach;	1323	static cpumask_var_t cpus_attach;
1324		1324
1325	/* Called by cgroups to determine if a cpuset is usable; cgroup_mutex held */	1325	/* Called by cgroups to determine if a cpuset is usable; cgroup_mutex held */
1326	static int cpuset_can_attach(struct cgroup_subsys *ss,	1326	static int cpuset_can_attach(struct cgroup_subsys *ss,
1327	struct cgroup cont, struct task_struct tsk)	1327	struct cgroup cont, struct task_struct tsk)
1328	{	1328	{
1329	struct cpuset *cs = cgroup_cs(cont);	1329	struct cpuset *cs = cgroup_cs(cont);
1330	int ret = 0;	1330	int ret = 0;
1331		1331
1332	if (cpumask_empty(cs->cpus_allowed) \|\| nodes_empty(cs->mems_allowed))	1332	if (cpumask_empty(cs->cpus_allowed) \|\| nodes_empty(cs->mems_allowed))
1333	return -ENOSPC;	1333	return -ENOSPC;
1334		1334
1335	if (tsk->flags & PF_THREAD_BOUND) {	1335	if (tsk->flags & PF_THREAD_BOUND) {
1336	mutex_lock(&callback_mutex);	1336	mutex_lock(&callback_mutex);
1337	if (!cpumask_equal(&tsk->cpus_allowed, cs->cpus_allowed))	1337	if (!cpumask_equal(&tsk->cpus_allowed, cs->cpus_allowed))
1338	ret = -EINVAL;	1338	ret = -EINVAL;
1339	mutex_unlock(&callback_mutex);	1339	mutex_unlock(&callback_mutex);
1340	}	1340	}
1341		1341
1342	return ret < 0 ? ret : security_task_setscheduler(tsk, 0, NULL);	1342	return ret < 0 ? ret : security_task_setscheduler(tsk, 0, NULL);
1343	}	1343	}
1344		1344
1345	static void cpuset_attach(struct cgroup_subsys *ss,	1345	static void cpuset_attach(struct cgroup_subsys *ss,
1346	struct cgroup cont, struct cgroup oldcont,	1346	struct cgroup cont, struct cgroup oldcont,
1347	struct task_struct *tsk)	1347	struct task_struct *tsk)
1348	{	1348	{
1349	nodemask_t from, to;	1349	nodemask_t from, to;
1350	struct mm_struct *mm;	1350	struct mm_struct *mm;
1351	struct cpuset *cs = cgroup_cs(cont);	1351	struct cpuset *cs = cgroup_cs(cont);
1352	struct cpuset *oldcs = cgroup_cs(oldcont);	1352	struct cpuset *oldcs = cgroup_cs(oldcont);
1353	int err;	1353	int err;
1354		1354
1355	if (cs == &top_cpuset) {	1355	if (cs == &top_cpuset) {
1356	cpumask_copy(cpus_attach, cpu_possible_mask);	1356	cpumask_copy(cpus_attach, cpu_possible_mask);
1357	} else {	1357	} else {
1358	mutex_lock(&callback_mutex);	1358	mutex_lock(&callback_mutex);
1359	guarantee_online_cpus(cs, cpus_attach);	1359	guarantee_online_cpus(cs, cpus_attach);
1360	mutex_unlock(&callback_mutex);	1360	mutex_unlock(&callback_mutex);
1361	}	1361	}
1362	err = set_cpus_allowed_ptr(tsk, cpus_attach);	1362	err = set_cpus_allowed_ptr(tsk, cpus_attach);
1363	if (err)	1363	if (err)
1364	return;	1364	return;
1365		1365
1366	from = oldcs->mems_allowed;	1366	from = oldcs->mems_allowed;
1367	to = cs->mems_allowed;	1367	to = cs->mems_allowed;
1368	mm = get_task_mm(tsk);	1368	mm = get_task_mm(tsk);
1369	if (mm) {	1369	if (mm) {
1370	mpol_rebind_mm(mm, &to);	1370	mpol_rebind_mm(mm, &to);
1371	if (is_memory_migrate(cs))	1371	if (is_memory_migrate(cs))
1372	cpuset_migrate_mm(mm, &from, &to);	1372	cpuset_migrate_mm(mm, &from, &to);
1373	mmput(mm);	1373	mmput(mm);
1374	}	1374	}
1375	}	1375	}
1376		1376
1377	/* The various types of files and directories in a cpuset file system */	1377	/* The various types of files and directories in a cpuset file system */
1378		1378
1379	typedef enum {	1379	typedef enum {
1380	FILE_MEMORY_MIGRATE,	1380	FILE_MEMORY_MIGRATE,
1381	FILE_CPULIST,	1381	FILE_CPULIST,
1382	FILE_MEMLIST,	1382	FILE_MEMLIST,
1383	FILE_CPU_EXCLUSIVE,	1383	FILE_CPU_EXCLUSIVE,
1384	FILE_MEM_EXCLUSIVE,	1384	FILE_MEM_EXCLUSIVE,
1385	FILE_MEM_HARDWALL,	1385	FILE_MEM_HARDWALL,
1386	FILE_SCHED_LOAD_BALANCE,	1386	FILE_SCHED_LOAD_BALANCE,
1387	FILE_SCHED_RELAX_DOMAIN_LEVEL,	1387	FILE_SCHED_RELAX_DOMAIN_LEVEL,
1388	FILE_MEMORY_PRESSURE_ENABLED,	1388	FILE_MEMORY_PRESSURE_ENABLED,
1389	FILE_MEMORY_PRESSURE,	1389	FILE_MEMORY_PRESSURE,
1390	FILE_SPREAD_PAGE,	1390	FILE_SPREAD_PAGE,
1391	FILE_SPREAD_SLAB,	1391	FILE_SPREAD_SLAB,
1392	} cpuset_filetype_t;	1392	} cpuset_filetype_t;
1393		1393
1394	static int cpuset_write_u64(struct cgroup cgrp, struct cftype cft, u64 val)	1394	static int cpuset_write_u64(struct cgroup cgrp, struct cftype cft, u64 val)
1395	{	1395	{
1396	int retval = 0;	1396	int retval = 0;
1397	struct cpuset *cs = cgroup_cs(cgrp);	1397	struct cpuset *cs = cgroup_cs(cgrp);
1398	cpuset_filetype_t type = cft->private;	1398	cpuset_filetype_t type = cft->private;
1399		1399
1400	if (!cgroup_lock_live_group(cgrp))	1400	if (!cgroup_lock_live_group(cgrp))
1401	return -ENODEV;	1401	return -ENODEV;
1402		1402
1403	switch (type) {	1403	switch (type) {
1404	case FILE_CPU_EXCLUSIVE:	1404	case FILE_CPU_EXCLUSIVE:
1405	retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);	1405	retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
1406	break;	1406	break;
1407	case FILE_MEM_EXCLUSIVE:	1407	case FILE_MEM_EXCLUSIVE:
1408	retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);	1408	retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
1409	break;	1409	break;
1410	case FILE_MEM_HARDWALL:	1410	case FILE_MEM_HARDWALL:
1411	retval = update_flag(CS_MEM_HARDWALL, cs, val);	1411	retval = update_flag(CS_MEM_HARDWALL, cs, val);
1412	break;	1412	break;
1413	case FILE_SCHED_LOAD_BALANCE:	1413	case FILE_SCHED_LOAD_BALANCE:
1414	retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);	1414	retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
1415	break;	1415	break;
1416	case FILE_MEMORY_MIGRATE:	1416	case FILE_MEMORY_MIGRATE:
1417	retval = update_flag(CS_MEMORY_MIGRATE, cs, val);	1417	retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
1418	break;	1418	break;
1419	case FILE_MEMORY_PRESSURE_ENABLED:	1419	case FILE_MEMORY_PRESSURE_ENABLED:
1420	cpuset_memory_pressure_enabled = !!val;	1420	cpuset_memory_pressure_enabled = !!val;
1421	break;	1421	break;
1422	case FILE_MEMORY_PRESSURE:	1422	case FILE_MEMORY_PRESSURE:
1423	retval = -EACCES;	1423	retval = -EACCES;
1424	break;	1424	break;
1425	case FILE_SPREAD_PAGE:	1425	case FILE_SPREAD_PAGE:
1426	retval = update_flag(CS_SPREAD_PAGE, cs, val);	1426	retval = update_flag(CS_SPREAD_PAGE, cs, val);
1427	cs->mems_generation = cpuset_mems_generation++;	1427	cs->mems_generation = cpuset_mems_generation++;
1428	break;	1428	break;
1429	case FILE_SPREAD_SLAB:	1429	case FILE_SPREAD_SLAB:
1430	retval = update_flag(CS_SPREAD_SLAB, cs, val);	1430	retval = update_flag(CS_SPREAD_SLAB, cs, val);
1431	cs->mems_generation = cpuset_mems_generation++;	1431	cs->mems_generation = cpuset_mems_generation++;
1432	break;	1432	break;
1433	default:	1433	default:
1434	retval = -EINVAL;	1434	retval = -EINVAL;
1435	break;	1435	break;
1436	}	1436	}
1437	cgroup_unlock();	1437	cgroup_unlock();
1438	return retval;	1438	return retval;
1439	}	1439	}
1440		1440
1441	static int cpuset_write_s64(struct cgroup cgrp, struct cftype cft, s64 val)	1441	static int cpuset_write_s64(struct cgroup cgrp, struct cftype cft, s64 val)
1442	{	1442	{
1443	int retval = 0;	1443	int retval = 0;
1444	struct cpuset *cs = cgroup_cs(cgrp);	1444	struct cpuset *cs = cgroup_cs(cgrp);
1445	cpuset_filetype_t type = cft->private;	1445	cpuset_filetype_t type = cft->private;
1446		1446
1447	if (!cgroup_lock_live_group(cgrp))	1447	if (!cgroup_lock_live_group(cgrp))
1448	return -ENODEV;	1448	return -ENODEV;
1449		1449
1450	switch (type) {	1450	switch (type) {
1451	case FILE_SCHED_RELAX_DOMAIN_LEVEL:	1451	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1452	retval = update_relax_domain_level(cs, val);	1452	retval = update_relax_domain_level(cs, val);
1453	break;	1453	break;
1454	default:	1454	default:
1455	retval = -EINVAL;	1455	retval = -EINVAL;
1456	break;	1456	break;
1457	}	1457	}
1458	cgroup_unlock();	1458	cgroup_unlock();
1459	return retval;	1459	return retval;
1460	}	1460	}
1461		1461
1462	/*	1462	/*
1463	* Common handling for a write to a "cpus" or "mems" file.	1463	* Common handling for a write to a "cpus" or "mems" file.
1464	*/	1464	*/
1465	static int cpuset_write_resmask(struct cgroup cgrp, struct cftype cft,	1465	static int cpuset_write_resmask(struct cgroup cgrp, struct cftype cft,
1466	const char *buf)	1466	const char *buf)
1467	{	1467	{
1468	int retval = 0;	1468	int retval = 0;
1469	struct cpuset *cs = cgroup_cs(cgrp);	1469	struct cpuset *cs = cgroup_cs(cgrp);
1470	struct cpuset *trialcs;	1470	struct cpuset *trialcs;
1471		1471
1472	if (!cgroup_lock_live_group(cgrp))	1472	if (!cgroup_lock_live_group(cgrp))
1473	return -ENODEV;	1473	return -ENODEV;
1474		1474
1475	trialcs = alloc_trial_cpuset(cs);	1475	trialcs = alloc_trial_cpuset(cs);
1476	if (!trialcs)	1476	if (!trialcs)
1477	return -ENOMEM;	1477	return -ENOMEM;
1478		1478
1479	switch (cft->private) {	1479	switch (cft->private) {
1480	case FILE_CPULIST:	1480	case FILE_CPULIST:
1481	retval = update_cpumask(cs, trialcs, buf);	1481	retval = update_cpumask(cs, trialcs, buf);
1482	break;	1482	break;
1483	case FILE_MEMLIST:	1483	case FILE_MEMLIST:
1484	retval = update_nodemask(cs, trialcs, buf);	1484	retval = update_nodemask(cs, trialcs, buf);
1485	break;	1485	break;
1486	default:	1486	default:
1487	retval = -EINVAL;	1487	retval = -EINVAL;
1488	break;	1488	break;
1489	}	1489	}
1490		1490
1491	free_trial_cpuset(trialcs);	1491	free_trial_cpuset(trialcs);
1492	cgroup_unlock();	1492	cgroup_unlock();
1493	return retval;	1493	return retval;
1494	}	1494	}
1495		1495
1496	/*	1496	/*
1497	* These ascii lists should be read in a single call, by using a user	1497	* These ascii lists should be read in a single call, by using a user
1498	* buffer large enough to hold the entire map. If read in smaller	1498	* buffer large enough to hold the entire map. If read in smaller
1499	* chunks, there is no guarantee of atomicity. Since the display format	1499	* chunks, there is no guarantee of atomicity. Since the display format
1500	* used, list of ranges of sequential numbers, is variable length,	1500	* used, list of ranges of sequential numbers, is variable length,
1501	* and since these maps can change value dynamically, one could read	1501	* and since these maps can change value dynamically, one could read
1502	* gibberish by doing partial reads while a list was changing.	1502	* gibberish by doing partial reads while a list was changing.
1503	* A single large read to a buffer that crosses a page boundary is	1503	* A single large read to a buffer that crosses a page boundary is
1504	* ok, because the result being copied to user land is not recomputed	1504	* ok, because the result being copied to user land is not recomputed
1505	* across a page fault.	1505	* across a page fault.
1506	*/	1506	*/
1507		1507
1508	static int cpuset_sprintf_cpulist(char page, struct cpuset cs)	1508	static int cpuset_sprintf_cpulist(char page, struct cpuset cs)
1509	{	1509	{
1510	int ret;	1510	int ret;
1511		1511
1512	mutex_lock(&callback_mutex);	1512	mutex_lock(&callback_mutex);
1513	ret = cpulist_scnprintf(page, PAGE_SIZE, cs->cpus_allowed);	1513	ret = cpulist_scnprintf(page, PAGE_SIZE, cs->cpus_allowed);
1514	mutex_unlock(&callback_mutex);	1514	mutex_unlock(&callback_mutex);
1515		1515
1516	return ret;	1516	return ret;
1517	}	1517	}
1518		1518
1519	static int cpuset_sprintf_memlist(char page, struct cpuset cs)	1519	static int cpuset_sprintf_memlist(char page, struct cpuset cs)
1520	{	1520	{
1521	nodemask_t mask;	1521	nodemask_t mask;
1522		1522
1523	mutex_lock(&callback_mutex);	1523	mutex_lock(&callback_mutex);
1524	mask = cs->mems_allowed;	1524	mask = cs->mems_allowed;
1525	mutex_unlock(&callback_mutex);	1525	mutex_unlock(&callback_mutex);
1526		1526
1527	return nodelist_scnprintf(page, PAGE_SIZE, mask);	1527	return nodelist_scnprintf(page, PAGE_SIZE, mask);
1528	}	1528	}
1529		1529
1530	static ssize_t cpuset_common_file_read(struct cgroup *cont,	1530	static ssize_t cpuset_common_file_read(struct cgroup *cont,
1531	struct cftype *cft,	1531	struct cftype *cft,
1532	struct file *file,	1532	struct file *file,
1533	char __user *buf,	1533	char __user *buf,
1534	size_t nbytes, loff_t *ppos)	1534	size_t nbytes, loff_t *ppos)
1535	{	1535	{
1536	struct cpuset *cs = cgroup_cs(cont);	1536	struct cpuset *cs = cgroup_cs(cont);
1537	cpuset_filetype_t type = cft->private;	1537	cpuset_filetype_t type = cft->private;
1538	char *page;	1538	char *page;
1539	ssize_t retval = 0;	1539	ssize_t retval = 0;
1540	char *s;	1540	char *s;
1541		1541
1542	if (!(page = (char *)__get_free_page(GFP_TEMPORARY)))	1542	if (!(page = (char *)__get_free_page(GFP_TEMPORARY)))
1543	return -ENOMEM;	1543	return -ENOMEM;
1544		1544
1545	s = page;	1545	s = page;
1546		1546
1547	switch (type) {	1547	switch (type) {
1548	case FILE_CPULIST:	1548	case FILE_CPULIST:
1549	s += cpuset_sprintf_cpulist(s, cs);	1549	s += cpuset_sprintf_cpulist(s, cs);
1550	break;	1550	break;
1551	case FILE_MEMLIST:	1551	case FILE_MEMLIST:
1552	s += cpuset_sprintf_memlist(s, cs);	1552	s += cpuset_sprintf_memlist(s, cs);
1553	break;	1553	break;
1554	default:	1554	default:
1555	retval = -EINVAL;	1555	retval = -EINVAL;
1556	goto out;	1556	goto out;
1557	}	1557	}
1558	*s++ = '\n';	1558	*s++ = '\n';
1559		1559
1560	retval = simple_read_from_buffer(buf, nbytes, ppos, page, s - page);	1560	retval = simple_read_from_buffer(buf, nbytes, ppos, page, s - page);
1561	out:	1561	out:
1562	free_page((unsigned long)page);	1562	free_page((unsigned long)page);
1563	return retval;	1563	return retval;
1564	}	1564	}
1565		1565
1566	static u64 cpuset_read_u64(struct cgroup cont, struct cftype cft)	1566	static u64 cpuset_read_u64(struct cgroup cont, struct cftype cft)
1567	{	1567	{
1568	struct cpuset *cs = cgroup_cs(cont);	1568	struct cpuset *cs = cgroup_cs(cont);
1569	cpuset_filetype_t type = cft->private;	1569	cpuset_filetype_t type = cft->private;
1570	switch (type) {	1570	switch (type) {
1571	case FILE_CPU_EXCLUSIVE:	1571	case FILE_CPU_EXCLUSIVE:
1572	return is_cpu_exclusive(cs);	1572	return is_cpu_exclusive(cs);
1573	case FILE_MEM_EXCLUSIVE:	1573	case FILE_MEM_EXCLUSIVE:
1574	return is_mem_exclusive(cs);	1574	return is_mem_exclusive(cs);
1575	case FILE_MEM_HARDWALL:	1575	case FILE_MEM_HARDWALL:
1576	return is_mem_hardwall(cs);	1576	return is_mem_hardwall(cs);
1577	case FILE_SCHED_LOAD_BALANCE:	1577	case FILE_SCHED_LOAD_BALANCE:
1578	return is_sched_load_balance(cs);	1578	return is_sched_load_balance(cs);
1579	case FILE_MEMORY_MIGRATE:	1579	case FILE_MEMORY_MIGRATE:
1580	return is_memory_migrate(cs);	1580	return is_memory_migrate(cs);
1581	case FILE_MEMORY_PRESSURE_ENABLED:	1581	case FILE_MEMORY_PRESSURE_ENABLED:
1582	return cpuset_memory_pressure_enabled;	1582	return cpuset_memory_pressure_enabled;
1583	case FILE_MEMORY_PRESSURE:	1583	case FILE_MEMORY_PRESSURE:
1584	return fmeter_getrate(&cs->fmeter);	1584	return fmeter_getrate(&cs->fmeter);
1585	case FILE_SPREAD_PAGE:	1585	case FILE_SPREAD_PAGE:
1586	return is_spread_page(cs);	1586	return is_spread_page(cs);
1587	case FILE_SPREAD_SLAB:	1587	case FILE_SPREAD_SLAB:
1588	return is_spread_slab(cs);	1588	return is_spread_slab(cs);
1589	default:	1589	default:
1590	BUG();	1590	BUG();
1591	}	1591	}
1592		1592
1593	/* Unreachable but makes gcc happy */	1593	/* Unreachable but makes gcc happy */
1594	return 0;	1594	return 0;
1595	}	1595	}
1596		1596
1597	static s64 cpuset_read_s64(struct cgroup cont, struct cftype cft)	1597	static s64 cpuset_read_s64(struct cgroup cont, struct cftype cft)
1598	{	1598	{
1599	struct cpuset *cs = cgroup_cs(cont);	1599	struct cpuset *cs = cgroup_cs(cont);
1600	cpuset_filetype_t type = cft->private;	1600	cpuset_filetype_t type = cft->private;
1601	switch (type) {	1601	switch (type) {
1602	case FILE_SCHED_RELAX_DOMAIN_LEVEL:	1602	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1603	return cs->relax_domain_level;	1603	return cs->relax_domain_level;
1604	default:	1604	default:
1605	BUG();	1605	BUG();
1606	}	1606	}
1607		1607
1608	/* Unrechable but makes gcc happy */	1608	/* Unrechable but makes gcc happy */
1609	return 0;	1609	return 0;
1610	}	1610	}
1611		1611
1612		1612
1613	/*	1613	/*
1614	* for the common functions, 'private' gives the type of file	1614	* for the common functions, 'private' gives the type of file
1615	*/	1615	*/
1616		1616
1617	static struct cftype files[] = {	1617	static struct cftype files[] = {
1618	{	1618	{
1619	.name = "cpus",	1619	.name = "cpus",
1620	.read = cpuset_common_file_read,	1620	.read = cpuset_common_file_read,
1621	.write_string = cpuset_write_resmask,	1621	.write_string = cpuset_write_resmask,
1622	.max_write_len = (100U + 6 * NR_CPUS),	1622	.max_write_len = (100U + 6 * NR_CPUS),
1623	.private = FILE_CPULIST,	1623	.private = FILE_CPULIST,
1624	},	1624	},
1625		1625
1626	{	1626	{
1627	.name = "mems",	1627	.name = "mems",
1628	.read = cpuset_common_file_read,	1628	.read = cpuset_common_file_read,
1629	.write_string = cpuset_write_resmask,	1629	.write_string = cpuset_write_resmask,
1630	.max_write_len = (100U + 6 * MAX_NUMNODES),	1630	.max_write_len = (100U + 6 * MAX_NUMNODES),
1631	.private = FILE_MEMLIST,	1631	.private = FILE_MEMLIST,
1632	},	1632	},
1633		1633
1634	{	1634	{
1635	.name = "cpu_exclusive",	1635	.name = "cpu_exclusive",
1636	.read_u64 = cpuset_read_u64,	1636	.read_u64 = cpuset_read_u64,
1637	.write_u64 = cpuset_write_u64,	1637	.write_u64 = cpuset_write_u64,
1638	.private = FILE_CPU_EXCLUSIVE,	1638	.private = FILE_CPU_EXCLUSIVE,
1639	},	1639	},
1640		1640
1641	{	1641	{
1642	.name = "mem_exclusive",	1642	.name = "mem_exclusive",
1643	.read_u64 = cpuset_read_u64,	1643	.read_u64 = cpuset_read_u64,
1644	.write_u64 = cpuset_write_u64,	1644	.write_u64 = cpuset_write_u64,
1645	.private = FILE_MEM_EXCLUSIVE,	1645	.private = FILE_MEM_EXCLUSIVE,
1646	},	1646	},
1647		1647
1648	{	1648	{
1649	.name = "mem_hardwall",	1649	.name = "mem_hardwall",
1650	.read_u64 = cpuset_read_u64,	1650	.read_u64 = cpuset_read_u64,
1651	.write_u64 = cpuset_write_u64,	1651	.write_u64 = cpuset_write_u64,
1652	.private = FILE_MEM_HARDWALL,	1652	.private = FILE_MEM_HARDWALL,
1653	},	1653	},
1654		1654
1655	{	1655	{
1656	.name = "sched_load_balance",	1656	.name = "sched_load_balance",
1657	.read_u64 = cpuset_read_u64,	1657	.read_u64 = cpuset_read_u64,
1658	.write_u64 = cpuset_write_u64,	1658	.write_u64 = cpuset_write_u64,
1659	.private = FILE_SCHED_LOAD_BALANCE,	1659	.private = FILE_SCHED_LOAD_BALANCE,
1660	},	1660	},
1661		1661
1662	{	1662	{
1663	.name = "sched_relax_domain_level",	1663	.name = "sched_relax_domain_level",
1664	.read_s64 = cpuset_read_s64,	1664	.read_s64 = cpuset_read_s64,
1665	.write_s64 = cpuset_write_s64,	1665	.write_s64 = cpuset_write_s64,
1666	.private = FILE_SCHED_RELAX_DOMAIN_LEVEL,	1666	.private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
1667	},	1667	},
1668		1668
1669	{	1669	{
1670	.name = "memory_migrate",	1670	.name = "memory_migrate",
1671	.read_u64 = cpuset_read_u64,	1671	.read_u64 = cpuset_read_u64,
1672	.write_u64 = cpuset_write_u64,	1672	.write_u64 = cpuset_write_u64,
1673	.private = FILE_MEMORY_MIGRATE,	1673	.private = FILE_MEMORY_MIGRATE,
1674	},	1674	},
1675		1675
1676	{	1676	{
1677	.name = "memory_pressure",	1677	.name = "memory_pressure",
1678	.read_u64 = cpuset_read_u64,	1678	.read_u64 = cpuset_read_u64,
1679	.write_u64 = cpuset_write_u64,	1679	.write_u64 = cpuset_write_u64,
1680	.private = FILE_MEMORY_PRESSURE,	1680	.private = FILE_MEMORY_PRESSURE,
1681	.mode = S_IRUGO,	1681	.mode = S_IRUGO,
1682	},	1682	},
1683		1683
1684	{	1684	{
1685	.name = "memory_spread_page",	1685	.name = "memory_spread_page",
1686	.read_u64 = cpuset_read_u64,	1686	.read_u64 = cpuset_read_u64,
1687	.write_u64 = cpuset_write_u64,	1687	.write_u64 = cpuset_write_u64,
1688	.private = FILE_SPREAD_PAGE,	1688	.private = FILE_SPREAD_PAGE,
1689	},	1689	},
1690		1690
1691	{	1691	{
1692	.name = "memory_spread_slab",	1692	.name = "memory_spread_slab",
1693	.read_u64 = cpuset_read_u64,	1693	.read_u64 = cpuset_read_u64,
1694	.write_u64 = cpuset_write_u64,	1694	.write_u64 = cpuset_write_u64,
1695	.private = FILE_SPREAD_SLAB,	1695	.private = FILE_SPREAD_SLAB,
1696	},	1696	},
1697	};	1697	};
1698		1698
1699	static struct cftype cft_memory_pressure_enabled = {	1699	static struct cftype cft_memory_pressure_enabled = {
1700	.name = "memory_pressure_enabled",	1700	.name = "memory_pressure_enabled",
1701	.read_u64 = cpuset_read_u64,	1701	.read_u64 = cpuset_read_u64,
1702	.write_u64 = cpuset_write_u64,	1702	.write_u64 = cpuset_write_u64,
1703	.private = FILE_MEMORY_PRESSURE_ENABLED,	1703	.private = FILE_MEMORY_PRESSURE_ENABLED,
1704	};	1704	};
1705		1705
1706	static int cpuset_populate(struct cgroup_subsys ss, struct cgroup cont)	1706	static int cpuset_populate(struct cgroup_subsys ss, struct cgroup cont)
1707	{	1707	{
1708	int err;	1708	int err;
1709		1709
1710	err = cgroup_add_files(cont, ss, files, ARRAY_SIZE(files));	1710	err = cgroup_add_files(cont, ss, files, ARRAY_SIZE(files));
1711	if (err)	1711	if (err)
1712	return err;	1712	return err;
1713	/* memory_pressure_enabled is in root cpuset only */	1713	/* memory_pressure_enabled is in root cpuset only */
1714	if (!cont->parent)	1714	if (!cont->parent)
1715	err = cgroup_add_file(cont, ss,	1715	err = cgroup_add_file(cont, ss,
1716	&cft_memory_pressure_enabled);	1716	&cft_memory_pressure_enabled);
1717	return err;	1717	return err;
1718	}	1718	}
1719		1719
1720	/*	1720	/*
1721	* post_clone() is called at the end of cgroup_clone().	1721	* post_clone() is called at the end of cgroup_clone().
1722	* 'cgroup' was just created automatically as a result of	1722	* 'cgroup' was just created automatically as a result of
1723	* a cgroup_clone(), and the current task is about to	1723	* a cgroup_clone(), and the current task is about to
1724	* be moved into 'cgroup'.	1724	* be moved into 'cgroup'.
1725	*	1725	*
1726	* Currently we refuse to set up the cgroup - thereby	1726	* Currently we refuse to set up the cgroup - thereby
1727	* refusing the task to be entered, and as a result refusing	1727	* refusing the task to be entered, and as a result refusing
1728	* the sys_unshare() or clone() which initiated it - if any	1728	* the sys_unshare() or clone() which initiated it - if any
1729	* sibling cpusets have exclusive cpus or mem.	1729	* sibling cpusets have exclusive cpus or mem.
1730	*	1730	*
1731	* If this becomes a problem for some users who wish to	1731	* If this becomes a problem for some users who wish to
1732	* allow that scenario, then cpuset_post_clone() could be	1732	* allow that scenario, then cpuset_post_clone() could be
1733	* changed to grant parent->cpus_allowed-sibling_cpus_exclusive	1733	* changed to grant parent->cpus_allowed-sibling_cpus_exclusive
1734	* (and likewise for mems) to the new cgroup. Called with cgroup_mutex	1734	* (and likewise for mems) to the new cgroup. Called with cgroup_mutex
1735	* held.	1735	* held.
1736	*/	1736	*/
1737	static void cpuset_post_clone(struct cgroup_subsys *ss,	1737	static void cpuset_post_clone(struct cgroup_subsys *ss,
1738	struct cgroup *cgroup)	1738	struct cgroup *cgroup)
1739	{	1739	{
1740	struct cgroup parent, child;	1740	struct cgroup parent, child;
1741	struct cpuset cs, parent_cs;	1741	struct cpuset cs, parent_cs;
1742		1742
1743	parent = cgroup->parent;	1743	parent = cgroup->parent;
1744	list_for_each_entry(child, &parent->children, sibling) {	1744	list_for_each_entry(child, &parent->children, sibling) {
1745	cs = cgroup_cs(child);	1745	cs = cgroup_cs(child);
1746	if (is_mem_exclusive(cs) \|\| is_cpu_exclusive(cs))	1746	if (is_mem_exclusive(cs) \|\| is_cpu_exclusive(cs))
1747	return;	1747	return;
1748	}	1748	}
1749	cs = cgroup_cs(cgroup);	1749	cs = cgroup_cs(cgroup);
1750	parent_cs = cgroup_cs(parent);	1750	parent_cs = cgroup_cs(parent);
1751		1751
1752	cs->mems_allowed = parent_cs->mems_allowed;	1752	cs->mems_allowed = parent_cs->mems_allowed;
1753	cpumask_copy(cs->cpus_allowed, parent_cs->cpus_allowed);	1753	cpumask_copy(cs->cpus_allowed, parent_cs->cpus_allowed);
1754	return;	1754	return;
1755	}	1755	}
1756		1756
1757	/*	1757	/*
1758	* cpuset_create - create a cpuset	1758	* cpuset_create - create a cpuset
1759	* ss: cpuset cgroup subsystem	1759	* ss: cpuset cgroup subsystem
1760	* cont: control group that the new cpuset will be part of	1760	* cont: control group that the new cpuset will be part of
1761	*/	1761	*/
1762		1762
1763	static struct cgroup_subsys_state *cpuset_create(	1763	static struct cgroup_subsys_state *cpuset_create(
1764	struct cgroup_subsys *ss,	1764	struct cgroup_subsys *ss,
1765	struct cgroup *cont)	1765	struct cgroup *cont)
1766	{	1766	{
1767	struct cpuset *cs;	1767	struct cpuset *cs;
1768	struct cpuset *parent;	1768	struct cpuset *parent;
1769		1769
1770	if (!cont->parent) {	1770	if (!cont->parent) {
1771	/* This is early initialization for the top cgroup */	1771	/* This is early initialization for the top cgroup */
1772	top_cpuset.mems_generation = cpuset_mems_generation++;	1772	top_cpuset.mems_generation = cpuset_mems_generation++;
1773	return &top_cpuset.css;	1773	return &top_cpuset.css;
1774	}	1774	}
1775	parent = cgroup_cs(cont->parent);	1775	parent = cgroup_cs(cont->parent);
1776	cs = kmalloc(sizeof(*cs), GFP_KERNEL);	1776	cs = kmalloc(sizeof(*cs), GFP_KERNEL);
1777	if (!cs)	1777	if (!cs)
1778	return ERR_PTR(-ENOMEM);	1778	return ERR_PTR(-ENOMEM);
1779	if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) {	1779	if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) {
1780	kfree(cs);	1780	kfree(cs);
1781	return ERR_PTR(-ENOMEM);	1781	return ERR_PTR(-ENOMEM);
1782	}	1782	}
1783		1783
1784	cpuset_update_task_memory_state();	1784	cpuset_update_task_memory_state();
1785	cs->flags = 0;	1785	cs->flags = 0;
1786	if (is_spread_page(parent))	1786	if (is_spread_page(parent))
1787	set_bit(CS_SPREAD_PAGE, &cs->flags);	1787	set_bit(CS_SPREAD_PAGE, &cs->flags);
1788	if (is_spread_slab(parent))	1788	if (is_spread_slab(parent))
1789	set_bit(CS_SPREAD_SLAB, &cs->flags);	1789	set_bit(CS_SPREAD_SLAB, &cs->flags);
1790	set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);	1790	set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
1791	cpumask_clear(cs->cpus_allowed);	1791	cpumask_clear(cs->cpus_allowed);
1792	nodes_clear(cs->mems_allowed);	1792	nodes_clear(cs->mems_allowed);
1793	cs->mems_generation = cpuset_mems_generation++;	1793	cs->mems_generation = cpuset_mems_generation++;
1794	fmeter_init(&cs->fmeter);	1794	fmeter_init(&cs->fmeter);
1795	cs->relax_domain_level = -1;	1795	cs->relax_domain_level = -1;
1796		1796
1797	cs->parent = parent;	1797	cs->parent = parent;
1798	number_of_cpusets++;	1798	number_of_cpusets++;
1799	return &cs->css ;	1799	return &cs->css ;
1800	}	1800	}
1801		1801
1802	/*	1802	/*
1803	* If the cpuset being removed has its flag 'sched_load_balance'	1803	* If the cpuset being removed has its flag 'sched_load_balance'
1804	* enabled, then simulate turning sched_load_balance off, which	1804	* enabled, then simulate turning sched_load_balance off, which
1805	* will call async_rebuild_sched_domains().	1805	* will call async_rebuild_sched_domains().
1806	*/	1806	*/
1807		1807
1808	static void cpuset_destroy(struct cgroup_subsys ss, struct cgroup cont)	1808	static void cpuset_destroy(struct cgroup_subsys ss, struct cgroup cont)
1809	{	1809	{
1810	struct cpuset *cs = cgroup_cs(cont);	1810	struct cpuset *cs = cgroup_cs(cont);
1811		1811
1812	cpuset_update_task_memory_state();	1812	cpuset_update_task_memory_state();
1813		1813
1814	if (is_sched_load_balance(cs))	1814	if (is_sched_load_balance(cs))
1815	update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);	1815	update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
1816		1816
1817	number_of_cpusets--;	1817	number_of_cpusets--;
1818	free_cpumask_var(cs->cpus_allowed);	1818	free_cpumask_var(cs->cpus_allowed);
1819	kfree(cs);	1819	kfree(cs);
1820	}	1820	}
1821		1821
1822	struct cgroup_subsys cpuset_subsys = {	1822	struct cgroup_subsys cpuset_subsys = {
1823	.name = "cpuset",	1823	.name = "cpuset",
1824	.create = cpuset_create,	1824	.create = cpuset_create,
1825	.destroy = cpuset_destroy,	1825	.destroy = cpuset_destroy,
1826	.can_attach = cpuset_can_attach,	1826	.can_attach = cpuset_can_attach,
1827	.attach = cpuset_attach,	1827	.attach = cpuset_attach,
1828	.populate = cpuset_populate,	1828	.populate = cpuset_populate,
1829	.post_clone = cpuset_post_clone,	1829	.post_clone = cpuset_post_clone,
1830	.subsys_id = cpuset_subsys_id,	1830	.subsys_id = cpuset_subsys_id,
1831	.early_init = 1,	1831	.early_init = 1,
1832	};	1832	};
1833		1833
1834	/*	1834	/*
1835	* cpuset_init_early - just enough so that the calls to	1835	* cpuset_init_early - just enough so that the calls to
1836	* cpuset_update_task_memory_state() in early init code	1836	* cpuset_update_task_memory_state() in early init code
1837	* are harmless.	1837	* are harmless.
1838	*/	1838	*/
1839		1839
1840	int __init cpuset_init_early(void)	1840	int __init cpuset_init_early(void)
1841	{	1841	{
1842	alloc_bootmem_cpumask_var(&top_cpuset.cpus_allowed);	1842	alloc_bootmem_cpumask_var(&top_cpuset.cpus_allowed);
1843		1843
1844	top_cpuset.mems_generation = cpuset_mems_generation++;	1844	top_cpuset.mems_generation = cpuset_mems_generation++;
1845	return 0;	1845	return 0;
1846	}	1846	}
1847		1847
1848		1848
1849	/**	1849	/**
1850	* cpuset_init - initialize cpusets at system boot	1850	* cpuset_init - initialize cpusets at system boot
1851	*	1851	*
1852	* Description: Initialize top_cpuset and the cpuset internal file system,	1852	* Description: Initialize top_cpuset and the cpuset internal file system,
1853	**/	1853	**/
1854		1854
1855	int __init cpuset_init(void)	1855	int __init cpuset_init(void)
1856	{	1856	{
1857	int err = 0;	1857	int err = 0;
1858		1858
1859	cpumask_setall(top_cpuset.cpus_allowed);	1859	cpumask_setall(top_cpuset.cpus_allowed);
1860	nodes_setall(top_cpuset.mems_allowed);	1860	nodes_setall(top_cpuset.mems_allowed);
1861		1861
1862	fmeter_init(&top_cpuset.fmeter);	1862	fmeter_init(&top_cpuset.fmeter);
1863	top_cpuset.mems_generation = cpuset_mems_generation++;	1863	top_cpuset.mems_generation = cpuset_mems_generation++;
1864	set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);	1864	set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
1865	top_cpuset.relax_domain_level = -1;	1865	top_cpuset.relax_domain_level = -1;
1866		1866
1867	err = register_filesystem(&cpuset_fs_type);	1867	err = register_filesystem(&cpuset_fs_type);
1868	if (err < 0)	1868	if (err < 0)
1869	return err;	1869	return err;
1870		1870
1871	if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))	1871	if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
1872	BUG();	1872	BUG();
1873		1873
1874	number_of_cpusets = 1;	1874	number_of_cpusets = 1;
1875	return 0;	1875	return 0;
1876	}	1876	}
1877		1877
1878	/**	1878	/**
1879	* cpuset_do_move_task - move a given task to another cpuset	1879	* cpuset_do_move_task - move a given task to another cpuset
1880	* @tsk: pointer to task_struct the task to move	1880	* @tsk: pointer to task_struct the task to move
1881	* @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner	1881	* @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner
1882	*	1882	*
1883	* Called by cgroup_scan_tasks() for each task in a cgroup.	1883	* Called by cgroup_scan_tasks() for each task in a cgroup.
1884	* Return nonzero to stop the walk through the tasks.	1884	* Return nonzero to stop the walk through the tasks.
1885	*/	1885	*/
1886	static void cpuset_do_move_task(struct task_struct *tsk,	1886	static void cpuset_do_move_task(struct task_struct *tsk,
1887	struct cgroup_scanner *scan)	1887	struct cgroup_scanner *scan)
1888	{	1888	{
1889	struct cgroup *new_cgroup = scan->data;	1889	struct cgroup *new_cgroup = scan->data;
1890		1890
1891	cgroup_attach_task(new_cgroup, tsk);	1891	cgroup_attach_task(new_cgroup, tsk);
1892	}	1892	}
1893		1893
1894	/**	1894	/**
1895	* move_member_tasks_to_cpuset - move tasks from one cpuset to another	1895	* move_member_tasks_to_cpuset - move tasks from one cpuset to another
1896	* @from: cpuset in which the tasks currently reside	1896	* @from: cpuset in which the tasks currently reside
1897	* @to: cpuset to which the tasks will be moved	1897	* @to: cpuset to which the tasks will be moved
1898	*	1898	*
1899	* Called with cgroup_mutex held	1899	* Called with cgroup_mutex held
1900	* callback_mutex must not be held, as cpuset_attach() will take it.	1900	* callback_mutex must not be held, as cpuset_attach() will take it.
1901	*	1901	*
1902	* The cgroup_scan_tasks() function will scan all the tasks in a cgroup,	1902	* The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
1903	* calling callback functions for each.	1903	* calling callback functions for each.
1904	*/	1904	*/
1905	static void move_member_tasks_to_cpuset(struct cpuset from, struct cpuset to)	1905	static void move_member_tasks_to_cpuset(struct cpuset from, struct cpuset to)
1906	{	1906	{
1907	struct cgroup_scanner scan;	1907	struct cgroup_scanner scan;
1908		1908
1909	scan.cg = from->css.cgroup;	1909	scan.cg = from->css.cgroup;
1910	scan.test_task = NULL; /* select all tasks in cgroup */	1910	scan.test_task = NULL; /* select all tasks in cgroup */
1911	scan.process_task = cpuset_do_move_task;	1911	scan.process_task = cpuset_do_move_task;
1912	scan.heap = NULL;	1912	scan.heap = NULL;
1913	scan.data = to->css.cgroup;	1913	scan.data = to->css.cgroup;
1914		1914
1915	if (cgroup_scan_tasks(&scan))	1915	if (cgroup_scan_tasks(&scan))
1916	printk(KERN_ERR "move_member_tasks_to_cpuset: "	1916	printk(KERN_ERR "move_member_tasks_to_cpuset: "
1917	"cgroup_scan_tasks failed\n");	1917	"cgroup_scan_tasks failed\n");
1918	}	1918	}
1919		1919
1920	/*	1920	/*
1921	* If CPU and/or memory hotplug handlers, below, unplug any CPUs	1921	* If CPU and/or memory hotplug handlers, below, unplug any CPUs
1922	* or memory nodes, we need to walk over the cpuset hierarchy,	1922	* or memory nodes, we need to walk over the cpuset hierarchy,
1923	* removing that CPU or node from all cpusets. If this removes the	1923	* removing that CPU or node from all cpusets. If this removes the
1924	* last CPU or node from a cpuset, then move the tasks in the empty	1924	* last CPU or node from a cpuset, then move the tasks in the empty
1925	* cpuset to its next-highest non-empty parent.	1925	* cpuset to its next-highest non-empty parent.
1926	*	1926	*
1927	* Called with cgroup_mutex held	1927	* Called with cgroup_mutex held
1928	* callback_mutex must not be held, as cpuset_attach() will take it.	1928	* callback_mutex must not be held, as cpuset_attach() will take it.
1929	*/	1929	*/
1930	static void remove_tasks_in_empty_cpuset(struct cpuset *cs)	1930	static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
1931	{	1931	{
1932	struct cpuset *parent;	1932	struct cpuset *parent;
1933		1933
1934	/*	1934	/*
1935	* The cgroup's css_sets list is in use if there are tasks	1935	* The cgroup's css_sets list is in use if there are tasks
1936	* in the cpuset; the list is empty if there are none;	1936	* in the cpuset; the list is empty if there are none;
1937	* the cs->css.refcnt seems always 0.	1937	* the cs->css.refcnt seems always 0.
1938	*/	1938	*/
1939	if (list_empty(&cs->css.cgroup->css_sets))	1939	if (list_empty(&cs->css.cgroup->css_sets))
1940	return;	1940	return;
1941		1941
1942	/*	1942	/*
1943	* Find its next-highest non-empty parent, (top cpuset	1943	* Find its next-highest non-empty parent, (top cpuset
1944	* has online cpus, so can't be empty).	1944	* has online cpus, so can't be empty).
1945	*/	1945	*/
1946	parent = cs->parent;	1946	parent = cs->parent;
1947	while (cpumask_empty(parent->cpus_allowed) \|\|	1947	while (cpumask_empty(parent->cpus_allowed) \|\|
1948	nodes_empty(parent->mems_allowed))	1948	nodes_empty(parent->mems_allowed))
1949	parent = parent->parent;	1949	parent = parent->parent;
1950		1950
1951	move_member_tasks_to_cpuset(cs, parent);	1951	move_member_tasks_to_cpuset(cs, parent);
1952	}	1952	}
1953		1953
1954	/*	1954	/*
1955	* Walk the specified cpuset subtree and look for empty cpusets.	1955	* Walk the specified cpuset subtree and look for empty cpusets.
1956	* The tasks of such cpuset must be moved to a parent cpuset.	1956	* The tasks of such cpuset must be moved to a parent cpuset.
1957	*	1957	*
1958	* Called with cgroup_mutex held. We take callback_mutex to modify	1958	* Called with cgroup_mutex held. We take callback_mutex to modify
1959	* cpus_allowed and mems_allowed.	1959	* cpus_allowed and mems_allowed.
1960	*	1960	*
1961	* This walk processes the tree from top to bottom, completing one layer	1961	* This walk processes the tree from top to bottom, completing one layer
1962	* before dropping down to the next. It always processes a node before	1962	* before dropping down to the next. It always processes a node before
1963	* any of its children.	1963	* any of its children.
1964	*	1964	*
1965	* For now, since we lack memory hot unplug, we'll never see a cpuset	1965	* For now, since we lack memory hot unplug, we'll never see a cpuset
1966	* that has tasks along with an empty 'mems'. But if we did see such	1966	* that has tasks along with an empty 'mems'. But if we did see such
1967	* a cpuset, we'd handle it just like we do if its 'cpus' was empty.	1967	* a cpuset, we'd handle it just like we do if its 'cpus' was empty.
1968	*/	1968	*/
1969	static void scan_for_empty_cpusets(struct cpuset *root)	1969	static void scan_for_empty_cpusets(struct cpuset *root)
1970	{	1970	{
1971	LIST_HEAD(queue);	1971	LIST_HEAD(queue);
1972	struct cpuset cp; / scans cpusets being updated */	1972	struct cpuset cp; / scans cpusets being updated */
1973	struct cpuset child; / scans child cpusets of cp */	1973	struct cpuset child; / scans child cpusets of cp */
1974	struct cgroup *cont;	1974	struct cgroup *cont;
1975	nodemask_t oldmems;	1975	nodemask_t oldmems;
1976		1976
1977	list_add_tail((struct list_head *)&root->stack_list, &queue);	1977	list_add_tail((struct list_head *)&root->stack_list, &queue);
1978		1978
1979	while (!list_empty(&queue)) {	1979	while (!list_empty(&queue)) {
1980	cp = list_first_entry(&queue, struct cpuset, stack_list);	1980	cp = list_first_entry(&queue, struct cpuset, stack_list);
1981	list_del(queue.next);	1981	list_del(queue.next);
1982	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {	1982	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {
1983	child = cgroup_cs(cont);	1983	child = cgroup_cs(cont);
1984	list_add_tail(&child->stack_list, &queue);	1984	list_add_tail(&child->stack_list, &queue);
1985	}	1985	}
1986		1986
1987	/* Continue past cpusets with all cpus, mems online */	1987	/* Continue past cpusets with all cpus, mems online */
1988	if (cpumask_subset(cp->cpus_allowed, cpu_online_mask) &&	1988	if (cpumask_subset(cp->cpus_allowed, cpu_online_mask) &&
1989	nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY]))	1989	nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY]))
1990	continue;	1990	continue;
1991		1991
1992	oldmems = cp->mems_allowed;	1992	oldmems = cp->mems_allowed;
1993		1993
1994	/* Remove offline cpus and mems from this cpuset. */	1994	/* Remove offline cpus and mems from this cpuset. */
1995	mutex_lock(&callback_mutex);	1995	mutex_lock(&callback_mutex);
1996	cpumask_and(cp->cpus_allowed, cp->cpus_allowed,	1996	cpumask_and(cp->cpus_allowed, cp->cpus_allowed,
1997	cpu_online_mask);	1997	cpu_online_mask);
1998	nodes_and(cp->mems_allowed, cp->mems_allowed,	1998	nodes_and(cp->mems_allowed, cp->mems_allowed,
1999	node_states[N_HIGH_MEMORY]);	1999	node_states[N_HIGH_MEMORY]);
2000	mutex_unlock(&callback_mutex);	2000	mutex_unlock(&callback_mutex);
2001		2001
2002	/* Move tasks from the empty cpuset to a parent */	2002	/* Move tasks from the empty cpuset to a parent */
2003	if (cpumask_empty(cp->cpus_allowed) \|\|	2003	if (cpumask_empty(cp->cpus_allowed) \|\|
2004	nodes_empty(cp->mems_allowed))	2004	nodes_empty(cp->mems_allowed))
2005	remove_tasks_in_empty_cpuset(cp);	2005	remove_tasks_in_empty_cpuset(cp);
2006	else {	2006	else {
2007	update_tasks_cpumask(cp, NULL);	2007	update_tasks_cpumask(cp, NULL);
2008	update_tasks_nodemask(cp, &oldmems, NULL);	2008	update_tasks_nodemask(cp, &oldmems, NULL);
2009	}	2009	}
2010	}	2010	}
2011	}	2011	}
2012		2012
2013	/*	2013	/*
2014	* The top_cpuset tracks what CPUs and Memory Nodes are online,	2014	* The top_cpuset tracks what CPUs and Memory Nodes are online,
2015	* period. This is necessary in order to make cpusets transparent	2015	* period. This is necessary in order to make cpusets transparent
2016	* (of no affect) on systems that are actively using CPU hotplug	2016	* (of no affect) on systems that are actively using CPU hotplug
2017	* but making no active use of cpusets.	2017	* but making no active use of cpusets.
2018	*	2018	*
2019	* This routine ensures that top_cpuset.cpus_allowed tracks	2019	* This routine ensures that top_cpuset.cpus_allowed tracks
2020	* cpu_online_map on each CPU hotplug (cpuhp) event.	2020	* cpu_online_map on each CPU hotplug (cpuhp) event.
2021	*	2021	*
2022	* Called within get_online_cpus(). Needs to call cgroup_lock()	2022	* Called within get_online_cpus(). Needs to call cgroup_lock()
2023	* before calling generate_sched_domains().	2023	* before calling generate_sched_domains().
2024	*/	2024	*/
2025	static int cpuset_track_online_cpus(struct notifier_block *unused_nb,	2025	static int cpuset_track_online_cpus(struct notifier_block *unused_nb,
2026	unsigned long phase, void *unused_cpu)	2026	unsigned long phase, void *unused_cpu)
2027	{	2027	{
2028	struct sched_domain_attr *attr;	2028	struct sched_domain_attr *attr;
2029	struct cpumask *doms;	2029	struct cpumask *doms;
2030	int ndoms;	2030	int ndoms;
2031		2031
2032	switch (phase) {	2032	switch (phase) {
2033	case CPU_ONLINE:	2033	case CPU_ONLINE:
2034	case CPU_ONLINE_FROZEN:	2034	case CPU_ONLINE_FROZEN:
2035	case CPU_DEAD:	2035	case CPU_DEAD:
2036	case CPU_DEAD_FROZEN:	2036	case CPU_DEAD_FROZEN:
2037	break;	2037	break;
2038		2038
2039	default:	2039	default:
2040	return NOTIFY_DONE;	2040	return NOTIFY_DONE;
2041	}	2041	}
2042		2042
2043	cgroup_lock();	2043	cgroup_lock();
2044	mutex_lock(&callback_mutex);	2044	mutex_lock(&callback_mutex);
2045	cpumask_copy(top_cpuset.cpus_allowed, cpu_online_mask);	2045	cpumask_copy(top_cpuset.cpus_allowed, cpu_online_mask);
2046	mutex_unlock(&callback_mutex);	2046	mutex_unlock(&callback_mutex);
2047	scan_for_empty_cpusets(&top_cpuset);	2047	scan_for_empty_cpusets(&top_cpuset);
2048	ndoms = generate_sched_domains(&doms, &attr);	2048	ndoms = generate_sched_domains(&doms, &attr);
2049	cgroup_unlock();	2049	cgroup_unlock();
2050		2050
2051	/* Have scheduler rebuild the domains */	2051	/* Have scheduler rebuild the domains */
2052	partition_sched_domains(ndoms, doms, attr);	2052	partition_sched_domains(ndoms, doms, attr);
2053		2053
2054	return NOTIFY_OK;	2054	return NOTIFY_OK;
2055	}	2055	}
2056		2056
2057	#ifdef CONFIG_MEMORY_HOTPLUG	2057	#ifdef CONFIG_MEMORY_HOTPLUG
2058	/*	2058	/*
2059	* Keep top_cpuset.mems_allowed tracking node_states[N_HIGH_MEMORY].	2059	* Keep top_cpuset.mems_allowed tracking node_states[N_HIGH_MEMORY].
2060	* Call this routine anytime after node_states[N_HIGH_MEMORY] changes.	2060	* Call this routine anytime after node_states[N_HIGH_MEMORY] changes.
2061	* See also the previous routine cpuset_track_online_cpus().	2061	* See also the previous routine cpuset_track_online_cpus().
2062	*/	2062	*/
2063	static int cpuset_track_online_nodes(struct notifier_block *self,	2063	static int cpuset_track_online_nodes(struct notifier_block *self,
2064	unsigned long action, void *arg)	2064	unsigned long action, void *arg)
2065	{	2065	{
2066	cgroup_lock();	2066	cgroup_lock();
2067	switch (action) {	2067	switch (action) {
2068	case MEM_ONLINE:	2068	case MEM_ONLINE:
2069	case MEM_OFFLINE:	2069	case MEM_OFFLINE:
2070	mutex_lock(&callback_mutex);	2070	mutex_lock(&callback_mutex);
2071	top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];	2071	top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
2072	mutex_unlock(&callback_mutex);	2072	mutex_unlock(&callback_mutex);
2073	if (action == MEM_OFFLINE)	2073	if (action == MEM_OFFLINE)
2074	scan_for_empty_cpusets(&top_cpuset);	2074	scan_for_empty_cpusets(&top_cpuset);
2075	break;	2075	break;
2076	default:	2076	default:
2077	break;	2077	break;
2078	}	2078	}
2079	cgroup_unlock();	2079	cgroup_unlock();
2080	return NOTIFY_OK;	2080	return NOTIFY_OK;
2081	}	2081	}
2082	#endif	2082	#endif
2083		2083
2084	/**	2084	/**
2085	* cpuset_init_smp - initialize cpus_allowed	2085	* cpuset_init_smp - initialize cpus_allowed
2086	*	2086	*
2087	* Description: Finish top cpuset after cpu, node maps are initialized	2087	* Description: Finish top cpuset after cpu, node maps are initialized
2088	**/	2088	**/
2089		2089
2090	void __init cpuset_init_smp(void)	2090	void __init cpuset_init_smp(void)
2091	{	2091	{
2092	cpumask_copy(top_cpuset.cpus_allowed, cpu_online_mask);	2092	cpumask_copy(top_cpuset.cpus_allowed, cpu_online_mask);
2093	top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];	2093	top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
2094		2094
2095	hotcpu_notifier(cpuset_track_online_cpus, 0);	2095	hotcpu_notifier(cpuset_track_online_cpus, 0);
2096	hotplug_memory_notifier(cpuset_track_online_nodes, 10);	2096	hotplug_memory_notifier(cpuset_track_online_nodes, 10);
2097		2097
2098	cpuset_wq = create_singlethread_workqueue("cpuset");	2098	cpuset_wq = create_singlethread_workqueue("cpuset");
2099	BUG_ON(!cpuset_wq);	2099	BUG_ON(!cpuset_wq);
2100	}	2100	}
2101		2101
2102	/**	2102	/**
2103	* cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.	2103	* cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
2104	* @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.	2104	* @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
2105	* @pmask: pointer to struct cpumask variable to receive cpus_allowed set.	2105	* @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
2106	*	2106	*
2107	* Description: Returns the cpumask_var_t cpus_allowed of the cpuset	2107	* Description: Returns the cpumask_var_t cpus_allowed of the cpuset
2108	* attached to the specified @tsk. Guaranteed to return some non-empty	2108	* attached to the specified @tsk. Guaranteed to return some non-empty
2109	* subset of cpu_online_map, even if this means going outside the	2109	* subset of cpu_online_map, even if this means going outside the
2110	* tasks cpuset.	2110	* tasks cpuset.
2111	**/	2111	**/
2112		2112
2113	void cpuset_cpus_allowed(struct task_struct tsk, struct cpumask pmask)	2113	void cpuset_cpus_allowed(struct task_struct tsk, struct cpumask pmask)
2114	{	2114	{
2115	mutex_lock(&callback_mutex);	2115	mutex_lock(&callback_mutex);
2116	cpuset_cpus_allowed_locked(tsk, pmask);	2116	cpuset_cpus_allowed_locked(tsk, pmask);
2117	mutex_unlock(&callback_mutex);	2117	mutex_unlock(&callback_mutex);
2118	}	2118	}
2119		2119
2120	/**	2120	/**
2121	* cpuset_cpus_allowed_locked - return cpus_allowed mask from a tasks cpuset.	2121	* cpuset_cpus_allowed_locked - return cpus_allowed mask from a tasks cpuset.
2122	* Must be called with callback_mutex held.	2122	* Must be called with callback_mutex held.
2123	**/	2123	**/
2124	void cpuset_cpus_allowed_locked(struct task_struct tsk, struct cpumask pmask)	2124	void cpuset_cpus_allowed_locked(struct task_struct tsk, struct cpumask pmask)
2125	{	2125	{
2126	task_lock(tsk);	2126	task_lock(tsk);
2127	guarantee_online_cpus(task_cs(tsk), pmask);	2127	guarantee_online_cpus(task_cs(tsk), pmask);
2128	task_unlock(tsk);	2128	task_unlock(tsk);
2129	}	2129	}
2130		2130
2131	void cpuset_init_current_mems_allowed(void)	2131	void cpuset_init_current_mems_allowed(void)
2132	{	2132	{
2133	nodes_setall(current->mems_allowed);	2133	nodes_setall(current->mems_allowed);
2134	}	2134	}
2135		2135
2136	/**	2136	/**
2137	* cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.	2137	* cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
2138	* @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.	2138	* @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
2139	*	2139	*
2140	* Description: Returns the nodemask_t mems_allowed of the cpuset	2140	* Description: Returns the nodemask_t mems_allowed of the cpuset
2141	* attached to the specified @tsk. Guaranteed to return some non-empty	2141	* attached to the specified @tsk. Guaranteed to return some non-empty
2142	* subset of node_states[N_HIGH_MEMORY], even if this means going outside the	2142	* subset of node_states[N_HIGH_MEMORY], even if this means going outside the
2143	* tasks cpuset.	2143	* tasks cpuset.
2144	**/	2144	**/
2145		2145
2146	nodemask_t cpuset_mems_allowed(struct task_struct *tsk)	2146	nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
2147	{	2147	{
2148	nodemask_t mask;	2148	nodemask_t mask;
2149		2149
2150	mutex_lock(&callback_mutex);	2150	mutex_lock(&callback_mutex);
2151	task_lock(tsk);	2151	task_lock(tsk);
2152	guarantee_online_mems(task_cs(tsk), &mask);	2152	guarantee_online_mems(task_cs(tsk), &mask);
2153	task_unlock(tsk);	2153	task_unlock(tsk);
2154	mutex_unlock(&callback_mutex);	2154	mutex_unlock(&callback_mutex);
2155		2155
2156	return mask;	2156	return mask;
2157	}	2157	}
2158		2158
2159	/**	2159	/**
2160	* cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed	2160	* cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
2161	* @nodemask: the nodemask to be checked	2161	* @nodemask: the nodemask to be checked
2162	*	2162	*
2163	* Are any of the nodes in the nodemask allowed in current->mems_allowed?	2163	* Are any of the nodes in the nodemask allowed in current->mems_allowed?
2164	*/	2164	*/
2165	int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)	2165	int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
2166	{	2166	{
2167	return nodes_intersects(*nodemask, current->mems_allowed);	2167	return nodes_intersects(*nodemask, current->mems_allowed);
2168	}	2168	}
2169		2169
2170	/*	2170	/*
2171	* nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or	2171	* nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
2172	* mem_hardwall ancestor to the specified cpuset. Call holding	2172	* mem_hardwall ancestor to the specified cpuset. Call holding
2173	* callback_mutex. If no ancestor is mem_exclusive or mem_hardwall	2173	* callback_mutex. If no ancestor is mem_exclusive or mem_hardwall
2174	* (an unusual configuration), then returns the root cpuset.	2174	* (an unusual configuration), then returns the root cpuset.
2175	*/	2175	*/
2176	static const struct cpuset nearest_hardwall_ancestor(const struct cpuset cs)	2176	static const struct cpuset nearest_hardwall_ancestor(const struct cpuset cs)
2177	{	2177	{
2178	while (!(is_mem_exclusive(cs) \|\| is_mem_hardwall(cs)) && cs->parent)	2178	while (!(is_mem_exclusive(cs) \|\| is_mem_hardwall(cs)) && cs->parent)
2179	cs = cs->parent;	2179	cs = cs->parent;
2180	return cs;	2180	return cs;
2181	}	2181	}
2182		2182
2183	/**	2183	/**
2184	* cpuset_zone_allowed_softwall - Can we allocate on zone z's memory node?	2184	* cpuset_node_allowed_softwall - Can we allocate on a memory node?
2185	* @z: is this zone on an allowed node?	2185	* @node: is this an allowed node?
2186	* @gfp_mask: memory allocation flags	2186	* @gfp_mask: memory allocation flags
2187	*	2187	*
2188	* If we're in interrupt, yes, we can always allocate. If	2188	* If we're in interrupt, yes, we can always allocate. If __GFP_THISNODE is
2189	* __GFP_THISNODE is set, yes, we can always allocate. If zone	2189	* set, yes, we can always allocate. If node is in our task's mems_allowed,
2190	* z's node is in our tasks mems_allowed, yes. If it's not a	2190	* yes. If it's not a __GFP_HARDWALL request and this node is in the nearest
2191	* __GFP_HARDWALL request and this zone's nodes is in the nearest	2191	* hardwalled cpuset ancestor to this task's cpuset, yes. If the task has been
2192	* hardwalled cpuset ancestor to this tasks cpuset, yes.	2192	* OOM killed and has access to memory reserves as specified by the TIF_MEMDIE
2193	* If the task has been OOM killed and has access to memory reserves	2193	* flag, yes.
2194	* as specified by the TIF_MEMDIE flag, yes.
2195	* Otherwise, no.	2194	* Otherwise, no.
2196	*	2195	*
2197	* If __GFP_HARDWALL is set, cpuset_zone_allowed_softwall()	2196	* If __GFP_HARDWALL is set, cpuset_node_allowed_softwall() reduces to
2198	* reduces to cpuset_zone_allowed_hardwall(). Otherwise,	2197	* cpuset_node_allowed_hardwall(). Otherwise, cpuset_node_allowed_softwall()
2199	* cpuset_zone_allowed_softwall() might sleep, and might allow a zone	2198	* might sleep, and might allow a node from an enclosing cpuset.
2200	* from an enclosing cpuset.
2201	*	2199	*
2202	* cpuset_zone_allowed_hardwall() only handles the simpler case of	2200	* cpuset_node_allowed_hardwall() only handles the simpler case of hardwall
2203	* hardwall cpusets, and never sleeps.	2201	* cpusets, and never sleeps.
2204	*	2202	*
2205	* The __GFP_THISNODE placement logic is really handled elsewhere,	2203	* The __GFP_THISNODE placement logic is really handled elsewhere,
2206	* by forcibly using a zonelist starting at a specified node, and by	2204	* by forcibly using a zonelist starting at a specified node, and by
2207	* (in get_page_from_freelist()) refusing to consider the zones for	2205	* (in get_page_from_freelist()) refusing to consider the zones for
2208	* any node on the zonelist except the first. By the time any such	2206	* any node on the zonelist except the first. By the time any such
2209	* calls get to this routine, we should just shut up and say 'yes'.	2207	* calls get to this routine, we should just shut up and say 'yes'.
2210	*	2208	*
2211	* GFP_USER allocations are marked with the __GFP_HARDWALL bit,	2209	* GFP_USER allocations are marked with the __GFP_HARDWALL bit,
2212	* and do not allow allocations outside the current tasks cpuset	2210	* and do not allow allocations outside the current tasks cpuset
2213	* unless the task has been OOM killed as is marked TIF_MEMDIE.	2211	* unless the task has been OOM killed as is marked TIF_MEMDIE.
2214	* GFP_KERNEL allocations are not so marked, so can escape to the	2212	* GFP_KERNEL allocations are not so marked, so can escape to the
2215	* nearest enclosing hardwalled ancestor cpuset.	2213	* nearest enclosing hardwalled ancestor cpuset.
2216	*	2214	*
2217	* Scanning up parent cpusets requires callback_mutex. The	2215	* Scanning up parent cpusets requires callback_mutex. The
2218	* __alloc_pages() routine only calls here with __GFP_HARDWALL bit	2216	* __alloc_pages() routine only calls here with __GFP_HARDWALL bit
2219	* _not_ set if it's a GFP_KERNEL allocation, and all nodes in the	2217	* _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
2220	* current tasks mems_allowed came up empty on the first pass over	2218	* current tasks mems_allowed came up empty on the first pass over
2221	* the zonelist. So only GFP_KERNEL allocations, if all nodes in the	2219	* the zonelist. So only GFP_KERNEL allocations, if all nodes in the
2222	* cpuset are short of memory, might require taking the callback_mutex	2220	* cpuset are short of memory, might require taking the callback_mutex
2223	* mutex.	2221	* mutex.
2224	*	2222	*
2225	* The first call here from mm/page_alloc:get_page_from_freelist()	2223	* The first call here from mm/page_alloc:get_page_from_freelist()
2226	* has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,	2224	* has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
2227	* so no allocation on a node outside the cpuset is allowed (unless	2225	* so no allocation on a node outside the cpuset is allowed (unless
2228	* in interrupt, of course).	2226	* in interrupt, of course).
2229	*	2227	*
2230	* The second pass through get_page_from_freelist() doesn't even call	2228	* The second pass through get_page_from_freelist() doesn't even call
2231	* here for GFP_ATOMIC calls. For those calls, the __alloc_pages()	2229	* here for GFP_ATOMIC calls. For those calls, the __alloc_pages()
2232	* variable 'wait' is not set, and the bit ALLOC_CPUSET is not set	2230	* variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
2233	* in alloc_flags. That logic and the checks below have the combined	2231	* in alloc_flags. That logic and the checks below have the combined
2234	* affect that:	2232	* affect that:
2235	* in_interrupt - any node ok (current task context irrelevant)	2233	* in_interrupt - any node ok (current task context irrelevant)
2236	* GFP_ATOMIC - any node ok	2234	* GFP_ATOMIC - any node ok
2237	* TIF_MEMDIE - any node ok	2235	* TIF_MEMDIE - any node ok
2238	* GFP_KERNEL - any node in enclosing hardwalled cpuset ok	2236	* GFP_KERNEL - any node in enclosing hardwalled cpuset ok
2239	* GFP_USER - only nodes in current tasks mems allowed ok.	2237	* GFP_USER - only nodes in current tasks mems allowed ok.
2240	*	2238	*
2241	* Rule:	2239	* Rule:
2242	* Don't call cpuset_zone_allowed_softwall if you can't sleep, unless you	2240	* Don't call cpuset_node_allowed_softwall if you can't sleep, unless you
2243	* pass in the __GFP_HARDWALL flag set in gfp_flag, which disables	2241	* pass in the __GFP_HARDWALL flag set in gfp_flag, which disables
2244	* the code that might scan up ancestor cpusets and sleep.	2242	* the code that might scan up ancestor cpusets and sleep.
2245	*/	2243	*/
2246		2244	int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
2247	int __cpuset_zone_allowed_softwall(struct zone *z, gfp_t gfp_mask)
2248	{	2245	{
2249	int node; /* node that zone z is on */
2250	const struct cpuset cs; / current cpuset ancestors */	2246	const struct cpuset cs; / current cpuset ancestors */
2251	int allowed; /* is allocation in zone z allowed? */	2247	int allowed; /* is allocation in zone z allowed? */
2252		2248
2253	if (in_interrupt() \|\| (gfp_mask & __GFP_THISNODE))	2249	if (in_interrupt() \|\| (gfp_mask & __GFP_THISNODE))
2254	return 1;	2250	return 1;
2255	node = zone_to_nid(z);
2256	might_sleep_if(!(gfp_mask & __GFP_HARDWALL));	2251	might_sleep_if(!(gfp_mask & __GFP_HARDWALL));
2257	if (node_isset(node, current->mems_allowed))	2252	if (node_isset(node, current->mems_allowed))
2258	return 1;	2253	return 1;
2259	/*	2254	/*
2260	* Allow tasks that have access to memory reserves because they have	2255	* Allow tasks that have access to memory reserves because they have
2261	* been OOM killed to get memory anywhere.	2256	* been OOM killed to get memory anywhere.
2262	*/	2257	*/
2263	if (unlikely(test_thread_flag(TIF_MEMDIE)))	2258	if (unlikely(test_thread_flag(TIF_MEMDIE)))
2264	return 1;	2259	return 1;
2265	if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */	2260	if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
2266	return 0;	2261	return 0;
2267		2262
2268	if (current->flags & PF_EXITING) /* Let dying task have memory */	2263	if (current->flags & PF_EXITING) /* Let dying task have memory */
2269	return 1;	2264	return 1;
2270		2265
2271	/* Not hardwall and node outside mems_allowed: scan up cpusets */	2266	/* Not hardwall and node outside mems_allowed: scan up cpusets */
2272	mutex_lock(&callback_mutex);	2267	mutex_lock(&callback_mutex);
2273		2268
2274	task_lock(current);	2269	task_lock(current);
2275	cs = nearest_hardwall_ancestor(task_cs(current));	2270	cs = nearest_hardwall_ancestor(task_cs(current));
2276	task_unlock(current);	2271	task_unlock(current);
2277		2272
2278	allowed = node_isset(node, cs->mems_allowed);	2273	allowed = node_isset(node, cs->mems_allowed);
2279	mutex_unlock(&callback_mutex);	2274	mutex_unlock(&callback_mutex);
2280	return allowed;	2275	return allowed;
2281	}	2276	}
2282		2277
2283	/*	2278	/*
2284	* cpuset_zone_allowed_hardwall - Can we allocate on zone z's memory node?	2279	* cpuset_node_allowed_hardwall - Can we allocate on a memory node?
2285	* @z: is this zone on an allowed node?	2280	* @node: is this an allowed node?
2286	* @gfp_mask: memory allocation flags	2281	* @gfp_mask: memory allocation flags
2287	*	2282	*
2288	* If we're in interrupt, yes, we can always allocate.	2283	* If we're in interrupt, yes, we can always allocate. If __GFP_THISNODE is
2289	* If __GFP_THISNODE is set, yes, we can always allocate. If zone	2284	* set, yes, we can always allocate. If node is in our task's mems_allowed,
2290	* z's node is in our tasks mems_allowed, yes. If the task has been	2285	* yes. If the task has been OOM killed and has access to memory reserves as
2291	* OOM killed and has access to memory reserves as specified by the	2286	* specified by the TIF_MEMDIE flag, yes.
2292	* TIF_MEMDIE flag, yes. Otherwise, no.	2287	* Otherwise, no.
2293	*	2288	*
2294	* The __GFP_THISNODE placement logic is really handled elsewhere,	2289	* The __GFP_THISNODE placement logic is really handled elsewhere,
2295	* by forcibly using a zonelist starting at a specified node, and by	2290	* by forcibly using a zonelist starting at a specified node, and by
2296	* (in get_page_from_freelist()) refusing to consider the zones for	2291	* (in get_page_from_freelist()) refusing to consider the zones for
2297	* any node on the zonelist except the first. By the time any such	2292	* any node on the zonelist except the first. By the time any such
2298	* calls get to this routine, we should just shut up and say 'yes'.	2293	* calls get to this routine, we should just shut up and say 'yes'.
2299	*	2294	*
2300	* Unlike the cpuset_zone_allowed_softwall() variant, above,	2295	* Unlike the cpuset_node_allowed_softwall() variant, above,
2301	* this variant requires that the zone be in the current tasks	2296	* this variant requires that the node be in the current task's
2302	* mems_allowed or that we're in interrupt. It does not scan up the	2297	* mems_allowed or that we're in interrupt. It does not scan up the
2303	* cpuset hierarchy for the nearest enclosing mem_exclusive cpuset.	2298	* cpuset hierarchy for the nearest enclosing mem_exclusive cpuset.
2304	* It never sleeps.	2299	* It never sleeps.
2305	*/	2300	*/
2306		2301	int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
2307	int __cpuset_zone_allowed_hardwall(struct zone *z, gfp_t gfp_mask)
2308	{	2302	{
2309	int node; /* node that zone z is on */
2310
2311	if (in_interrupt() \|\| (gfp_mask & __GFP_THISNODE))	2303	if (in_interrupt() \|\| (gfp_mask & __GFP_THISNODE))
2312	return 1;	2304	return 1;
2313	node = zone_to_nid(z);
2314	if (node_isset(node, current->mems_allowed))	2305	if (node_isset(node, current->mems_allowed))
2315	return 1;	2306	return 1;
2316	/*	2307	/*
2317	* Allow tasks that have access to memory reserves because they have	2308	* Allow tasks that have access to memory reserves because they have
2318	* been OOM killed to get memory anywhere.	2309	* been OOM killed to get memory anywhere.
2319	*/	2310	*/
2320	if (unlikely(test_thread_flag(TIF_MEMDIE)))	2311	if (unlikely(test_thread_flag(TIF_MEMDIE)))
2321	return 1;	2312	return 1;
2322	return 0;	2313	return 0;
2323	}	2314	}
2324		2315
2325	/**	2316	/**
2326	* cpuset_lock - lock out any changes to cpuset structures	2317	* cpuset_lock - lock out any changes to cpuset structures
2327	*	2318	*
2328	* The out of memory (oom) code needs to mutex_lock cpusets	2319	* The out of memory (oom) code needs to mutex_lock cpusets
2329	* from being changed while it scans the tasklist looking for a	2320	* from being changed while it scans the tasklist looking for a
2330	* task in an overlapping cpuset. Expose callback_mutex via this	2321	* task in an overlapping cpuset. Expose callback_mutex via this
2331	* cpuset_lock() routine, so the oom code can lock it, before	2322	* cpuset_lock() routine, so the oom code can lock it, before
2332	* locking the task list. The tasklist_lock is a spinlock, so	2323	* locking the task list. The tasklist_lock is a spinlock, so
2333	* must be taken inside callback_mutex.	2324	* must be taken inside callback_mutex.
2334	*/	2325	*/
2335		2326
2336	void cpuset_lock(void)	2327	void cpuset_lock(void)
2337	{	2328	{
2338	mutex_lock(&callback_mutex);	2329	mutex_lock(&callback_mutex);
2339	}	2330	}
2340		2331
2341	/**	2332	/**
2342	* cpuset_unlock - release lock on cpuset changes	2333	* cpuset_unlock - release lock on cpuset changes
2343	*	2334	*
2344	* Undo the lock taken in a previous cpuset_lock() call.	2335	* Undo the lock taken in a previous cpuset_lock() call.
2345	*/	2336	*/
2346		2337
2347	void cpuset_unlock(void)	2338	void cpuset_unlock(void)
2348	{	2339	{
2349	mutex_unlock(&callback_mutex);	2340	mutex_unlock(&callback_mutex);
2350	}	2341	}
2351		2342
2352	/**	2343	/**
2353	* cpuset_mem_spread_node() - On which node to begin search for a page	2344	* cpuset_mem_spread_node() - On which node to begin search for a page
2354	*	2345	*
2355	* If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for	2346	* If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
2356	* tasks in a cpuset with is_spread_page or is_spread_slab set),	2347	* tasks in a cpuset with is_spread_page or is_spread_slab set),
2357	* and if the memory allocation used cpuset_mem_spread_node()	2348	* and if the memory allocation used cpuset_mem_spread_node()
2358	* to determine on which node to start looking, as it will for	2349	* to determine on which node to start looking, as it will for
2359	* certain page cache or slab cache pages such as used for file	2350	* certain page cache or slab cache pages such as used for file
2360	* system buffers and inode caches, then instead of starting on the	2351	* system buffers and inode caches, then instead of starting on the
2361	* local node to look for a free page, rather spread the starting	2352	* local node to look for a free page, rather spread the starting
2362	* node around the tasks mems_allowed nodes.	2353	* node around the tasks mems_allowed nodes.
2363	*	2354	*
2364	* We don't have to worry about the returned node being offline	2355	* We don't have to worry about the returned node being offline
2365	* because "it can't happen", and even if it did, it would be ok.	2356	* because "it can't happen", and even if it did, it would be ok.
2366	*	2357	*
2367	* The routines calling guarantee_online_mems() are careful to	2358	* The routines calling guarantee_online_mems() are careful to
2368	* only set nodes in task->mems_allowed that are online. So it	2359	* only set nodes in task->mems_allowed that are online. So it
2369	* should not be possible for the following code to return an	2360	* should not be possible for the following code to return an
2370	* offline node. But if it did, that would be ok, as this routine	2361	* offline node. But if it did, that would be ok, as this routine
2371	* is not returning the node where the allocation must be, only	2362	* is not returning the node where the allocation must be, only
2372	* the node where the search should start. The zonelist passed to	2363	* the node where the search should start. The zonelist passed to
2373	* __alloc_pages() will include all nodes. If the slab allocator	2364	* __alloc_pages() will include all nodes. If the slab allocator
2374	* is passed an offline node, it will fall back to the local node.	2365	* is passed an offline node, it will fall back to the local node.
2375	* See kmem_cache_alloc_node().	2366	* See kmem_cache_alloc_node().
2376	*/	2367	*/
2377		2368
2378	int cpuset_mem_spread_node(void)	2369	int cpuset_mem_spread_node(void)
2379	{	2370	{
2380	int node;	2371	int node;
2381		2372
2382	node = next_node(current->cpuset_mem_spread_rotor, current->mems_allowed);	2373	node = next_node(current->cpuset_mem_spread_rotor, current->mems_allowed);
2383	if (node == MAX_NUMNODES)	2374	if (node == MAX_NUMNODES)
2384	node = first_node(current->mems_allowed);	2375	node = first_node(current->mems_allowed);
2385	current->cpuset_mem_spread_rotor = node;	2376	current->cpuset_mem_spread_rotor = node;
2386	return node;	2377	return node;
2387	}	2378	}
2388	EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);	2379	EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);
2389		2380
2390	/**	2381	/**
2391	* cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?	2382	* cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
2392	* @tsk1: pointer to task_struct of some task.	2383	* @tsk1: pointer to task_struct of some task.
2393	* @tsk2: pointer to task_struct of some other task.	2384	* @tsk2: pointer to task_struct of some other task.
2394	*	2385	*
2395	* Description: Return true if @tsk1's mems_allowed intersects the	2386	* Description: Return true if @tsk1's mems_allowed intersects the
2396	* mems_allowed of @tsk2. Used by the OOM killer to determine if	2387	* mems_allowed of @tsk2. Used by the OOM killer to determine if
2397	* one of the task's memory usage might impact the memory available	2388	* one of the task's memory usage might impact the memory available
2398	* to the other.	2389	* to the other.
2399	**/	2390	**/
2400		2391
2401	int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,	2392	int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
2402	const struct task_struct *tsk2)	2393	const struct task_struct *tsk2)
2403	{	2394	{
2404	return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);	2395	return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
2405	}	2396	}
2406		2397
2407	/**	2398	/**
2408	* cpuset_print_task_mems_allowed - prints task's cpuset and mems_allowed	2399	* cpuset_print_task_mems_allowed - prints task's cpuset and mems_allowed
2409	* @task: pointer to task_struct of some task.	2400	* @task: pointer to task_struct of some task.
2410	*	2401	*
2411	* Description: Prints @task's name, cpuset name, and cached copy of its	2402	* Description: Prints @task's name, cpuset name, and cached copy of its
2412	* mems_allowed to the kernel log. Must hold task_lock(task) to allow	2403	* mems_allowed to the kernel log. Must hold task_lock(task) to allow
2413	* dereferencing task_cs(task).	2404	* dereferencing task_cs(task).
2414	*/	2405	*/
2415	void cpuset_print_task_mems_allowed(struct task_struct *tsk)	2406	void cpuset_print_task_mems_allowed(struct task_struct *tsk)
2416	{	2407	{
2417	struct dentry *dentry;	2408	struct dentry *dentry;
2418		2409
2419	dentry = task_cs(tsk)->css.cgroup->dentry;	2410	dentry = task_cs(tsk)->css.cgroup->dentry;
2420	spin_lock(&cpuset_buffer_lock);	2411	spin_lock(&cpuset_buffer_lock);
2421	snprintf(cpuset_name, CPUSET_NAME_LEN,	2412	snprintf(cpuset_name, CPUSET_NAME_LEN,
2422	dentry ? (const char *)dentry->d_name.name : "/");	2413	dentry ? (const char *)dentry->d_name.name : "/");
2423	nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,	2414	nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,
2424	tsk->mems_allowed);	2415	tsk->mems_allowed);
2425	printk(KERN_INFO "%s cpuset=%s mems_allowed=%s\n",	2416	printk(KERN_INFO "%s cpuset=%s mems_allowed=%s\n",
2426	tsk->comm, cpuset_name, cpuset_nodelist);	2417	tsk->comm, cpuset_name, cpuset_nodelist);
2427	spin_unlock(&cpuset_buffer_lock);	2418	spin_unlock(&cpuset_buffer_lock);
2428	}	2419	}
2429		2420
2430	/*	2421	/*
2431	* Collection of memory_pressure is suppressed unless	2422	* Collection of memory_pressure is suppressed unless
2432	* this flag is enabled by writing "1" to the special	2423	* this flag is enabled by writing "1" to the special
2433	* cpuset file 'memory_pressure_enabled' in the root cpuset.	2424	* cpuset file 'memory_pressure_enabled' in the root cpuset.
2434	*/	2425	*/
2435		2426
2436	int cpuset_memory_pressure_enabled __read_mostly;	2427	int cpuset_memory_pressure_enabled __read_mostly;
2437		2428
2438	/**	2429	/**
2439	* cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.	2430	* cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
2440	*	2431	*
2441	* Keep a running average of the rate of synchronous (direct)	2432	* Keep a running average of the rate of synchronous (direct)
2442	* page reclaim efforts initiated by tasks in each cpuset.	2433	* page reclaim efforts initiated by tasks in each cpuset.
2443	*	2434	*
2444	* This represents the rate at which some task in the cpuset	2435	* This represents the rate at which some task in the cpuset
2445	* ran low on memory on all nodes it was allowed to use, and	2436	* ran low on memory on all nodes it was allowed to use, and
2446	* had to enter the kernels page reclaim code in an effort to	2437	* had to enter the kernels page reclaim code in an effort to
2447	* create more free memory by tossing clean pages or swapping	2438	* create more free memory by tossing clean pages or swapping
2448	* or writing dirty pages.	2439	* or writing dirty pages.
2449	*	2440	*
2450	* Display to user space in the per-cpuset read-only file	2441	* Display to user space in the per-cpuset read-only file
2451	* "memory_pressure". Value displayed is an integer	2442	* "memory_pressure". Value displayed is an integer
2452	* representing the recent rate of entry into the synchronous	2443	* representing the recent rate of entry into the synchronous
2453	* (direct) page reclaim by any task attached to the cpuset.	2444	* (direct) page reclaim by any task attached to the cpuset.
2454	**/	2445	**/
2455		2446
2456	void __cpuset_memory_pressure_bump(void)	2447	void __cpuset_memory_pressure_bump(void)
2457	{	2448	{
2458	task_lock(current);	2449	task_lock(current);
2459	fmeter_markevent(&task_cs(current)->fmeter);	2450	fmeter_markevent(&task_cs(current)->fmeter);
2460	task_unlock(current);	2451	task_unlock(current);
2461	}	2452	}
2462		2453
2463	#ifdef CONFIG_PROC_PID_CPUSET	2454	#ifdef CONFIG_PROC_PID_CPUSET
2464	/*	2455	/*
2465	* proc_cpuset_show()	2456	* proc_cpuset_show()
2466	* - Print tasks cpuset path into seq_file.	2457	* - Print tasks cpuset path into seq_file.
2467	* - Used for /proc/<pid>/cpuset.	2458	* - Used for /proc/<pid>/cpuset.
2468	* - No need to task_lock(tsk) on this tsk->cpuset reference, as it	2459	* - No need to task_lock(tsk) on this tsk->cpuset reference, as it
2469	* doesn't really matter if tsk->cpuset changes after we read it,	2460	* doesn't really matter if tsk->cpuset changes after we read it,
2470	* and we take cgroup_mutex, keeping cpuset_attach() from changing it	2461	* and we take cgroup_mutex, keeping cpuset_attach() from changing it
2471	* anyway.	2462	* anyway.
2472	*/	2463	*/
2473	static int proc_cpuset_show(struct seq_file m, void unused_v)	2464	static int proc_cpuset_show(struct seq_file m, void unused_v)
2474	{	2465	{
2475	struct pid *pid;	2466	struct pid *pid;
2476	struct task_struct *tsk;	2467	struct task_struct *tsk;
2477	char *buf;	2468	char *buf;
2478	struct cgroup_subsys_state *css;	2469	struct cgroup_subsys_state *css;
2479	int retval;	2470	int retval;
2480		2471
2481	retval = -ENOMEM;	2472	retval = -ENOMEM;
2482	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);	2473	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
2483	if (!buf)	2474	if (!buf)
2484	goto out;	2475	goto out;
2485		2476
2486	retval = -ESRCH;	2477	retval = -ESRCH;
2487	pid = m->private;	2478	pid = m->private;
2488	tsk = get_pid_task(pid, PIDTYPE_PID);	2479	tsk = get_pid_task(pid, PIDTYPE_PID);
2489	if (!tsk)	2480	if (!tsk)
2490	goto out_free;	2481	goto out_free;
2491		2482
2492	retval = -EINVAL;	2483	retval = -EINVAL;
2493	cgroup_lock();	2484	cgroup_lock();
2494	css = task_subsys_state(tsk, cpuset_subsys_id);	2485	css = task_subsys_state(tsk, cpuset_subsys_id);
2495	retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);	2486	retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);
2496	if (retval < 0)	2487	if (retval < 0)
2497	goto out_unlock;	2488	goto out_unlock;
2498	seq_puts(m, buf);	2489	seq_puts(m, buf);
2499	seq_putc(m, '\n');	2490	seq_putc(m, '\n');
2500	out_unlock:	2491	out_unlock:
2501	cgroup_unlock();	2492	cgroup_unlock();
2502	put_task_struct(tsk);	2493	put_task_struct(tsk);
2503	out_free:	2494	out_free:
2504	kfree(buf);	2495	kfree(buf);
2505	out:	2496	out:
2506	return retval;	2497	return retval;
2507	}	2498	}
2508		2499
2509	static int cpuset_open(struct inode inode, struct file file)	2500	static int cpuset_open(struct inode inode, struct file file)
2510	{	2501	{
2511	struct pid *pid = PROC_I(inode)->pid;	2502	struct pid *pid = PROC_I(inode)->pid;
2512	return single_open(file, proc_cpuset_show, pid);	2503	return single_open(file, proc_cpuset_show, pid);
2513	}	2504	}
2514		2505
2515	const struct file_operations proc_cpuset_operations = {	2506	const struct file_operations proc_cpuset_operations = {
2516	.open = cpuset_open,	2507	.open = cpuset_open,
2517	.read = seq_read,	2508	.read = seq_read,
2518	.llseek = seq_lseek,	2509	.llseek = seq_lseek,
2519	.release = single_release,	2510	.release = single_release,
2520	};	2511	};
2521	#endif /* CONFIG_PROC_PID_CPUSET */	2512	#endif /* CONFIG_PROC_PID_CPUSET */
2522		2513
2523	/* Display task cpus_allowed, mems_allowed in /proc/<pid>/status file. */	2514	/* Display task cpus_allowed, mems_allowed in /proc/<pid>/status file. */
2524	void cpuset_task_status_allowed(struct seq_file m, struct task_struct task)	2515	void cpuset_task_status_allowed(struct seq_file m, struct task_struct task)
2525	{	2516	{
2526	seq_printf(m, "Cpus_allowed:\t");	2517	seq_printf(m, "Cpus_allowed:\t");
2527	seq_cpumask(m, &task->cpus_allowed);	2518	seq_cpumask(m, &task->cpus_allowed);
2528	seq_printf(m, "\n");	2519	seq_printf(m, "\n");
2529	seq_printf(m, "Cpus_allowed_list:\t");	2520	seq_printf(m, "Cpus_allowed_list:\t");
2530	seq_cpumask_list(m, &task->cpus_allowed);	2521	seq_cpumask_list(m, &task->cpus_allowed);
2531	seq_printf(m, "\n");	2522	seq_printf(m, "\n");
2532	seq_printf(m, "Mems_allowed:\t");	2523	seq_printf(m, "Mems_allowed:\t");
2533	seq_nodemask(m, &task->mems_allowed);	2524	seq_nodemask(m, &task->mems_allowed);
2534	seq_printf(m, "\n");	2525	seq_printf(m, "\n");
2535	seq_printf(m, "Mems_allowed_list:\t");	2526	seq_printf(m, "Mems_allowed_list:\t");
2536	seq_nodemask_list(m, &task->mems_allowed);	2527	seq_nodemask_list(m, &task->mems_allowed);
2537	seq_printf(m, "\n");	2528	seq_printf(m, "\n");
2538	}	2529	}
2539		2530