Blame view

kernel/cpu.c 18.2 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
3
4
5
6
7
8
9
10
11
12
  /* CPU control.
   * (C) 2001, 2002, 2003, 2004 Rusty Russell
   *
   * This code is licenced under the GPL.
   */
  #include <linux/proc_fs.h>
  #include <linux/smp.h>
  #include <linux/init.h>
  #include <linux/notifier.h>
  #include <linux/sched.h>
  #include <linux/unistd.h>
  #include <linux/cpu.h>
cb79295e2   Anton Vorontsov   cpu: introduce cl...
13
14
  #include <linux/oom.h>
  #include <linux/rcupdate.h>
9984de1a5   Paul Gortmaker   kernel: Map most ...
15
  #include <linux/export.h>
e4cc2f873   Anton Vorontsov   kernel/cpu.c: doc...
16
  #include <linux/bug.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
17
18
  #include <linux/kthread.h>
  #include <linux/stop_machine.h>
81615b624   Ingo Molnar   [PATCH] Convert k...
19
  #include <linux/mutex.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
20
  #include <linux/gfp.h>
79cfbdfa8   Srivatsa S. Bhat   PM / Sleep: Fix r...
21
  #include <linux/suspend.h>
a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
22
  #include <linux/lockdep.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
23

38498a67a   Thomas Gleixner   smp: Add generic ...
24
  #include "smpboot.h"
98a79d6a5   Rusty Russell   cpumask: centrali...
25
  #ifdef CONFIG_SMP
b3199c025   Rusty Russell   cpumask: switch o...
26
  /* Serializes the updates to cpu_online_mask, cpu_present_mask */
aa9538777   Linus Torvalds   cpu hotplug: simp...
27
  static DEFINE_MUTEX(cpu_add_remove_lock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
28

79a6cdeb7   Lai Jiangshan   cpuhotplug: do no...
29
  /*
93ae4f978   Srivatsa S. Bhat   CPU hotplug: Prov...
30
31
32
33
34
   * The following two APIs (cpu_maps_update_begin/done) must be used when
   * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
   * The APIs cpu_notifier_register_begin/done() must be used to protect CPU
   * hotplug callback (un)registration performed using __register_cpu_notifier()
   * or __unregister_cpu_notifier().
79a6cdeb7   Lai Jiangshan   cpuhotplug: do no...
35
36
37
38
39
   */
  void cpu_maps_update_begin(void)
  {
  	mutex_lock(&cpu_add_remove_lock);
  }
93ae4f978   Srivatsa S. Bhat   CPU hotplug: Prov...
40
  EXPORT_SYMBOL(cpu_notifier_register_begin);
79a6cdeb7   Lai Jiangshan   cpuhotplug: do no...
41
42
43
44
45
  
  void cpu_maps_update_done(void)
  {
  	mutex_unlock(&cpu_add_remove_lock);
  }
93ae4f978   Srivatsa S. Bhat   CPU hotplug: Prov...
46
  EXPORT_SYMBOL(cpu_notifier_register_done);
79a6cdeb7   Lai Jiangshan   cpuhotplug: do no...
47

5c113fbee   Daniel J Blueman   fix cpu_chain sec...
48
  static RAW_NOTIFIER_HEAD(cpu_chain);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
49

e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
50
51
52
53
  /* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
   * Should always be manipulated under cpu_add_remove_lock
   */
  static int cpu_hotplug_disabled;
79a6cdeb7   Lai Jiangshan   cpuhotplug: do no...
54
  #ifdef CONFIG_HOTPLUG_CPU
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
55
56
57
58
59
60
61
62
  static struct {
  	struct task_struct *active_writer;
  	struct mutex lock; /* Synchronizes accesses to refcount, */
  	/*
  	 * Also blocks the new readers during
  	 * an ongoing cpu hotplug operation.
  	 */
  	int refcount;
a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
63
64
65
66
  
  #ifdef CONFIG_DEBUG_LOCK_ALLOC
  	struct lockdep_map dep_map;
  #endif
31950eb66   Linus Torvalds   mm/init: cpu_hotp...
67
68
69
70
  } cpu_hotplug = {
  	.active_writer = NULL,
  	.lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
  	.refcount = 0,
a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
71
72
73
  #ifdef CONFIG_DEBUG_LOCK_ALLOC
  	.dep_map = {.name = "cpu_hotplug.lock" },
  #endif
31950eb66   Linus Torvalds   mm/init: cpu_hotp...
74
  };
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
75

a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
76
77
78
79
  /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
  #define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
  #define cpuhp_lock_acquire()      lock_map_acquire(&cpu_hotplug.dep_map)
  #define cpuhp_lock_release()      lock_map_release(&cpu_hotplug.dep_map)
86ef5c9a8   Gautham R Shenoy   cpu-hotplug: repl...
80
  void get_online_cpus(void)
a9d9baa1e   Ashok Raj   [PATCH] clean up ...
81
  {
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
82
83
  	might_sleep();
  	if (cpu_hotplug.active_writer == current)
aa9538777   Linus Torvalds   cpu hotplug: simp...
84
  		return;
a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
85
  	cpuhp_lock_acquire_read();
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
86
87
88
  	mutex_lock(&cpu_hotplug.lock);
  	cpu_hotplug.refcount++;
  	mutex_unlock(&cpu_hotplug.lock);
a9d9baa1e   Ashok Raj   [PATCH] clean up ...
89
  }
86ef5c9a8   Gautham R Shenoy   cpu-hotplug: repl...
90
  EXPORT_SYMBOL_GPL(get_online_cpus);
90d45d17f   Ashok Raj   [PATCH] cpu hotpl...
91

86ef5c9a8   Gautham R Shenoy   cpu-hotplug: repl...
92
  void put_online_cpus(void)
a9d9baa1e   Ashok Raj   [PATCH] clean up ...
93
  {
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
94
  	if (cpu_hotplug.active_writer == current)
aa9538777   Linus Torvalds   cpu hotplug: simp...
95
  		return;
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
96
  	mutex_lock(&cpu_hotplug.lock);
075663d19   Srivatsa S. Bhat   CPU hotplug, debu...
97
98
99
  
  	if (WARN_ON(!cpu_hotplug.refcount))
  		cpu_hotplug.refcount++; /* try to fix things up */
d2ba7e2ae   Oleg Nesterov   simplify cpu_hotp...
100
101
  	if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
  		wake_up_process(cpu_hotplug.active_writer);
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
102
  	mutex_unlock(&cpu_hotplug.lock);
a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
103
  	cpuhp_lock_release();
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
104

a9d9baa1e   Ashok Raj   [PATCH] clean up ...
105
  }
86ef5c9a8   Gautham R Shenoy   cpu-hotplug: repl...
106
  EXPORT_SYMBOL_GPL(put_online_cpus);
a9d9baa1e   Ashok Raj   [PATCH] clean up ...
107

d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
108
109
110
111
112
113
114
  /*
   * This ensures that the hotplug operation can begin only when the
   * refcount goes to zero.
   *
   * Note that during a cpu-hotplug operation, the new readers, if any,
   * will be blocked by the cpu_hotplug.lock
   *
d2ba7e2ae   Oleg Nesterov   simplify cpu_hotp...
115
116
   * Since cpu_hotplug_begin() is always called after invoking
   * cpu_maps_update_begin(), we can be sure that only one writer is active.
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
117
118
119
120
121
122
123
124
125
126
   *
   * Note that theoretically, there is a possibility of a livelock:
   * - Refcount goes to zero, last reader wakes up the sleeping
   *   writer.
   * - Last reader unlocks the cpu_hotplug.lock.
   * - A new reader arrives at this moment, bumps up the refcount.
   * - The writer acquires the cpu_hotplug.lock finds the refcount
   *   non zero and goes to sleep again.
   *
   * However, this is very difficult to achieve in practice since
86ef5c9a8   Gautham R Shenoy   cpu-hotplug: repl...
127
   * get_online_cpus() not an api which is called all that often.
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
128
129
   *
   */
b9d10be7a   Toshi Kani   ACPI / processor:...
130
  void cpu_hotplug_begin(void)
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
131
  {
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
132
  	cpu_hotplug.active_writer = current;
d2ba7e2ae   Oleg Nesterov   simplify cpu_hotp...
133

a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
134
  	cpuhp_lock_acquire();
d2ba7e2ae   Oleg Nesterov   simplify cpu_hotp...
135
136
137
138
139
  	for (;;) {
  		mutex_lock(&cpu_hotplug.lock);
  		if (likely(!cpu_hotplug.refcount))
  			break;
  		__set_current_state(TASK_UNINTERRUPTIBLE);
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
140
141
  		mutex_unlock(&cpu_hotplug.lock);
  		schedule();
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
142
  	}
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
143
  }
b9d10be7a   Toshi Kani   ACPI / processor:...
144
  void cpu_hotplug_done(void)
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
145
146
147
  {
  	cpu_hotplug.active_writer = NULL;
  	mutex_unlock(&cpu_hotplug.lock);
a19423b98   Gautham R. Shenoy   CPU hotplug: Add ...
148
  	cpuhp_lock_release();
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
149
  }
79a6cdeb7   Lai Jiangshan   cpuhotplug: do no...
150

16e53dbf1   Srivatsa S. Bhat   CPU hotplug: prov...
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
  /*
   * Wait for currently running CPU hotplug operations to complete (if any) and
   * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
   * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
   * hotplug path before performing hotplug operations. So acquiring that lock
   * guarantees mutual exclusion from any currently running hotplug operations.
   */
  void cpu_hotplug_disable(void)
  {
  	cpu_maps_update_begin();
  	cpu_hotplug_disabled = 1;
  	cpu_maps_update_done();
  }
  
  void cpu_hotplug_enable(void)
  {
  	cpu_maps_update_begin();
  	cpu_hotplug_disabled = 0;
  	cpu_maps_update_done();
  }
b9d10be7a   Toshi Kani   ACPI / processor:...
171
  #endif	/* CONFIG_HOTPLUG_CPU */
79a6cdeb7   Lai Jiangshan   cpuhotplug: do no...
172

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
173
  /* Need to know about CPUs going up/down? */
f7b16c108   Sam Ravnborg   cpu: fix section ...
174
  int __ref register_cpu_notifier(struct notifier_block *nb)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
175
  {
bd5349cfd   Neil Brown   [PATCH] Convert c...
176
  	int ret;
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
177
  	cpu_maps_update_begin();
bd5349cfd   Neil Brown   [PATCH] Convert c...
178
  	ret = raw_notifier_chain_register(&cpu_chain, nb);
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
179
  	cpu_maps_update_done();
bd5349cfd   Neil Brown   [PATCH] Convert c...
180
  	return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
181
  }
65edc68c3   Chandra Seetharaman   [PATCH] cpu hotpl...
182

93ae4f978   Srivatsa S. Bhat   CPU hotplug: Prov...
183
184
185
186
  int __ref __register_cpu_notifier(struct notifier_block *nb)
  {
  	return raw_notifier_chain_register(&cpu_chain, nb);
  }
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
187
188
189
  static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
  			int *nr_calls)
  {
e6bde73b0   Akinobu Mita   cpu-hotplug: retu...
190
191
192
  	int ret;
  
  	ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
193
  					nr_calls);
e6bde73b0   Akinobu Mita   cpu-hotplug: retu...
194
195
  
  	return notifier_to_errno(ret);
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
196
197
198
199
200
201
  }
  
  static int cpu_notify(unsigned long val, void *v)
  {
  	return __cpu_notify(val, v, -1, NULL);
  }
00b9b0af5   Linus Torvalds   Avoid warning whe...
202
  #ifdef CONFIG_HOTPLUG_CPU
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
203
204
  static void cpu_notify_nofail(unsigned long val, void *v)
  {
00b9b0af5   Linus Torvalds   Avoid warning whe...
205
  	BUG_ON(cpu_notify(val, v));
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
206
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
207
  EXPORT_SYMBOL(register_cpu_notifier);
93ae4f978   Srivatsa S. Bhat   CPU hotplug: Prov...
208
  EXPORT_SYMBOL(__register_cpu_notifier);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
209

9647155ff   Sam Ravnborg   cpu: fix section ...
210
  void __ref unregister_cpu_notifier(struct notifier_block *nb)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
211
  {
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
212
  	cpu_maps_update_begin();
bd5349cfd   Neil Brown   [PATCH] Convert c...
213
  	raw_notifier_chain_unregister(&cpu_chain, nb);
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
214
  	cpu_maps_update_done();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
215
216
  }
  EXPORT_SYMBOL(unregister_cpu_notifier);
93ae4f978   Srivatsa S. Bhat   CPU hotplug: Prov...
217
218
219
220
221
  void __ref __unregister_cpu_notifier(struct notifier_block *nb)
  {
  	raw_notifier_chain_unregister(&cpu_chain, nb);
  }
  EXPORT_SYMBOL(__unregister_cpu_notifier);
e4cc2f873   Anton Vorontsov   kernel/cpu.c: doc...
222
223
224
225
226
227
228
229
230
231
232
233
  /**
   * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
   * @cpu: a CPU id
   *
   * This function walks all processes, finds a valid mm struct for each one and
   * then clears a corresponding bit in mm's cpumask.  While this all sounds
   * trivial, there are various non-obvious corner cases, which this function
   * tries to solve in a safe manner.
   *
   * Also note that the function uses a somewhat relaxed locking scheme, so it may
   * be called only for an already offlined CPU.
   */
cb79295e2   Anton Vorontsov   cpu: introduce cl...
234
235
236
237
238
239
240
241
242
243
244
  void clear_tasks_mm_cpumask(int cpu)
  {
  	struct task_struct *p;
  
  	/*
  	 * This function is called after the cpu is taken down and marked
  	 * offline, so its not like new tasks will ever get this cpu set in
  	 * their mm mask. -- Peter Zijlstra
  	 * Thus, we may use rcu_read_lock() here, instead of grabbing
  	 * full-fledged tasklist_lock.
  	 */
e4cc2f873   Anton Vorontsov   kernel/cpu.c: doc...
245
  	WARN_ON(cpu_online(cpu));
cb79295e2   Anton Vorontsov   cpu: introduce cl...
246
247
248
  	rcu_read_lock();
  	for_each_process(p) {
  		struct task_struct *t;
e4cc2f873   Anton Vorontsov   kernel/cpu.c: doc...
249
250
251
252
  		/*
  		 * Main thread might exit, but other threads may still have
  		 * a valid mm. Find one.
  		 */
cb79295e2   Anton Vorontsov   cpu: introduce cl...
253
254
255
256
257
258
259
260
  		t = find_lock_task_mm(p);
  		if (!t)
  			continue;
  		cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
  		task_unlock(t);
  	}
  	rcu_read_unlock();
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
261
262
263
  static inline void check_for_tasks(int cpu)
  {
  	struct task_struct *p;
6fac4829c   Frederic Weisbecker   cputime: Use acce...
264
  	cputime_t utime, stime;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
265
266
267
  
  	write_lock_irq(&tasklist_lock);
  	for_each_process(p) {
6fac4829c   Frederic Weisbecker   cputime: Use acce...
268
  		task_cputime(p, &utime, &stime);
11854247e   Peter Zijlstra   sched: Fix incorr...
269
  		if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
6fac4829c   Frederic Weisbecker   cputime: Use acce...
270
  		    (utime || stime))
9d3cfc4c1   Frans Pop   sched: Correct pr...
271
272
273
274
275
  			printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
  				"(state = %ld, flags = %x)
  ",
  				p->comm, task_pid_nr(p), cpu,
  				p->state, p->flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
276
277
278
  	}
  	write_unlock_irq(&tasklist_lock);
  }
db912f963   Avi Kivity   HOTPLUG: Add CPU_...
279
280
281
282
  struct take_cpu_down_param {
  	unsigned long mod;
  	void *hcpu;
  };
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
283
  /* Take this CPU down. */
514a20a5d   Sam Ravnborg   cpu: fix section ...
284
  static int __ref take_cpu_down(void *_param)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
285
  {
db912f963   Avi Kivity   HOTPLUG: Add CPU_...
286
  	struct take_cpu_down_param *param = _param;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
287
  	int err;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
288
289
290
  	/* Ensure this CPU doesn't handle any more interrupts. */
  	err = __cpu_disable();
  	if (err < 0)
f37051364   Zwane Mwaikambo   [PATCH] i386 CPU ...
291
  		return err;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
292

e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
293
  	cpu_notify(CPU_DYING | param->mod, param->hcpu);
14e568e78   Thomas Gleixner   stop_machine: Use...
294
295
  	/* Park the stopper thread */
  	kthread_park(current);
f37051364   Zwane Mwaikambo   [PATCH] i386 CPU ...
296
  	return 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
297
  }
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
298
  /* Requires cpu_add_remove_lock to be held */
514a20a5d   Sam Ravnborg   cpu: fix section ...
299
  static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
300
  {
e7407dcc6   Heiko Carstens   call cpu_chain wi...
301
  	int err, nr_calls = 0;
e7407dcc6   Heiko Carstens   call cpu_chain wi...
302
  	void *hcpu = (void *)(long)cpu;
8bb784428   Rafael J. Wysocki   Add suspend-relat...
303
  	unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
db912f963   Avi Kivity   HOTPLUG: Add CPU_...
304
305
306
307
  	struct take_cpu_down_param tcd_param = {
  		.mod = mod,
  		.hcpu = hcpu,
  	};
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
308

e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
309
310
  	if (num_online_cpus() == 1)
  		return -EBUSY;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
311

e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
312
313
  	if (!cpu_online(cpu))
  		return -EINVAL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
314

d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
315
  	cpu_hotplug_begin();
4d51985e4   Michael Rodriguez   kernel/cpu.c: fix...
316

e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
317
  	err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
e6bde73b0   Akinobu Mita   cpu-hotplug: retu...
318
  	if (err) {
a0d8cdb65   Akinobu Mita   cpu hotplug: cpu:...
319
  		nr_calls--;
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
320
  		__cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
321
322
  		printk("%s: attempt to take down CPU %u failed
  ",
af1f16d08   Harvey Harrison   kernel: replace r...
323
  				__func__, cpu);
baaca49f4   Gautham R Shenoy   Define and use ne...
324
  		goto out_release;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
325
  	}
6acce3ef8   Peter Zijlstra   sched: Remove get...
326
327
328
329
330
331
332
  	/*
  	 * By now we've cleared cpu_active_mask, wait for all preempt-disabled
  	 * and RCU users of this state to go away such that all new such users
  	 * will observe it.
  	 *
  	 * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
  	 * not imply sync_sched(), so explicitly call both.
106dd5afd   Michael wang   sched: Fix endles...
333
334
  	 *
  	 * Do sync before park smpboot threads to take care the rcu boost case.
6acce3ef8   Peter Zijlstra   sched: Remove get...
335
336
337
338
339
  	 */
  #ifdef CONFIG_PREEMPT
  	synchronize_sched();
  #endif
  	synchronize_rcu();
106dd5afd   Michael wang   sched: Fix endles...
340
  	smpboot_park_threads(cpu);
6acce3ef8   Peter Zijlstra   sched: Remove get...
341
342
343
  	/*
  	 * So now all preempt/rcu users must observe !cpu_active().
  	 */
e0b582ec5   Rusty Russell   cpumask: convert ...
344
  	err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
043215875   Rusty Russell   Hotplug CPU: don'...
345
  	if (err) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
346
  		/* CPU didn't die: tell everyone.  Can't complain. */
f97f8f06a   Thomas Gleixner   smpboot: Provide ...
347
  		smpboot_unpark_threads(cpu);
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
348
  		cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
6a1bdc1b5   Oleg Nesterov   sched: _cpu_down(...
349
  		goto out_release;
8fa1d7d3b   Satoru Takeuchi   [PATCH] cpu-hotpl...
350
  	}
043215875   Rusty Russell   Hotplug CPU: don'...
351
  	BUG_ON(cpu_online(cpu));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
352

48c5ccae8   Peter Zijlstra   sched: Simplify c...
353
354
355
356
  	/*
  	 * The migration_call() CPU_DYING callback will have removed all
  	 * runnable tasks from the cpu, there's only the idle task left now
  	 * that the migration thread is done doing the stop_machine thing.
51a96c778   Peter Zijlstra   cpu: Remove incor...
357
358
  	 *
  	 * Wait for the stop thread to go away.
48c5ccae8   Peter Zijlstra   sched: Simplify c...
359
  	 */
51a96c778   Peter Zijlstra   cpu: Remove incor...
360
361
  	while (!idle_cpu(cpu))
  		cpu_relax();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
362
363
364
  
  	/* This actually kills the CPU. */
  	__cpu_die(cpu);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
365
  	/* CPU is completely dead: tell everyone.  Too late to complain. */
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
366
  	cpu_notify_nofail(CPU_DEAD | mod, hcpu);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
367
368
  
  	check_for_tasks(cpu);
baaca49f4   Gautham R Shenoy   Define and use ne...
369
  out_release:
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
370
  	cpu_hotplug_done();
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
371
372
  	if (!err)
  		cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
373
374
  	return err;
  }
514a20a5d   Sam Ravnborg   cpu: fix section ...
375
  int __ref cpu_down(unsigned int cpu)
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
376
  {
9ea09af3b   Heiko Carstens   stop_machine: int...
377
  	int err;
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
378

d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
379
  	cpu_maps_update_begin();
e761b7725   Max Krasnyansky   cpu hotplug, sche...
380
381
  
  	if (cpu_hotplug_disabled) {
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
382
  		err = -EBUSY;
e761b7725   Max Krasnyansky   cpu hotplug, sche...
383
384
  		goto out;
  	}
e761b7725   Max Krasnyansky   cpu hotplug, sche...
385
  	err = _cpu_down(cpu, 0);
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
386

e761b7725   Max Krasnyansky   cpu hotplug, sche...
387
  out:
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
388
  	cpu_maps_update_done();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
389
390
  	return err;
  }
b62b8ef90   Zhang Rui   force offline the...
391
  EXPORT_SYMBOL(cpu_down);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
392
  #endif /*CONFIG_HOTPLUG_CPU*/
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
393
  /* Requires cpu_add_remove_lock to be held */
0db0628d9   Paul Gortmaker   kernel: delete __...
394
  static int _cpu_up(unsigned int cpu, int tasks_frozen)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
395
  {
baaca49f4   Gautham R Shenoy   Define and use ne...
396
  	int ret, nr_calls = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
397
  	void *hcpu = (void *)(long)cpu;
8bb784428   Rafael J. Wysocki   Add suspend-relat...
398
  	unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
3bb5d2ee3   Suresh Siddha   smp, idle: Alloca...
399
  	struct task_struct *idle;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
400

d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
401
  	cpu_hotplug_begin();
38498a67a   Thomas Gleixner   smp: Add generic ...
402

5e5041f35   Yasuaki Ishimatsu   ACPI / processor:...
403
404
405
406
  	if (cpu_online(cpu) || !cpu_present(cpu)) {
  		ret = -EINVAL;
  		goto out;
  	}
3bb5d2ee3   Suresh Siddha   smp, idle: Alloca...
407
408
409
  	idle = idle_thread_get(cpu);
  	if (IS_ERR(idle)) {
  		ret = PTR_ERR(idle);
38498a67a   Thomas Gleixner   smp: Add generic ...
410
  		goto out;
3bb5d2ee3   Suresh Siddha   smp, idle: Alloca...
411
  	}
38498a67a   Thomas Gleixner   smp: Add generic ...
412

f97f8f06a   Thomas Gleixner   smpboot: Provide ...
413
414
415
  	ret = smpboot_create_threads(cpu);
  	if (ret)
  		goto out;
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
416
  	ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
e6bde73b0   Akinobu Mita   cpu-hotplug: retu...
417
  	if (ret) {
a0d8cdb65   Akinobu Mita   cpu hotplug: cpu:...
418
  		nr_calls--;
4d51985e4   Michael Rodriguez   kernel/cpu.c: fix...
419
420
  		printk(KERN_WARNING "%s: attempt to bring up CPU %u failed
  ",
af1f16d08   Harvey Harrison   kernel: replace r...
421
  				__func__, cpu);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
422
423
424
425
  		goto out_notify;
  	}
  
  	/* Arch-specific enabling code. */
3bb5d2ee3   Suresh Siddha   smp, idle: Alloca...
426
  	ret = __cpu_up(cpu, idle);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
427
428
  	if (ret != 0)
  		goto out_notify;
6978c7052   Eric Sesterhenn   BUG_ON() Conversi...
429
  	BUG_ON(!cpu_online(cpu));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
430

f97f8f06a   Thomas Gleixner   smpboot: Provide ...
431
432
  	/* Wake the per cpu threads */
  	smpboot_unpark_threads(cpu);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
433
  	/* Now call notifier in preparation. */
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
434
  	cpu_notify(CPU_ONLINE | mod, hcpu);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
435
436
437
  
  out_notify:
  	if (ret != 0)
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
438
  		__cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
38498a67a   Thomas Gleixner   smp: Add generic ...
439
  out:
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
440
  	cpu_hotplug_done();
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
441
442
443
  
  	return ret;
  }
0db0628d9   Paul Gortmaker   kernel: delete __...
444
  int cpu_up(unsigned int cpu)
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
445
446
  {
  	int err = 0;
cf23422b9   minskey guo   cpu/mem hotplug: ...
447

e0b582ec5   Rusty Russell   cpumask: convert ...
448
  	if (!cpu_possible(cpu)) {
73e753a50   KAMEZAWA Hiroyuki   CPU HOTPLUG: avoi...
449
450
451
  		printk(KERN_ERR "can't online cpu %d because it is not "
  			"configured as may-hotadd at boot time
  ", cpu);
87d5e0236   Chen Gong   kernel/cpu.c: del...
452
  #if defined(CONFIG_IA64)
73e753a50   KAMEZAWA Hiroyuki   CPU HOTPLUG: avoi...
453
454
455
456
457
458
  		printk(KERN_ERR "please check additional_cpus= boot "
  				"parameter
  ");
  #endif
  		return -EINVAL;
  	}
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
459

01b0f1970   Toshi Kani   cpu/mem hotplug: ...
460
461
462
  	err = try_online_node(cpu_to_node(cpu));
  	if (err)
  		return err;
cf23422b9   minskey guo   cpu/mem hotplug: ...
463

d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
464
  	cpu_maps_update_begin();
e761b7725   Max Krasnyansky   cpu hotplug, sche...
465
466
  
  	if (cpu_hotplug_disabled) {
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
467
  		err = -EBUSY;
e761b7725   Max Krasnyansky   cpu hotplug, sche...
468
469
470
471
  		goto out;
  	}
  
  	err = _cpu_up(cpu, 0);
e761b7725   Max Krasnyansky   cpu hotplug, sche...
472
  out:
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
473
  	cpu_maps_update_done();
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
474
475
  	return err;
  }
a513f6bab   Paul E. McKenney   cpu: Export cpu_up()
476
  EXPORT_SYMBOL_GPL(cpu_up);
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
477

f3de4be9d   Rafael J. Wysocki   PM: Fix dependenc...
478
  #ifdef CONFIG_PM_SLEEP_SMP
e0b582ec5   Rusty Russell   cpumask: convert ...
479
  static cpumask_var_t frozen_cpus;
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
480
481
482
  
  int disable_nonboot_cpus(void)
  {
e9a5f426b   Rafael J. Wysocki   CPU: Avoid using ...
483
  	int cpu, first_cpu, error = 0;
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
484

d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
485
  	cpu_maps_update_begin();
e0b582ec5   Rusty Russell   cpumask: convert ...
486
  	first_cpu = cpumask_first(cpu_online_mask);
9ee349ad6   Xiaotian Feng   sched: Fix set_cp...
487
488
  	/*
  	 * We take down all of the non-boot CPUs in one shot to avoid races
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
489
490
  	 * with the userspace trying to use the CPU hotplug at the same time
  	 */
e0b582ec5   Rusty Russell   cpumask: convert ...
491
  	cpumask_clear(frozen_cpus);
6ad4c1888   Peter Zijlstra   sched: Fix balanc...
492

e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
493
494
495
496
497
  	printk("Disabling non-boot CPUs ...
  ");
  	for_each_online_cpu(cpu) {
  		if (cpu == first_cpu)
  			continue;
8bb784428   Rafael J. Wysocki   Add suspend-relat...
498
  		error = _cpu_down(cpu, 1);
feae3203d   Mike Travis   timers, init: Lim...
499
  		if (!error)
e0b582ec5   Rusty Russell   cpumask: convert ...
500
  			cpumask_set_cpu(cpu, frozen_cpus);
feae3203d   Mike Travis   timers, init: Lim...
501
  		else {
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
502
503
504
505
506
507
  			printk(KERN_ERR "Error taking CPU%d down: %d
  ",
  				cpu, error);
  			break;
  		}
  	}
86886e55b   Joseph Cihula   x86, intel_txt: I...
508

e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
509
510
511
512
513
  	if (!error) {
  		BUG_ON(num_online_cpus() > 1);
  		/* Make sure the CPUs won't be enabled by someone else */
  		cpu_hotplug_disabled = 1;
  	} else {
e1d9fd2e3   Ingo Molnar   [PATCH] suspend: ...
514
515
  		printk(KERN_ERR "Non-boot CPUs are not disabled
  ");
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
516
  	}
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
517
  	cpu_maps_update_done();
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
518
519
  	return error;
  }
d0af9eed5   Suresh Siddha   x86, pat/mtrr: Re...
520
521
522
523
524
525
526
  void __weak arch_enable_nonboot_cpus_begin(void)
  {
  }
  
  void __weak arch_enable_nonboot_cpus_end(void)
  {
  }
fa7303e22   Sam Ravnborg   cpu: fix section ...
527
  void __ref enable_nonboot_cpus(void)
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
528
529
530
531
  {
  	int cpu, error;
  
  	/* Allow everyone to use the CPU hotplug again */
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
532
  	cpu_maps_update_begin();
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
533
  	cpu_hotplug_disabled = 0;
e0b582ec5   Rusty Russell   cpumask: convert ...
534
  	if (cpumask_empty(frozen_cpus))
1d64b9cb1   Rafael J. Wysocki   [PATCH] Fix micro...
535
  		goto out;
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
536

4d51985e4   Michael Rodriguez   kernel/cpu.c: fix...
537
538
  	printk(KERN_INFO "Enabling non-boot CPUs ...
  ");
d0af9eed5   Suresh Siddha   x86, pat/mtrr: Re...
539
540
  
  	arch_enable_nonboot_cpus_begin();
e0b582ec5   Rusty Russell   cpumask: convert ...
541
  	for_each_cpu(cpu, frozen_cpus) {
8bb784428   Rafael J. Wysocki   Add suspend-relat...
542
  		error = _cpu_up(cpu, 1);
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
543
  		if (!error) {
4d51985e4   Michael Rodriguez   kernel/cpu.c: fix...
544
545
  			printk(KERN_INFO "CPU%d is up
  ", cpu);
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
546
547
  			continue;
  		}
1d64b9cb1   Rafael J. Wysocki   [PATCH] Fix micro...
548
549
  		printk(KERN_WARNING "Error taking CPU%d up: %d
  ", cpu, error);
e3920fb42   Rafael J. Wysocki   [PATCH] Disable C...
550
  	}
d0af9eed5   Suresh Siddha   x86, pat/mtrr: Re...
551
552
  
  	arch_enable_nonboot_cpus_end();
e0b582ec5   Rusty Russell   cpumask: convert ...
553
  	cpumask_clear(frozen_cpus);
1d64b9cb1   Rafael J. Wysocki   [PATCH] Fix micro...
554
  out:
d221938c0   Gautham R Shenoy   cpu-hotplug: refc...
555
  	cpu_maps_update_done();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
556
  }
e0b582ec5   Rusty Russell   cpumask: convert ...
557

d7268a31c   Fenghua Yu   CPU: Add right qu...
558
  static int __init alloc_frozen_cpus(void)
e0b582ec5   Rusty Russell   cpumask: convert ...
559
560
561
562
563
564
  {
  	if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
  		return -ENOMEM;
  	return 0;
  }
  core_initcall(alloc_frozen_cpus);
79cfbdfa8   Srivatsa S. Bhat   PM / Sleep: Fix r...
565
566
  
  /*
79cfbdfa8   Srivatsa S. Bhat   PM / Sleep: Fix r...
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
   * When callbacks for CPU hotplug notifications are being executed, we must
   * ensure that the state of the system with respect to the tasks being frozen
   * or not, as reported by the notification, remains unchanged *throughout the
   * duration* of the execution of the callbacks.
   * Hence we need to prevent the freezer from racing with regular CPU hotplug.
   *
   * This synchronization is implemented by mutually excluding regular CPU
   * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
   * Hibernate notifications.
   */
  static int
  cpu_hotplug_pm_callback(struct notifier_block *nb,
  			unsigned long action, void *ptr)
  {
  	switch (action) {
  
  	case PM_SUSPEND_PREPARE:
  	case PM_HIBERNATION_PREPARE:
16e53dbf1   Srivatsa S. Bhat   CPU hotplug: prov...
585
  		cpu_hotplug_disable();
79cfbdfa8   Srivatsa S. Bhat   PM / Sleep: Fix r...
586
587
588
589
  		break;
  
  	case PM_POST_SUSPEND:
  	case PM_POST_HIBERNATION:
16e53dbf1   Srivatsa S. Bhat   CPU hotplug: prov...
590
  		cpu_hotplug_enable();
79cfbdfa8   Srivatsa S. Bhat   PM / Sleep: Fix r...
591
592
593
594
595
596
597
598
  		break;
  
  	default:
  		return NOTIFY_DONE;
  	}
  
  	return NOTIFY_OK;
  }
d7268a31c   Fenghua Yu   CPU: Add right qu...
599
  static int __init cpu_hotplug_pm_sync_init(void)
79cfbdfa8   Srivatsa S. Bhat   PM / Sleep: Fix r...
600
  {
6e32d479d   Fenghua Yu   kernel/cpu.c: Add...
601
602
603
604
605
  	/*
  	 * cpu_hotplug_pm_callback has higher priority than x86
  	 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
  	 * to disable cpu hotplug to avoid cpu hotplug race.
  	 */
79cfbdfa8   Srivatsa S. Bhat   PM / Sleep: Fix r...
606
607
608
609
  	pm_notifier(cpu_hotplug_pm_callback, 0);
  	return 0;
  }
  core_initcall(cpu_hotplug_pm_sync_init);
f3de4be9d   Rafael J. Wysocki   PM: Fix dependenc...
610
  #endif /* CONFIG_PM_SLEEP_SMP */
68f4f1ec0   Max Krasnyansky   sched: Move cpu m...
611

e545a6140   Manfred Spraul   kernel/cpu.c: cre...
612
613
614
615
616
617
618
619
  /**
   * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
   * @cpu: cpu that just started
   *
   * This function calls the cpu_chain notifiers with CPU_STARTING.
   * It must be called by the arch code on the new cpu, before the new cpu
   * enables interrupts and before the "boot" cpu returns from __cpu_up().
   */
0db0628d9   Paul Gortmaker   kernel: delete __...
620
  void notify_cpu_starting(unsigned int cpu)
e545a6140   Manfred Spraul   kernel/cpu.c: cre...
621
622
623
624
  {
  	unsigned long val = CPU_STARTING;
  
  #ifdef CONFIG_PM_SLEEP_SMP
e0b582ec5   Rusty Russell   cpumask: convert ...
625
  	if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
e545a6140   Manfred Spraul   kernel/cpu.c: cre...
626
627
  		val = CPU_STARTING_FROZEN;
  #endif /* CONFIG_PM_SLEEP_SMP */
e9fb7631e   Akinobu Mita   cpu-hotplug: intr...
628
  	cpu_notify(val, (void *)(long)cpu);
e545a6140   Manfred Spraul   kernel/cpu.c: cre...
629
  }
68f4f1ec0   Max Krasnyansky   sched: Move cpu m...
630
  #endif /* CONFIG_SMP */
b8d317d10   Mike Travis   cpumask: make cpu...
631

e56b3bc79   Linus Torvalds   cpu masks: optimi...
632
633
634
635
  /*
   * cpu_bit_bitmap[] is a special, "compressed" data structure that
   * represents all NR_CPUS bits binary values of 1<<nr.
   *
e0b582ec5   Rusty Russell   cpumask: convert ...
636
   * It is used by cpumask_of() to get a constant address to a CPU
e56b3bc79   Linus Torvalds   cpu masks: optimi...
637
638
   * mask value that has a single bit set only.
   */
b8d317d10   Mike Travis   cpumask: make cpu...
639

e56b3bc79   Linus Torvalds   cpu masks: optimi...
640
  /* cpu_bit_bitmap[0] is empty - so we can back into it */
4d51985e4   Michael Rodriguez   kernel/cpu.c: fix...
641
  #define MASK_DECLARE_1(x)	[x+1][0] = (1UL << (x))
e56b3bc79   Linus Torvalds   cpu masks: optimi...
642
643
644
  #define MASK_DECLARE_2(x)	MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
  #define MASK_DECLARE_4(x)	MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
  #define MASK_DECLARE_8(x)	MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
b8d317d10   Mike Travis   cpumask: make cpu...
645

e56b3bc79   Linus Torvalds   cpu masks: optimi...
646
647
648
649
650
651
652
  const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
  
  	MASK_DECLARE_8(0),	MASK_DECLARE_8(8),
  	MASK_DECLARE_8(16),	MASK_DECLARE_8(24),
  #if BITS_PER_LONG > 32
  	MASK_DECLARE_8(32),	MASK_DECLARE_8(40),
  	MASK_DECLARE_8(48),	MASK_DECLARE_8(56),
b8d317d10   Mike Travis   cpumask: make cpu...
653
654
  #endif
  };
e56b3bc79   Linus Torvalds   cpu masks: optimi...
655
  EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2d3854a37   Rusty Russell   cpumask: introduc...
656
657
658
  
  const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
  EXPORT_SYMBOL(cpu_all_bits);
b3199c025   Rusty Russell   cpumask: switch o...
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
  
  #ifdef CONFIG_INIT_ALL_POSSIBLE
  static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
  	= CPU_BITS_ALL;
  #else
  static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
  #endif
  const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
  EXPORT_SYMBOL(cpu_possible_mask);
  
  static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
  const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
  EXPORT_SYMBOL(cpu_online_mask);
  
  static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
  const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
  EXPORT_SYMBOL(cpu_present_mask);
  
  static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
  const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
  EXPORT_SYMBOL(cpu_active_mask);
3fa415206   Rusty Russell   cpumask: make set...
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
  
  void set_cpu_possible(unsigned int cpu, bool possible)
  {
  	if (possible)
  		cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
  	else
  		cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
  }
  
  void set_cpu_present(unsigned int cpu, bool present)
  {
  	if (present)
  		cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
  	else
  		cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
  }
  
  void set_cpu_online(unsigned int cpu, bool online)
  {
  	if (online)
  		cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
  	else
  		cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
  }
  
  void set_cpu_active(unsigned int cpu, bool active)
  {
  	if (active)
  		cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
  	else
  		cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
  }
  
  void init_cpu_present(const struct cpumask *src)
  {
  	cpumask_copy(to_cpumask(cpu_present_bits), src);
  }
  
  void init_cpu_possible(const struct cpumask *src)
  {
  	cpumask_copy(to_cpumask(cpu_possible_bits), src);
  }
  
  void init_cpu_online(const struct cpumask *src)
  {
  	cpumask_copy(to_cpumask(cpu_online_bits), src);
  }