Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 16e53dbf10a2d7e228709a7286310e629ede5e45

Authored by Srivatsa S. Bhat
Committed by Linus Torvalds
1 parent 77293e215e
Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga

CPU hotplug: provide a generic helper to disable/enable CPU hotplug 

There are instances in the kernel where we would like to disable CPU
hotplug (from sysfs) during some important operation.  Today the freezer
code depends on this and the code to do it was kinda tailor-made for
that.

Restructure the code and make it generic enough to be useful for other
usecases too.

Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Signed-off-by: Robin Holt <holt@sgi.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Russ Anderson <rja@sgi.com>
Cc: Robin Holt <holt@sgi.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Guan Xuetao <gxt@mprc.pku.edu.cn>
Cc: Shawn Guo <shawn.guo@linaro.org>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 2 changed files with 27 additions and 32 deletions Inline Diff

1 /* 1 /*
2 * include/linux/cpu.h - generic cpu definition 2 * include/linux/cpu.h - generic cpu definition
3 * 3 *
4 * This is mainly for topological representation. We define the 4 * This is mainly for topological representation. We define the
5 * basic 'struct cpu' here, which can be embedded in per-arch 5 * basic 'struct cpu' here, which can be embedded in per-arch
6 * definitions of processors. 6 * definitions of processors.
7 * 7 *
8 * Basic handling of the devices is done in drivers/base/cpu.c 8 * Basic handling of the devices is done in drivers/base/cpu.c
9 * and system devices are handled in drivers/base/sys.c. 9 * and system devices are handled in drivers/base/sys.c.
10 * 10 *
11 * CPUs are exported via sysfs in the class/cpu/devices/ 11 * CPUs are exported via sysfs in the class/cpu/devices/
12 * directory. 12 * directory.
13 */ 13 */
14 #ifndef _LINUX_CPU_H_ 14 #ifndef _LINUX_CPU_H_
15 #define _LINUX_CPU_H_ 15 #define _LINUX_CPU_H_
16 16
17 #include <linux/node.h> 17 #include <linux/node.h>
18 #include <linux/compiler.h> 18 #include <linux/compiler.h>
19 #include <linux/cpumask.h> 19 #include <linux/cpumask.h>
20 20
21 struct device; 21 struct device;
22 22
23 struct cpu { 23 struct cpu {
24 int node_id; /* The node which contains the CPU */ 24 int node_id; /* The node which contains the CPU */
25 int hotpluggable; /* creates sysfs control file if hotpluggable */ 25 int hotpluggable; /* creates sysfs control file if hotpluggable */
26 struct device dev; 26 struct device dev;
27 }; 27 };
28 28
29 extern int register_cpu(struct cpu *cpu, int num); 29 extern int register_cpu(struct cpu *cpu, int num);
30 extern struct device *get_cpu_device(unsigned cpu); 30 extern struct device *get_cpu_device(unsigned cpu);
31 extern bool cpu_is_hotpluggable(unsigned cpu); 31 extern bool cpu_is_hotpluggable(unsigned cpu);
32 32
33 extern int cpu_add_dev_attr(struct device_attribute *attr); 33 extern int cpu_add_dev_attr(struct device_attribute *attr);
34 extern void cpu_remove_dev_attr(struct device_attribute *attr); 34 extern void cpu_remove_dev_attr(struct device_attribute *attr);
35 35
36 extern int cpu_add_dev_attr_group(struct attribute_group *attrs); 36 extern int cpu_add_dev_attr_group(struct attribute_group *attrs);
37 extern void cpu_remove_dev_attr_group(struct attribute_group *attrs); 37 extern void cpu_remove_dev_attr_group(struct attribute_group *attrs);
38 38
39 #ifdef CONFIG_HOTPLUG_CPU 39 #ifdef CONFIG_HOTPLUG_CPU
40 extern void unregister_cpu(struct cpu *cpu); 40 extern void unregister_cpu(struct cpu *cpu);
41 extern ssize_t arch_cpu_probe(const char *, size_t); 41 extern ssize_t arch_cpu_probe(const char *, size_t);
42 extern ssize_t arch_cpu_release(const char *, size_t); 42 extern ssize_t arch_cpu_release(const char *, size_t);
43 #endif 43 #endif
44 struct notifier_block; 44 struct notifier_block;
45 45
46 #ifdef CONFIG_ARCH_HAS_CPU_AUTOPROBE 46 #ifdef CONFIG_ARCH_HAS_CPU_AUTOPROBE
47 extern int arch_cpu_uevent(struct device *dev, struct kobj_uevent_env *env); 47 extern int arch_cpu_uevent(struct device *dev, struct kobj_uevent_env *env);
48 extern ssize_t arch_print_cpu_modalias(struct device *dev, 48 extern ssize_t arch_print_cpu_modalias(struct device *dev,
49 struct device_attribute *attr, 49 struct device_attribute *attr,
50 char *bufptr); 50 char *bufptr);
51 #endif 51 #endif
52 52
53 /* 53 /*
54 * CPU notifier priorities. 54 * CPU notifier priorities.
55 */ 55 */
56 enum { 56 enum {
57 /* 57 /*
58 * SCHED_ACTIVE marks a cpu which is coming up active during 58 * SCHED_ACTIVE marks a cpu which is coming up active during
59 * CPU_ONLINE and CPU_DOWN_FAILED and must be the first 59 * CPU_ONLINE and CPU_DOWN_FAILED and must be the first
60 * notifier. CPUSET_ACTIVE adjusts cpuset according to 60 * notifier. CPUSET_ACTIVE adjusts cpuset according to
61 * cpu_active mask right after SCHED_ACTIVE. During 61 * cpu_active mask right after SCHED_ACTIVE. During
62 * CPU_DOWN_PREPARE, SCHED_INACTIVE and CPUSET_INACTIVE are 62 * CPU_DOWN_PREPARE, SCHED_INACTIVE and CPUSET_INACTIVE are
63 * ordered in the similar way. 63 * ordered in the similar way.
64 * 64 *
65 * This ordering guarantees consistent cpu_active mask and 65 * This ordering guarantees consistent cpu_active mask and
66 * migration behavior to all cpu notifiers. 66 * migration behavior to all cpu notifiers.
67 */ 67 */
68 CPU_PRI_SCHED_ACTIVE = INT_MAX, 68 CPU_PRI_SCHED_ACTIVE = INT_MAX,
69 CPU_PRI_CPUSET_ACTIVE = INT_MAX - 1, 69 CPU_PRI_CPUSET_ACTIVE = INT_MAX - 1,
70 CPU_PRI_SCHED_INACTIVE = INT_MIN + 1, 70 CPU_PRI_SCHED_INACTIVE = INT_MIN + 1,
71 CPU_PRI_CPUSET_INACTIVE = INT_MIN, 71 CPU_PRI_CPUSET_INACTIVE = INT_MIN,
72 72
73 /* migration should happen before other stuff but after perf */ 73 /* migration should happen before other stuff but after perf */
74 CPU_PRI_PERF = 20, 74 CPU_PRI_PERF = 20,
75 CPU_PRI_MIGRATION = 10, 75 CPU_PRI_MIGRATION = 10,
76 /* bring up workqueues before normal notifiers and down after */ 76 /* bring up workqueues before normal notifiers and down after */
77 CPU_PRI_WORKQUEUE_UP = 5, 77 CPU_PRI_WORKQUEUE_UP = 5,
78 CPU_PRI_WORKQUEUE_DOWN = -5, 78 CPU_PRI_WORKQUEUE_DOWN = -5,
79 }; 79 };
80 80
81 #define CPU_ONLINE 0x0002 /* CPU (unsigned)v is up */ 81 #define CPU_ONLINE 0x0002 /* CPU (unsigned)v is up */
82 #define CPU_UP_PREPARE 0x0003 /* CPU (unsigned)v coming up */ 82 #define CPU_UP_PREPARE 0x0003 /* CPU (unsigned)v coming up */
83 #define CPU_UP_CANCELED 0x0004 /* CPU (unsigned)v NOT coming up */ 83 #define CPU_UP_CANCELED 0x0004 /* CPU (unsigned)v NOT coming up */
84 #define CPU_DOWN_PREPARE 0x0005 /* CPU (unsigned)v going down */ 84 #define CPU_DOWN_PREPARE 0x0005 /* CPU (unsigned)v going down */
85 #define CPU_DOWN_FAILED 0x0006 /* CPU (unsigned)v NOT going down */ 85 #define CPU_DOWN_FAILED 0x0006 /* CPU (unsigned)v NOT going down */
86 #define CPU_DEAD 0x0007 /* CPU (unsigned)v dead */ 86 #define CPU_DEAD 0x0007 /* CPU (unsigned)v dead */
87 #define CPU_DYING 0x0008 /* CPU (unsigned)v not running any task, 87 #define CPU_DYING 0x0008 /* CPU (unsigned)v not running any task,
88 * not handling interrupts, soon dead. 88 * not handling interrupts, soon dead.
89 * Called on the dying cpu, interrupts 89 * Called on the dying cpu, interrupts
90 * are already disabled. Must not 90 * are already disabled. Must not
91 * sleep, must not fail */ 91 * sleep, must not fail */
92 #define CPU_POST_DEAD 0x0009 /* CPU (unsigned)v dead, cpu_hotplug 92 #define CPU_POST_DEAD 0x0009 /* CPU (unsigned)v dead, cpu_hotplug
93 * lock is dropped */ 93 * lock is dropped */
94 #define CPU_STARTING 0x000A /* CPU (unsigned)v soon running. 94 #define CPU_STARTING 0x000A /* CPU (unsigned)v soon running.
95 * Called on the new cpu, just before 95 * Called on the new cpu, just before
96 * enabling interrupts. Must not sleep, 96 * enabling interrupts. Must not sleep,
97 * must not fail */ 97 * must not fail */
98 98
99 /* Used for CPU hotplug events occurring while tasks are frozen due to a suspend 99 /* Used for CPU hotplug events occurring while tasks are frozen due to a suspend
100 * operation in progress 100 * operation in progress
101 */ 101 */
102 #define CPU_TASKS_FROZEN 0x0010 102 #define CPU_TASKS_FROZEN 0x0010
103 103
104 #define CPU_ONLINE_FROZEN (CPU_ONLINE | CPU_TASKS_FROZEN) 104 #define CPU_ONLINE_FROZEN (CPU_ONLINE | CPU_TASKS_FROZEN)
105 #define CPU_UP_PREPARE_FROZEN (CPU_UP_PREPARE | CPU_TASKS_FROZEN) 105 #define CPU_UP_PREPARE_FROZEN (CPU_UP_PREPARE | CPU_TASKS_FROZEN)
106 #define CPU_UP_CANCELED_FROZEN (CPU_UP_CANCELED | CPU_TASKS_FROZEN) 106 #define CPU_UP_CANCELED_FROZEN (CPU_UP_CANCELED | CPU_TASKS_FROZEN)
107 #define CPU_DOWN_PREPARE_FROZEN (CPU_DOWN_PREPARE | CPU_TASKS_FROZEN) 107 #define CPU_DOWN_PREPARE_FROZEN (CPU_DOWN_PREPARE | CPU_TASKS_FROZEN)
108 #define CPU_DOWN_FAILED_FROZEN (CPU_DOWN_FAILED | CPU_TASKS_FROZEN) 108 #define CPU_DOWN_FAILED_FROZEN (CPU_DOWN_FAILED | CPU_TASKS_FROZEN)
109 #define CPU_DEAD_FROZEN (CPU_DEAD | CPU_TASKS_FROZEN) 109 #define CPU_DEAD_FROZEN (CPU_DEAD | CPU_TASKS_FROZEN)
110 #define CPU_DYING_FROZEN (CPU_DYING | CPU_TASKS_FROZEN) 110 #define CPU_DYING_FROZEN (CPU_DYING | CPU_TASKS_FROZEN)
111 #define CPU_STARTING_FROZEN (CPU_STARTING | CPU_TASKS_FROZEN) 111 #define CPU_STARTING_FROZEN (CPU_STARTING | CPU_TASKS_FROZEN)
112 112
113 113
114 #ifdef CONFIG_SMP 114 #ifdef CONFIG_SMP
115 /* Need to know about CPUs going up/down? */ 115 /* Need to know about CPUs going up/down? */
116 #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) 116 #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE)
117 #define cpu_notifier(fn, pri) { \ 117 #define cpu_notifier(fn, pri) { \
118 static struct notifier_block fn##_nb __cpuinitdata = \ 118 static struct notifier_block fn##_nb __cpuinitdata = \
119 { .notifier_call = fn, .priority = pri }; \ 119 { .notifier_call = fn, .priority = pri }; \
120 register_cpu_notifier(&fn##_nb); \ 120 register_cpu_notifier(&fn##_nb); \
121 } 121 }
122 #else /* #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ 122 #else /* #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */
123 #define cpu_notifier(fn, pri) do { (void)(fn); } while (0) 123 #define cpu_notifier(fn, pri) do { (void)(fn); } while (0)
124 #endif /* #else #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ 124 #endif /* #else #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */
125 #ifdef CONFIG_HOTPLUG_CPU 125 #ifdef CONFIG_HOTPLUG_CPU
126 extern int register_cpu_notifier(struct notifier_block *nb); 126 extern int register_cpu_notifier(struct notifier_block *nb);
127 extern void unregister_cpu_notifier(struct notifier_block *nb); 127 extern void unregister_cpu_notifier(struct notifier_block *nb);
128 #else 128 #else
129 129
130 #ifndef MODULE 130 #ifndef MODULE
131 extern int register_cpu_notifier(struct notifier_block *nb); 131 extern int register_cpu_notifier(struct notifier_block *nb);
132 #else 132 #else
133 static inline int register_cpu_notifier(struct notifier_block *nb) 133 static inline int register_cpu_notifier(struct notifier_block *nb)
134 { 134 {
135 return 0; 135 return 0;
136 } 136 }
137 #endif 137 #endif
138 138
139 static inline void unregister_cpu_notifier(struct notifier_block *nb) 139 static inline void unregister_cpu_notifier(struct notifier_block *nb)
140 { 140 {
141 } 141 }
142 #endif 142 #endif
143 143
144 int cpu_up(unsigned int cpu); 144 int cpu_up(unsigned int cpu);
145 void notify_cpu_starting(unsigned int cpu); 145 void notify_cpu_starting(unsigned int cpu);
146 extern void cpu_maps_update_begin(void); 146 extern void cpu_maps_update_begin(void);
147 extern void cpu_maps_update_done(void); 147 extern void cpu_maps_update_done(void);
148 148
149 #else /* CONFIG_SMP */ 149 #else /* CONFIG_SMP */
150 150
151 #define cpu_notifier(fn, pri) do { (void)(fn); } while (0) 151 #define cpu_notifier(fn, pri) do { (void)(fn); } while (0)
152 152
153 static inline int register_cpu_notifier(struct notifier_block *nb) 153 static inline int register_cpu_notifier(struct notifier_block *nb)
154 { 154 {
155 return 0; 155 return 0;
156 } 156 }
157 157
158 static inline void unregister_cpu_notifier(struct notifier_block *nb) 158 static inline void unregister_cpu_notifier(struct notifier_block *nb)
159 { 159 {
160 } 160 }
161 161
162 static inline void cpu_maps_update_begin(void) 162 static inline void cpu_maps_update_begin(void)
163 { 163 {
164 } 164 }
165 165
166 static inline void cpu_maps_update_done(void) 166 static inline void cpu_maps_update_done(void)
167 { 167 {
168 } 168 }
169 169
170 #endif /* CONFIG_SMP */ 170 #endif /* CONFIG_SMP */
171 extern struct bus_type cpu_subsys; 171 extern struct bus_type cpu_subsys;
172 172
173 #ifdef CONFIG_HOTPLUG_CPU 173 #ifdef CONFIG_HOTPLUG_CPU
174 /* Stop CPUs going up and down. */ 174 /* Stop CPUs going up and down. */
175 175
176 extern void get_online_cpus(void); 176 extern void get_online_cpus(void);
177 extern void put_online_cpus(void); 177 extern void put_online_cpus(void);
178 extern void cpu_hotplug_disable(void);
179 extern void cpu_hotplug_enable(void);
178 #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) 180 #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri)
179 #define register_hotcpu_notifier(nb) register_cpu_notifier(nb) 181 #define register_hotcpu_notifier(nb) register_cpu_notifier(nb)
180 #define unregister_hotcpu_notifier(nb) unregister_cpu_notifier(nb) 182 #define unregister_hotcpu_notifier(nb) unregister_cpu_notifier(nb)
181 void clear_tasks_mm_cpumask(int cpu); 183 void clear_tasks_mm_cpumask(int cpu);
182 int cpu_down(unsigned int cpu); 184 int cpu_down(unsigned int cpu);
183 185
184 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE 186 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
185 extern void cpu_hotplug_driver_lock(void); 187 extern void cpu_hotplug_driver_lock(void);
186 extern void cpu_hotplug_driver_unlock(void); 188 extern void cpu_hotplug_driver_unlock(void);
187 #else 189 #else
188 static inline void cpu_hotplug_driver_lock(void) 190 static inline void cpu_hotplug_driver_lock(void)
189 { 191 {
190 } 192 }
191 193
192 static inline void cpu_hotplug_driver_unlock(void) 194 static inline void cpu_hotplug_driver_unlock(void)
193 { 195 {
194 } 196 }
195 #endif 197 #endif
196 198
197 #else /* CONFIG_HOTPLUG_CPU */ 199 #else /* CONFIG_HOTPLUG_CPU */
198 200
199 #define get_online_cpus() do { } while (0) 201 #define get_online_cpus() do { } while (0)
200 #define put_online_cpus() do { } while (0) 202 #define put_online_cpus() do { } while (0)
203 #define cpu_hotplug_disable() do { } while (0)
204 #define cpu_hotplug_enable() do { } while (0)
201 #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) 205 #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
202 /* These aren't inline functions due to a GCC bug. */ 206 /* These aren't inline functions due to a GCC bug. */
203 #define register_hotcpu_notifier(nb) ({ (void)(nb); 0; }) 207 #define register_hotcpu_notifier(nb) ({ (void)(nb); 0; })
204 #define unregister_hotcpu_notifier(nb) ({ (void)(nb); }) 208 #define unregister_hotcpu_notifier(nb) ({ (void)(nb); })
205 #endif /* CONFIG_HOTPLUG_CPU */ 209 #endif /* CONFIG_HOTPLUG_CPU */
206 210
207 #ifdef CONFIG_PM_SLEEP_SMP 211 #ifdef CONFIG_PM_SLEEP_SMP
208 extern int disable_nonboot_cpus(void); 212 extern int disable_nonboot_cpus(void);
209 extern void enable_nonboot_cpus(void); 213 extern void enable_nonboot_cpus(void);
210 #else /* !CONFIG_PM_SLEEP_SMP */ 214 #else /* !CONFIG_PM_SLEEP_SMP */
211 static inline int disable_nonboot_cpus(void) { return 0; } 215 static inline int disable_nonboot_cpus(void) { return 0; }
212 static inline void enable_nonboot_cpus(void) {} 216 static inline void enable_nonboot_cpus(void) {}
213 #endif /* !CONFIG_PM_SLEEP_SMP */ 217 #endif /* !CONFIG_PM_SLEEP_SMP */
214 218
215 enum cpuhp_state { 219 enum cpuhp_state {
216 CPUHP_OFFLINE, 220 CPUHP_OFFLINE,
217 CPUHP_ONLINE, 221 CPUHP_ONLINE,
218 }; 222 };
219 223
220 void cpu_startup_entry(enum cpuhp_state state); 224 void cpu_startup_entry(enum cpuhp_state state);
221 void cpu_idle(void); 225 void cpu_idle(void);
222 226
223 void cpu_idle_poll_ctrl(bool enable); 227 void cpu_idle_poll_ctrl(bool enable);
224 228
225 void arch_cpu_idle(void); 229 void arch_cpu_idle(void);
226 void arch_cpu_idle_prepare(void); 230 void arch_cpu_idle_prepare(void);
227 void arch_cpu_idle_enter(void); 231 void arch_cpu_idle_enter(void);
228 void arch_cpu_idle_exit(void); 232 void arch_cpu_idle_exit(void);
229 void arch_cpu_idle_dead(void); 233 void arch_cpu_idle_dead(void);
230 234
231 #endif /* _LINUX_CPU_H_ */ 235 #endif /* _LINUX_CPU_H_ */
232 236
1 /* CPU control. 1 /* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell 2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 * 3 *
4 * This code is licenced under the GPL. 4 * This code is licenced under the GPL.
5 */ 5 */
6 #include <linux/proc_fs.h> 6 #include <linux/proc_fs.h>
7 #include <linux/smp.h> 7 #include <linux/smp.h>
8 #include <linux/init.h> 8 #include <linux/init.h>
9 #include <linux/notifier.h> 9 #include <linux/notifier.h>
10 #include <linux/sched.h> 10 #include <linux/sched.h>
11 #include <linux/unistd.h> 11 #include <linux/unistd.h>
12 #include <linux/cpu.h> 12 #include <linux/cpu.h>
13 #include <linux/oom.h> 13 #include <linux/oom.h>
14 #include <linux/rcupdate.h> 14 #include <linux/rcupdate.h>
15 #include <linux/export.h> 15 #include <linux/export.h>
16 #include <linux/bug.h> 16 #include <linux/bug.h>
17 #include <linux/kthread.h> 17 #include <linux/kthread.h>
18 #include <linux/stop_machine.h> 18 #include <linux/stop_machine.h>
19 #include <linux/mutex.h> 19 #include <linux/mutex.h>
20 #include <linux/gfp.h> 20 #include <linux/gfp.h>
21 #include <linux/suspend.h> 21 #include <linux/suspend.h>
22 22
23 #include "smpboot.h" 23 #include "smpboot.h"
24 24
25 #ifdef CONFIG_SMP 25 #ifdef CONFIG_SMP
26 /* Serializes the updates to cpu_online_mask, cpu_present_mask */ 26 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
27 static DEFINE_MUTEX(cpu_add_remove_lock); 27 static DEFINE_MUTEX(cpu_add_remove_lock);
28 28
29 /* 29 /*
30 * The following two API's must be used when attempting 30 * The following two API's must be used when attempting
31 * to serialize the updates to cpu_online_mask, cpu_present_mask. 31 * to serialize the updates to cpu_online_mask, cpu_present_mask.
32 */ 32 */
33 void cpu_maps_update_begin(void) 33 void cpu_maps_update_begin(void)
34 { 34 {
35 mutex_lock(&cpu_add_remove_lock); 35 mutex_lock(&cpu_add_remove_lock);
36 } 36 }
37 37
38 void cpu_maps_update_done(void) 38 void cpu_maps_update_done(void)
39 { 39 {
40 mutex_unlock(&cpu_add_remove_lock); 40 mutex_unlock(&cpu_add_remove_lock);
41 } 41 }
42 42
43 static RAW_NOTIFIER_HEAD(cpu_chain); 43 static RAW_NOTIFIER_HEAD(cpu_chain);
44 44
45 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. 45 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
46 * Should always be manipulated under cpu_add_remove_lock 46 * Should always be manipulated under cpu_add_remove_lock
47 */ 47 */
48 static int cpu_hotplug_disabled; 48 static int cpu_hotplug_disabled;
49 49
50 #ifdef CONFIG_HOTPLUG_CPU 50 #ifdef CONFIG_HOTPLUG_CPU
51 51
52 static struct { 52 static struct {
53 struct task_struct *active_writer; 53 struct task_struct *active_writer;
54 struct mutex lock; /* Synchronizes accesses to refcount, */ 54 struct mutex lock; /* Synchronizes accesses to refcount, */
55 /* 55 /*
56 * Also blocks the new readers during 56 * Also blocks the new readers during
57 * an ongoing cpu hotplug operation. 57 * an ongoing cpu hotplug operation.
58 */ 58 */
59 int refcount; 59 int refcount;
60 } cpu_hotplug = { 60 } cpu_hotplug = {
61 .active_writer = NULL, 61 .active_writer = NULL,
62 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), 62 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
63 .refcount = 0, 63 .refcount = 0,
64 }; 64 };
65 65
66 void get_online_cpus(void) 66 void get_online_cpus(void)
67 { 67 {
68 might_sleep(); 68 might_sleep();
69 if (cpu_hotplug.active_writer == current) 69 if (cpu_hotplug.active_writer == current)
70 return; 70 return;
71 mutex_lock(&cpu_hotplug.lock); 71 mutex_lock(&cpu_hotplug.lock);
72 cpu_hotplug.refcount++; 72 cpu_hotplug.refcount++;
73 mutex_unlock(&cpu_hotplug.lock); 73 mutex_unlock(&cpu_hotplug.lock);
74 74
75 } 75 }
76 EXPORT_SYMBOL_GPL(get_online_cpus); 76 EXPORT_SYMBOL_GPL(get_online_cpus);
77 77
78 void put_online_cpus(void) 78 void put_online_cpus(void)
79 { 79 {
80 if (cpu_hotplug.active_writer == current) 80 if (cpu_hotplug.active_writer == current)
81 return; 81 return;
82 mutex_lock(&cpu_hotplug.lock); 82 mutex_lock(&cpu_hotplug.lock);
83 83
84 if (WARN_ON(!cpu_hotplug.refcount)) 84 if (WARN_ON(!cpu_hotplug.refcount))
85 cpu_hotplug.refcount++; /* try to fix things up */ 85 cpu_hotplug.refcount++; /* try to fix things up */
86 86
87 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer)) 87 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
88 wake_up_process(cpu_hotplug.active_writer); 88 wake_up_process(cpu_hotplug.active_writer);
89 mutex_unlock(&cpu_hotplug.lock); 89 mutex_unlock(&cpu_hotplug.lock);
90 90
91 } 91 }
92 EXPORT_SYMBOL_GPL(put_online_cpus); 92 EXPORT_SYMBOL_GPL(put_online_cpus);
93 93
94 /* 94 /*
95 * This ensures that the hotplug operation can begin only when the 95 * This ensures that the hotplug operation can begin only when the
96 * refcount goes to zero. 96 * refcount goes to zero.
97 * 97 *
98 * Note that during a cpu-hotplug operation, the new readers, if any, 98 * Note that during a cpu-hotplug operation, the new readers, if any,
99 * will be blocked by the cpu_hotplug.lock 99 * will be blocked by the cpu_hotplug.lock
100 * 100 *
101 * Since cpu_hotplug_begin() is always called after invoking 101 * Since cpu_hotplug_begin() is always called after invoking
102 * cpu_maps_update_begin(), we can be sure that only one writer is active. 102 * cpu_maps_update_begin(), we can be sure that only one writer is active.
103 * 103 *
104 * Note that theoretically, there is a possibility of a livelock: 104 * Note that theoretically, there is a possibility of a livelock:
105 * - Refcount goes to zero, last reader wakes up the sleeping 105 * - Refcount goes to zero, last reader wakes up the sleeping
106 * writer. 106 * writer.
107 * - Last reader unlocks the cpu_hotplug.lock. 107 * - Last reader unlocks the cpu_hotplug.lock.
108 * - A new reader arrives at this moment, bumps up the refcount. 108 * - A new reader arrives at this moment, bumps up the refcount.
109 * - The writer acquires the cpu_hotplug.lock finds the refcount 109 * - The writer acquires the cpu_hotplug.lock finds the refcount
110 * non zero and goes to sleep again. 110 * non zero and goes to sleep again.
111 * 111 *
112 * However, this is very difficult to achieve in practice since 112 * However, this is very difficult to achieve in practice since
113 * get_online_cpus() not an api which is called all that often. 113 * get_online_cpus() not an api which is called all that often.
114 * 114 *
115 */ 115 */
116 static void cpu_hotplug_begin(void) 116 static void cpu_hotplug_begin(void)
117 { 117 {
118 cpu_hotplug.active_writer = current; 118 cpu_hotplug.active_writer = current;
119 119
120 for (;;) { 120 for (;;) {
121 mutex_lock(&cpu_hotplug.lock); 121 mutex_lock(&cpu_hotplug.lock);
122 if (likely(!cpu_hotplug.refcount)) 122 if (likely(!cpu_hotplug.refcount))
123 break; 123 break;
124 __set_current_state(TASK_UNINTERRUPTIBLE); 124 __set_current_state(TASK_UNINTERRUPTIBLE);
125 mutex_unlock(&cpu_hotplug.lock); 125 mutex_unlock(&cpu_hotplug.lock);
126 schedule(); 126 schedule();
127 } 127 }
128 } 128 }
129 129
130 static void cpu_hotplug_done(void) 130 static void cpu_hotplug_done(void)
131 { 131 {
132 cpu_hotplug.active_writer = NULL; 132 cpu_hotplug.active_writer = NULL;
133 mutex_unlock(&cpu_hotplug.lock); 133 mutex_unlock(&cpu_hotplug.lock);
134 } 134 }
135 135
136 /*
137 * Wait for currently running CPU hotplug operations to complete (if any) and
138 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
139 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
140 * hotplug path before performing hotplug operations. So acquiring that lock
141 * guarantees mutual exclusion from any currently running hotplug operations.
142 */
143 void cpu_hotplug_disable(void)
144 {
145 cpu_maps_update_begin();
146 cpu_hotplug_disabled = 1;
147 cpu_maps_update_done();
148 }
149
150 void cpu_hotplug_enable(void)
151 {
152 cpu_maps_update_begin();
153 cpu_hotplug_disabled = 0;
154 cpu_maps_update_done();
155 }
156
136 #else /* #if CONFIG_HOTPLUG_CPU */ 157 #else /* #if CONFIG_HOTPLUG_CPU */
137 static void cpu_hotplug_begin(void) {} 158 static void cpu_hotplug_begin(void) {}
138 static void cpu_hotplug_done(void) {} 159 static void cpu_hotplug_done(void) {}
139 #endif /* #else #if CONFIG_HOTPLUG_CPU */ 160 #endif /* #else #if CONFIG_HOTPLUG_CPU */
140 161
141 /* Need to know about CPUs going up/down? */ 162 /* Need to know about CPUs going up/down? */
142 int __ref register_cpu_notifier(struct notifier_block *nb) 163 int __ref register_cpu_notifier(struct notifier_block *nb)
143 { 164 {
144 int ret; 165 int ret;
145 cpu_maps_update_begin(); 166 cpu_maps_update_begin();
146 ret = raw_notifier_chain_register(&cpu_chain, nb); 167 ret = raw_notifier_chain_register(&cpu_chain, nb);
147 cpu_maps_update_done(); 168 cpu_maps_update_done();
148 return ret; 169 return ret;
149 } 170 }
150 171
151 static int __cpu_notify(unsigned long val, void *v, int nr_to_call, 172 static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
152 int *nr_calls) 173 int *nr_calls)
153 { 174 {
154 int ret; 175 int ret;
155 176
156 ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call, 177 ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
157 nr_calls); 178 nr_calls);
158 179
159 return notifier_to_errno(ret); 180 return notifier_to_errno(ret);
160 } 181 }
161 182
162 static int cpu_notify(unsigned long val, void *v) 183 static int cpu_notify(unsigned long val, void *v)
163 { 184 {
164 return __cpu_notify(val, v, -1, NULL); 185 return __cpu_notify(val, v, -1, NULL);
165 } 186 }
166 187
167 #ifdef CONFIG_HOTPLUG_CPU 188 #ifdef CONFIG_HOTPLUG_CPU
168 189
169 static void cpu_notify_nofail(unsigned long val, void *v) 190 static void cpu_notify_nofail(unsigned long val, void *v)
170 { 191 {
171 BUG_ON(cpu_notify(val, v)); 192 BUG_ON(cpu_notify(val, v));
172 } 193 }
173 EXPORT_SYMBOL(register_cpu_notifier); 194 EXPORT_SYMBOL(register_cpu_notifier);
174 195
175 void __ref unregister_cpu_notifier(struct notifier_block *nb) 196 void __ref unregister_cpu_notifier(struct notifier_block *nb)
176 { 197 {
177 cpu_maps_update_begin(); 198 cpu_maps_update_begin();
178 raw_notifier_chain_unregister(&cpu_chain, nb); 199 raw_notifier_chain_unregister(&cpu_chain, nb);
179 cpu_maps_update_done(); 200 cpu_maps_update_done();
180 } 201 }
181 EXPORT_SYMBOL(unregister_cpu_notifier); 202 EXPORT_SYMBOL(unregister_cpu_notifier);
182 203
183 /** 204 /**
184 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU 205 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
185 * @cpu: a CPU id 206 * @cpu: a CPU id
186 * 207 *
187 * This function walks all processes, finds a valid mm struct for each one and 208 * This function walks all processes, finds a valid mm struct for each one and
188 * then clears a corresponding bit in mm's cpumask. While this all sounds 209 * then clears a corresponding bit in mm's cpumask. While this all sounds
189 * trivial, there are various non-obvious corner cases, which this function 210 * trivial, there are various non-obvious corner cases, which this function
190 * tries to solve in a safe manner. 211 * tries to solve in a safe manner.
191 * 212 *
192 * Also note that the function uses a somewhat relaxed locking scheme, so it may 213 * Also note that the function uses a somewhat relaxed locking scheme, so it may
193 * be called only for an already offlined CPU. 214 * be called only for an already offlined CPU.
194 */ 215 */
195 void clear_tasks_mm_cpumask(int cpu) 216 void clear_tasks_mm_cpumask(int cpu)
196 { 217 {
197 struct task_struct *p; 218 struct task_struct *p;
198 219
199 /* 220 /*
200 * This function is called after the cpu is taken down and marked 221 * This function is called after the cpu is taken down and marked
201 * offline, so its not like new tasks will ever get this cpu set in 222 * offline, so its not like new tasks will ever get this cpu set in
202 * their mm mask. -- Peter Zijlstra 223 * their mm mask. -- Peter Zijlstra
203 * Thus, we may use rcu_read_lock() here, instead of grabbing 224 * Thus, we may use rcu_read_lock() here, instead of grabbing
204 * full-fledged tasklist_lock. 225 * full-fledged tasklist_lock.
205 */ 226 */
206 WARN_ON(cpu_online(cpu)); 227 WARN_ON(cpu_online(cpu));
207 rcu_read_lock(); 228 rcu_read_lock();
208 for_each_process(p) { 229 for_each_process(p) {
209 struct task_struct *t; 230 struct task_struct *t;
210 231
211 /* 232 /*
212 * Main thread might exit, but other threads may still have 233 * Main thread might exit, but other threads may still have
213 * a valid mm. Find one. 234 * a valid mm. Find one.
214 */ 235 */
215 t = find_lock_task_mm(p); 236 t = find_lock_task_mm(p);
216 if (!t) 237 if (!t)
217 continue; 238 continue;
218 cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); 239 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
219 task_unlock(t); 240 task_unlock(t);
220 } 241 }
221 rcu_read_unlock(); 242 rcu_read_unlock();
222 } 243 }
223 244
224 static inline void check_for_tasks(int cpu) 245 static inline void check_for_tasks(int cpu)
225 { 246 {
226 struct task_struct *p; 247 struct task_struct *p;
227 cputime_t utime, stime; 248 cputime_t utime, stime;
228 249
229 write_lock_irq(&tasklist_lock); 250 write_lock_irq(&tasklist_lock);
230 for_each_process(p) { 251 for_each_process(p) {
231 task_cputime(p, &utime, &stime); 252 task_cputime(p, &utime, &stime);
232 if (task_cpu(p) == cpu && p->state == TASK_RUNNING && 253 if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
233 (utime || stime)) 254 (utime || stime))
234 printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d " 255 printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
235 "(state = %ld, flags = %x)\n", 256 "(state = %ld, flags = %x)\n",
236 p->comm, task_pid_nr(p), cpu, 257 p->comm, task_pid_nr(p), cpu,
237 p->state, p->flags); 258 p->state, p->flags);
238 } 259 }
239 write_unlock_irq(&tasklist_lock); 260 write_unlock_irq(&tasklist_lock);
240 } 261 }
241 262
242 struct take_cpu_down_param { 263 struct take_cpu_down_param {
243 unsigned long mod; 264 unsigned long mod;
244 void *hcpu; 265 void *hcpu;
245 }; 266 };
246 267
247 /* Take this CPU down. */ 268 /* Take this CPU down. */
248 static int __ref take_cpu_down(void *_param) 269 static int __ref take_cpu_down(void *_param)
249 { 270 {
250 struct take_cpu_down_param *param = _param; 271 struct take_cpu_down_param *param = _param;
251 int err; 272 int err;
252 273
253 /* Ensure this CPU doesn't handle any more interrupts. */ 274 /* Ensure this CPU doesn't handle any more interrupts. */
254 err = __cpu_disable(); 275 err = __cpu_disable();
255 if (err < 0) 276 if (err < 0)
256 return err; 277 return err;
257 278
258 cpu_notify(CPU_DYING | param->mod, param->hcpu); 279 cpu_notify(CPU_DYING | param->mod, param->hcpu);
259 /* Park the stopper thread */ 280 /* Park the stopper thread */
260 kthread_park(current); 281 kthread_park(current);
261 return 0; 282 return 0;
262 } 283 }
263 284
264 /* Requires cpu_add_remove_lock to be held */ 285 /* Requires cpu_add_remove_lock to be held */
265 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) 286 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
266 { 287 {
267 int err, nr_calls = 0; 288 int err, nr_calls = 0;
268 void *hcpu = (void *)(long)cpu; 289 void *hcpu = (void *)(long)cpu;
269 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; 290 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
270 struct take_cpu_down_param tcd_param = { 291 struct take_cpu_down_param tcd_param = {
271 .mod = mod, 292 .mod = mod,
272 .hcpu = hcpu, 293 .hcpu = hcpu,
273 }; 294 };
274 295
275 if (num_online_cpus() == 1) 296 if (num_online_cpus() == 1)
276 return -EBUSY; 297 return -EBUSY;
277 298
278 if (!cpu_online(cpu)) 299 if (!cpu_online(cpu))
279 return -EINVAL; 300 return -EINVAL;
280 301
281 cpu_hotplug_begin(); 302 cpu_hotplug_begin();
282 303
283 err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); 304 err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
284 if (err) { 305 if (err) {
285 nr_calls--; 306 nr_calls--;
286 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL); 307 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
287 printk("%s: attempt to take down CPU %u failed\n", 308 printk("%s: attempt to take down CPU %u failed\n",
288 __func__, cpu); 309 __func__, cpu);
289 goto out_release; 310 goto out_release;
290 } 311 }
291 smpboot_park_threads(cpu); 312 smpboot_park_threads(cpu);
292 313
293 err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); 314 err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
294 if (err) { 315 if (err) {
295 /* CPU didn't die: tell everyone. Can't complain. */ 316 /* CPU didn't die: tell everyone. Can't complain. */
296 smpboot_unpark_threads(cpu); 317 smpboot_unpark_threads(cpu);
297 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu); 318 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
298 goto out_release; 319 goto out_release;
299 } 320 }
300 BUG_ON(cpu_online(cpu)); 321 BUG_ON(cpu_online(cpu));
301 322
302 /* 323 /*
303 * The migration_call() CPU_DYING callback will have removed all 324 * The migration_call() CPU_DYING callback will have removed all
304 * runnable tasks from the cpu, there's only the idle task left now 325 * runnable tasks from the cpu, there's only the idle task left now
305 * that the migration thread is done doing the stop_machine thing. 326 * that the migration thread is done doing the stop_machine thing.
306 * 327 *
307 * Wait for the stop thread to go away. 328 * Wait for the stop thread to go away.
308 */ 329 */
309 while (!idle_cpu(cpu)) 330 while (!idle_cpu(cpu))
310 cpu_relax(); 331 cpu_relax();
311 332
312 /* This actually kills the CPU. */ 333 /* This actually kills the CPU. */
313 __cpu_die(cpu); 334 __cpu_die(cpu);
314 335
315 /* CPU is completely dead: tell everyone. Too late to complain. */ 336 /* CPU is completely dead: tell everyone. Too late to complain. */
316 cpu_notify_nofail(CPU_DEAD | mod, hcpu); 337 cpu_notify_nofail(CPU_DEAD | mod, hcpu);
317 338
318 check_for_tasks(cpu); 339 check_for_tasks(cpu);
319 340
320 out_release: 341 out_release:
321 cpu_hotplug_done(); 342 cpu_hotplug_done();
322 if (!err) 343 if (!err)
323 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); 344 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
324 return err; 345 return err;
325 } 346 }
326 347
327 int __ref cpu_down(unsigned int cpu) 348 int __ref cpu_down(unsigned int cpu)
328 { 349 {
329 int err; 350 int err;
330 351
331 cpu_maps_update_begin(); 352 cpu_maps_update_begin();
332 353
333 if (cpu_hotplug_disabled) { 354 if (cpu_hotplug_disabled) {
334 err = -EBUSY; 355 err = -EBUSY;
335 goto out; 356 goto out;
336 } 357 }
337 358
338 err = _cpu_down(cpu, 0); 359 err = _cpu_down(cpu, 0);
339 360
340 out: 361 out:
341 cpu_maps_update_done(); 362 cpu_maps_update_done();
342 return err; 363 return err;
343 } 364 }
344 EXPORT_SYMBOL(cpu_down); 365 EXPORT_SYMBOL(cpu_down);
345 #endif /*CONFIG_HOTPLUG_CPU*/ 366 #endif /*CONFIG_HOTPLUG_CPU*/
346 367
347 /* Requires cpu_add_remove_lock to be held */ 368 /* Requires cpu_add_remove_lock to be held */
348 static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) 369 static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
349 { 370 {
350 int ret, nr_calls = 0; 371 int ret, nr_calls = 0;
351 void *hcpu = (void *)(long)cpu; 372 void *hcpu = (void *)(long)cpu;
352 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; 373 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
353 struct task_struct *idle; 374 struct task_struct *idle;
354 375
355 cpu_hotplug_begin(); 376 cpu_hotplug_begin();
356 377
357 if (cpu_online(cpu) || !cpu_present(cpu)) { 378 if (cpu_online(cpu) || !cpu_present(cpu)) {
358 ret = -EINVAL; 379 ret = -EINVAL;
359 goto out; 380 goto out;
360 } 381 }
361 382
362 idle = idle_thread_get(cpu); 383 idle = idle_thread_get(cpu);
363 if (IS_ERR(idle)) { 384 if (IS_ERR(idle)) {
364 ret = PTR_ERR(idle); 385 ret = PTR_ERR(idle);
365 goto out; 386 goto out;
366 } 387 }
367 388
368 ret = smpboot_create_threads(cpu); 389 ret = smpboot_create_threads(cpu);
369 if (ret) 390 if (ret)
370 goto out; 391 goto out;
371 392
372 ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls); 393 ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
373 if (ret) { 394 if (ret) {
374 nr_calls--; 395 nr_calls--;
375 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n", 396 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
376 __func__, cpu); 397 __func__, cpu);
377 goto out_notify; 398 goto out_notify;
378 } 399 }
379 400
380 /* Arch-specific enabling code. */ 401 /* Arch-specific enabling code. */
381 ret = __cpu_up(cpu, idle); 402 ret = __cpu_up(cpu, idle);
382 if (ret != 0) 403 if (ret != 0)
383 goto out_notify; 404 goto out_notify;
384 BUG_ON(!cpu_online(cpu)); 405 BUG_ON(!cpu_online(cpu));
385 406
386 /* Wake the per cpu threads */ 407 /* Wake the per cpu threads */
387 smpboot_unpark_threads(cpu); 408 smpboot_unpark_threads(cpu);
388 409
389 /* Now call notifier in preparation. */ 410 /* Now call notifier in preparation. */
390 cpu_notify(CPU_ONLINE | mod, hcpu); 411 cpu_notify(CPU_ONLINE | mod, hcpu);
391 412
392 out_notify: 413 out_notify:
393 if (ret != 0) 414 if (ret != 0)
394 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL); 415 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
395 out: 416 out:
396 cpu_hotplug_done(); 417 cpu_hotplug_done();
397 418
398 return ret; 419 return ret;
399 } 420 }
400 421
401 int __cpuinit cpu_up(unsigned int cpu) 422 int __cpuinit cpu_up(unsigned int cpu)
402 { 423 {
403 int err = 0; 424 int err = 0;
404 425
405 #ifdef CONFIG_MEMORY_HOTPLUG 426 #ifdef CONFIG_MEMORY_HOTPLUG
406 int nid; 427 int nid;
407 pg_data_t *pgdat; 428 pg_data_t *pgdat;
408 #endif 429 #endif
409 430
410 if (!cpu_possible(cpu)) { 431 if (!cpu_possible(cpu)) {
411 printk(KERN_ERR "can't online cpu %d because it is not " 432 printk(KERN_ERR "can't online cpu %d because it is not "
412 "configured as may-hotadd at boot time\n", cpu); 433 "configured as may-hotadd at boot time\n", cpu);
413 #if defined(CONFIG_IA64) 434 #if defined(CONFIG_IA64)
414 printk(KERN_ERR "please check additional_cpus= boot " 435 printk(KERN_ERR "please check additional_cpus= boot "
415 "parameter\n"); 436 "parameter\n");
416 #endif 437 #endif
417 return -EINVAL; 438 return -EINVAL;
418 } 439 }
419 440
420 #ifdef CONFIG_MEMORY_HOTPLUG 441 #ifdef CONFIG_MEMORY_HOTPLUG
421 nid = cpu_to_node(cpu); 442 nid = cpu_to_node(cpu);
422 if (!node_online(nid)) { 443 if (!node_online(nid)) {
423 err = mem_online_node(nid); 444 err = mem_online_node(nid);
424 if (err) 445 if (err)
425 return err; 446 return err;
426 } 447 }
427 448
428 pgdat = NODE_DATA(nid); 449 pgdat = NODE_DATA(nid);
429 if (!pgdat) { 450 if (!pgdat) {
430 printk(KERN_ERR 451 printk(KERN_ERR
431 "Can't online cpu %d due to NULL pgdat\n", cpu); 452 "Can't online cpu %d due to NULL pgdat\n", cpu);
432 return -ENOMEM; 453 return -ENOMEM;
433 } 454 }
434 455
435 if (pgdat->node_zonelists->_zonerefs->zone == NULL) { 456 if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
436 mutex_lock(&zonelists_mutex); 457 mutex_lock(&zonelists_mutex);
437 build_all_zonelists(NULL, NULL); 458 build_all_zonelists(NULL, NULL);
438 mutex_unlock(&zonelists_mutex); 459 mutex_unlock(&zonelists_mutex);
439 } 460 }
440 #endif 461 #endif
441 462
442 cpu_maps_update_begin(); 463 cpu_maps_update_begin();
443 464
444 if (cpu_hotplug_disabled) { 465 if (cpu_hotplug_disabled) {
445 err = -EBUSY; 466 err = -EBUSY;
446 goto out; 467 goto out;
447 } 468 }
448 469
449 err = _cpu_up(cpu, 0); 470 err = _cpu_up(cpu, 0);
450 471
451 out: 472 out:
452 cpu_maps_update_done(); 473 cpu_maps_update_done();
453 return err; 474 return err;
454 } 475 }
455 EXPORT_SYMBOL_GPL(cpu_up); 476 EXPORT_SYMBOL_GPL(cpu_up);
456 477
457 #ifdef CONFIG_PM_SLEEP_SMP 478 #ifdef CONFIG_PM_SLEEP_SMP
458 static cpumask_var_t frozen_cpus; 479 static cpumask_var_t frozen_cpus;
459 480
460 int disable_nonboot_cpus(void) 481 int disable_nonboot_cpus(void)
461 { 482 {
462 int cpu, first_cpu, error = 0; 483 int cpu, first_cpu, error = 0;
463 484
464 cpu_maps_update_begin(); 485 cpu_maps_update_begin();
465 first_cpu = cpumask_first(cpu_online_mask); 486 first_cpu = cpumask_first(cpu_online_mask);
466 /* 487 /*
467 * We take down all of the non-boot CPUs in one shot to avoid races 488 * We take down all of the non-boot CPUs in one shot to avoid races
468 * with the userspace trying to use the CPU hotplug at the same time 489 * with the userspace trying to use the CPU hotplug at the same time
469 */ 490 */
470 cpumask_clear(frozen_cpus); 491 cpumask_clear(frozen_cpus);
471 492
472 printk("Disabling non-boot CPUs ...\n"); 493 printk("Disabling non-boot CPUs ...\n");
473 for_each_online_cpu(cpu) { 494 for_each_online_cpu(cpu) {
474 if (cpu == first_cpu) 495 if (cpu == first_cpu)
475 continue; 496 continue;
476 error = _cpu_down(cpu, 1); 497 error = _cpu_down(cpu, 1);
477 if (!error) 498 if (!error)
478 cpumask_set_cpu(cpu, frozen_cpus); 499 cpumask_set_cpu(cpu, frozen_cpus);
479 else { 500 else {
480 printk(KERN_ERR "Error taking CPU%d down: %d\n", 501 printk(KERN_ERR "Error taking CPU%d down: %d\n",
481 cpu, error); 502 cpu, error);
482 break; 503 break;
483 } 504 }
484 } 505 }
485 506
486 if (!error) { 507 if (!error) {
487 BUG_ON(num_online_cpus() > 1); 508 BUG_ON(num_online_cpus() > 1);
488 /* Make sure the CPUs won't be enabled by someone else */ 509 /* Make sure the CPUs won't be enabled by someone else */
489 cpu_hotplug_disabled = 1; 510 cpu_hotplug_disabled = 1;
490 } else { 511 } else {
491 printk(KERN_ERR "Non-boot CPUs are not disabled\n"); 512 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
492 } 513 }
493 cpu_maps_update_done(); 514 cpu_maps_update_done();
494 return error; 515 return error;
495 } 516 }
496 517
497 void __weak arch_enable_nonboot_cpus_begin(void) 518 void __weak arch_enable_nonboot_cpus_begin(void)
498 { 519 {
499 } 520 }
500 521
501 void __weak arch_enable_nonboot_cpus_end(void) 522 void __weak arch_enable_nonboot_cpus_end(void)
502 { 523 {
503 } 524 }
504 525
505 void __ref enable_nonboot_cpus(void) 526 void __ref enable_nonboot_cpus(void)
506 { 527 {
507 int cpu, error; 528 int cpu, error;
508 529
509 /* Allow everyone to use the CPU hotplug again */ 530 /* Allow everyone to use the CPU hotplug again */
510 cpu_maps_update_begin(); 531 cpu_maps_update_begin();
511 cpu_hotplug_disabled = 0; 532 cpu_hotplug_disabled = 0;
512 if (cpumask_empty(frozen_cpus)) 533 if (cpumask_empty(frozen_cpus))
513 goto out; 534 goto out;
514 535
515 printk(KERN_INFO "Enabling non-boot CPUs ...\n"); 536 printk(KERN_INFO "Enabling non-boot CPUs ...\n");
516 537
517 arch_enable_nonboot_cpus_begin(); 538 arch_enable_nonboot_cpus_begin();
518 539
519 for_each_cpu(cpu, frozen_cpus) { 540 for_each_cpu(cpu, frozen_cpus) {
520 error = _cpu_up(cpu, 1); 541 error = _cpu_up(cpu, 1);
521 if (!error) { 542 if (!error) {
522 printk(KERN_INFO "CPU%d is up\n", cpu); 543 printk(KERN_INFO "CPU%d is up\n", cpu);
523 continue; 544 continue;
524 } 545 }
525 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error); 546 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
526 } 547 }
527 548
528 arch_enable_nonboot_cpus_end(); 549 arch_enable_nonboot_cpus_end();
529 550
530 cpumask_clear(frozen_cpus); 551 cpumask_clear(frozen_cpus);
531 out: 552 out:
532 cpu_maps_update_done(); 553 cpu_maps_update_done();
533 } 554 }
534 555
535 static int __init alloc_frozen_cpus(void) 556 static int __init alloc_frozen_cpus(void)
536 { 557 {
537 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) 558 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
538 return -ENOMEM; 559 return -ENOMEM;
539 return 0; 560 return 0;
540 } 561 }
541 core_initcall(alloc_frozen_cpus); 562 core_initcall(alloc_frozen_cpus);
542 563
543 /* 564 /*
544 * Prevent regular CPU hotplug from racing with the freezer, by disabling CPU
545 * hotplug when tasks are about to be frozen. Also, don't allow the freezer
546 * to continue until any currently running CPU hotplug operation gets
547 * completed.
548 * To modify the 'cpu_hotplug_disabled' flag, we need to acquire the
549 * 'cpu_add_remove_lock'. And this same lock is also taken by the regular
550 * CPU hotplug path and released only after it is complete. Thus, we
551 * (and hence the freezer) will block here until any currently running CPU
552 * hotplug operation gets completed.
553 */
554 void cpu_hotplug_disable_before_freeze(void)
555 {
556 cpu_maps_update_begin();
557 cpu_hotplug_disabled = 1;
558 cpu_maps_update_done();
559 }
560
561
562 /*
563 * When tasks have been thawed, re-enable regular CPU hotplug (which had been
564 * disabled while beginning to freeze tasks).
565 */
566 void cpu_hotplug_enable_after_thaw(void)
567 {
568 cpu_maps_update_begin();
569 cpu_hotplug_disabled = 0;
570 cpu_maps_update_done();
571 }
572
573 /*
574 * When callbacks for CPU hotplug notifications are being executed, we must 565 * When callbacks for CPU hotplug notifications are being executed, we must
575 * ensure that the state of the system with respect to the tasks being frozen 566 * ensure that the state of the system with respect to the tasks being frozen
576 * or not, as reported by the notification, remains unchanged *throughout the 567 * or not, as reported by the notification, remains unchanged *throughout the
577 * duration* of the execution of the callbacks. 568 * duration* of the execution of the callbacks.
578 * Hence we need to prevent the freezer from racing with regular CPU hotplug. 569 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
579 * 570 *
580 * This synchronization is implemented by mutually excluding regular CPU 571 * This synchronization is implemented by mutually excluding regular CPU
581 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ 572 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
582 * Hibernate notifications. 573 * Hibernate notifications.
583 */ 574 */
584 static int 575 static int
585 cpu_hotplug_pm_callback(struct notifier_block *nb, 576 cpu_hotplug_pm_callback(struct notifier_block *nb,
586 unsigned long action, void *ptr) 577 unsigned long action, void *ptr)
587 { 578 {
588 switch (action) { 579 switch (action) {
589 580
590 case PM_SUSPEND_PREPARE: 581 case PM_SUSPEND_PREPARE:
591 case PM_HIBERNATION_PREPARE: 582 case PM_HIBERNATION_PREPARE:
592 cpu_hotplug_disable_before_freeze(); 583 cpu_hotplug_disable();
593 break; 584 break;
594 585
595 case PM_POST_SUSPEND: 586 case PM_POST_SUSPEND:
596 case PM_POST_HIBERNATION: 587 case PM_POST_HIBERNATION:
597 cpu_hotplug_enable_after_thaw(); 588 cpu_hotplug_enable();
598 break; 589 break;
599 590
600 default: 591 default:
601 return NOTIFY_DONE; 592 return NOTIFY_DONE;
602 } 593 }
603 594
604 return NOTIFY_OK; 595 return NOTIFY_OK;
605 } 596 }
606 597
607 598
608 static int __init cpu_hotplug_pm_sync_init(void) 599 static int __init cpu_hotplug_pm_sync_init(void)
609 { 600 {
610 /* 601 /*
611 * cpu_hotplug_pm_callback has higher priority than x86 602 * cpu_hotplug_pm_callback has higher priority than x86
612 * bsp_pm_callback which depends on cpu_hotplug_pm_callback 603 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
613 * to disable cpu hotplug to avoid cpu hotplug race. 604 * to disable cpu hotplug to avoid cpu hotplug race.
614 */ 605 */
615 pm_notifier(cpu_hotplug_pm_callback, 0); 606 pm_notifier(cpu_hotplug_pm_callback, 0);
616 return 0; 607 return 0;
617 } 608 }
618 core_initcall(cpu_hotplug_pm_sync_init); 609 core_initcall(cpu_hotplug_pm_sync_init);
619 610
620 #endif /* CONFIG_PM_SLEEP_SMP */ 611 #endif /* CONFIG_PM_SLEEP_SMP */
621 612
622 /** 613 /**
623 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers 614 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
624 * @cpu: cpu that just started 615 * @cpu: cpu that just started
625 * 616 *
626 * This function calls the cpu_chain notifiers with CPU_STARTING. 617 * This function calls the cpu_chain notifiers with CPU_STARTING.
627 * It must be called by the arch code on the new cpu, before the new cpu 618 * It must be called by the arch code on the new cpu, before the new cpu
628 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 619 * enables interrupts and before the "boot" cpu returns from __cpu_up().
629 */ 620 */
630 void __cpuinit notify_cpu_starting(unsigned int cpu) 621 void __cpuinit notify_cpu_starting(unsigned int cpu)
631 { 622 {
632 unsigned long val = CPU_STARTING; 623 unsigned long val = CPU_STARTING;
633 624
634 #ifdef CONFIG_PM_SLEEP_SMP 625 #ifdef CONFIG_PM_SLEEP_SMP
635 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus)) 626 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
636 val = CPU_STARTING_FROZEN; 627 val = CPU_STARTING_FROZEN;
637 #endif /* CONFIG_PM_SLEEP_SMP */ 628 #endif /* CONFIG_PM_SLEEP_SMP */
638 cpu_notify(val, (void *)(long)cpu); 629 cpu_notify(val, (void *)(long)cpu);
639 } 630 }
640 631
641 #endif /* CONFIG_SMP */ 632 #endif /* CONFIG_SMP */
642 633
643 /* 634 /*
644 * cpu_bit_bitmap[] is a special, "compressed" data structure that 635 * cpu_bit_bitmap[] is a special, "compressed" data structure that
645 * represents all NR_CPUS bits binary values of 1<<nr. 636 * represents all NR_CPUS bits binary values of 1<<nr.
646 * 637 *
647 * It is used by cpumask_of() to get a constant address to a CPU 638 * It is used by cpumask_of() to get a constant address to a CPU
648 * mask value that has a single bit set only. 639 * mask value that has a single bit set only.
649 */ 640 */
650 641
651 /* cpu_bit_bitmap[0] is empty - so we can back into it */ 642 /* cpu_bit_bitmap[0] is empty - so we can back into it */
652 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) 643 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
653 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) 644 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
654 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) 645 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
655 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) 646 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
656 647
657 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { 648 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
658 649
659 MASK_DECLARE_8(0), MASK_DECLARE_8(8), 650 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
660 MASK_DECLARE_8(16), MASK_DECLARE_8(24), 651 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
661 #if BITS_PER_LONG > 32 652 #if BITS_PER_LONG > 32
662 MASK_DECLARE_8(32), MASK_DECLARE_8(40), 653 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
663 MASK_DECLARE_8(48), MASK_DECLARE_8(56), 654 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
664 #endif 655 #endif
665 }; 656 };
666 EXPORT_SYMBOL_GPL(cpu_bit_bitmap); 657 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
667 658
668 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 659 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
669 EXPORT_SYMBOL(cpu_all_bits); 660 EXPORT_SYMBOL(cpu_all_bits);
670 661
671 #ifdef CONFIG_INIT_ALL_POSSIBLE 662 #ifdef CONFIG_INIT_ALL_POSSIBLE
672 static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly 663 static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
673 = CPU_BITS_ALL; 664 = CPU_BITS_ALL;
674 #else 665 #else
675 static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly; 666 static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
676 #endif 667 #endif
677 const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits); 668 const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
678 EXPORT_SYMBOL(cpu_possible_mask); 669 EXPORT_SYMBOL(cpu_possible_mask);
679 670
680 static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly; 671 static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
681 const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits); 672 const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
682 EXPORT_SYMBOL(cpu_online_mask); 673 EXPORT_SYMBOL(cpu_online_mask);
683 674
684 static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly; 675 static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
685 const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits); 676 const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
686 EXPORT_SYMBOL(cpu_present_mask); 677 EXPORT_SYMBOL(cpu_present_mask);
687 678
688 static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly; 679 static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
689 const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits); 680 const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
690 EXPORT_SYMBOL(cpu_active_mask); 681 EXPORT_SYMBOL(cpu_active_mask);
691 682
692 void set_cpu_possible(unsigned int cpu, bool possible) 683 void set_cpu_possible(unsigned int cpu, bool possible)
693 { 684 {
694 if (possible) 685 if (possible)
695 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits)); 686 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
696 else 687 else
697 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits)); 688 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
698 } 689 }
699 690
700 void set_cpu_present(unsigned int cpu, bool present) 691 void set_cpu_present(unsigned int cpu, bool present)
701 { 692 {
702 if (present) 693 if (present)
703 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits)); 694 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
704 else 695 else
705 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits)); 696 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
706 } 697 }
707 698
708 void set_cpu_online(unsigned int cpu, bool online) 699 void set_cpu_online(unsigned int cpu, bool online)
709 { 700 {
710 if (online) 701 if (online)
711 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits)); 702 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
712 else 703 else
713 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits)); 704 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
714 } 705 }
715 706
716 void set_cpu_active(unsigned int cpu, bool active) 707 void set_cpu_active(unsigned int cpu, bool active)
717 { 708 {