Eric Lee / smarc-ti-linux-kernel

39
Download as
Browse Code ยป

Commit 81c98869faa5f3a9457c93efef908ef476326b31

Authored by Nishanth Aravamudan
Committed by Linus Torvalds
1 parent 5acda9d12d
Exists in master and in 13 other branches dlt-processor-sdk-linux-03.00.00.04, processor-sdk-linux-02.00.01, smarc-ti-linux-3.15.y, smarc-ti-lsk-linux-4.1.y, smarct3x-processor-sdk-04.01.00.06, smarct3x-processor-sdk-linux-02.00.01, smarct3x-processor-sdk-linux-03.00.00.04, smarct4x-800-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-04.01.00.06, smarct4x-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-linux-03.00.00.04, ti-linux-3.15.y, ti-lsk-linux-4.1.y

kthread: ensure locality of task_struct allocations 

In the presence of memoryless nodes, numa_node_id() will return the
current CPU's NUMA node, but that may not be where we expect to allocate
from memory from.  Instead, we should rely on the fallback code in the
memory allocator itself, by using NUMA_NO_NODE.  Also, when calling
kthread_create_on_node(), use the nearest node with memory to the cpu in
question, rather than the node it is running on.

Signed-off-by: Nishanth Aravamudan <nacc@linux.vnet.ibm.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Acked-by: David Rientjes <rientjes@google.com>
Cc: Anton Blanchard <anton@samba.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Wanpeng Li <liwanp@linux.vnet.ibm.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Ben Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 1 changed file with 2 additions and 2 deletions Inline Diff

1 /* Kernel thread helper functions. 1 /* Kernel thread helper functions.
2 * Copyright (C) 2004 IBM Corporation, Rusty Russell. 2 * Copyright (C) 2004 IBM Corporation, Rusty Russell.
3 * 3 *
4 * Creation is done via kthreadd, so that we get a clean environment 4 * Creation is done via kthreadd, so that we get a clean environment
5 * even if we're invoked from userspace (think modprobe, hotplug cpu, 5 * even if we're invoked from userspace (think modprobe, hotplug cpu,
6 * etc.). 6 * etc.).
7 */ 7 */
8 #include <linux/sched.h> 8 #include <linux/sched.h>
9 #include <linux/kthread.h> 9 #include <linux/kthread.h>
10 #include <linux/completion.h> 10 #include <linux/completion.h>
11 #include <linux/err.h> 11 #include <linux/err.h>
12 #include <linux/cpuset.h> 12 #include <linux/cpuset.h>
13 #include <linux/unistd.h> 13 #include <linux/unistd.h>
14 #include <linux/file.h> 14 #include <linux/file.h>
15 #include <linux/export.h> 15 #include <linux/export.h>
16 #include <linux/mutex.h> 16 #include <linux/mutex.h>
17 #include <linux/slab.h> 17 #include <linux/slab.h>
18 #include <linux/freezer.h> 18 #include <linux/freezer.h>
19 #include <linux/ptrace.h> 19 #include <linux/ptrace.h>
20 #include <linux/uaccess.h> 20 #include <linux/uaccess.h>
21 #include <trace/events/sched.h> 21 #include <trace/events/sched.h>
22 22
23 static DEFINE_SPINLOCK(kthread_create_lock); 23 static DEFINE_SPINLOCK(kthread_create_lock);
24 static LIST_HEAD(kthread_create_list); 24 static LIST_HEAD(kthread_create_list);
25 struct task_struct *kthreadd_task; 25 struct task_struct *kthreadd_task;
26 26
27 struct kthread_create_info 27 struct kthread_create_info
28 { 28 {
29 /* Information passed to kthread() from kthreadd. */ 29 /* Information passed to kthread() from kthreadd. */
30 int (*threadfn)(void *data); 30 int (*threadfn)(void *data);
31 void *data; 31 void *data;
32 int node; 32 int node;
33 33
34 /* Result passed back to kthread_create() from kthreadd. */ 34 /* Result passed back to kthread_create() from kthreadd. */
35 struct task_struct *result; 35 struct task_struct *result;
36 struct completion *done; 36 struct completion *done;
37 37
38 struct list_head list; 38 struct list_head list;
39 }; 39 };
40 40
41 struct kthread { 41 struct kthread {
42 unsigned long flags; 42 unsigned long flags;
43 unsigned int cpu; 43 unsigned int cpu;
44 void *data; 44 void *data;
45 struct completion parked; 45 struct completion parked;
46 struct completion exited; 46 struct completion exited;
47 }; 47 };
48 48
49 enum KTHREAD_BITS { 49 enum KTHREAD_BITS {
50 KTHREAD_IS_PER_CPU = 0, 50 KTHREAD_IS_PER_CPU = 0,
51 KTHREAD_SHOULD_STOP, 51 KTHREAD_SHOULD_STOP,
52 KTHREAD_SHOULD_PARK, 52 KTHREAD_SHOULD_PARK,
53 KTHREAD_IS_PARKED, 53 KTHREAD_IS_PARKED,
54 }; 54 };
55 55
56 #define __to_kthread(vfork) \ 56 #define __to_kthread(vfork) \
57 container_of(vfork, struct kthread, exited) 57 container_of(vfork, struct kthread, exited)
58 58
59 static inline struct kthread *to_kthread(struct task_struct *k) 59 static inline struct kthread *to_kthread(struct task_struct *k)
60 { 60 {
61 return __to_kthread(k->vfork_done); 61 return __to_kthread(k->vfork_done);
62 } 62 }
63 63
64 static struct kthread *to_live_kthread(struct task_struct *k) 64 static struct kthread *to_live_kthread(struct task_struct *k)
65 { 65 {
66 struct completion *vfork = ACCESS_ONCE(k->vfork_done); 66 struct completion *vfork = ACCESS_ONCE(k->vfork_done);
67 if (likely(vfork)) 67 if (likely(vfork))
68 return __to_kthread(vfork); 68 return __to_kthread(vfork);
69 return NULL; 69 return NULL;
70 } 70 }
71 71
72 /** 72 /**
73 * kthread_should_stop - should this kthread return now? 73 * kthread_should_stop - should this kthread return now?
74 * 74 *
75 * When someone calls kthread_stop() on your kthread, it will be woken 75 * When someone calls kthread_stop() on your kthread, it will be woken
76 * and this will return true. You should then return, and your return 76 * and this will return true. You should then return, and your return
77 * value will be passed through to kthread_stop(). 77 * value will be passed through to kthread_stop().
78 */ 78 */
79 bool kthread_should_stop(void) 79 bool kthread_should_stop(void)
80 { 80 {
81 return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); 81 return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
82 } 82 }
83 EXPORT_SYMBOL(kthread_should_stop); 83 EXPORT_SYMBOL(kthread_should_stop);
84 84
85 /** 85 /**
86 * kthread_should_park - should this kthread park now? 86 * kthread_should_park - should this kthread park now?
87 * 87 *
88 * When someone calls kthread_park() on your kthread, it will be woken 88 * When someone calls kthread_park() on your kthread, it will be woken
89 * and this will return true. You should then do the necessary 89 * and this will return true. You should then do the necessary
90 * cleanup and call kthread_parkme() 90 * cleanup and call kthread_parkme()
91 * 91 *
92 * Similar to kthread_should_stop(), but this keeps the thread alive 92 * Similar to kthread_should_stop(), but this keeps the thread alive
93 * and in a park position. kthread_unpark() "restarts" the thread and 93 * and in a park position. kthread_unpark() "restarts" the thread and
94 * calls the thread function again. 94 * calls the thread function again.
95 */ 95 */
96 bool kthread_should_park(void) 96 bool kthread_should_park(void)
97 { 97 {
98 return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(current)->flags); 98 return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(current)->flags);
99 } 99 }
100 100
101 /** 101 /**
102 * kthread_freezable_should_stop - should this freezable kthread return now? 102 * kthread_freezable_should_stop - should this freezable kthread return now?
103 * @was_frozen: optional out parameter, indicates whether %current was frozen 103 * @was_frozen: optional out parameter, indicates whether %current was frozen
104 * 104 *
105 * kthread_should_stop() for freezable kthreads, which will enter 105 * kthread_should_stop() for freezable kthreads, which will enter
106 * refrigerator if necessary. This function is safe from kthread_stop() / 106 * refrigerator if necessary. This function is safe from kthread_stop() /
107 * freezer deadlock and freezable kthreads should use this function instead 107 * freezer deadlock and freezable kthreads should use this function instead
108 * of calling try_to_freeze() directly. 108 * of calling try_to_freeze() directly.
109 */ 109 */
110 bool kthread_freezable_should_stop(bool *was_frozen) 110 bool kthread_freezable_should_stop(bool *was_frozen)
111 { 111 {
112 bool frozen = false; 112 bool frozen = false;
113 113
114 might_sleep(); 114 might_sleep();
115 115
116 if (unlikely(freezing(current))) 116 if (unlikely(freezing(current)))
117 frozen = __refrigerator(true); 117 frozen = __refrigerator(true);
118 118
119 if (was_frozen) 119 if (was_frozen)
120 *was_frozen = frozen; 120 *was_frozen = frozen;
121 121
122 return kthread_should_stop(); 122 return kthread_should_stop();
123 } 123 }
124 EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); 124 EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
125 125
126 /** 126 /**
127 * kthread_data - return data value specified on kthread creation 127 * kthread_data - return data value specified on kthread creation
128 * @task: kthread task in question 128 * @task: kthread task in question
129 * 129 *
130 * Return the data value specified when kthread @task was created. 130 * Return the data value specified when kthread @task was created.
131 * The caller is responsible for ensuring the validity of @task when 131 * The caller is responsible for ensuring the validity of @task when
132 * calling this function. 132 * calling this function.
133 */ 133 */
134 void *kthread_data(struct task_struct *task) 134 void *kthread_data(struct task_struct *task)
135 { 135 {
136 return to_kthread(task)->data; 136 return to_kthread(task)->data;
137 } 137 }
138 138
139 /** 139 /**
140 * probe_kthread_data - speculative version of kthread_data() 140 * probe_kthread_data - speculative version of kthread_data()
141 * @task: possible kthread task in question 141 * @task: possible kthread task in question
142 * 142 *
143 * @task could be a kthread task. Return the data value specified when it 143 * @task could be a kthread task. Return the data value specified when it
144 * was created if accessible. If @task isn't a kthread task or its data is 144 * was created if accessible. If @task isn't a kthread task or its data is
145 * inaccessible for any reason, %NULL is returned. This function requires 145 * inaccessible for any reason, %NULL is returned. This function requires
146 * that @task itself is safe to dereference. 146 * that @task itself is safe to dereference.
147 */ 147 */
148 void *probe_kthread_data(struct task_struct *task) 148 void *probe_kthread_data(struct task_struct *task)
149 { 149 {
150 struct kthread *kthread = to_kthread(task); 150 struct kthread *kthread = to_kthread(task);
151 void *data = NULL; 151 void *data = NULL;
152 152
153 probe_kernel_read(&data, &kthread->data, sizeof(data)); 153 probe_kernel_read(&data, &kthread->data, sizeof(data));
154 return data; 154 return data;
155 } 155 }
156 156
157 static void __kthread_parkme(struct kthread *self) 157 static void __kthread_parkme(struct kthread *self)
158 { 158 {
159 __set_current_state(TASK_PARKED); 159 __set_current_state(TASK_PARKED);
160 while (test_bit(KTHREAD_SHOULD_PARK, &self->flags)) { 160 while (test_bit(KTHREAD_SHOULD_PARK, &self->flags)) {
161 if (!test_and_set_bit(KTHREAD_IS_PARKED, &self->flags)) 161 if (!test_and_set_bit(KTHREAD_IS_PARKED, &self->flags))
162 complete(&self->parked); 162 complete(&self->parked);
163 schedule(); 163 schedule();
164 __set_current_state(TASK_PARKED); 164 __set_current_state(TASK_PARKED);
165 } 165 }
166 clear_bit(KTHREAD_IS_PARKED, &self->flags); 166 clear_bit(KTHREAD_IS_PARKED, &self->flags);
167 __set_current_state(TASK_RUNNING); 167 __set_current_state(TASK_RUNNING);
168 } 168 }
169 169
170 void kthread_parkme(void) 170 void kthread_parkme(void)
171 { 171 {
172 __kthread_parkme(to_kthread(current)); 172 __kthread_parkme(to_kthread(current));
173 } 173 }
174 174
175 static int kthread(void *_create) 175 static int kthread(void *_create)
176 { 176 {
177 /* Copy data: it's on kthread's stack */ 177 /* Copy data: it's on kthread's stack */
178 struct kthread_create_info *create = _create; 178 struct kthread_create_info *create = _create;
179 int (*threadfn)(void *data) = create->threadfn; 179 int (*threadfn)(void *data) = create->threadfn;
180 void *data = create->data; 180 void *data = create->data;
181 struct completion *done; 181 struct completion *done;
182 struct kthread self; 182 struct kthread self;
183 int ret; 183 int ret;
184 184
185 self.flags = 0; 185 self.flags = 0;
186 self.data = data; 186 self.data = data;
187 init_completion(&self.exited); 187 init_completion(&self.exited);
188 init_completion(&self.parked); 188 init_completion(&self.parked);
189 current->vfork_done = &self.exited; 189 current->vfork_done = &self.exited;
190 190
191 /* If user was SIGKILLed, I release the structure. */ 191 /* If user was SIGKILLed, I release the structure. */
192 done = xchg(&create->done, NULL); 192 done = xchg(&create->done, NULL);
193 if (!done) { 193 if (!done) {
194 kfree(create); 194 kfree(create);
195 do_exit(-EINTR); 195 do_exit(-EINTR);
196 } 196 }
197 /* OK, tell user we're spawned, wait for stop or wakeup */ 197 /* OK, tell user we're spawned, wait for stop or wakeup */
198 __set_current_state(TASK_UNINTERRUPTIBLE); 198 __set_current_state(TASK_UNINTERRUPTIBLE);
199 create->result = current; 199 create->result = current;
200 complete(done); 200 complete(done);
201 schedule(); 201 schedule();
202 202
203 ret = -EINTR; 203 ret = -EINTR;
204 204
205 if (!test_bit(KTHREAD_SHOULD_STOP, &self.flags)) { 205 if (!test_bit(KTHREAD_SHOULD_STOP, &self.flags)) {
206 __kthread_parkme(&self); 206 __kthread_parkme(&self);
207 ret = threadfn(data); 207 ret = threadfn(data);
208 } 208 }
209 /* we can't just return, we must preserve "self" on stack */ 209 /* we can't just return, we must preserve "self" on stack */
210 do_exit(ret); 210 do_exit(ret);
211 } 211 }
212 212
213 /* called from do_fork() to get node information for about to be created task */ 213 /* called from do_fork() to get node information for about to be created task */
214 int tsk_fork_get_node(struct task_struct *tsk) 214 int tsk_fork_get_node(struct task_struct *tsk)
215 { 215 {
216 #ifdef CONFIG_NUMA 216 #ifdef CONFIG_NUMA
217 if (tsk == kthreadd_task) 217 if (tsk == kthreadd_task)
218 return tsk->pref_node_fork; 218 return tsk->pref_node_fork;
219 #endif 219 #endif
220 return numa_node_id(); 220 return NUMA_NO_NODE;
221 } 221 }
222 222
223 static void create_kthread(struct kthread_create_info *create) 223 static void create_kthread(struct kthread_create_info *create)
224 { 224 {
225 int pid; 225 int pid;
226 226
227 #ifdef CONFIG_NUMA 227 #ifdef CONFIG_NUMA
228 current->pref_node_fork = create->node; 228 current->pref_node_fork = create->node;
229 #endif 229 #endif
230 /* We want our own signal handler (we take no signals by default). */ 230 /* We want our own signal handler (we take no signals by default). */
231 pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD); 231 pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
232 if (pid < 0) { 232 if (pid < 0) {
233 /* If user was SIGKILLed, I release the structure. */ 233 /* If user was SIGKILLed, I release the structure. */
234 struct completion *done = xchg(&create->done, NULL); 234 struct completion *done = xchg(&create->done, NULL);
235 235
236 if (!done) { 236 if (!done) {
237 kfree(create); 237 kfree(create);
238 return; 238 return;
239 } 239 }
240 create->result = ERR_PTR(pid); 240 create->result = ERR_PTR(pid);
241 complete(done); 241 complete(done);
242 } 242 }
243 } 243 }
244 244
245 /** 245 /**
246 * kthread_create_on_node - create a kthread. 246 * kthread_create_on_node - create a kthread.
247 * @threadfn: the function to run until signal_pending(current). 247 * @threadfn: the function to run until signal_pending(current).
248 * @data: data ptr for @threadfn. 248 * @data: data ptr for @threadfn.
249 * @node: memory node number. 249 * @node: memory node number.
250 * @namefmt: printf-style name for the thread. 250 * @namefmt: printf-style name for the thread.
251 * 251 *
252 * Description: This helper function creates and names a kernel 252 * Description: This helper function creates and names a kernel
253 * thread. The thread will be stopped: use wake_up_process() to start 253 * thread. The thread will be stopped: use wake_up_process() to start
254 * it. See also kthread_run(). 254 * it. See also kthread_run().
255 * 255 *
256 * If thread is going to be bound on a particular cpu, give its node 256 * If thread is going to be bound on a particular cpu, give its node
257 * in @node, to get NUMA affinity for kthread stack, or else give -1. 257 * in @node, to get NUMA affinity for kthread stack, or else give -1.
258 * When woken, the thread will run @threadfn() with @data as its 258 * When woken, the thread will run @threadfn() with @data as its
259 * argument. @threadfn() can either call do_exit() directly if it is a 259 * argument. @threadfn() can either call do_exit() directly if it is a
260 * standalone thread for which no one will call kthread_stop(), or 260 * standalone thread for which no one will call kthread_stop(), or
261 * return when 'kthread_should_stop()' is true (which means 261 * return when 'kthread_should_stop()' is true (which means
262 * kthread_stop() has been called). The return value should be zero 262 * kthread_stop() has been called). The return value should be zero
263 * or a negative error number; it will be passed to kthread_stop(). 263 * or a negative error number; it will be passed to kthread_stop().
264 * 264 *
265 * Returns a task_struct or ERR_PTR(-ENOMEM). 265 * Returns a task_struct or ERR_PTR(-ENOMEM).
266 */ 266 */
267 struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), 267 struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
268 void *data, int node, 268 void *data, int node,
269 const char namefmt[], 269 const char namefmt[],
270 ...) 270 ...)
271 { 271 {
272 DECLARE_COMPLETION_ONSTACK(done); 272 DECLARE_COMPLETION_ONSTACK(done);
273 struct task_struct *task; 273 struct task_struct *task;
274 struct kthread_create_info *create = kmalloc(sizeof(*create), 274 struct kthread_create_info *create = kmalloc(sizeof(*create),
275 GFP_KERNEL); 275 GFP_KERNEL);
276 276
277 if (!create) 277 if (!create)
278 return ERR_PTR(-ENOMEM); 278 return ERR_PTR(-ENOMEM);
279 create->threadfn = threadfn; 279 create->threadfn = threadfn;
280 create->data = data; 280 create->data = data;
281 create->node = node; 281 create->node = node;
282 create->done = &done; 282 create->done = &done;
283 283
284 spin_lock(&kthread_create_lock); 284 spin_lock(&kthread_create_lock);
285 list_add_tail(&create->list, &kthread_create_list); 285 list_add_tail(&create->list, &kthread_create_list);
286 spin_unlock(&kthread_create_lock); 286 spin_unlock(&kthread_create_lock);
287 287
288 wake_up_process(kthreadd_task); 288 wake_up_process(kthreadd_task);
289 /* 289 /*
290 * Wait for completion in killable state, for I might be chosen by 290 * Wait for completion in killable state, for I might be chosen by
291 * the OOM killer while kthreadd is trying to allocate memory for 291 * the OOM killer while kthreadd is trying to allocate memory for
292 * new kernel thread. 292 * new kernel thread.
293 */ 293 */
294 if (unlikely(wait_for_completion_killable(&done))) { 294 if (unlikely(wait_for_completion_killable(&done))) {
295 /* 295 /*
296 * If I was SIGKILLed before kthreadd (or new kernel thread) 296 * If I was SIGKILLed before kthreadd (or new kernel thread)
297 * calls complete(), leave the cleanup of this structure to 297 * calls complete(), leave the cleanup of this structure to
298 * that thread. 298 * that thread.
299 */ 299 */
300 if (xchg(&create->done, NULL)) 300 if (xchg(&create->done, NULL))
301 return ERR_PTR(-ENOMEM); 301 return ERR_PTR(-ENOMEM);
302 /* 302 /*
303 * kthreadd (or new kernel thread) will call complete() 303 * kthreadd (or new kernel thread) will call complete()
304 * shortly. 304 * shortly.
305 */ 305 */
306 wait_for_completion(&done); 306 wait_for_completion(&done);
307 } 307 }
308 task = create->result; 308 task = create->result;
309 if (!IS_ERR(task)) { 309 if (!IS_ERR(task)) {
310 static const struct sched_param param = { .sched_priority = 0 }; 310 static const struct sched_param param = { .sched_priority = 0 };
311 va_list args; 311 va_list args;
312 312
313 va_start(args, namefmt); 313 va_start(args, namefmt);
314 vsnprintf(task->comm, sizeof(task->comm), namefmt, args); 314 vsnprintf(task->comm, sizeof(task->comm), namefmt, args);
315 va_end(args); 315 va_end(args);
316 /* 316 /*
317 * root may have changed our (kthreadd's) priority or CPU mask. 317 * root may have changed our (kthreadd's) priority or CPU mask.
318 * The kernel thread should not inherit these properties. 318 * The kernel thread should not inherit these properties.
319 */ 319 */
320 sched_setscheduler_nocheck(task, SCHED_NORMAL, &param); 320 sched_setscheduler_nocheck(task, SCHED_NORMAL, &param);
321 set_cpus_allowed_ptr(task, cpu_all_mask); 321 set_cpus_allowed_ptr(task, cpu_all_mask);
322 } 322 }
323 kfree(create); 323 kfree(create);
324 return task; 324 return task;
325 } 325 }
326 EXPORT_SYMBOL(kthread_create_on_node); 326 EXPORT_SYMBOL(kthread_create_on_node);
327 327
328 static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state) 328 static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state)
329 { 329 {
330 /* Must have done schedule() in kthread() before we set_task_cpu */ 330 /* Must have done schedule() in kthread() before we set_task_cpu */
331 if (!wait_task_inactive(p, state)) { 331 if (!wait_task_inactive(p, state)) {
332 WARN_ON(1); 332 WARN_ON(1);
333 return; 333 return;
334 } 334 }
335 /* It's safe because the task is inactive. */ 335 /* It's safe because the task is inactive. */
336 do_set_cpus_allowed(p, cpumask_of(cpu)); 336 do_set_cpus_allowed(p, cpumask_of(cpu));
337 p->flags |= PF_NO_SETAFFINITY; 337 p->flags |= PF_NO_SETAFFINITY;
338 } 338 }
339 339
340 /** 340 /**
341 * kthread_bind - bind a just-created kthread to a cpu. 341 * kthread_bind - bind a just-created kthread to a cpu.
342 * @p: thread created by kthread_create(). 342 * @p: thread created by kthread_create().
343 * @cpu: cpu (might not be online, must be possible) for @k to run on. 343 * @cpu: cpu (might not be online, must be possible) for @k to run on.
344 * 344 *
345 * Description: This function is equivalent to set_cpus_allowed(), 345 * Description: This function is equivalent to set_cpus_allowed(),
346 * except that @cpu doesn't need to be online, and the thread must be 346 * except that @cpu doesn't need to be online, and the thread must be
347 * stopped (i.e., just returned from kthread_create()). 347 * stopped (i.e., just returned from kthread_create()).
348 */ 348 */
349 void kthread_bind(struct task_struct *p, unsigned int cpu) 349 void kthread_bind(struct task_struct *p, unsigned int cpu)
350 { 350 {
351 __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE); 351 __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
352 } 352 }
353 EXPORT_SYMBOL(kthread_bind); 353 EXPORT_SYMBOL(kthread_bind);
354 354
355 /** 355 /**
356 * kthread_create_on_cpu - Create a cpu bound kthread 356 * kthread_create_on_cpu - Create a cpu bound kthread
357 * @threadfn: the function to run until signal_pending(current). 357 * @threadfn: the function to run until signal_pending(current).
358 * @data: data ptr for @threadfn. 358 * @data: data ptr for @threadfn.
359 * @cpu: The cpu on which the thread should be bound, 359 * @cpu: The cpu on which the thread should be bound,
360 * @namefmt: printf-style name for the thread. Format is restricted 360 * @namefmt: printf-style name for the thread. Format is restricted
361 * to "name.*%u". Code fills in cpu number. 361 * to "name.*%u". Code fills in cpu number.
362 * 362 *
363 * Description: This helper function creates and names a kernel thread 363 * Description: This helper function creates and names a kernel thread
364 * The thread will be woken and put into park mode. 364 * The thread will be woken and put into park mode.
365 */ 365 */
366 struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), 366 struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
367 void *data, unsigned int cpu, 367 void *data, unsigned int cpu,
368 const char *namefmt) 368 const char *namefmt)
369 { 369 {
370 struct task_struct *p; 370 struct task_struct *p;
371 371
372 p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt, 372 p = kthread_create_on_node(threadfn, data, cpu_to_mem(cpu), namefmt,
373 cpu); 373 cpu);
374 if (IS_ERR(p)) 374 if (IS_ERR(p))
375 return p; 375 return p;
376 set_bit(KTHREAD_IS_PER_CPU, &to_kthread(p)->flags); 376 set_bit(KTHREAD_IS_PER_CPU, &to_kthread(p)->flags);
377 to_kthread(p)->cpu = cpu; 377 to_kthread(p)->cpu = cpu;
378 /* Park the thread to get it out of TASK_UNINTERRUPTIBLE state */ 378 /* Park the thread to get it out of TASK_UNINTERRUPTIBLE state */
379 kthread_park(p); 379 kthread_park(p);
380 return p; 380 return p;
381 } 381 }
382 382
383 static void __kthread_unpark(struct task_struct *k, struct kthread *kthread) 383 static void __kthread_unpark(struct task_struct *k, struct kthread *kthread)
384 { 384 {
385 clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 385 clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
386 /* 386 /*
387 * We clear the IS_PARKED bit here as we don't wait 387 * We clear the IS_PARKED bit here as we don't wait
388 * until the task has left the park code. So if we'd 388 * until the task has left the park code. So if we'd
389 * park before that happens we'd see the IS_PARKED bit 389 * park before that happens we'd see the IS_PARKED bit
390 * which might be about to be cleared. 390 * which might be about to be cleared.
391 */ 391 */
392 if (test_and_clear_bit(KTHREAD_IS_PARKED, &kthread->flags)) { 392 if (test_and_clear_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
393 if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags)) 393 if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
394 __kthread_bind(k, kthread->cpu, TASK_PARKED); 394 __kthread_bind(k, kthread->cpu, TASK_PARKED);
395 wake_up_state(k, TASK_PARKED); 395 wake_up_state(k, TASK_PARKED);
396 } 396 }
397 } 397 }
398 398
399 /** 399 /**
400 * kthread_unpark - unpark a thread created by kthread_create(). 400 * kthread_unpark - unpark a thread created by kthread_create().
401 * @k: thread created by kthread_create(). 401 * @k: thread created by kthread_create().
402 * 402 *
403 * Sets kthread_should_park() for @k to return false, wakes it, and 403 * Sets kthread_should_park() for @k to return false, wakes it, and
404 * waits for it to return. If the thread is marked percpu then its 404 * waits for it to return. If the thread is marked percpu then its
405 * bound to the cpu again. 405 * bound to the cpu again.
406 */ 406 */
407 void kthread_unpark(struct task_struct *k) 407 void kthread_unpark(struct task_struct *k)
408 { 408 {
409 struct kthread *kthread = to_live_kthread(k); 409 struct kthread *kthread = to_live_kthread(k);
410 410
411 if (kthread) 411 if (kthread)
412 __kthread_unpark(k, kthread); 412 __kthread_unpark(k, kthread);
413 } 413 }
414 414
415 /** 415 /**
416 * kthread_park - park a thread created by kthread_create(). 416 * kthread_park - park a thread created by kthread_create().
417 * @k: thread created by kthread_create(). 417 * @k: thread created by kthread_create().
418 * 418 *
419 * Sets kthread_should_park() for @k to return true, wakes it, and 419 * Sets kthread_should_park() for @k to return true, wakes it, and
420 * waits for it to return. This can also be called after kthread_create() 420 * waits for it to return. This can also be called after kthread_create()
421 * instead of calling wake_up_process(): the thread will park without 421 * instead of calling wake_up_process(): the thread will park without
422 * calling threadfn(). 422 * calling threadfn().
423 * 423 *
424 * Returns 0 if the thread is parked, -ENOSYS if the thread exited. 424 * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
425 * If called by the kthread itself just the park bit is set. 425 * If called by the kthread itself just the park bit is set.
426 */ 426 */
427 int kthread_park(struct task_struct *k) 427 int kthread_park(struct task_struct *k)
428 { 428 {
429 struct kthread *kthread = to_live_kthread(k); 429 struct kthread *kthread = to_live_kthread(k);
430 int ret = -ENOSYS; 430 int ret = -ENOSYS;
431 431
432 if (kthread) { 432 if (kthread) {
433 if (!test_bit(KTHREAD_IS_PARKED, &kthread->flags)) { 433 if (!test_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
434 set_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 434 set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
435 if (k != current) { 435 if (k != current) {
436 wake_up_process(k); 436 wake_up_process(k);
437 wait_for_completion(&kthread->parked); 437 wait_for_completion(&kthread->parked);
438 } 438 }
439 } 439 }
440 ret = 0; 440 ret = 0;
441 } 441 }
442 return ret; 442 return ret;
443 } 443 }
444 444
445 /** 445 /**
446 * kthread_stop - stop a thread created by kthread_create(). 446 * kthread_stop - stop a thread created by kthread_create().
447 * @k: thread created by kthread_create(). 447 * @k: thread created by kthread_create().
448 * 448 *
449 * Sets kthread_should_stop() for @k to return true, wakes it, and 449 * Sets kthread_should_stop() for @k to return true, wakes it, and
450 * waits for it to exit. This can also be called after kthread_create() 450 * waits for it to exit. This can also be called after kthread_create()
451 * instead of calling wake_up_process(): the thread will exit without 451 * instead of calling wake_up_process(): the thread will exit without
452 * calling threadfn(). 452 * calling threadfn().
453 * 453 *
454 * If threadfn() may call do_exit() itself, the caller must ensure 454 * If threadfn() may call do_exit() itself, the caller must ensure
455 * task_struct can't go away. 455 * task_struct can't go away.
456 * 456 *
457 * Returns the result of threadfn(), or %-EINTR if wake_up_process() 457 * Returns the result of threadfn(), or %-EINTR if wake_up_process()
458 * was never called. 458 * was never called.
459 */ 459 */
460 int kthread_stop(struct task_struct *k) 460 int kthread_stop(struct task_struct *k)
461 { 461 {
462 struct kthread *kthread; 462 struct kthread *kthread;
463 int ret; 463 int ret;
464 464
465 trace_sched_kthread_stop(k); 465 trace_sched_kthread_stop(k);
466 466
467 get_task_struct(k); 467 get_task_struct(k);
468 kthread = to_live_kthread(k); 468 kthread = to_live_kthread(k);
469 if (kthread) { 469 if (kthread) {
470 set_bit(KTHREAD_SHOULD_STOP, &kthread->flags); 470 set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
471 __kthread_unpark(k, kthread); 471 __kthread_unpark(k, kthread);
472 wake_up_process(k); 472 wake_up_process(k);
473 wait_for_completion(&kthread->exited); 473 wait_for_completion(&kthread->exited);
474 } 474 }
475 ret = k->exit_code; 475 ret = k->exit_code;
476 put_task_struct(k); 476 put_task_struct(k);
477 477
478 trace_sched_kthread_stop_ret(ret); 478 trace_sched_kthread_stop_ret(ret);
479 return ret; 479 return ret;
480 } 480 }
481 EXPORT_SYMBOL(kthread_stop); 481 EXPORT_SYMBOL(kthread_stop);
482 482
483 int kthreadd(void *unused) 483 int kthreadd(void *unused)
484 { 484 {
485 struct task_struct *tsk = current; 485 struct task_struct *tsk = current;
486 486
487 /* Setup a clean context for our children to inherit. */ 487 /* Setup a clean context for our children to inherit. */
488 set_task_comm(tsk, "kthreadd"); 488 set_task_comm(tsk, "kthreadd");
489 ignore_signals(tsk); 489 ignore_signals(tsk);
490 set_cpus_allowed_ptr(tsk, cpu_all_mask); 490 set_cpus_allowed_ptr(tsk, cpu_all_mask);
491 set_mems_allowed(node_states[N_MEMORY]); 491 set_mems_allowed(node_states[N_MEMORY]);
492 492
493 current->flags |= PF_NOFREEZE; 493 current->flags |= PF_NOFREEZE;
494 494
495 for (;;) { 495 for (;;) {
496 set_current_state(TASK_INTERRUPTIBLE); 496 set_current_state(TASK_INTERRUPTIBLE);
497 if (list_empty(&kthread_create_list)) 497 if (list_empty(&kthread_create_list))
498 schedule(); 498 schedule();
499 __set_current_state(TASK_RUNNING); 499 __set_current_state(TASK_RUNNING);
500 500
501 spin_lock(&kthread_create_lock); 501 spin_lock(&kthread_create_lock);
502 while (!list_empty(&kthread_create_list)) { 502 while (!list_empty(&kthread_create_list)) {
503 struct kthread_create_info *create; 503 struct kthread_create_info *create;
504 504
505 create = list_entry(kthread_create_list.next, 505 create = list_entry(kthread_create_list.next,
506 struct kthread_create_info, list); 506 struct kthread_create_info, list);
507 list_del_init(&create->list); 507 list_del_init(&create->list);
508 spin_unlock(&kthread_create_lock); 508 spin_unlock(&kthread_create_lock);
509 509
510 create_kthread(create); 510 create_kthread(create);
511 511
512 spin_lock(&kthread_create_lock); 512 spin_lock(&kthread_create_lock);
513 } 513 }
514 spin_unlock(&kthread_create_lock); 514 spin_unlock(&kthread_create_lock);
515 } 515 }
516 516
517 return 0; 517 return 0;
518 } 518 }
519 519
520 void __init_kthread_worker(struct kthread_worker *worker, 520 void __init_kthread_worker(struct kthread_worker *worker,
521 const char *name, 521 const char *name,
522 struct lock_class_key *key) 522 struct lock_class_key *key)
523 { 523 {
524 spin_lock_init(&worker->lock); 524 spin_lock_init(&worker->lock);
525 lockdep_set_class_and_name(&worker->lock, key, name); 525 lockdep_set_class_and_name(&worker->lock, key, name);
526 INIT_LIST_HEAD(&worker->work_list); 526 INIT_LIST_HEAD(&worker->work_list);
527 worker->task = NULL; 527 worker->task = NULL;
528 } 528 }
529 EXPORT_SYMBOL_GPL(__init_kthread_worker); 529 EXPORT_SYMBOL_GPL(__init_kthread_worker);
530 530
531 /** 531 /**
532 * kthread_worker_fn - kthread function to process kthread_worker 532 * kthread_worker_fn - kthread function to process kthread_worker
533 * @worker_ptr: pointer to initialized kthread_worker 533 * @worker_ptr: pointer to initialized kthread_worker
534 * 534 *
535 * This function can be used as @threadfn to kthread_create() or 535 * This function can be used as @threadfn to kthread_create() or
536 * kthread_run() with @worker_ptr argument pointing to an initialized 536 * kthread_run() with @worker_ptr argument pointing to an initialized
537 * kthread_worker. The started kthread will process work_list until 537 * kthread_worker. The started kthread will process work_list until
538 * the it is stopped with kthread_stop(). A kthread can also call 538 * the it is stopped with kthread_stop(). A kthread can also call
539 * this function directly after extra initialization. 539 * this function directly after extra initialization.
540 * 540 *
541 * Different kthreads can be used for the same kthread_worker as long 541 * Different kthreads can be used for the same kthread_worker as long
542 * as there's only one kthread attached to it at any given time. A 542 * as there's only one kthread attached to it at any given time. A
543 * kthread_worker without an attached kthread simply collects queued 543 * kthread_worker without an attached kthread simply collects queued
544 * kthread_works. 544 * kthread_works.
545 */ 545 */
546 int kthread_worker_fn(void *worker_ptr) 546 int kthread_worker_fn(void *worker_ptr)
547 { 547 {
548 struct kthread_worker *worker = worker_ptr; 548 struct kthread_worker *worker = worker_ptr;
549 struct kthread_work *work; 549 struct kthread_work *work;
550 550
551 WARN_ON(worker->task); 551 WARN_ON(worker->task);
552 worker->task = current; 552 worker->task = current;
553 repeat: 553 repeat:
554 set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */ 554 set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */
555 555
556 if (kthread_should_stop()) { 556 if (kthread_should_stop()) {
557 __set_current_state(TASK_RUNNING); 557 __set_current_state(TASK_RUNNING);
558 spin_lock_irq(&worker->lock); 558 spin_lock_irq(&worker->lock);
559 worker->task = NULL; 559 worker->task = NULL;
560 spin_unlock_irq(&worker->lock); 560 spin_unlock_irq(&worker->lock);
561 return 0; 561 return 0;
562 } 562 }
563 563
564 work = NULL; 564 work = NULL;
565 spin_lock_irq(&worker->lock); 565 spin_lock_irq(&worker->lock);
566 if (!list_empty(&worker->work_list)) { 566 if (!list_empty(&worker->work_list)) {
567 work = list_first_entry(&worker->work_list, 567 work = list_first_entry(&worker->work_list,
568 struct kthread_work, node); 568 struct kthread_work, node);
569 list_del_init(&work->node); 569 list_del_init(&work->node);
570 } 570 }
571 worker->current_work = work; 571 worker->current_work = work;
572 spin_unlock_irq(&worker->lock); 572 spin_unlock_irq(&worker->lock);
573 573
574 if (work) { 574 if (work) {
575 __set_current_state(TASK_RUNNING); 575 __set_current_state(TASK_RUNNING);
576 work->func(work); 576 work->func(work);
577 } else if (!freezing(current)) 577 } else if (!freezing(current))
578 schedule(); 578 schedule();
579 579
580 try_to_freeze(); 580 try_to_freeze();
581 goto repeat; 581 goto repeat;
582 } 582 }
583 EXPORT_SYMBOL_GPL(kthread_worker_fn); 583 EXPORT_SYMBOL_GPL(kthread_worker_fn);
584 584
585 /* insert @work before @pos in @worker */ 585 /* insert @work before @pos in @worker */
586 static void insert_kthread_work(struct kthread_worker *worker, 586 static void insert_kthread_work(struct kthread_worker *worker,
587 struct kthread_work *work, 587 struct kthread_work *work,
588 struct list_head *pos) 588 struct list_head *pos)
589 { 589 {
590 lockdep_assert_held(&worker->lock); 590 lockdep_assert_held(&worker->lock);
591 591
592 list_add_tail(&work->node, pos); 592 list_add_tail(&work->node, pos);
593 work->worker = worker; 593 work->worker = worker;
594 if (likely(worker->task)) 594 if (likely(worker->task))
595 wake_up_process(worker->task); 595 wake_up_process(worker->task);
596 } 596 }
597 597
598 /** 598 /**
599 * queue_kthread_work - queue a kthread_work 599 * queue_kthread_work - queue a kthread_work
600 * @worker: target kthread_worker 600 * @worker: target kthread_worker
601 * @work: kthread_work to queue 601 * @work: kthread_work to queue
602 * 602 *
603 * Queue @work to work processor @task for async execution. @task 603 * Queue @work to work processor @task for async execution. @task
604 * must have been created with kthread_worker_create(). Returns %true 604 * must have been created with kthread_worker_create(). Returns %true
605 * if @work was successfully queued, %false if it was already pending. 605 * if @work was successfully queued, %false if it was already pending.
606 */ 606 */
607 bool queue_kthread_work(struct kthread_worker *worker, 607 bool queue_kthread_work(struct kthread_worker *worker,
608 struct kthread_work *work) 608 struct kthread_work *work)
609 { 609 {
610 bool ret = false; 610 bool ret = false;
611 unsigned long flags; 611 unsigned long flags;
612 612
613 spin_lock_irqsave(&worker->lock, flags); 613 spin_lock_irqsave(&worker->lock, flags);
614 if (list_empty(&work->node)) { 614 if (list_empty(&work->node)) {
615 insert_kthread_work(worker, work, &worker->work_list); 615 insert_kthread_work(worker, work, &worker->work_list);
616 ret = true; 616 ret = true;
617 } 617 }
618 spin_unlock_irqrestore(&worker->lock, flags); 618 spin_unlock_irqrestore(&worker->lock, flags);
619 return ret; 619 return ret;
620 } 620 }
621 EXPORT_SYMBOL_GPL(queue_kthread_work); 621 EXPORT_SYMBOL_GPL(queue_kthread_work);
622 622
623 struct kthread_flush_work { 623 struct kthread_flush_work {
624 struct kthread_work work; 624 struct kthread_work work;
625 struct completion done; 625 struct completion done;
626 }; 626 };
627 627
628 static void kthread_flush_work_fn(struct kthread_work *work) 628 static void kthread_flush_work_fn(struct kthread_work *work)
629 { 629 {
630 struct kthread_flush_work *fwork = 630 struct kthread_flush_work *fwork =
631 container_of(work, struct kthread_flush_work, work); 631 container_of(work, struct kthread_flush_work, work);
632 complete(&fwork->done); 632 complete(&fwork->done);
633 } 633 }
634 634
635 /** 635 /**
636 * flush_kthread_work - flush a kthread_work 636 * flush_kthread_work - flush a kthread_work
637 * @work: work to flush 637 * @work: work to flush
638 * 638 *
639 * If @work is queued or executing, wait for it to finish execution. 639 * If @work is queued or executing, wait for it to finish execution.
640 */ 640 */
641 void flush_kthread_work(struct kthread_work *work) 641 void flush_kthread_work(struct kthread_work *work)
642 { 642 {
643 struct kthread_flush_work fwork = { 643 struct kthread_flush_work fwork = {
644 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 644 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
645 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 645 COMPLETION_INITIALIZER_ONSTACK(fwork.done),
646 }; 646 };
647 struct kthread_worker *worker; 647 struct kthread_worker *worker;
648 bool noop = false; 648 bool noop = false;
649 649
650 retry: 650 retry:
651 worker = work->worker; 651 worker = work->worker;
652 if (!worker) 652 if (!worker)
653 return; 653 return;
654 654
655 spin_lock_irq(&worker->lock); 655 spin_lock_irq(&worker->lock);
656 if (work->worker != worker) { 656 if (work->worker != worker) {
657 spin_unlock_irq(&worker->lock); 657 spin_unlock_irq(&worker->lock);
658 goto retry; 658 goto retry;
659 } 659 }
660 660
661 if (!list_empty(&work->node)) 661 if (!list_empty(&work->node))
662 insert_kthread_work(worker, &fwork.work, work->node.next); 662 insert_kthread_work(worker, &fwork.work, work->node.next);
663 else if (worker->current_work == work) 663 else if (worker->current_work == work)
664 insert_kthread_work(worker, &fwork.work, worker->work_list.next); 664 insert_kthread_work(worker, &fwork.work, worker->work_list.next);
665 else 665 else
666 noop = true; 666 noop = true;
667 667
668 spin_unlock_irq(&worker->lock); 668 spin_unlock_irq(&worker->lock);
669 669
670 if (!noop) 670 if (!noop)
671 wait_for_completion(&fwork.done); 671 wait_for_completion(&fwork.done);
672 } 672 }
673 EXPORT_SYMBOL_GPL(flush_kthread_work); 673 EXPORT_SYMBOL_GPL(flush_kthread_work);
674 674
675 /** 675 /**
676 * flush_kthread_worker - flush all current works on a kthread_worker 676 * flush_kthread_worker - flush all current works on a kthread_worker
677 * @worker: worker to flush 677 * @worker: worker to flush
678 * 678 *
679 * Wait until all currently executing or pending works on @worker are 679 * Wait until all currently executing or pending works on @worker are
680 * finished. 680 * finished.
681 */ 681 */
682 void flush_kthread_worker(struct kthread_worker *worker) 682 void flush_kthread_worker(struct kthread_worker *worker)
683 { 683 {
684 struct kthread_flush_work fwork = { 684 struct kthread_flush_work fwork = {
685 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 685 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
686 COMPLETION_INITIALIZER_ONSTACK(fwork.done), 686 COMPLETION_INITIALIZER_ONSTACK(fwork.done),
687 }; 687 };
688 688
689 queue_kthread_work(worker, &fwork.work); 689 queue_kthread_work(worker, &fwork.work);
690 wait_for_completion(&fwork.done); 690 wait_for_completion(&fwork.done);
691 } 691 }
692 EXPORT_SYMBOL_GPL(flush_kthread_worker); 692 EXPORT_SYMBOL_GPL(flush_kthread_worker);
693 693