Blame view
kernel/workqueue.c
104 KB
1da177e4c Linux-2.6.12-rc2 |
1 |
/* |
c54fce6ef workqueue: add do... |
2 |
* kernel/workqueue.c - generic async execution with shared worker pool |
1da177e4c Linux-2.6.12-rc2 |
3 |
* |
c54fce6ef workqueue: add do... |
4 |
* Copyright (C) 2002 Ingo Molnar |
1da177e4c Linux-2.6.12-rc2 |
5 |
* |
c54fce6ef workqueue: add do... |
6 7 8 9 10 |
* Derived from the taskqueue/keventd code by: * David Woodhouse <dwmw2@infradead.org> * Andrew Morton * Kai Petzke <wpp@marie.physik.tu-berlin.de> * Theodore Ts'o <tytso@mit.edu> |
1da177e4c Linux-2.6.12-rc2 |
11 |
* |
c54fce6ef workqueue: add do... |
12 |
* Made to use alloc_percpu by Christoph Lameter. |
1da177e4c Linux-2.6.12-rc2 |
13 |
* |
c54fce6ef workqueue: add do... |
14 15 |
* Copyright (C) 2010 SUSE Linux Products GmbH * Copyright (C) 2010 Tejun Heo <tj@kernel.org> |
89ada6791 [PATCH] Use alloc... |
16 |
* |
c54fce6ef workqueue: add do... |
17 18 19 20 21 22 23 |
* This is the generic async execution mechanism. Work items as are * executed in process context. The worker pool is shared and * automatically managed. There is one worker pool for each CPU and * one extra for works which are better served by workers which are * not bound to any specific CPU. * * Please read Documentation/workqueue.txt for details. |
1da177e4c Linux-2.6.12-rc2 |
24 |
*/ |
9984de1a5 kernel: Map most ... |
25 |
#include <linux/export.h> |
1da177e4c Linux-2.6.12-rc2 |
26 27 28 29 30 31 32 33 34 35 |
#include <linux/kernel.h> #include <linux/sched.h> #include <linux/init.h> #include <linux/signal.h> #include <linux/completion.h> #include <linux/workqueue.h> #include <linux/slab.h> #include <linux/cpu.h> #include <linux/notifier.h> #include <linux/kthread.h> |
1fa44ecad [SCSI] add execut... |
36 |
#include <linux/hardirq.h> |
469340236 [PATCH] mm: keven... |
37 |
#include <linux/mempolicy.h> |
341a59585 [PATCH] Support f... |
38 |
#include <linux/freezer.h> |
d5abe6691 [PATCH] debug: wo... |
39 40 |
#include <linux/kallsyms.h> #include <linux/debug_locks.h> |
4e6045f13 workqueue: debug ... |
41 |
#include <linux/lockdep.h> |
c34056a3f workqueue: introd... |
42 |
#include <linux/idr.h> |
e22bee782 workqueue: implem... |
43 44 |
#include "workqueue_sched.h" |
1da177e4c Linux-2.6.12-rc2 |
45 |
|
c8e55f360 workqueue: implem... |
46 |
enum { |
db7bccf45 workqueue: reimpl... |
47 |
/* global_cwq flags */ |
e22bee782 workqueue: implem... |
48 49 50 |
GCWQ_MANAGE_WORKERS = 1 << 0, /* need to manage workers */ GCWQ_MANAGING_WORKERS = 1 << 1, /* managing workers */ GCWQ_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */ |
db7bccf45 workqueue: reimpl... |
51 |
GCWQ_FREEZING = 1 << 3, /* freeze in progress */ |
649027d73 workqueue: implem... |
52 |
GCWQ_HIGHPRI_PENDING = 1 << 4, /* highpri works on queue */ |
db7bccf45 workqueue: reimpl... |
53 |
|
c8e55f360 workqueue: implem... |
54 55 56 57 |
/* worker flags */ WORKER_STARTED = 1 << 0, /* started */ WORKER_DIE = 1 << 1, /* die die die */ WORKER_IDLE = 1 << 2, /* is idle */ |
e22bee782 workqueue: implem... |
58 |
WORKER_PREP = 1 << 3, /* preparing to run works */ |
db7bccf45 workqueue: reimpl... |
59 |
WORKER_ROGUE = 1 << 4, /* not bound to any cpu */ |
e22bee782 workqueue: implem... |
60 |
WORKER_REBIND = 1 << 5, /* mom is home, come back */ |
fb0e7beb5 workqueue: implem... |
61 |
WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */ |
f34217977 workqueue: implem... |
62 |
WORKER_UNBOUND = 1 << 7, /* worker is unbound */ |
e22bee782 workqueue: implem... |
63 |
|
fb0e7beb5 workqueue: implem... |
64 |
WORKER_NOT_RUNNING = WORKER_PREP | WORKER_ROGUE | WORKER_REBIND | |
f34217977 workqueue: implem... |
65 |
WORKER_CPU_INTENSIVE | WORKER_UNBOUND, |
db7bccf45 workqueue: reimpl... |
66 67 68 69 70 71 72 |
/* gcwq->trustee_state */ TRUSTEE_START = 0, /* start */ TRUSTEE_IN_CHARGE = 1, /* trustee in charge of gcwq */ TRUSTEE_BUTCHER = 2, /* butcher workers */ TRUSTEE_RELEASE = 3, /* release workers */ TRUSTEE_DONE = 4, /* trustee is done */ |
c8e55f360 workqueue: implem... |
73 74 75 76 |
BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */ BUSY_WORKER_HASH_SIZE = 1 << BUSY_WORKER_HASH_ORDER, BUSY_WORKER_HASH_MASK = BUSY_WORKER_HASH_SIZE - 1, |
db7bccf45 workqueue: reimpl... |
77 |
|
e22bee782 workqueue: implem... |
78 79 |
MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */ IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */ |
3233cdbd9 workqueue: make s... |
80 81 82 |
MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2, /* call for help after 10ms (min two ticks) */ |
e22bee782 workqueue: implem... |
83 84 |
MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ CREATE_COOLDOWN = HZ, /* time to breath after fail */ |
db7bccf45 workqueue: reimpl... |
85 |
TRUSTEE_COOLDOWN = HZ / 10, /* for trustee draining */ |
e22bee782 workqueue: implem... |
86 87 88 89 90 91 |
/* * Rescue workers are used only on emergencies and shared by * all cpus. Give -20. */ RESCUER_NICE_LEVEL = -20, |
c8e55f360 workqueue: implem... |
92 |
}; |
1da177e4c Linux-2.6.12-rc2 |
93 94 |
/* |
4690c4ab5 workqueue: misc/c... |
95 96 |
* Structure fields follow one of the following exclusion rules. * |
e41e704bc workqueue: improv... |
97 98 |
* I: Modifiable by initialization/destruction paths and read-only for * everyone else. |
4690c4ab5 workqueue: misc/c... |
99 |
* |
e22bee782 workqueue: implem... |
100 101 102 |
* P: Preemption protected. Disabling preemption is enough and should * only be modified and accessed from the local cpu. * |
8b03ae3cd workqueue: introd... |
103 |
* L: gcwq->lock protected. Access with gcwq->lock held. |
4690c4ab5 workqueue: misc/c... |
104 |
* |
e22bee782 workqueue: implem... |
105 106 107 |
* X: During normal operation, modification requires gcwq->lock and * should be done only from local cpu. Either disabling preemption * on local cpu or grabbing gcwq->lock is enough for read access. |
f34217977 workqueue: implem... |
108 |
* If GCWQ_DISASSOCIATED is set, it's identical to L. |
e22bee782 workqueue: implem... |
109 |
* |
73f53c4aa workqueue: reimpl... |
110 111 |
* F: wq->flush_mutex protected. * |
4690c4ab5 workqueue: misc/c... |
112 |
* W: workqueue_lock protected. |
1da177e4c Linux-2.6.12-rc2 |
113 |
*/ |
1da177e4c Linux-2.6.12-rc2 |
114 |
|
8b03ae3cd workqueue: introd... |
115 |
struct global_cwq; |
1da177e4c Linux-2.6.12-rc2 |
116 |
|
e22bee782 workqueue: implem... |
117 118 119 120 |
/* * The poor guys doing the actual heavy lifting. All on-duty workers * are either serving the manager role, on idle list or on busy hash. */ |
c34056a3f workqueue: introd... |
121 |
struct worker { |
c8e55f360 workqueue: implem... |
122 123 124 125 126 |
/* on idle list while idle, on busy hash table while busy */ union { struct list_head entry; /* L: while idle */ struct hlist_node hentry; /* L: while busy */ }; |
1da177e4c Linux-2.6.12-rc2 |
127 |
|
c34056a3f workqueue: introd... |
128 |
struct work_struct *current_work; /* L: work being processed */ |
8cca0eea3 workqueue: add fi... |
129 |
struct cpu_workqueue_struct *current_cwq; /* L: current_work's cwq */ |
affee4b29 workqueue: reimpl... |
130 |
struct list_head scheduled; /* L: scheduled works */ |
c34056a3f workqueue: introd... |
131 |
struct task_struct *task; /* I: worker task */ |
8b03ae3cd workqueue: introd... |
132 |
struct global_cwq *gcwq; /* I: the associated gcwq */ |
e22bee782 workqueue: implem... |
133 134 135 |
/* 64 bytes boundary on 64bit, 32 on 32bit */ unsigned long last_active; /* L: last active timestamp */ unsigned int flags; /* X: flags */ |
c34056a3f workqueue: introd... |
136 |
int id; /* I: worker id */ |
e22bee782 workqueue: implem... |
137 |
struct work_struct rebind_work; /* L: rebind worker to cpu */ |
c34056a3f workqueue: introd... |
138 |
}; |
4690c4ab5 workqueue: misc/c... |
139 |
/* |
e22bee782 workqueue: implem... |
140 141 142 |
* Global per-cpu workqueue. There's one and only one for each cpu * and all works are queued and processed here regardless of their * target workqueues. |
8b03ae3cd workqueue: introd... |
143 144 145 |
*/ struct global_cwq { spinlock_t lock; /* the gcwq lock */ |
7e11629d0 workqueue: use sh... |
146 |
struct list_head worklist; /* L: list of pending works */ |
8b03ae3cd workqueue: introd... |
147 |
unsigned int cpu; /* I: the associated cpu */ |
db7bccf45 workqueue: reimpl... |
148 |
unsigned int flags; /* L: GCWQ_* flags */ |
c8e55f360 workqueue: implem... |
149 150 151 152 153 |
int nr_workers; /* L: total number of workers */ int nr_idle; /* L: currently idle ones */ /* workers are chained either in the idle_list or busy_hash */ |
e22bee782 workqueue: implem... |
154 |
struct list_head idle_list; /* X: list of idle workers */ |
c8e55f360 workqueue: implem... |
155 156 |
struct hlist_head busy_hash[BUSY_WORKER_HASH_SIZE]; /* L: hash of busy workers */ |
e22bee782 workqueue: implem... |
157 158 |
struct timer_list idle_timer; /* L: worker idle timeout */ struct timer_list mayday_timer; /* L: SOS timer for dworkers */ |
8b03ae3cd workqueue: introd... |
159 |
struct ida worker_ida; /* L: for worker IDs */ |
db7bccf45 workqueue: reimpl... |
160 161 162 163 |
struct task_struct *trustee; /* L: for gcwq shutdown */ unsigned int trustee_state; /* L: trustee state */ wait_queue_head_t trustee_wait; /* trustee wait */ |
e22bee782 workqueue: implem... |
164 |
struct worker *first_idle; /* L: first idle worker */ |
8b03ae3cd workqueue: introd... |
165 166 167 |
} ____cacheline_aligned_in_smp; /* |
502ca9d81 workqueue: make s... |
168 |
* The per-CPU workqueue. The lower WORK_STRUCT_FLAG_BITS of |
0f900049c workqueue: update... |
169 170 |
* work_struct->data are used for flags and thus cwqs need to be * aligned at two's power of the number of flag bits. |
1da177e4c Linux-2.6.12-rc2 |
171 172 |
*/ struct cpu_workqueue_struct { |
8b03ae3cd workqueue: introd... |
173 |
struct global_cwq *gcwq; /* I: the associated gcwq */ |
4690c4ab5 workqueue: misc/c... |
174 |
struct workqueue_struct *wq; /* I: the owning workqueue */ |
73f53c4aa workqueue: reimpl... |
175 176 177 178 |
int work_color; /* L: current color */ int flush_color; /* L: flushing color */ int nr_in_flight[WORK_NR_COLORS]; /* L: nr of in_flight works */ |
1e19ffc63 workqueue: implem... |
179 |
int nr_active; /* L: nr of active works */ |
a0a1a5fd4 workqueue: reimpl... |
180 |
int max_active; /* L: max active works */ |
1e19ffc63 workqueue: implem... |
181 |
struct list_head delayed_works; /* L: delayed works */ |
0f900049c workqueue: update... |
182 |
}; |
1da177e4c Linux-2.6.12-rc2 |
183 184 |
/* |
73f53c4aa workqueue: reimpl... |
185 186 187 188 189 190 191 192 193 |
* Structure used to wait for workqueue flush. */ struct wq_flusher { struct list_head list; /* F: list of flushers */ int flush_color; /* F: flush color waiting for */ struct completion done; /* flush completion */ }; /* |
f2e005aaf workqueue: fix ma... |
194 195 196 197 198 199 200 201 202 |
* All cpumasks are assumed to be always set on UP and thus can't be * used to determine whether there's something to be done. */ #ifdef CONFIG_SMP typedef cpumask_var_t mayday_mask_t; #define mayday_test_and_set_cpu(cpu, mask) \ cpumask_test_and_set_cpu((cpu), (mask)) #define mayday_clear_cpu(cpu, mask) cpumask_clear_cpu((cpu), (mask)) #define for_each_mayday_cpu(cpu, mask) for_each_cpu((cpu), (mask)) |
9c37547ab workqueue: use za... |
203 |
#define alloc_mayday_mask(maskp, gfp) zalloc_cpumask_var((maskp), (gfp)) |
f2e005aaf workqueue: fix ma... |
204 205 206 207 208 209 210 211 212 |
#define free_mayday_mask(mask) free_cpumask_var((mask)) #else typedef unsigned long mayday_mask_t; #define mayday_test_and_set_cpu(cpu, mask) test_and_set_bit(0, &(mask)) #define mayday_clear_cpu(cpu, mask) clear_bit(0, &(mask)) #define for_each_mayday_cpu(cpu, mask) if ((cpu) = 0, (mask)) #define alloc_mayday_mask(maskp, gfp) true #define free_mayday_mask(mask) do { } while (0) #endif |
1da177e4c Linux-2.6.12-rc2 |
213 214 215 216 217 218 |
/* * The externally visible workqueue abstraction is an array of * per-CPU workqueues: */ struct workqueue_struct { |
9c5a2ba70 workqueue: separa... |
219 |
unsigned int flags; /* W: WQ_* flags */ |
bdbc5dd7d workqueue: prepar... |
220 221 222 223 224 |
union { struct cpu_workqueue_struct __percpu *pcpu; struct cpu_workqueue_struct *single; unsigned long v; } cpu_wq; /* I: cwq's */ |
4690c4ab5 workqueue: misc/c... |
225 |
struct list_head list; /* W: list of all workqueues */ |
73f53c4aa workqueue: reimpl... |
226 227 228 229 230 231 232 233 |
struct mutex flush_mutex; /* protects wq flushing */ int work_color; /* F: current work color */ int flush_color; /* F: current flush color */ atomic_t nr_cwqs_to_flush; /* flush in progress */ struct wq_flusher *first_flusher; /* F: first flusher */ struct list_head flusher_queue; /* F: flush waiters */ struct list_head flusher_overflow; /* F: flush overflow list */ |
f2e005aaf workqueue: fix ma... |
234 |
mayday_mask_t mayday_mask; /* cpus requesting rescue */ |
e22bee782 workqueue: implem... |
235 |
struct worker *rescuer; /* I: rescue worker */ |
9c5a2ba70 workqueue: separa... |
236 |
int nr_drainers; /* W: drain in progress */ |
dcd989cb7 workqueue: implem... |
237 |
int saved_max_active; /* W: saved cwq max_active */ |
4690c4ab5 workqueue: misc/c... |
238 |
const char *name; /* I: workqueue name */ |
4e6045f13 workqueue: debug ... |
239 |
#ifdef CONFIG_LOCKDEP |
4690c4ab5 workqueue: misc/c... |
240 |
struct lockdep_map lockdep_map; |
4e6045f13 workqueue: debug ... |
241 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
242 |
}; |
d320c0383 workqueue: s/__cr... |
243 244 245 |
struct workqueue_struct *system_wq __read_mostly; struct workqueue_struct *system_long_wq __read_mostly; struct workqueue_struct *system_nrt_wq __read_mostly; |
f34217977 workqueue: implem... |
246 |
struct workqueue_struct *system_unbound_wq __read_mostly; |
24d51add7 workqueue: fix bu... |
247 |
struct workqueue_struct *system_freezable_wq __read_mostly; |
d320c0383 workqueue: s/__cr... |
248 249 250 |
EXPORT_SYMBOL_GPL(system_wq); EXPORT_SYMBOL_GPL(system_long_wq); EXPORT_SYMBOL_GPL(system_nrt_wq); |
f34217977 workqueue: implem... |
251 |
EXPORT_SYMBOL_GPL(system_unbound_wq); |
24d51add7 workqueue: fix bu... |
252 |
EXPORT_SYMBOL_GPL(system_freezable_wq); |
d320c0383 workqueue: s/__cr... |
253 |
|
97bd23470 workqueue: prepar... |
254 255 |
#define CREATE_TRACE_POINTS #include <trace/events/workqueue.h> |
db7bccf45 workqueue: reimpl... |
256 257 258 |
#define for_each_busy_worker(worker, i, pos, gcwq) \ for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) \ hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry) |
f34217977 workqueue: implem... |
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 |
static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask, unsigned int sw) { if (cpu < nr_cpu_ids) { if (sw & 1) { cpu = cpumask_next(cpu, mask); if (cpu < nr_cpu_ids) return cpu; } if (sw & 2) return WORK_CPU_UNBOUND; } return WORK_CPU_NONE; } static inline int __next_wq_cpu(int cpu, const struct cpumask *mask, struct workqueue_struct *wq) { return __next_gcwq_cpu(cpu, mask, !(wq->flags & WQ_UNBOUND) ? 1 : 2); } |
098849516 workqueue: explai... |
279 280 281 282 283 284 285 286 287 288 289 290 291 |
/* * CPU iterators * * An extra gcwq is defined for an invalid cpu number * (WORK_CPU_UNBOUND) to host workqueues which are not bound to any * specific CPU. The following iterators are similar to * for_each_*_cpu() iterators but also considers the unbound gcwq. * * for_each_gcwq_cpu() : possible CPUs + WORK_CPU_UNBOUND * for_each_online_gcwq_cpu() : online CPUs + WORK_CPU_UNBOUND * for_each_cwq_cpu() : possible CPUs for bound workqueues, * WORK_CPU_UNBOUND for unbound workqueues */ |
f34217977 workqueue: implem... |
292 293 294 295 296 297 298 299 300 301 302 303 304 305 |
#define for_each_gcwq_cpu(cpu) \ for ((cpu) = __next_gcwq_cpu(-1, cpu_possible_mask, 3); \ (cpu) < WORK_CPU_NONE; \ (cpu) = __next_gcwq_cpu((cpu), cpu_possible_mask, 3)) #define for_each_online_gcwq_cpu(cpu) \ for ((cpu) = __next_gcwq_cpu(-1, cpu_online_mask, 3); \ (cpu) < WORK_CPU_NONE; \ (cpu) = __next_gcwq_cpu((cpu), cpu_online_mask, 3)) #define for_each_cwq_cpu(cpu, wq) \ for ((cpu) = __next_wq_cpu(-1, cpu_possible_mask, (wq)); \ (cpu) < WORK_CPU_NONE; \ (cpu) = __next_wq_cpu((cpu), cpu_possible_mask, (wq))) |
dc186ad74 workqueue: Add de... |
306 307 308 |
#ifdef CONFIG_DEBUG_OBJECTS_WORK static struct debug_obj_descr work_debug_descr; |
997772884 debugobjects: Add... |
309 310 311 312 |
static void *work_debug_hint(void *addr) { return ((struct work_struct *) addr)->func; } |
dc186ad74 workqueue: Add de... |
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 |
/* * fixup_init is called when: * - an active object is initialized */ static int work_fixup_init(void *addr, enum debug_obj_state state) { struct work_struct *work = addr; switch (state) { case ODEBUG_STATE_ACTIVE: cancel_work_sync(work); debug_object_init(work, &work_debug_descr); return 1; default: return 0; } } /* * fixup_activate is called when: * - an active object is activated * - an unknown object is activated (might be a statically initialized object) */ static int work_fixup_activate(void *addr, enum debug_obj_state state) { struct work_struct *work = addr; switch (state) { case ODEBUG_STATE_NOTAVAILABLE: /* * This is not really a fixup. The work struct was * statically initialized. We just make sure that it * is tracked in the object tracker. */ |
22df02bb3 workqueue: define... |
348 |
if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) { |
dc186ad74 workqueue: Add de... |
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 |
debug_object_init(work, &work_debug_descr); debug_object_activate(work, &work_debug_descr); return 0; } WARN_ON_ONCE(1); return 0; case ODEBUG_STATE_ACTIVE: WARN_ON(1); default: return 0; } } /* * fixup_free is called when: * - an active object is freed */ static int work_fixup_free(void *addr, enum debug_obj_state state) { struct work_struct *work = addr; switch (state) { case ODEBUG_STATE_ACTIVE: cancel_work_sync(work); debug_object_free(work, &work_debug_descr); return 1; default: return 0; } } static struct debug_obj_descr work_debug_descr = { .name = "work_struct", |
997772884 debugobjects: Add... |
384 |
.debug_hint = work_debug_hint, |
dc186ad74 workqueue: Add de... |
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 |
.fixup_init = work_fixup_init, .fixup_activate = work_fixup_activate, .fixup_free = work_fixup_free, }; static inline void debug_work_activate(struct work_struct *work) { debug_object_activate(work, &work_debug_descr); } static inline void debug_work_deactivate(struct work_struct *work) { debug_object_deactivate(work, &work_debug_descr); } void __init_work(struct work_struct *work, int onstack) { if (onstack) debug_object_init_on_stack(work, &work_debug_descr); else debug_object_init(work, &work_debug_descr); } EXPORT_SYMBOL_GPL(__init_work); void destroy_work_on_stack(struct work_struct *work) { debug_object_free(work, &work_debug_descr); } EXPORT_SYMBOL_GPL(destroy_work_on_stack); #else static inline void debug_work_activate(struct work_struct *work) { } static inline void debug_work_deactivate(struct work_struct *work) { } #endif |
95402b382 cpu-hotplug: repl... |
419 420 |
/* Serializes the accesses to the list of workqueues. */ static DEFINE_SPINLOCK(workqueue_lock); |
1da177e4c Linux-2.6.12-rc2 |
421 |
static LIST_HEAD(workqueues); |
a0a1a5fd4 workqueue: reimpl... |
422 |
static bool workqueue_freezing; /* W: have wqs started freezing? */ |
c34056a3f workqueue: introd... |
423 |
|
e22bee782 workqueue: implem... |
424 425 426 427 428 |
/* * The almighty global cpu workqueues. nr_running is the only field * which is expected to be used frequently by other cpus via * try_to_wake_up(). Put it in a separate cacheline. */ |
8b03ae3cd workqueue: introd... |
429 |
static DEFINE_PER_CPU(struct global_cwq, global_cwq); |
e22bee782 workqueue: implem... |
430 |
static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running); |
8b03ae3cd workqueue: introd... |
431 |
|
f34217977 workqueue: implem... |
432 433 434 435 436 437 438 |
/* * Global cpu workqueue and nr_running counter for unbound gcwq. The * gcwq is always online, has GCWQ_DISASSOCIATED set, and all its * workers have WORKER_UNBOUND set. */ static struct global_cwq unbound_global_cwq; static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT(0); /* always 0 */ |
c34056a3f workqueue: introd... |
439 |
static int worker_thread(void *__worker); |
1da177e4c Linux-2.6.12-rc2 |
440 |
|
8b03ae3cd workqueue: introd... |
441 442 |
static struct global_cwq *get_gcwq(unsigned int cpu) { |
f34217977 workqueue: implem... |
443 444 445 446 |
if (cpu != WORK_CPU_UNBOUND) return &per_cpu(global_cwq, cpu); else return &unbound_global_cwq; |
8b03ae3cd workqueue: introd... |
447 |
} |
e22bee782 workqueue: implem... |
448 449 |
static atomic_t *get_gcwq_nr_running(unsigned int cpu) { |
f34217977 workqueue: implem... |
450 451 452 453 |
if (cpu != WORK_CPU_UNBOUND) return &per_cpu(gcwq_nr_running, cpu); else return &unbound_gcwq_nr_running; |
e22bee782 workqueue: implem... |
454 |
} |
1537663f5 workqueue: kill c... |
455 456 |
static struct cpu_workqueue_struct *get_cwq(unsigned int cpu, struct workqueue_struct *wq) |
b1f4ec172 workqueue: introd... |
457 |
{ |
f34217977 workqueue: implem... |
458 459 460 461 |
if (!(wq->flags & WQ_UNBOUND)) { if (likely(cpu < nr_cpu_ids)) { #ifdef CONFIG_SMP return per_cpu_ptr(wq->cpu_wq.pcpu, cpu); |
bdbc5dd7d workqueue: prepar... |
462 |
#else |
f34217977 workqueue: implem... |
463 |
return wq->cpu_wq.single; |
bdbc5dd7d workqueue: prepar... |
464 |
#endif |
f34217977 workqueue: implem... |
465 466 467 468 |
} } else if (likely(cpu == WORK_CPU_UNBOUND)) return wq->cpu_wq.single; return NULL; |
b1f4ec172 workqueue: introd... |
469 |
} |
73f53c4aa workqueue: reimpl... |
470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 |
static unsigned int work_color_to_flags(int color) { return color << WORK_STRUCT_COLOR_SHIFT; } static int get_work_color(struct work_struct *work) { return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) & ((1 << WORK_STRUCT_COLOR_BITS) - 1); } static int work_next_color(int color) { return (color + 1) % WORK_NR_COLORS; } |
1da177e4c Linux-2.6.12-rc2 |
485 |
|
14441960e simplify cleanup_... |
486 |
/* |
e120153dd workqueue: fix ho... |
487 488 489 |
* A work's data points to the cwq with WORK_STRUCT_CWQ set while the * work is on queue. Once execution starts, WORK_STRUCT_CWQ is * cleared and the work data contains the cpu number it was last on. |
7a22ad757 workqueue: carry ... |
490 491 492 493 494 495 496 497 498 |
* * set_work_{cwq|cpu}() and clear_work_data() can be used to set the * cwq, cpu or clear work->data. These functions should only be * called while the work is owned - ie. while the PENDING bit is set. * * get_work_[g]cwq() can be used to obtain the gcwq or cwq * corresponding to a work. gcwq is available once the work has been * queued anywhere after initialization. cwq is available only from * queueing until execution starts. |
14441960e simplify cleanup_... |
499 |
*/ |
7a22ad757 workqueue: carry ... |
500 501 |
static inline void set_work_data(struct work_struct *work, unsigned long data, unsigned long flags) |
365970a1e WorkStruct: Merge... |
502 |
{ |
4594bf159 [PATCH] WorkStruc... |
503 |
BUG_ON(!work_pending(work)); |
7a22ad757 workqueue: carry ... |
504 505 |
atomic_long_set(&work->data, data | flags | work_static(work)); } |
365970a1e WorkStruct: Merge... |
506 |
|
7a22ad757 workqueue: carry ... |
507 508 509 510 511 |
static void set_work_cwq(struct work_struct *work, struct cpu_workqueue_struct *cwq, unsigned long extra_flags) { set_work_data(work, (unsigned long)cwq, |
e120153dd workqueue: fix ho... |
512 |
WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags); |
365970a1e WorkStruct: Merge... |
513 |
} |
7a22ad757 workqueue: carry ... |
514 515 516 517 |
static void set_work_cpu(struct work_struct *work, unsigned int cpu) { set_work_data(work, cpu << WORK_STRUCT_FLAG_BITS, WORK_STRUCT_PENDING); } |
f756d5e25 [PATCH] fix workq... |
518 |
|
7a22ad757 workqueue: carry ... |
519 |
static void clear_work_data(struct work_struct *work) |
1da177e4c Linux-2.6.12-rc2 |
520 |
{ |
7a22ad757 workqueue: carry ... |
521 |
set_work_data(work, WORK_STRUCT_NO_CPU, 0); |
1da177e4c Linux-2.6.12-rc2 |
522 |
} |
7a22ad757 workqueue: carry ... |
523 |
static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work) |
b1f4ec172 workqueue: introd... |
524 |
{ |
e120153dd workqueue: fix ho... |
525 |
unsigned long data = atomic_long_read(&work->data); |
7a22ad757 workqueue: carry ... |
526 |
|
e120153dd workqueue: fix ho... |
527 528 529 530 |
if (data & WORK_STRUCT_CWQ) return (void *)(data & WORK_STRUCT_WQ_DATA_MASK); else return NULL; |
4d707b9f4 workqueue: change... |
531 |
} |
7a22ad757 workqueue: carry ... |
532 |
static struct global_cwq *get_work_gcwq(struct work_struct *work) |
365970a1e WorkStruct: Merge... |
533 |
{ |
e120153dd workqueue: fix ho... |
534 |
unsigned long data = atomic_long_read(&work->data); |
7a22ad757 workqueue: carry ... |
535 |
unsigned int cpu; |
e120153dd workqueue: fix ho... |
536 537 538 |
if (data & WORK_STRUCT_CWQ) return ((struct cpu_workqueue_struct *) (data & WORK_STRUCT_WQ_DATA_MASK))->gcwq; |
7a22ad757 workqueue: carry ... |
539 540 |
cpu = data >> WORK_STRUCT_FLAG_BITS; |
bdbc5dd7d workqueue: prepar... |
541 |
if (cpu == WORK_CPU_NONE) |
7a22ad757 workqueue: carry ... |
542 |
return NULL; |
f34217977 workqueue: implem... |
543 |
BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND); |
7a22ad757 workqueue: carry ... |
544 |
return get_gcwq(cpu); |
b1f4ec172 workqueue: introd... |
545 |
} |
e22bee782 workqueue: implem... |
546 547 548 549 550 |
/* * Policy functions. These define the policies on how the global * worker pool is managed. Unless noted otherwise, these functions * assume that they're being called with gcwq->lock held. */ |
649027d73 workqueue: implem... |
551 |
static bool __need_more_worker(struct global_cwq *gcwq) |
a848e3b67 workqueue: introd... |
552 |
{ |
649027d73 workqueue: implem... |
553 554 |
return !atomic_read(get_gcwq_nr_running(gcwq->cpu)) || gcwq->flags & GCWQ_HIGHPRI_PENDING; |
a848e3b67 workqueue: introd... |
555 |
} |
4594bf159 [PATCH] WorkStruc... |
556 |
/* |
e22bee782 workqueue: implem... |
557 558 |
* Need to wake up a worker? Called from anything but currently * running workers. |
4594bf159 [PATCH] WorkStruc... |
559 |
*/ |
e22bee782 workqueue: implem... |
560 |
static bool need_more_worker(struct global_cwq *gcwq) |
365970a1e WorkStruct: Merge... |
561 |
{ |
649027d73 workqueue: implem... |
562 |
return !list_empty(&gcwq->worklist) && __need_more_worker(gcwq); |
e22bee782 workqueue: implem... |
563 |
} |
4594bf159 [PATCH] WorkStruc... |
564 |
|
e22bee782 workqueue: implem... |
565 566 567 568 569 570 571 572 573 574 |
/* Can I start working? Called from busy but !running workers. */ static bool may_start_working(struct global_cwq *gcwq) { return gcwq->nr_idle; } /* Do I need to keep working? Called from currently running workers. */ static bool keep_working(struct global_cwq *gcwq) { atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); |
30310045d workqueue: fix HI... |
575 576 577 |
return !list_empty(&gcwq->worklist) && (atomic_read(nr_running) <= 1 || gcwq->flags & GCWQ_HIGHPRI_PENDING); |
e22bee782 workqueue: implem... |
578 579 580 581 582 583 584 |
} /* Do we need a new worker? Called from manager. */ static bool need_to_create_worker(struct global_cwq *gcwq) { return need_more_worker(gcwq) && !may_start_working(gcwq); } |
365970a1e WorkStruct: Merge... |
585 |
|
e22bee782 workqueue: implem... |
586 587 588 589 590 591 592 593 594 595 596 597 598 599 |
/* Do I need to be the manager? */ static bool need_to_manage_workers(struct global_cwq *gcwq) { return need_to_create_worker(gcwq) || gcwq->flags & GCWQ_MANAGE_WORKERS; } /* Do we have too many workers and should some go away? */ static bool too_many_workers(struct global_cwq *gcwq) { bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS; int nr_idle = gcwq->nr_idle + managing; /* manager is considered idle */ int nr_busy = gcwq->nr_workers - nr_idle; return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; |
365970a1e WorkStruct: Merge... |
600 |
} |
4d707b9f4 workqueue: change... |
601 |
/* |
e22bee782 workqueue: implem... |
602 603 |
* Wake up functions. */ |
7e11629d0 workqueue: use sh... |
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 |
/* Return the first worker. Safe with preemption disabled */ static struct worker *first_worker(struct global_cwq *gcwq) { if (unlikely(list_empty(&gcwq->idle_list))) return NULL; return list_first_entry(&gcwq->idle_list, struct worker, entry); } /** * wake_up_worker - wake up an idle worker * @gcwq: gcwq to wake worker for * * Wake up the first idle worker of @gcwq. * * CONTEXT: * spin_lock_irq(gcwq->lock). */ static void wake_up_worker(struct global_cwq *gcwq) { struct worker *worker = first_worker(gcwq); if (likely(worker)) wake_up_process(worker->task); } |
4690c4ab5 workqueue: misc/c... |
629 |
/** |
e22bee782 workqueue: implem... |
630 631 632 633 634 635 636 637 638 639 640 641 642 |
* wq_worker_waking_up - a worker is waking up * @task: task waking up * @cpu: CPU @task is waking up to * * This function is called during try_to_wake_up() when a worker is * being awoken. * * CONTEXT: * spin_lock_irq(rq->lock) */ void wq_worker_waking_up(struct task_struct *task, unsigned int cpu) { struct worker *worker = kthread_data(task); |
2d64672ed workqueue: It is ... |
643 |
if (!(worker->flags & WORKER_NOT_RUNNING)) |
e22bee782 workqueue: implem... |
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 |
atomic_inc(get_gcwq_nr_running(cpu)); } /** * wq_worker_sleeping - a worker is going to sleep * @task: task going to sleep * @cpu: CPU in question, must be the current CPU number * * This function is called during schedule() when a busy worker is * going to sleep. Worker on the same cpu can be woken up by * returning pointer to its task. * * CONTEXT: * spin_lock_irq(rq->lock) * * RETURNS: * Worker task on @cpu to wake up, %NULL if none. */ struct task_struct *wq_worker_sleeping(struct task_struct *task, unsigned int cpu) { struct worker *worker = kthread_data(task), *to_wakeup = NULL; struct global_cwq *gcwq = get_gcwq(cpu); atomic_t *nr_running = get_gcwq_nr_running(cpu); |
2d64672ed workqueue: It is ... |
668 |
if (worker->flags & WORKER_NOT_RUNNING) |
e22bee782 workqueue: implem... |
669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 |
return NULL; /* this can only happen on the local cpu */ BUG_ON(cpu != raw_smp_processor_id()); /* * The counterpart of the following dec_and_test, implied mb, * worklist not empty test sequence is in insert_work(). * Please read comment there. * * NOT_RUNNING is clear. This means that trustee is not in * charge and we're running on the local cpu w/ rq lock held * and preemption disabled, which in turn means that none else * could be manipulating idle_list, so dereferencing idle_list * without gcwq lock is safe. */ if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist)) to_wakeup = first_worker(gcwq); return to_wakeup ? to_wakeup->task : NULL; } /** * worker_set_flags - set worker flags and adjust nr_running accordingly |
cb4447669 workqueue: use wo... |
692 |
* @worker: self |
d302f0178 workqueue: implem... |
693 694 695 |
* @flags: flags to set * @wakeup: wakeup an idle worker if necessary * |
e22bee782 workqueue: implem... |
696 697 698 |
* Set @flags in @worker->flags and adjust nr_running accordingly. If * nr_running becomes zero and @wakeup is %true, an idle worker is * woken up. |
d302f0178 workqueue: implem... |
699 |
* |
cb4447669 workqueue: use wo... |
700 701 |
* CONTEXT: * spin_lock_irq(gcwq->lock) |
d302f0178 workqueue: implem... |
702 703 704 705 |
*/ static inline void worker_set_flags(struct worker *worker, unsigned int flags, bool wakeup) { |
e22bee782 workqueue: implem... |
706 |
struct global_cwq *gcwq = worker->gcwq; |
cb4447669 workqueue: use wo... |
707 |
WARN_ON_ONCE(worker->task != current); |
e22bee782 workqueue: implem... |
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 |
/* * If transitioning into NOT_RUNNING, adjust nr_running and * wake up an idle worker as necessary if requested by * @wakeup. */ if ((flags & WORKER_NOT_RUNNING) && !(worker->flags & WORKER_NOT_RUNNING)) { atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu); if (wakeup) { if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist)) wake_up_worker(gcwq); } else atomic_dec(nr_running); } |
d302f0178 workqueue: implem... |
724 725 726 727 |
worker->flags |= flags; } /** |
e22bee782 workqueue: implem... |
728 |
* worker_clr_flags - clear worker flags and adjust nr_running accordingly |
cb4447669 workqueue: use wo... |
729 |
* @worker: self |
d302f0178 workqueue: implem... |
730 731 |
* @flags: flags to clear * |
e22bee782 workqueue: implem... |
732 |
* Clear @flags in @worker->flags and adjust nr_running accordingly. |
d302f0178 workqueue: implem... |
733 |
* |
cb4447669 workqueue: use wo... |
734 735 |
* CONTEXT: * spin_lock_irq(gcwq->lock) |
d302f0178 workqueue: implem... |
736 737 738 |
*/ static inline void worker_clr_flags(struct worker *worker, unsigned int flags) { |
e22bee782 workqueue: implem... |
739 740 |
struct global_cwq *gcwq = worker->gcwq; unsigned int oflags = worker->flags; |
cb4447669 workqueue: use wo... |
741 |
WARN_ON_ONCE(worker->task != current); |
d302f0178 workqueue: implem... |
742 |
worker->flags &= ~flags; |
e22bee782 workqueue: implem... |
743 |
|
42c025f3d workqueue: note t... |
744 745 746 747 748 |
/* * If transitioning out of NOT_RUNNING, increment nr_running. Note * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask * of multiple flags, not a single flag. */ |
e22bee782 workqueue: implem... |
749 750 751 |
if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING)) if (!(worker->flags & WORKER_NOT_RUNNING)) atomic_inc(get_gcwq_nr_running(gcwq->cpu)); |
d302f0178 workqueue: implem... |
752 753 754 |
} /** |
c8e55f360 workqueue: implem... |
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 |
* busy_worker_head - return the busy hash head for a work * @gcwq: gcwq of interest * @work: work to be hashed * * Return hash head of @gcwq for @work. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to the hash head. */ static struct hlist_head *busy_worker_head(struct global_cwq *gcwq, struct work_struct *work) { const int base_shift = ilog2(sizeof(struct work_struct)); unsigned long v = (unsigned long)work; /* simple shift and fold hash, do we need something better? */ v >>= base_shift; v += v >> BUSY_WORKER_HASH_ORDER; v &= BUSY_WORKER_HASH_MASK; return &gcwq->busy_hash[v]; } /** |
8cca0eea3 workqueue: add fi... |
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 |
* __find_worker_executing_work - find worker which is executing a work * @gcwq: gcwq of interest * @bwh: hash head as returned by busy_worker_head() * @work: work to find worker for * * Find a worker which is executing @work on @gcwq. @bwh should be * the hash head obtained by calling busy_worker_head() with the same * work. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to worker which is executing @work if found, NULL * otherwise. */ static struct worker *__find_worker_executing_work(struct global_cwq *gcwq, struct hlist_head *bwh, struct work_struct *work) { struct worker *worker; struct hlist_node *tmp; hlist_for_each_entry(worker, tmp, bwh, hentry) if (worker->current_work == work) return worker; return NULL; } /** * find_worker_executing_work - find worker which is executing a work * @gcwq: gcwq of interest * @work: work to find worker for * * Find a worker which is executing @work on @gcwq. This function is * identical to __find_worker_executing_work() except that this * function calculates @bwh itself. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to worker which is executing @work if found, NULL * otherwise. |
4d707b9f4 workqueue: change... |
826 |
*/ |
8cca0eea3 workqueue: add fi... |
827 828 |
static struct worker *find_worker_executing_work(struct global_cwq *gcwq, struct work_struct *work) |
4d707b9f4 workqueue: change... |
829 |
{ |
8cca0eea3 workqueue: add fi... |
830 831 |
return __find_worker_executing_work(gcwq, busy_worker_head(gcwq, work), work); |
4d707b9f4 workqueue: change... |
832 |
} |
8cca0eea3 workqueue: add fi... |
833 |
/** |
649027d73 workqueue: implem... |
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 |
* gcwq_determine_ins_pos - find insertion position * @gcwq: gcwq of interest * @cwq: cwq a work is being queued for * * A work for @cwq is about to be queued on @gcwq, determine insertion * position for the work. If @cwq is for HIGHPRI wq, the work is * queued at the head of the queue but in FIFO order with respect to * other HIGHPRI works; otherwise, at the end of the queue. This * function also sets GCWQ_HIGHPRI_PENDING flag to hint @gcwq that * there are HIGHPRI works pending. * * CONTEXT: * spin_lock_irq(gcwq->lock). * * RETURNS: * Pointer to inserstion position. */ static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq, struct cpu_workqueue_struct *cwq) |
365970a1e WorkStruct: Merge... |
853 |
{ |
649027d73 workqueue: implem... |
854 855 856 857 858 859 860 861 862 863 864 865 866 867 |
struct work_struct *twork; if (likely(!(cwq->wq->flags & WQ_HIGHPRI))) return &gcwq->worklist; list_for_each_entry(twork, &gcwq->worklist, entry) { struct cpu_workqueue_struct *tcwq = get_work_cwq(twork); if (!(tcwq->wq->flags & WQ_HIGHPRI)) break; } gcwq->flags |= GCWQ_HIGHPRI_PENDING; return &twork->entry; |
365970a1e WorkStruct: Merge... |
868 |
} |
649027d73 workqueue: implem... |
869 |
/** |
7e11629d0 workqueue: use sh... |
870 |
* insert_work - insert a work into gcwq |
4690c4ab5 workqueue: misc/c... |
871 872 873 874 875 |
* @cwq: cwq @work belongs to * @work: work to insert * @head: insertion point * @extra_flags: extra WORK_STRUCT_* flags to set * |
7e11629d0 workqueue: use sh... |
876 877 |
* Insert @work which belongs to @cwq into @gcwq after @head. * @extra_flags is or'd to work_struct flags. |
4690c4ab5 workqueue: misc/c... |
878 879 |
* * CONTEXT: |
8b03ae3cd workqueue: introd... |
880 |
* spin_lock_irq(gcwq->lock). |
4690c4ab5 workqueue: misc/c... |
881 |
*/ |
b89deed32 implement flush_w... |
882 |
static void insert_work(struct cpu_workqueue_struct *cwq, |
4690c4ab5 workqueue: misc/c... |
883 884 |
struct work_struct *work, struct list_head *head, unsigned int extra_flags) |
b89deed32 implement flush_w... |
885 |
{ |
e22bee782 workqueue: implem... |
886 |
struct global_cwq *gcwq = cwq->gcwq; |
4690c4ab5 workqueue: misc/c... |
887 |
/* we own @work, set data and link */ |
7a22ad757 workqueue: carry ... |
888 |
set_work_cwq(work, cwq, extra_flags); |
e1d8aa9f1 tracing: add a ne... |
889 |
|
6e84d644b make cancel_rearm... |
890 891 892 893 894 |
/* * Ensure that we get the right work->data if we see the * result of list_add() below, see try_to_grab_pending(). */ smp_wmb(); |
4690c4ab5 workqueue: misc/c... |
895 |
|
1a4d9b0aa workqueues: inser... |
896 |
list_add_tail(&work->entry, head); |
e22bee782 workqueue: implem... |
897 898 899 900 901 902 903 |
/* * Ensure either worker_sched_deactivated() sees the above * list_add_tail() or we see zero nr_running to avoid workers * lying around lazily while there are works to be processed. */ smp_mb(); |
649027d73 workqueue: implem... |
904 |
if (__need_more_worker(gcwq)) |
e22bee782 workqueue: implem... |
905 |
wake_up_worker(gcwq); |
b89deed32 implement flush_w... |
906 |
} |
c8efcc258 workqueue: allow ... |
907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 |
/* * Test whether @work is being queued from another work executing on the * same workqueue. This is rather expensive and should only be used from * cold paths. */ static bool is_chained_work(struct workqueue_struct *wq) { unsigned long flags; unsigned int cpu; for_each_gcwq_cpu(cpu) { struct global_cwq *gcwq = get_gcwq(cpu); struct worker *worker; struct hlist_node *pos; int i; spin_lock_irqsave(&gcwq->lock, flags); for_each_busy_worker(worker, i, pos, gcwq) { if (worker->task != current) continue; spin_unlock_irqrestore(&gcwq->lock, flags); /* * I'm @worker, no locking necessary. See if @work * is headed to the same workqueue. */ return worker->current_cwq->wq == wq; } spin_unlock_irqrestore(&gcwq->lock, flags); } return false; } |
4690c4ab5 workqueue: misc/c... |
938 |
static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, |
1da177e4c Linux-2.6.12-rc2 |
939 940 |
struct work_struct *work) { |
502ca9d81 workqueue: make s... |
941 942 |
struct global_cwq *gcwq; struct cpu_workqueue_struct *cwq; |
1e19ffc63 workqueue: implem... |
943 |
struct list_head *worklist; |
8a2e8e5de workqueue: fix cw... |
944 |
unsigned int work_flags; |
1da177e4c Linux-2.6.12-rc2 |
945 |
unsigned long flags; |
dc186ad74 workqueue: Add de... |
946 |
debug_work_activate(work); |
1e19ffc63 workqueue: implem... |
947 |
|
c8efcc258 workqueue: allow ... |
948 |
/* if dying, only works from the same workqueue are allowed */ |
9c5a2ba70 workqueue: separa... |
949 |
if (unlikely(wq->flags & WQ_DRAINING) && |
c8efcc258 workqueue: allow ... |
950 |
WARN_ON_ONCE(!is_chained_work(wq))) |
e41e704bc workqueue: improv... |
951 |
return; |
c7fc77f78 workqueue: remove... |
952 953 |
/* determine gcwq to use */ if (!(wq->flags & WQ_UNBOUND)) { |
18aa9effa workqueue: implem... |
954 |
struct global_cwq *last_gcwq; |
c7fc77f78 workqueue: remove... |
955 956 |
if (unlikely(cpu == WORK_CPU_UNBOUND)) cpu = raw_smp_processor_id(); |
18aa9effa workqueue: implem... |
957 958 959 960 961 962 |
/* * It's multi cpu. If @wq is non-reentrant and @work * was previously on a different cpu, it might still * be running there, in which case the work needs to * be queued on that cpu to guarantee non-reentrance. */ |
502ca9d81 workqueue: make s... |
963 |
gcwq = get_gcwq(cpu); |
18aa9effa workqueue: implem... |
964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 |
if (wq->flags & WQ_NON_REENTRANT && (last_gcwq = get_work_gcwq(work)) && last_gcwq != gcwq) { struct worker *worker; spin_lock_irqsave(&last_gcwq->lock, flags); worker = find_worker_executing_work(last_gcwq, work); if (worker && worker->current_cwq->wq == wq) gcwq = last_gcwq; else { /* meh... not running there, queue here */ spin_unlock_irqrestore(&last_gcwq->lock, flags); spin_lock_irqsave(&gcwq->lock, flags); } } else spin_lock_irqsave(&gcwq->lock, flags); |
f34217977 workqueue: implem... |
981 982 983 |
} else { gcwq = get_gcwq(WORK_CPU_UNBOUND); spin_lock_irqsave(&gcwq->lock, flags); |
502ca9d81 workqueue: make s... |
984 985 986 987 |
} /* gcwq determined, get cwq and queue */ cwq = get_cwq(gcwq->cpu, wq); |
cdadf0097 workqueue: add qu... |
988 |
trace_workqueue_queue_work(cpu, cwq, work); |
502ca9d81 workqueue: make s... |
989 |
|
4690c4ab5 workqueue: misc/c... |
990 |
BUG_ON(!list_empty(&work->entry)); |
1e19ffc63 workqueue: implem... |
991 |
|
73f53c4aa workqueue: reimpl... |
992 |
cwq->nr_in_flight[cwq->work_color]++; |
8a2e8e5de workqueue: fix cw... |
993 |
work_flags = work_color_to_flags(cwq->work_color); |
1e19ffc63 workqueue: implem... |
994 995 |
if (likely(cwq->nr_active < cwq->max_active)) { |
cdadf0097 workqueue: add qu... |
996 |
trace_workqueue_activate_work(work); |
1e19ffc63 workqueue: implem... |
997 |
cwq->nr_active++; |
649027d73 workqueue: implem... |
998 |
worklist = gcwq_determine_ins_pos(gcwq, cwq); |
8a2e8e5de workqueue: fix cw... |
999 1000 |
} else { work_flags |= WORK_STRUCT_DELAYED; |
1e19ffc63 workqueue: implem... |
1001 |
worklist = &cwq->delayed_works; |
8a2e8e5de workqueue: fix cw... |
1002 |
} |
1e19ffc63 workqueue: implem... |
1003 |
|
8a2e8e5de workqueue: fix cw... |
1004 |
insert_work(cwq, work, worklist, work_flags); |
1e19ffc63 workqueue: implem... |
1005 |
|
8b03ae3cd workqueue: introd... |
1006 |
spin_unlock_irqrestore(&gcwq->lock, flags); |
1da177e4c Linux-2.6.12-rc2 |
1007 |
} |
0fcb78c22 [PATCH] Add DocBo... |
1008 1009 1010 1011 1012 |
/** * queue_work - queue work on a workqueue * @wq: workqueue to use * @work: work to queue * |
057647fc4 [PATCH] workqueue... |
1013 |
* Returns 0 if @work was already on a queue, non-zero otherwise. |
1da177e4c Linux-2.6.12-rc2 |
1014 |
* |
00dfcaf74 workqueues: shrin... |
1015 1016 |
* We queue the work to the CPU on which it was submitted, but if the CPU dies * it can be processed by another CPU. |
1da177e4c Linux-2.6.12-rc2 |
1017 |
*/ |
7ad5b3a50 kernel: remove fa... |
1018 |
int queue_work(struct workqueue_struct *wq, struct work_struct *work) |
1da177e4c Linux-2.6.12-rc2 |
1019 |
{ |
ef1ca236b workqueues: queue... |
1020 1021 1022 1023 |
int ret; ret = queue_work_on(get_cpu(), wq, work); put_cpu(); |
1da177e4c Linux-2.6.12-rc2 |
1024 1025 |
return ret; } |
ae90dd5db Move workqueue ex... |
1026 |
EXPORT_SYMBOL_GPL(queue_work); |
1da177e4c Linux-2.6.12-rc2 |
1027 |
|
c1a220e7a pm: introduce new... |
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 |
/** * queue_work_on - queue work on specific cpu * @cpu: CPU number to execute work on * @wq: workqueue to use * @work: work to queue * * Returns 0 if @work was already on a queue, non-zero otherwise. * * We queue the work to a specific CPU, the caller must ensure it * can't go away. */ int queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work) { int ret = 0; |
22df02bb3 workqueue: define... |
1043 |
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { |
4690c4ab5 workqueue: misc/c... |
1044 |
__queue_work(cpu, wq, work); |
c1a220e7a pm: introduce new... |
1045 1046 1047 1048 1049 |
ret = 1; } return ret; } EXPORT_SYMBOL_GPL(queue_work_on); |
6d141c3ff workqueue: make d... |
1050 |
static void delayed_work_timer_fn(unsigned long __data) |
1da177e4c Linux-2.6.12-rc2 |
1051 |
{ |
52bad64d9 WorkStruct: Separ... |
1052 |
struct delayed_work *dwork = (struct delayed_work *)__data; |
7a22ad757 workqueue: carry ... |
1053 |
struct cpu_workqueue_struct *cwq = get_work_cwq(&dwork->work); |
1da177e4c Linux-2.6.12-rc2 |
1054 |
|
4690c4ab5 workqueue: misc/c... |
1055 |
__queue_work(smp_processor_id(), cwq->wq, &dwork->work); |
1da177e4c Linux-2.6.12-rc2 |
1056 |
} |
0fcb78c22 [PATCH] Add DocBo... |
1057 1058 1059 |
/** * queue_delayed_work - queue work on a workqueue after delay * @wq: workqueue to use |
af9997e42 [PATCH] fix kerne... |
1060 |
* @dwork: delayable work to queue |
0fcb78c22 [PATCH] Add DocBo... |
1061 1062 |
* @delay: number of jiffies to wait before queueing * |
057647fc4 [PATCH] workqueue... |
1063 |
* Returns 0 if @work was already on a queue, non-zero otherwise. |
0fcb78c22 [PATCH] Add DocBo... |
1064 |
*/ |
7ad5b3a50 kernel: remove fa... |
1065 |
int queue_delayed_work(struct workqueue_struct *wq, |
52bad64d9 WorkStruct: Separ... |
1066 |
struct delayed_work *dwork, unsigned long delay) |
1da177e4c Linux-2.6.12-rc2 |
1067 |
{ |
52bad64d9 WorkStruct: Separ... |
1068 |
if (delay == 0) |
63bc03625 unify queue_delay... |
1069 |
return queue_work(wq, &dwork->work); |
1da177e4c Linux-2.6.12-rc2 |
1070 |
|
63bc03625 unify queue_delay... |
1071 |
return queue_delayed_work_on(-1, wq, dwork, delay); |
1da177e4c Linux-2.6.12-rc2 |
1072 |
} |
ae90dd5db Move workqueue ex... |
1073 |
EXPORT_SYMBOL_GPL(queue_delayed_work); |
1da177e4c Linux-2.6.12-rc2 |
1074 |
|
0fcb78c22 [PATCH] Add DocBo... |
1075 1076 1077 1078 |
/** * queue_delayed_work_on - queue work on specific CPU after delay * @cpu: CPU number to execute work on * @wq: workqueue to use |
af9997e42 [PATCH] fix kerne... |
1079 |
* @dwork: work to queue |
0fcb78c22 [PATCH] Add DocBo... |
1080 1081 |
* @delay: number of jiffies to wait before queueing * |
057647fc4 [PATCH] workqueue... |
1082 |
* Returns 0 if @work was already on a queue, non-zero otherwise. |
0fcb78c22 [PATCH] Add DocBo... |
1083 |
*/ |
7a6bc1cdd [CPUFREQ] Add que... |
1084 |
int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, |
52bad64d9 WorkStruct: Separ... |
1085 |
struct delayed_work *dwork, unsigned long delay) |
7a6bc1cdd [CPUFREQ] Add que... |
1086 1087 |
{ int ret = 0; |
52bad64d9 WorkStruct: Separ... |
1088 1089 |
struct timer_list *timer = &dwork->timer; struct work_struct *work = &dwork->work; |
7a6bc1cdd [CPUFREQ] Add que... |
1090 |
|
22df02bb3 workqueue: define... |
1091 |
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { |
c7fc77f78 workqueue: remove... |
1092 |
unsigned int lcpu; |
7a22ad757 workqueue: carry ... |
1093 |
|
7a6bc1cdd [CPUFREQ] Add que... |
1094 1095 |
BUG_ON(timer_pending(timer)); BUG_ON(!list_empty(&work->entry)); |
8a3e77cc2 workqueue: remove... |
1096 |
timer_stats_timer_set_start_info(&dwork->timer); |
7a22ad757 workqueue: carry ... |
1097 1098 1099 1100 1101 |
/* * This stores cwq for the moment, for the timer_fn. * Note that the work's gcwq is preserved to allow * reentrance detection for delayed works. */ |
c7fc77f78 workqueue: remove... |
1102 1103 1104 1105 1106 1107 1108 1109 1110 |
if (!(wq->flags & WQ_UNBOUND)) { struct global_cwq *gcwq = get_work_gcwq(work); if (gcwq && gcwq->cpu != WORK_CPU_UNBOUND) lcpu = gcwq->cpu; else lcpu = raw_smp_processor_id(); } else lcpu = WORK_CPU_UNBOUND; |
7a22ad757 workqueue: carry ... |
1111 |
set_work_cwq(work, get_cwq(lcpu, wq), 0); |
c7fc77f78 workqueue: remove... |
1112 |
|
7a6bc1cdd [CPUFREQ] Add que... |
1113 |
timer->expires = jiffies + delay; |
52bad64d9 WorkStruct: Separ... |
1114 |
timer->data = (unsigned long)dwork; |
7a6bc1cdd [CPUFREQ] Add que... |
1115 |
timer->function = delayed_work_timer_fn; |
63bc03625 unify queue_delay... |
1116 1117 1118 1119 1120 |
if (unlikely(cpu >= 0)) add_timer_on(timer, cpu); else add_timer(timer); |
7a6bc1cdd [CPUFREQ] Add que... |
1121 1122 1123 1124 |
ret = 1; } return ret; } |
ae90dd5db Move workqueue ex... |
1125 |
EXPORT_SYMBOL_GPL(queue_delayed_work_on); |
1da177e4c Linux-2.6.12-rc2 |
1126 |
|
c8e55f360 workqueue: implem... |
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 |
/** * worker_enter_idle - enter idle state * @worker: worker which is entering idle state * * @worker is entering idle state. Update stats and idle timer if * necessary. * * LOCKING: * spin_lock_irq(gcwq->lock). */ static void worker_enter_idle(struct worker *worker) |
1da177e4c Linux-2.6.12-rc2 |
1138 |
{ |
c8e55f360 workqueue: implem... |
1139 1140 1141 1142 1143 |
struct global_cwq *gcwq = worker->gcwq; BUG_ON(worker->flags & WORKER_IDLE); BUG_ON(!list_empty(&worker->entry) && (worker->hentry.next || worker->hentry.pprev)); |
cb4447669 workqueue: use wo... |
1144 1145 |
/* can't use worker_set_flags(), also called from start_worker() */ worker->flags |= WORKER_IDLE; |
c8e55f360 workqueue: implem... |
1146 |
gcwq->nr_idle++; |
e22bee782 workqueue: implem... |
1147 |
worker->last_active = jiffies; |
c8e55f360 workqueue: implem... |
1148 1149 1150 |
/* idle_list is LIFO */ list_add(&worker->entry, &gcwq->idle_list); |
db7bccf45 workqueue: reimpl... |
1151 |
|
e22bee782 workqueue: implem... |
1152 1153 1154 1155 1156 |
if (likely(!(worker->flags & WORKER_ROGUE))) { if (too_many_workers(gcwq) && !timer_pending(&gcwq->idle_timer)) mod_timer(&gcwq->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); } else |
db7bccf45 workqueue: reimpl... |
1157 |
wake_up_all(&gcwq->trustee_wait); |
cb4447669 workqueue: use wo... |
1158 1159 1160 1161 |
/* sanity check nr_running */ WARN_ON_ONCE(gcwq->nr_workers == gcwq->nr_idle && atomic_read(get_gcwq_nr_running(gcwq->cpu))); |
c8e55f360 workqueue: implem... |
1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 |
} /** * worker_leave_idle - leave idle state * @worker: worker which is leaving idle state * * @worker is leaving idle state. Update stats. * * LOCKING: * spin_lock_irq(gcwq->lock). */ static void worker_leave_idle(struct worker *worker) { struct global_cwq *gcwq = worker->gcwq; BUG_ON(!(worker->flags & WORKER_IDLE)); |
d302f0178 workqueue: implem... |
1178 |
worker_clr_flags(worker, WORKER_IDLE); |
c8e55f360 workqueue: implem... |
1179 1180 1181 |
gcwq->nr_idle--; list_del_init(&worker->entry); } |
e22bee782 workqueue: implem... |
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 |
/** * worker_maybe_bind_and_lock - bind worker to its cpu if possible and lock gcwq * @worker: self * * Works which are scheduled while the cpu is online must at least be * scheduled to a worker which is bound to the cpu so that if they are * flushed from cpu callbacks while cpu is going down, they are * guaranteed to execute on the cpu. * * This function is to be used by rogue workers and rescuers to bind * themselves to the target cpu and may race with cpu going down or * coming online. kthread_bind() can't be used because it may put the * worker to already dead cpu and set_cpus_allowed_ptr() can't be used * verbatim as it's best effort and blocking and gcwq may be * [dis]associated in the meantime. * * This function tries set_cpus_allowed() and locks gcwq and verifies * the binding against GCWQ_DISASSOCIATED which is set during * CPU_DYING and cleared during CPU_ONLINE, so if the worker enters * idle state or fetches works without dropping lock, it can guarantee * the scheduling requirement described in the first paragraph. * * CONTEXT: * Might sleep. Called without any lock but returns with gcwq->lock * held. * * RETURNS: * %true if the associated gcwq is online (@worker is successfully * bound), %false if offline. */ static bool worker_maybe_bind_and_lock(struct worker *worker) |
972fa1c53 workqueue: mark l... |
1213 |
__acquires(&gcwq->lock) |
e22bee782 workqueue: implem... |
1214 1215 1216 1217 1218 |
{ struct global_cwq *gcwq = worker->gcwq; struct task_struct *task = worker->task; while (true) { |
4e6045f13 workqueue: debug ... |
1219 |
/* |
e22bee782 workqueue: implem... |
1220 1221 1222 1223 |
* The following call may fail, succeed or succeed * without actually migrating the task to the cpu if * it races with cpu hotunplug operation. Verify * against GCWQ_DISASSOCIATED. |
4e6045f13 workqueue: debug ... |
1224 |
*/ |
f34217977 workqueue: implem... |
1225 1226 |
if (!(gcwq->flags & GCWQ_DISASSOCIATED)) set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu)); |
e22bee782 workqueue: implem... |
1227 1228 1229 1230 1231 1232 1233 1234 1235 |
spin_lock_irq(&gcwq->lock); if (gcwq->flags & GCWQ_DISASSOCIATED) return false; if (task_cpu(task) == gcwq->cpu && cpumask_equal(¤t->cpus_allowed, get_cpu_mask(gcwq->cpu))) return true; spin_unlock_irq(&gcwq->lock); |
5035b20fa workqueue: fix de... |
1236 1237 1238 1239 1240 1241 |
/* * We've raced with CPU hot[un]plug. Give it a breather * and retry migration. cond_resched() is required here; * otherwise, we might deadlock against cpu_stop trying to * bring down the CPU on non-preemptive kernel. */ |
e22bee782 workqueue: implem... |
1242 |
cpu_relax(); |
5035b20fa workqueue: fix de... |
1243 |
cond_resched(); |
e22bee782 workqueue: implem... |
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 |
} } /* * Function for worker->rebind_work used to rebind rogue busy workers * to the associated cpu which is coming back online. This is * scheduled by cpu up but can race with other cpu hotplug operations * and may be executed twice without intervening cpu down. */ static void worker_rebind_fn(struct work_struct *work) { struct worker *worker = container_of(work, struct worker, rebind_work); struct global_cwq *gcwq = worker->gcwq; if (worker_maybe_bind_and_lock(worker)) worker_clr_flags(worker, WORKER_REBIND); spin_unlock_irq(&gcwq->lock); } |
c34056a3f workqueue: introd... |
1263 1264 1265 1266 1267 |
static struct worker *alloc_worker(void) { struct worker *worker; worker = kzalloc(sizeof(*worker), GFP_KERNEL); |
c8e55f360 workqueue: implem... |
1268 1269 |
if (worker) { INIT_LIST_HEAD(&worker->entry); |
affee4b29 workqueue: reimpl... |
1270 |
INIT_LIST_HEAD(&worker->scheduled); |
e22bee782 workqueue: implem... |
1271 1272 1273 |
INIT_WORK(&worker->rebind_work, worker_rebind_fn); /* on creation a worker is in !idle && prep state */ worker->flags = WORKER_PREP; |
c8e55f360 workqueue: implem... |
1274 |
} |
c34056a3f workqueue: introd... |
1275 1276 1277 1278 1279 |
return worker; } /** * create_worker - create a new workqueue worker |
7e11629d0 workqueue: use sh... |
1280 |
* @gcwq: gcwq the new worker will belong to |
c34056a3f workqueue: introd... |
1281 1282 |
* @bind: whether to set affinity to @cpu or not * |
7e11629d0 workqueue: use sh... |
1283 |
* Create a new worker which is bound to @gcwq. The returned worker |
c34056a3f workqueue: introd... |
1284 1285 1286 1287 1288 1289 1290 1291 1292 |
* can be started by calling start_worker() or destroyed using * destroy_worker(). * * CONTEXT: * Might sleep. Does GFP_KERNEL allocations. * * RETURNS: * Pointer to the newly created worker. */ |
7e11629d0 workqueue: use sh... |
1293 |
static struct worker *create_worker(struct global_cwq *gcwq, bool bind) |
c34056a3f workqueue: introd... |
1294 |
{ |
f34217977 workqueue: implem... |
1295 |
bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND; |
c34056a3f workqueue: introd... |
1296 |
struct worker *worker = NULL; |
f34217977 workqueue: implem... |
1297 |
int id = -1; |
c34056a3f workqueue: introd... |
1298 |
|
8b03ae3cd workqueue: introd... |
1299 1300 1301 1302 |
spin_lock_irq(&gcwq->lock); while (ida_get_new(&gcwq->worker_ida, &id)) { spin_unlock_irq(&gcwq->lock); if (!ida_pre_get(&gcwq->worker_ida, GFP_KERNEL)) |
c34056a3f workqueue: introd... |
1303 |
goto fail; |
8b03ae3cd workqueue: introd... |
1304 |
spin_lock_irq(&gcwq->lock); |
c34056a3f workqueue: introd... |
1305 |
} |
8b03ae3cd workqueue: introd... |
1306 |
spin_unlock_irq(&gcwq->lock); |
c34056a3f workqueue: introd... |
1307 1308 1309 1310 |
worker = alloc_worker(); if (!worker) goto fail; |
8b03ae3cd workqueue: introd... |
1311 |
worker->gcwq = gcwq; |
c34056a3f workqueue: introd... |
1312 |
worker->id = id; |
f34217977 workqueue: implem... |
1313 |
if (!on_unbound_cpu) |
94dcf29a1 kthread: use kthr... |
1314 1315 1316 1317 |
worker->task = kthread_create_on_node(worker_thread, worker, cpu_to_node(gcwq->cpu), "kworker/%u:%d", gcwq->cpu, id); |
f34217977 workqueue: implem... |
1318 1319 1320 |
else worker->task = kthread_create(worker_thread, worker, "kworker/u:%d", id); |
c34056a3f workqueue: introd... |
1321 1322 |
if (IS_ERR(worker->task)) goto fail; |
db7bccf45 workqueue: reimpl... |
1323 1324 1325 1326 1327 |
/* * A rogue worker will become a regular one if CPU comes * online later on. Make sure every worker has * PF_THREAD_BOUND set. */ |
f34217977 workqueue: implem... |
1328 |
if (bind && !on_unbound_cpu) |
8b03ae3cd workqueue: introd... |
1329 |
kthread_bind(worker->task, gcwq->cpu); |
f34217977 workqueue: implem... |
1330 |
else { |
db7bccf45 workqueue: reimpl... |
1331 |
worker->task->flags |= PF_THREAD_BOUND; |
f34217977 workqueue: implem... |
1332 1333 1334 |
if (on_unbound_cpu) worker->flags |= WORKER_UNBOUND; } |
c34056a3f workqueue: introd... |
1335 1336 1337 1338 |
return worker; fail: if (id >= 0) { |
8b03ae3cd workqueue: introd... |
1339 1340 1341 |
spin_lock_irq(&gcwq->lock); ida_remove(&gcwq->worker_ida, id); spin_unlock_irq(&gcwq->lock); |
c34056a3f workqueue: introd... |
1342 1343 1344 1345 1346 1347 1348 1349 1350 |
} kfree(worker); return NULL; } /** * start_worker - start a newly created worker * @worker: worker to start * |
c8e55f360 workqueue: implem... |
1351 |
* Make the gcwq aware of @worker and start it. |
c34056a3f workqueue: introd... |
1352 1353 |
* * CONTEXT: |
8b03ae3cd workqueue: introd... |
1354 |
* spin_lock_irq(gcwq->lock). |
c34056a3f workqueue: introd... |
1355 1356 1357 |
*/ static void start_worker(struct worker *worker) { |
cb4447669 workqueue: use wo... |
1358 |
worker->flags |= WORKER_STARTED; |
c8e55f360 workqueue: implem... |
1359 1360 |
worker->gcwq->nr_workers++; worker_enter_idle(worker); |
c34056a3f workqueue: introd... |
1361 1362 1363 1364 1365 1366 1367 |
wake_up_process(worker->task); } /** * destroy_worker - destroy a workqueue worker * @worker: worker to be destroyed * |
c8e55f360 workqueue: implem... |
1368 1369 1370 1371 |
* Destroy @worker and adjust @gcwq stats accordingly. * * CONTEXT: * spin_lock_irq(gcwq->lock) which is released and regrabbed. |
c34056a3f workqueue: introd... |
1372 1373 1374 |
*/ static void destroy_worker(struct worker *worker) { |
8b03ae3cd workqueue: introd... |
1375 |
struct global_cwq *gcwq = worker->gcwq; |
c34056a3f workqueue: introd... |
1376 1377 1378 1379 |
int id = worker->id; /* sanity check frenzy */ BUG_ON(worker->current_work); |
affee4b29 workqueue: reimpl... |
1380 |
BUG_ON(!list_empty(&worker->scheduled)); |
c34056a3f workqueue: introd... |
1381 |
|
c8e55f360 workqueue: implem... |
1382 1383 1384 1385 1386 1387 |
if (worker->flags & WORKER_STARTED) gcwq->nr_workers--; if (worker->flags & WORKER_IDLE) gcwq->nr_idle--; list_del_init(&worker->entry); |
cb4447669 workqueue: use wo... |
1388 |
worker->flags |= WORKER_DIE; |
c8e55f360 workqueue: implem... |
1389 1390 |
spin_unlock_irq(&gcwq->lock); |
c34056a3f workqueue: introd... |
1391 1392 |
kthread_stop(worker->task); kfree(worker); |
8b03ae3cd workqueue: introd... |
1393 1394 |
spin_lock_irq(&gcwq->lock); ida_remove(&gcwq->worker_ida, id); |
c34056a3f workqueue: introd... |
1395 |
} |
e22bee782 workqueue: implem... |
1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 |
static void idle_worker_timeout(unsigned long __gcwq) { struct global_cwq *gcwq = (void *)__gcwq; spin_lock_irq(&gcwq->lock); if (too_many_workers(gcwq)) { struct worker *worker; unsigned long expires; /* idle_list is kept in LIFO order, check the last one */ worker = list_entry(gcwq->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; if (time_before(jiffies, expires)) mod_timer(&gcwq->idle_timer, expires); else { /* it's been idle for too long, wake up manager */ gcwq->flags |= GCWQ_MANAGE_WORKERS; wake_up_worker(gcwq); |
d5abe6691 [PATCH] debug: wo... |
1416 |
} |
e22bee782 workqueue: implem... |
1417 1418 1419 1420 |
} spin_unlock_irq(&gcwq->lock); } |
d5abe6691 [PATCH] debug: wo... |
1421 |
|
e22bee782 workqueue: implem... |
1422 1423 1424 1425 |
static bool send_mayday(struct work_struct *work) { struct cpu_workqueue_struct *cwq = get_work_cwq(work); struct workqueue_struct *wq = cwq->wq; |
f34217977 workqueue: implem... |
1426 |
unsigned int cpu; |
e22bee782 workqueue: implem... |
1427 1428 1429 1430 1431 |
if (!(wq->flags & WQ_RESCUER)) return false; /* mayday mayday mayday */ |
f34217977 workqueue: implem... |
1432 1433 1434 1435 |
cpu = cwq->gcwq->cpu; /* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */ if (cpu == WORK_CPU_UNBOUND) cpu = 0; |
f2e005aaf workqueue: fix ma... |
1436 |
if (!mayday_test_and_set_cpu(cpu, wq->mayday_mask)) |
e22bee782 workqueue: implem... |
1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 |
wake_up_process(wq->rescuer->task); return true; } static void gcwq_mayday_timeout(unsigned long __gcwq) { struct global_cwq *gcwq = (void *)__gcwq; struct work_struct *work; spin_lock_irq(&gcwq->lock); if (need_to_create_worker(gcwq)) { /* * We've been trying to create a new worker but * haven't been successful. We might be hitting an * allocation deadlock. Send distress signals to * rescuers. */ list_for_each_entry(work, &gcwq->worklist, entry) send_mayday(work); |
1da177e4c Linux-2.6.12-rc2 |
1457 |
} |
e22bee782 workqueue: implem... |
1458 1459 1460 1461 |
spin_unlock_irq(&gcwq->lock); mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INTERVAL); |
1da177e4c Linux-2.6.12-rc2 |
1462 |
} |
e22bee782 workqueue: implem... |
1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 |
/** * maybe_create_worker - create a new worker if necessary * @gcwq: gcwq to create a new worker for * * Create a new worker for @gcwq if necessary. @gcwq is guaranteed to * have at least one idle worker on return from this function. If * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is * sent to all rescuers with works scheduled on @gcwq to resolve * possible allocation deadlock. * * On return, need_to_create_worker() is guaranteed to be false and * may_start_working() true. * * LOCKING: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Does GFP_KERNEL allocations. Called only from * manager. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */ static bool maybe_create_worker(struct global_cwq *gcwq) |
06bd6ebff workqueue: annota... |
1486 1487 |
__releases(&gcwq->lock) __acquires(&gcwq->lock) |
1da177e4c Linux-2.6.12-rc2 |
1488 |
{ |
e22bee782 workqueue: implem... |
1489 1490 1491 |
if (!need_to_create_worker(gcwq)) return false; restart: |
9f9c23644 workqueue: fix lo... |
1492 |
spin_unlock_irq(&gcwq->lock); |
e22bee782 workqueue: implem... |
1493 1494 1495 1496 1497 |
/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */ mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); while (true) { struct worker *worker; |
e22bee782 workqueue: implem... |
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 |
worker = create_worker(gcwq, true); if (worker) { del_timer_sync(&gcwq->mayday_timer); spin_lock_irq(&gcwq->lock); start_worker(worker); BUG_ON(need_to_create_worker(gcwq)); return true; } if (!need_to_create_worker(gcwq)) break; |
1da177e4c Linux-2.6.12-rc2 |
1509 |
|
e22bee782 workqueue: implem... |
1510 1511 |
__set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(CREATE_COOLDOWN); |
9f9c23644 workqueue: fix lo... |
1512 |
|
e22bee782 workqueue: implem... |
1513 1514 1515 |
if (!need_to_create_worker(gcwq)) break; } |
e22bee782 workqueue: implem... |
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 |
del_timer_sync(&gcwq->mayday_timer); spin_lock_irq(&gcwq->lock); if (need_to_create_worker(gcwq)) goto restart; return true; } /** * maybe_destroy_worker - destroy workers which have been idle for a while * @gcwq: gcwq to destroy workers for * * Destroy @gcwq workers which have been idle for longer than * IDLE_WORKER_TIMEOUT. * * LOCKING: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Called only from manager. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true * otherwise. */ static bool maybe_destroy_workers(struct global_cwq *gcwq) { bool ret = false; |
1da177e4c Linux-2.6.12-rc2 |
1541 |
|
e22bee782 workqueue: implem... |
1542 1543 1544 |
while (too_many_workers(gcwq)) { struct worker *worker; unsigned long expires; |
3af24433e workqueue: don't ... |
1545 |
|
e22bee782 workqueue: implem... |
1546 1547 |
worker = list_entry(gcwq->idle_list.prev, struct worker, entry); expires = worker->last_active + IDLE_WORKER_TIMEOUT; |
85f4186af worker_thread: fi... |
1548 |
|
e22bee782 workqueue: implem... |
1549 1550 |
if (time_before(jiffies, expires)) { mod_timer(&gcwq->idle_timer, expires); |
3af24433e workqueue: don't ... |
1551 |
break; |
e22bee782 workqueue: implem... |
1552 |
} |
1da177e4c Linux-2.6.12-rc2 |
1553 |
|
e22bee782 workqueue: implem... |
1554 1555 |
destroy_worker(worker); ret = true; |
1da177e4c Linux-2.6.12-rc2 |
1556 |
} |
3af24433e workqueue: don't ... |
1557 |
|
e22bee782 workqueue: implem... |
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 |
return ret; } /** * manage_workers - manage worker pool * @worker: self * * Assume the manager role and manage gcwq worker pool @worker belongs * to. At any given time, there can be only zero or one manager per * gcwq. The exclusion is handled automatically by this function. * * The caller can safely start processing works on false return. On * true return, it's guaranteed that need_to_create_worker() is false * and may_start_working() is true. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. Does GFP_KERNEL allocations. * * RETURNS: * false if no action was taken and gcwq->lock stayed locked, true if * some action was taken. */ static bool manage_workers(struct worker *worker) { struct global_cwq *gcwq = worker->gcwq; bool ret = false; if (gcwq->flags & GCWQ_MANAGING_WORKERS) return ret; gcwq->flags &= ~GCWQ_MANAGE_WORKERS; gcwq->flags |= GCWQ_MANAGING_WORKERS; /* * Destroy and then create so that may_start_working() is true * on return. */ ret |= maybe_destroy_workers(gcwq); ret |= maybe_create_worker(gcwq); gcwq->flags &= ~GCWQ_MANAGING_WORKERS; /* * The trustee might be waiting to take over the manager * position, tell it we're done. */ if (unlikely(gcwq->trustee)) wake_up_all(&gcwq->trustee_wait); return ret; } |
a62428c0a workqueue: separa... |
1610 |
/** |
affee4b29 workqueue: reimpl... |
1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 |
* move_linked_works - move linked works to a list * @work: start of series of works to be scheduled * @head: target list to append @work to * @nextp: out paramter for nested worklist walking * * Schedule linked works starting from @work to @head. Work series to * be scheduled starts at @work and includes any consecutive work with * WORK_STRUCT_LINKED set in its predecessor. * * If @nextp is not NULL, it's updated to point to the next work of * the last scheduled work. This allows move_linked_works() to be * nested inside outer list_for_each_entry_safe(). * * CONTEXT: |
8b03ae3cd workqueue: introd... |
1625 |
* spin_lock_irq(gcwq->lock). |
affee4b29 workqueue: reimpl... |
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 |
*/ static void move_linked_works(struct work_struct *work, struct list_head *head, struct work_struct **nextp) { struct work_struct *n; /* * Linked worklist will always end before the end of the list, * use NULL for list head. */ list_for_each_entry_safe_from(work, n, NULL, entry) { list_move_tail(&work->entry, head); if (!(*work_data_bits(work) & WORK_STRUCT_LINKED)) break; } /* * If we're already inside safe list traversal and have moved * multiple works to the scheduled queue, the next position * needs to be updated. */ if (nextp) *nextp = n; } |
1e19ffc63 workqueue: implem... |
1650 1651 1652 1653 |
static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq) { struct work_struct *work = list_first_entry(&cwq->delayed_works, struct work_struct, entry); |
649027d73 workqueue: implem... |
1654 |
struct list_head *pos = gcwq_determine_ins_pos(cwq->gcwq, cwq); |
1e19ffc63 workqueue: implem... |
1655 |
|
cdadf0097 workqueue: add qu... |
1656 |
trace_workqueue_activate_work(work); |
649027d73 workqueue: implem... |
1657 |
move_linked_works(work, pos, NULL); |
8a2e8e5de workqueue: fix cw... |
1658 |
__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work)); |
1e19ffc63 workqueue: implem... |
1659 1660 |
cwq->nr_active++; } |
affee4b29 workqueue: reimpl... |
1661 |
/** |
73f53c4aa workqueue: reimpl... |
1662 1663 1664 |
* cwq_dec_nr_in_flight - decrement cwq's nr_in_flight * @cwq: cwq of interest * @color: color of work which left the queue |
8a2e8e5de workqueue: fix cw... |
1665 |
* @delayed: for a delayed work |
73f53c4aa workqueue: reimpl... |
1666 1667 1668 1669 1670 |
* * A work either has completed or is removed from pending queue, * decrement nr_in_flight of its cwq and handle workqueue flushing. * * CONTEXT: |
8b03ae3cd workqueue: introd... |
1671 |
* spin_lock_irq(gcwq->lock). |
73f53c4aa workqueue: reimpl... |
1672 |
*/ |
8a2e8e5de workqueue: fix cw... |
1673 1674 |
static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color, bool delayed) |
73f53c4aa workqueue: reimpl... |
1675 1676 1677 1678 1679 1680 |
{ /* ignore uncolored works */ if (color == WORK_NO_COLOR) return; cwq->nr_in_flight[color]--; |
1e19ffc63 workqueue: implem... |
1681 |
|
8a2e8e5de workqueue: fix cw... |
1682 1683 1684 1685 1686 1687 1688 |
if (!delayed) { cwq->nr_active--; if (!list_empty(&cwq->delayed_works)) { /* one down, submit a delayed one */ if (cwq->nr_active < cwq->max_active) cwq_activate_first_delayed(cwq); } |
502ca9d81 workqueue: make s... |
1689 |
} |
73f53c4aa workqueue: reimpl... |
1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 |
/* is flush in progress and are we at the flushing tip? */ if (likely(cwq->flush_color != color)) return; /* are there still in-flight works? */ if (cwq->nr_in_flight[color]) return; /* this cwq is done, clear flush_color */ cwq->flush_color = -1; /* * If this was the last cwq, wake up the first flusher. It * will handle the rest. */ if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush)) complete(&cwq->wq->first_flusher->done); } /** |
a62428c0a workqueue: separa... |
1711 |
* process_one_work - process single work |
c34056a3f workqueue: introd... |
1712 |
* @worker: self |
a62428c0a workqueue: separa... |
1713 1714 1715 1716 1717 1718 1719 1720 1721 |
* @work: work to process * * Process @work. This function contains all the logics necessary to * process a single work including synchronization against and * interaction with other workers on the same cpu, queueing and * flushing. As long as context requirement is met, any worker can * call this function to process a work. * * CONTEXT: |
8b03ae3cd workqueue: introd... |
1722 |
* spin_lock_irq(gcwq->lock) which is released and regrabbed. |
a62428c0a workqueue: separa... |
1723 |
*/ |
c34056a3f workqueue: introd... |
1724 |
static void process_one_work(struct worker *worker, struct work_struct *work) |
06bd6ebff workqueue: annota... |
1725 1726 |
__releases(&gcwq->lock) __acquires(&gcwq->lock) |
a62428c0a workqueue: separa... |
1727 |
{ |
7e11629d0 workqueue: use sh... |
1728 |
struct cpu_workqueue_struct *cwq = get_work_cwq(work); |
8b03ae3cd workqueue: introd... |
1729 |
struct global_cwq *gcwq = cwq->gcwq; |
c8e55f360 workqueue: implem... |
1730 |
struct hlist_head *bwh = busy_worker_head(gcwq, work); |
fb0e7beb5 workqueue: implem... |
1731 |
bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE; |
a62428c0a workqueue: separa... |
1732 |
work_func_t f = work->func; |
73f53c4aa workqueue: reimpl... |
1733 |
int work_color; |
7e11629d0 workqueue: use sh... |
1734 |
struct worker *collision; |
a62428c0a workqueue: separa... |
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 |
#ifdef CONFIG_LOCKDEP /* * It is permissible to free the struct work_struct from * inside the function that is called from it, this we need to * take into account for lockdep too. To avoid bogus "held * lock freed" warnings as well as problems when looking into * work->lockdep_map, make a copy and use that here. */ struct lockdep_map lockdep_map = work->lockdep_map; #endif |
7e11629d0 workqueue: use sh... |
1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 |
/* * A single work shouldn't be executed concurrently by * multiple workers on a single cpu. Check whether anyone is * already processing the work. If so, defer the work to the * currently executing one. */ collision = __find_worker_executing_work(gcwq, bwh, work); if (unlikely(collision)) { move_linked_works(work, &collision->scheduled, NULL); return; } |
a62428c0a workqueue: separa... |
1756 |
/* claim and process */ |
a62428c0a workqueue: separa... |
1757 |
debug_work_deactivate(work); |
c8e55f360 workqueue: implem... |
1758 |
hlist_add_head(&worker->hentry, bwh); |
c34056a3f workqueue: introd... |
1759 |
worker->current_work = work; |
8cca0eea3 workqueue: add fi... |
1760 |
worker->current_cwq = cwq; |
73f53c4aa workqueue: reimpl... |
1761 |
work_color = get_work_color(work); |
7a22ad757 workqueue: carry ... |
1762 |
|
7a22ad757 workqueue: carry ... |
1763 1764 |
/* record the current cpu number in the work data and dequeue */ set_work_cpu(work, gcwq->cpu); |
a62428c0a workqueue: separa... |
1765 |
list_del_init(&work->entry); |
649027d73 workqueue: implem... |
1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 |
/* * If HIGHPRI_PENDING, check the next work, and, if HIGHPRI, * wake up another worker; otherwise, clear HIGHPRI_PENDING. */ if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) { struct work_struct *nwork = list_first_entry(&gcwq->worklist, struct work_struct, entry); if (!list_empty(&gcwq->worklist) && get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI) wake_up_worker(gcwq); else gcwq->flags &= ~GCWQ_HIGHPRI_PENDING; } |
fb0e7beb5 workqueue: implem... |
1780 1781 1782 1783 1784 1785 |
/* * CPU intensive works don't participate in concurrency * management. They're the scheduler's responsibility. */ if (unlikely(cpu_intensive)) worker_set_flags(worker, WORKER_CPU_INTENSIVE, true); |
8b03ae3cd workqueue: introd... |
1786 |
spin_unlock_irq(&gcwq->lock); |
a62428c0a workqueue: separa... |
1787 |
|
a62428c0a workqueue: separa... |
1788 |
work_clear_pending(work); |
e159489ba workqueue: relax ... |
1789 |
lock_map_acquire_read(&cwq->wq->lockdep_map); |
a62428c0a workqueue: separa... |
1790 |
lock_map_acquire(&lockdep_map); |
e36c886a0 workqueue: Add ba... |
1791 |
trace_workqueue_execute_start(work); |
a62428c0a workqueue: separa... |
1792 |
f(work); |
e36c886a0 workqueue: Add ba... |
1793 1794 1795 1796 1797 |
/* * While we must be careful to not use "work" after this, the trace * point will only record its address. */ trace_workqueue_execute_end(work); |
a62428c0a workqueue: separa... |
1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 |
lock_map_release(&lockdep_map); lock_map_release(&cwq->wq->lockdep_map); if (unlikely(in_atomic() || lockdep_depth(current) > 0)) { printk(KERN_ERR "BUG: workqueue leaked lock or atomic: " "%s/0x%08x/%d ", current->comm, preempt_count(), task_pid_nr(current)); printk(KERN_ERR " last function: "); print_symbol("%s ", (unsigned long)f); debug_show_held_locks(current); dump_stack(); } |
8b03ae3cd workqueue: introd... |
1812 |
spin_lock_irq(&gcwq->lock); |
a62428c0a workqueue: separa... |
1813 |
|
fb0e7beb5 workqueue: implem... |
1814 1815 1816 |
/* clear cpu intensive status */ if (unlikely(cpu_intensive)) worker_clr_flags(worker, WORKER_CPU_INTENSIVE); |
a62428c0a workqueue: separa... |
1817 |
/* we're done with it, release */ |
c8e55f360 workqueue: implem... |
1818 |
hlist_del_init(&worker->hentry); |
c34056a3f workqueue: introd... |
1819 |
worker->current_work = NULL; |
8cca0eea3 workqueue: add fi... |
1820 |
worker->current_cwq = NULL; |
8a2e8e5de workqueue: fix cw... |
1821 |
cwq_dec_nr_in_flight(cwq, work_color, false); |
a62428c0a workqueue: separa... |
1822 |
} |
affee4b29 workqueue: reimpl... |
1823 1824 1825 1826 1827 1828 1829 1830 1831 |
/** * process_scheduled_works - process scheduled works * @worker: self * * Process all scheduled works. Please note that the scheduled list * may change while processing a work, so this function repeatedly * fetches a work from the top and executes it. * * CONTEXT: |
8b03ae3cd workqueue: introd... |
1832 |
* spin_lock_irq(gcwq->lock) which may be released and regrabbed |
affee4b29 workqueue: reimpl... |
1833 1834 1835 |
* multiple times. */ static void process_scheduled_works(struct worker *worker) |
1da177e4c Linux-2.6.12-rc2 |
1836 |
{ |
affee4b29 workqueue: reimpl... |
1837 1838 |
while (!list_empty(&worker->scheduled)) { struct work_struct *work = list_first_entry(&worker->scheduled, |
1da177e4c Linux-2.6.12-rc2 |
1839 |
struct work_struct, entry); |
c34056a3f workqueue: introd... |
1840 |
process_one_work(worker, work); |
1da177e4c Linux-2.6.12-rc2 |
1841 |
} |
1da177e4c Linux-2.6.12-rc2 |
1842 |
} |
4690c4ab5 workqueue: misc/c... |
1843 1844 |
/** * worker_thread - the worker thread function |
c34056a3f workqueue: introd... |
1845 |
* @__worker: self |
4690c4ab5 workqueue: misc/c... |
1846 |
* |
e22bee782 workqueue: implem... |
1847 1848 1849 1850 1851 |
* The gcwq worker thread function. There's a single dynamic pool of * these per each cpu. These workers process all works regardless of * their specific target workqueue. The only exception is works which * belong to workqueues with a rescuer which will be explained in * rescuer_thread(). |
4690c4ab5 workqueue: misc/c... |
1852 |
*/ |
c34056a3f workqueue: introd... |
1853 |
static int worker_thread(void *__worker) |
1da177e4c Linux-2.6.12-rc2 |
1854 |
{ |
c34056a3f workqueue: introd... |
1855 |
struct worker *worker = __worker; |
8b03ae3cd workqueue: introd... |
1856 |
struct global_cwq *gcwq = worker->gcwq; |
1da177e4c Linux-2.6.12-rc2 |
1857 |
|
e22bee782 workqueue: implem... |
1858 1859 |
/* tell the scheduler that this is a workqueue worker */ worker->task->flags |= PF_WQ_WORKER; |
c8e55f360 workqueue: implem... |
1860 |
woke_up: |
c8e55f360 workqueue: implem... |
1861 |
spin_lock_irq(&gcwq->lock); |
1da177e4c Linux-2.6.12-rc2 |
1862 |
|
c8e55f360 workqueue: implem... |
1863 1864 1865 |
/* DIE can be set only while we're idle, checking here is enough */ if (worker->flags & WORKER_DIE) { spin_unlock_irq(&gcwq->lock); |
e22bee782 workqueue: implem... |
1866 |
worker->task->flags &= ~PF_WQ_WORKER; |
c8e55f360 workqueue: implem... |
1867 1868 |
return 0; } |
affee4b29 workqueue: reimpl... |
1869 |
|
c8e55f360 workqueue: implem... |
1870 |
worker_leave_idle(worker); |
db7bccf45 workqueue: reimpl... |
1871 |
recheck: |
e22bee782 workqueue: implem... |
1872 1873 1874 1875 1876 1877 1878 |
/* no more worker necessary? */ if (!need_more_worker(gcwq)) goto sleep; /* do we need to manage? */ if (unlikely(!may_start_working(gcwq)) && manage_workers(worker)) goto recheck; |
c8e55f360 workqueue: implem... |
1879 1880 1881 1882 1883 1884 |
/* * ->scheduled list can only be filled while a worker is * preparing to process a work or actually processing it. * Make sure nobody diddled with it while I was sleeping. */ BUG_ON(!list_empty(&worker->scheduled)); |
e22bee782 workqueue: implem... |
1885 1886 1887 1888 1889 1890 1891 1892 |
/* * When control reaches this point, we're guaranteed to have * at least one idle worker or that someone else has already * assumed the manager role. */ worker_clr_flags(worker, WORKER_PREP); do { |
c8e55f360 workqueue: implem... |
1893 |
struct work_struct *work = |
7e11629d0 workqueue: use sh... |
1894 |
list_first_entry(&gcwq->worklist, |
c8e55f360 workqueue: implem... |
1895 1896 1897 1898 1899 1900 |
struct work_struct, entry); if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) { /* optimization path, not strictly necessary */ process_one_work(worker, work); if (unlikely(!list_empty(&worker->scheduled))) |
affee4b29 workqueue: reimpl... |
1901 |
process_scheduled_works(worker); |
c8e55f360 workqueue: implem... |
1902 1903 1904 |
} else { move_linked_works(work, &worker->scheduled, NULL); process_scheduled_works(worker); |
affee4b29 workqueue: reimpl... |
1905 |
} |
e22bee782 workqueue: implem... |
1906 1907 1908 |
} while (keep_working(gcwq)); worker_set_flags(worker, WORKER_PREP, false); |
d313dd85a workqueue: fix wo... |
1909 |
sleep: |
e22bee782 workqueue: implem... |
1910 1911 |
if (unlikely(need_to_manage_workers(gcwq)) && manage_workers(worker)) goto recheck; |
d313dd85a workqueue: fix wo... |
1912 |
|
c8e55f360 workqueue: implem... |
1913 |
/* |
e22bee782 workqueue: implem... |
1914 1915 1916 1917 1918 |
* gcwq->lock is held and there's no work to process and no * need to manage, sleep. Workers are woken up only while * holding gcwq->lock or from local cpu, so setting the * current state before releasing gcwq->lock is enough to * prevent losing any event. |
c8e55f360 workqueue: implem... |
1919 1920 1921 1922 1923 1924 |
*/ worker_enter_idle(worker); __set_current_state(TASK_INTERRUPTIBLE); spin_unlock_irq(&gcwq->lock); schedule(); goto woke_up; |
1da177e4c Linux-2.6.12-rc2 |
1925 |
} |
e22bee782 workqueue: implem... |
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 |
/** * rescuer_thread - the rescuer thread function * @__wq: the associated workqueue * * Workqueue rescuer thread function. There's one rescuer for each * workqueue which has WQ_RESCUER set. * * Regular work processing on a gcwq may block trying to create a new * worker which uses GFP_KERNEL allocation which has slight chance of * developing into deadlock if some works currently on the same queue * need to be processed to satisfy the GFP_KERNEL allocation. This is * the problem rescuer solves. * * When such condition is possible, the gcwq summons rescuers of all * workqueues which have works queued on the gcwq and let them process * those works so that forward progress can be guaranteed. * * This should happen rarely. */ static int rescuer_thread(void *__wq) { struct workqueue_struct *wq = __wq; struct worker *rescuer = wq->rescuer; struct list_head *scheduled = &rescuer->scheduled; |
f34217977 workqueue: implem... |
1950 |
bool is_unbound = wq->flags & WQ_UNBOUND; |
e22bee782 workqueue: implem... |
1951 1952 1953 1954 1955 1956 1957 1958 |
unsigned int cpu; set_user_nice(current, RESCUER_NICE_LEVEL); repeat: set_current_state(TASK_INTERRUPTIBLE); if (kthread_should_stop()) return 0; |
f34217977 workqueue: implem... |
1959 1960 1961 1962 |
/* * See whether any cpu is asking for help. Unbounded * workqueues use cpu 0 in mayday_mask for CPU_UNBOUND. */ |
f2e005aaf workqueue: fix ma... |
1963 |
for_each_mayday_cpu(cpu, wq->mayday_mask) { |
f34217977 workqueue: implem... |
1964 1965 |
unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu; struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq); |
e22bee782 workqueue: implem... |
1966 1967 1968 1969 |
struct global_cwq *gcwq = cwq->gcwq; struct work_struct *work, *n; __set_current_state(TASK_RUNNING); |
f2e005aaf workqueue: fix ma... |
1970 |
mayday_clear_cpu(cpu, wq->mayday_mask); |
e22bee782 workqueue: implem... |
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 |
/* migrate to the target cpu if possible */ rescuer->gcwq = gcwq; worker_maybe_bind_and_lock(rescuer); /* * Slurp in all works issued via this workqueue and * process'em. */ BUG_ON(!list_empty(&rescuer->scheduled)); list_for_each_entry_safe(work, n, &gcwq->worklist, entry) if (get_work_cwq(work) == cwq) move_linked_works(work, scheduled, &n); process_scheduled_works(rescuer); |
7576958a9 workqueue: wake u... |
1986 1987 1988 1989 1990 1991 1992 1993 |
/* * Leave this gcwq. If keep_working() is %true, notify a * regular worker; otherwise, we end up with 0 concurrency * and stalling the execution. */ if (keep_working(gcwq)) wake_up_worker(gcwq); |
e22bee782 workqueue: implem... |
1994 1995 1996 1997 1998 |
spin_unlock_irq(&gcwq->lock); } schedule(); goto repeat; |
1da177e4c Linux-2.6.12-rc2 |
1999 |
} |
fc2e4d704 reimplement flush... |
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 |
struct wq_barrier { struct work_struct work; struct completion done; }; static void wq_barrier_func(struct work_struct *work) { struct wq_barrier *barr = container_of(work, struct wq_barrier, work); complete(&barr->done); } |
4690c4ab5 workqueue: misc/c... |
2010 2011 2012 2013 |
/** * insert_wq_barrier - insert a barrier work * @cwq: cwq to insert barrier into * @barr: wq_barrier to insert |
affee4b29 workqueue: reimpl... |
2014 2015 |
* @target: target work to attach @barr to * @worker: worker currently executing @target, NULL if @target is not executing |
4690c4ab5 workqueue: misc/c... |
2016 |
* |
affee4b29 workqueue: reimpl... |
2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 |
* @barr is linked to @target such that @barr is completed only after * @target finishes execution. Please note that the ordering * guarantee is observed only with respect to @target and on the local * cpu. * * Currently, a queued barrier can't be canceled. This is because * try_to_grab_pending() can't determine whether the work to be * grabbed is at the head of the queue and thus can't clear LINKED * flag of the previous work while there must be a valid next work * after a work with LINKED flag set. * * Note that when @worker is non-NULL, @target may be modified * underneath us, so we can't reliably determine cwq from @target. |
4690c4ab5 workqueue: misc/c... |
2030 2031 |
* * CONTEXT: |
8b03ae3cd workqueue: introd... |
2032 |
* spin_lock_irq(gcwq->lock). |
4690c4ab5 workqueue: misc/c... |
2033 |
*/ |
83c22520c flush_cpu_workque... |
2034 |
static void insert_wq_barrier(struct cpu_workqueue_struct *cwq, |
affee4b29 workqueue: reimpl... |
2035 2036 |
struct wq_barrier *barr, struct work_struct *target, struct worker *worker) |
fc2e4d704 reimplement flush... |
2037 |
{ |
affee4b29 workqueue: reimpl... |
2038 2039 |
struct list_head *head; unsigned int linked = 0; |
dc186ad74 workqueue: Add de... |
2040 |
/* |
8b03ae3cd workqueue: introd... |
2041 |
* debugobject calls are safe here even with gcwq->lock locked |
dc186ad74 workqueue: Add de... |
2042 2043 2044 2045 |
* as we know for sure that this will not trigger any of the * checks and call back into the fixup functions where we * might deadlock. */ |
ca1cab37d workqueues: s/ON_... |
2046 |
INIT_WORK_ONSTACK(&barr->work, wq_barrier_func); |
22df02bb3 workqueue: define... |
2047 |
__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); |
fc2e4d704 reimplement flush... |
2048 |
init_completion(&barr->done); |
83c22520c flush_cpu_workque... |
2049 |
|
affee4b29 workqueue: reimpl... |
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 |
/* * If @target is currently being executed, schedule the * barrier to the worker; otherwise, put it after @target. */ if (worker) head = worker->scheduled.next; else { unsigned long *bits = work_data_bits(target); head = target->entry.next; /* there can already be other linked works, inherit and set */ linked = *bits & WORK_STRUCT_LINKED; __set_bit(WORK_STRUCT_LINKED_BIT, bits); } |
dc186ad74 workqueue: Add de... |
2064 |
debug_work_activate(&barr->work); |
affee4b29 workqueue: reimpl... |
2065 2066 |
insert_work(cwq, &barr->work, head, work_color_to_flags(WORK_NO_COLOR) | linked); |
fc2e4d704 reimplement flush... |
2067 |
} |
73f53c4aa workqueue: reimpl... |
2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 |
/** * flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing * @wq: workqueue being flushed * @flush_color: new flush color, < 0 for no-op * @work_color: new work color, < 0 for no-op * * Prepare cwqs for workqueue flushing. * * If @flush_color is non-negative, flush_color on all cwqs should be * -1. If no cwq has in-flight commands at the specified color, all * cwq->flush_color's stay at -1 and %false is returned. If any cwq * has in flight commands, its cwq->flush_color is set to * @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq * wakeup logic is armed and %true is returned. * * The caller should have initialized @wq->first_flusher prior to * calling this function with non-negative @flush_color. If * @flush_color is negative, no flush color update is done and %false * is returned. * * If @work_color is non-negative, all cwqs should have the same * work_color which is previous to @work_color and all will be * advanced to @work_color. * * CONTEXT: * mutex_lock(wq->flush_mutex). * * RETURNS: * %true if @flush_color >= 0 and there's something to flush. %false * otherwise. */ static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq, int flush_color, int work_color) |
1da177e4c Linux-2.6.12-rc2 |
2101 |
{ |
73f53c4aa workqueue: reimpl... |
2102 2103 |
bool wait = false; unsigned int cpu; |
1da177e4c Linux-2.6.12-rc2 |
2104 |
|
73f53c4aa workqueue: reimpl... |
2105 2106 2107 |
if (flush_color >= 0) { BUG_ON(atomic_read(&wq->nr_cwqs_to_flush)); atomic_set(&wq->nr_cwqs_to_flush, 1); |
1da177e4c Linux-2.6.12-rc2 |
2108 |
} |
2355b70fd workqueue: avoid ... |
2109 |
|
f34217977 workqueue: implem... |
2110 |
for_each_cwq_cpu(cpu, wq) { |
73f53c4aa workqueue: reimpl... |
2111 |
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
8b03ae3cd workqueue: introd... |
2112 |
struct global_cwq *gcwq = cwq->gcwq; |
fc2e4d704 reimplement flush... |
2113 |
|
8b03ae3cd workqueue: introd... |
2114 |
spin_lock_irq(&gcwq->lock); |
83c22520c flush_cpu_workque... |
2115 |
|
73f53c4aa workqueue: reimpl... |
2116 2117 |
if (flush_color >= 0) { BUG_ON(cwq->flush_color != -1); |
fc2e4d704 reimplement flush... |
2118 |
|
73f53c4aa workqueue: reimpl... |
2119 2120 2121 2122 2123 2124 |
if (cwq->nr_in_flight[flush_color]) { cwq->flush_color = flush_color; atomic_inc(&wq->nr_cwqs_to_flush); wait = true; } } |
1da177e4c Linux-2.6.12-rc2 |
2125 |
|
73f53c4aa workqueue: reimpl... |
2126 2127 2128 2129 |
if (work_color >= 0) { BUG_ON(work_color != work_next_color(cwq->work_color)); cwq->work_color = work_color; } |
1da177e4c Linux-2.6.12-rc2 |
2130 |
|
8b03ae3cd workqueue: introd... |
2131 |
spin_unlock_irq(&gcwq->lock); |
1da177e4c Linux-2.6.12-rc2 |
2132 |
} |
2355b70fd workqueue: avoid ... |
2133 |
|
73f53c4aa workqueue: reimpl... |
2134 2135 |
if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush)) complete(&wq->first_flusher->done); |
14441960e simplify cleanup_... |
2136 |
|
73f53c4aa workqueue: reimpl... |
2137 |
return wait; |
1da177e4c Linux-2.6.12-rc2 |
2138 |
} |
0fcb78c22 [PATCH] Add DocBo... |
2139 |
/** |
1da177e4c Linux-2.6.12-rc2 |
2140 |
* flush_workqueue - ensure that any scheduled work has run to completion. |
0fcb78c22 [PATCH] Add DocBo... |
2141 |
* @wq: workqueue to flush |
1da177e4c Linux-2.6.12-rc2 |
2142 2143 2144 2145 |
* * Forces execution of the workqueue and blocks until its completion. * This is typically used in driver shutdown handlers. * |
fc2e4d704 reimplement flush... |
2146 2147 |
* We sleep until all works which were queued on entry have been handled, * but we are not livelocked by new incoming ones. |
1da177e4c Linux-2.6.12-rc2 |
2148 |
*/ |
7ad5b3a50 kernel: remove fa... |
2149 |
void flush_workqueue(struct workqueue_struct *wq) |
1da177e4c Linux-2.6.12-rc2 |
2150 |
{ |
73f53c4aa workqueue: reimpl... |
2151 2152 2153 2154 2155 2156 |
struct wq_flusher this_flusher = { .list = LIST_HEAD_INIT(this_flusher.list), .flush_color = -1, .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done), }; int next_color; |
1da177e4c Linux-2.6.12-rc2 |
2157 |
|
3295f0ef9 lockdep: rename m... |
2158 2159 |
lock_map_acquire(&wq->lockdep_map); lock_map_release(&wq->lockdep_map); |
73f53c4aa workqueue: reimpl... |
2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 |
mutex_lock(&wq->flush_mutex); /* * Start-to-wait phase */ next_color = work_next_color(wq->work_color); if (next_color != wq->flush_color) { /* * Color space is not full. The current work_color * becomes our flush_color and work_color is advanced * by one. */ BUG_ON(!list_empty(&wq->flusher_overflow)); this_flusher.flush_color = wq->work_color; wq->work_color = next_color; if (!wq->first_flusher) { /* no flush in progress, become the first flusher */ BUG_ON(wq->flush_color != this_flusher.flush_color); wq->first_flusher = &this_flusher; if (!flush_workqueue_prep_cwqs(wq, wq->flush_color, wq->work_color)) { /* nothing to flush, done */ wq->flush_color = next_color; wq->first_flusher = NULL; goto out_unlock; } } else { /* wait in queue */ BUG_ON(wq->flush_color == this_flusher.flush_color); list_add_tail(&this_flusher.list, &wq->flusher_queue); flush_workqueue_prep_cwqs(wq, -1, wq->work_color); } } else { /* * Oops, color space is full, wait on overflow queue. * The next flush completion will assign us * flush_color and transfer to flusher_queue. */ list_add_tail(&this_flusher.list, &wq->flusher_overflow); } mutex_unlock(&wq->flush_mutex); wait_for_completion(&this_flusher.done); /* * Wake-up-and-cascade phase * * First flushers are responsible for cascading flushes and * handling overflow. Non-first flushers can simply return. */ if (wq->first_flusher != &this_flusher) return; mutex_lock(&wq->flush_mutex); |
4ce48b37b workqueue: fix ra... |
2220 2221 2222 |
/* we might have raced, check again with mutex held */ if (wq->first_flusher != &this_flusher) goto out_unlock; |
73f53c4aa workqueue: reimpl... |
2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 |
wq->first_flusher = NULL; BUG_ON(!list_empty(&this_flusher.list)); BUG_ON(wq->flush_color != this_flusher.flush_color); while (true) { struct wq_flusher *next, *tmp; /* complete all the flushers sharing the current flush color */ list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { if (next->flush_color != wq->flush_color) break; list_del_init(&next->list); complete(&next->done); } BUG_ON(!list_empty(&wq->flusher_overflow) && wq->flush_color != work_next_color(wq->work_color)); /* this flush_color is finished, advance by one */ wq->flush_color = work_next_color(wq->flush_color); /* one color has been freed, handle overflow queue */ if (!list_empty(&wq->flusher_overflow)) { /* * Assign the same color to all overflowed * flushers, advance work_color and append to * flusher_queue. This is the start-to-wait * phase for these overflowed flushers. */ list_for_each_entry(tmp, &wq->flusher_overflow, list) tmp->flush_color = wq->work_color; wq->work_color = work_next_color(wq->work_color); list_splice_tail_init(&wq->flusher_overflow, &wq->flusher_queue); flush_workqueue_prep_cwqs(wq, -1, wq->work_color); } if (list_empty(&wq->flusher_queue)) { BUG_ON(wq->flush_color != wq->work_color); break; } /* * Need to flush more colors. Make the next flusher * the new first flusher and arm cwqs. */ BUG_ON(wq->flush_color == wq->work_color); BUG_ON(wq->flush_color != next->flush_color); list_del_init(&next->list); wq->first_flusher = next; if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1)) break; /* * Meh... this color is already done, clear first * flusher and repeat cascading. */ wq->first_flusher = NULL; } out_unlock: mutex_unlock(&wq->flush_mutex); |
1da177e4c Linux-2.6.12-rc2 |
2290 |
} |
ae90dd5db Move workqueue ex... |
2291 |
EXPORT_SYMBOL_GPL(flush_workqueue); |
1da177e4c Linux-2.6.12-rc2 |
2292 |
|
9c5a2ba70 workqueue: separa... |
2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 |
/** * drain_workqueue - drain a workqueue * @wq: workqueue to drain * * Wait until the workqueue becomes empty. While draining is in progress, * only chain queueing is allowed. IOW, only currently pending or running * work items on @wq can queue further work items on it. @wq is flushed * repeatedly until it becomes empty. The number of flushing is detemined * by the depth of chaining and should be relatively short. Whine if it * takes too long. */ void drain_workqueue(struct workqueue_struct *wq) { unsigned int flush_cnt = 0; unsigned int cpu; /* * __queue_work() needs to test whether there are drainers, is much * hotter than drain_workqueue() and already looks at @wq->flags. * Use WQ_DRAINING so that queue doesn't have to check nr_drainers. */ spin_lock(&workqueue_lock); if (!wq->nr_drainers++) wq->flags |= WQ_DRAINING; spin_unlock(&workqueue_lock); reflush: flush_workqueue(wq); for_each_cwq_cpu(cpu, wq) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
fa2563e41 workqueue: lock c... |
2323 |
bool drained; |
9c5a2ba70 workqueue: separa... |
2324 |
|
fa2563e41 workqueue: lock c... |
2325 2326 2327 2328 2329 |
spin_lock_irq(&cwq->gcwq->lock); drained = !cwq->nr_active && list_empty(&cwq->delayed_works); spin_unlock_irq(&cwq->gcwq->lock); if (drained) |
9c5a2ba70 workqueue: separa... |
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 |
continue; if (++flush_cnt == 10 || (flush_cnt % 100 == 0 && flush_cnt <= 1000)) pr_warning("workqueue %s: flush on destruction isn't complete after %u tries ", wq->name, flush_cnt); goto reflush; } spin_lock(&workqueue_lock); if (!--wq->nr_drainers) wq->flags &= ~WQ_DRAINING; spin_unlock(&workqueue_lock); } EXPORT_SYMBOL_GPL(drain_workqueue); |
baf59022c workqueue: factor... |
2346 2347 |
static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, bool wait_executing) |
db7008972 workqueues: imple... |
2348 |
{ |
affee4b29 workqueue: reimpl... |
2349 |
struct worker *worker = NULL; |
8b03ae3cd workqueue: introd... |
2350 |
struct global_cwq *gcwq; |
db7008972 workqueues: imple... |
2351 |
struct cpu_workqueue_struct *cwq; |
db7008972 workqueues: imple... |
2352 2353 |
might_sleep(); |
7a22ad757 workqueue: carry ... |
2354 2355 |
gcwq = get_work_gcwq(work); if (!gcwq) |
baf59022c workqueue: factor... |
2356 |
return false; |
db7008972 workqueues: imple... |
2357 |
|
8b03ae3cd workqueue: introd... |
2358 |
spin_lock_irq(&gcwq->lock); |
db7008972 workqueues: imple... |
2359 2360 2361 |
if (!list_empty(&work->entry)) { /* * See the comment near try_to_grab_pending()->smp_rmb(). |
7a22ad757 workqueue: carry ... |
2362 2363 |
* If it was re-queued to a different gcwq under us, we * are not going to wait. |
db7008972 workqueues: imple... |
2364 2365 |
*/ smp_rmb(); |
7a22ad757 workqueue: carry ... |
2366 2367 |
cwq = get_work_cwq(work); if (unlikely(!cwq || gcwq != cwq->gcwq)) |
4690c4ab5 workqueue: misc/c... |
2368 |
goto already_gone; |
baf59022c workqueue: factor... |
2369 |
} else if (wait_executing) { |
7a22ad757 workqueue: carry ... |
2370 |
worker = find_worker_executing_work(gcwq, work); |
affee4b29 workqueue: reimpl... |
2371 |
if (!worker) |
4690c4ab5 workqueue: misc/c... |
2372 |
goto already_gone; |
7a22ad757 workqueue: carry ... |
2373 |
cwq = worker->current_cwq; |
baf59022c workqueue: factor... |
2374 2375 |
} else goto already_gone; |
db7008972 workqueues: imple... |
2376 |
|
baf59022c workqueue: factor... |
2377 |
insert_wq_barrier(cwq, barr, work, worker); |
8b03ae3cd workqueue: introd... |
2378 |
spin_unlock_irq(&gcwq->lock); |
7a22ad757 workqueue: carry ... |
2379 |
|
e159489ba workqueue: relax ... |
2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 |
/* * If @max_active is 1 or rescuer is in use, flushing another work * item on the same workqueue may lead to deadlock. Make sure the * flusher is not running on the same workqueue by verifying write * access. */ if (cwq->wq->saved_max_active == 1 || cwq->wq->flags & WQ_RESCUER) lock_map_acquire(&cwq->wq->lockdep_map); else lock_map_acquire_read(&cwq->wq->lockdep_map); |
7a22ad757 workqueue: carry ... |
2390 |
lock_map_release(&cwq->wq->lockdep_map); |
e159489ba workqueue: relax ... |
2391 |
|
401a8d048 workqueue: cleanu... |
2392 |
return true; |
4690c4ab5 workqueue: misc/c... |
2393 |
already_gone: |
8b03ae3cd workqueue: introd... |
2394 |
spin_unlock_irq(&gcwq->lock); |
401a8d048 workqueue: cleanu... |
2395 |
return false; |
db7008972 workqueues: imple... |
2396 |
} |
baf59022c workqueue: factor... |
2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 |
/** * flush_work - wait for a work to finish executing the last queueing instance * @work: the work to flush * * Wait until @work has finished execution. This function considers * only the last queueing instance of @work. If @work has been * enqueued across different CPUs on a non-reentrant workqueue or on * multiple workqueues, @work might still be executing on return on * some of the CPUs from earlier queueing. * * If @work was queued only on a non-reentrant, ordered or unbound * workqueue, @work is guaranteed to be idle on return if it hasn't * been requeued since flush started. * * RETURNS: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. */ bool flush_work(struct work_struct *work) { struct wq_barrier barr; if (start_flush_work(work, &barr, true)) { wait_for_completion(&barr.done); destroy_work_on_stack(&barr.work); return true; } else return false; } |
db7008972 workqueues: imple... |
2427 |
EXPORT_SYMBOL_GPL(flush_work); |
401a8d048 workqueue: cleanu... |
2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 |
static bool wait_on_cpu_work(struct global_cwq *gcwq, struct work_struct *work) { struct wq_barrier barr; struct worker *worker; spin_lock_irq(&gcwq->lock); worker = find_worker_executing_work(gcwq, work); if (unlikely(worker)) insert_wq_barrier(worker->current_cwq, &barr, work, worker); spin_unlock_irq(&gcwq->lock); if (unlikely(worker)) { wait_for_completion(&barr.done); destroy_work_on_stack(&barr.work); return true; } else return false; } static bool wait_on_work(struct work_struct *work) { bool ret = false; int cpu; might_sleep(); lock_map_acquire(&work->lockdep_map); lock_map_release(&work->lockdep_map); for_each_gcwq_cpu(cpu) ret |= wait_on_cpu_work(get_gcwq(cpu), work); return ret; } |
09383498c workqueue: implem... |
2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 |
/** * flush_work_sync - wait until a work has finished execution * @work: the work to flush * * Wait until @work has finished execution. On return, it's * guaranteed that all queueing instances of @work which happened * before this function is called are finished. In other words, if * @work hasn't been requeued since this function was called, @work is * guaranteed to be idle on return. * * RETURNS: * %true if flush_work_sync() waited for the work to finish execution, * %false if it was already idle. */ bool flush_work_sync(struct work_struct *work) { struct wq_barrier barr; bool pending, waited; /* we'll wait for executions separately, queue barr only if pending */ pending = start_flush_work(work, &barr, false); /* wait for executions to finish */ waited = wait_on_work(work); /* wait for the pending one */ if (pending) { wait_for_completion(&barr.done); destroy_work_on_stack(&barr.work); } return pending || waited; } EXPORT_SYMBOL_GPL(flush_work_sync); |
6e84d644b make cancel_rearm... |
2497 |
/* |
1f1f642e2 make cancel_xxx_w... |
2498 |
* Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit, |
6e84d644b make cancel_rearm... |
2499 2500 2501 2502 |
* so this work can't be re-armed in any way. */ static int try_to_grab_pending(struct work_struct *work) { |
8b03ae3cd workqueue: introd... |
2503 |
struct global_cwq *gcwq; |
1f1f642e2 make cancel_xxx_w... |
2504 |
int ret = -1; |
6e84d644b make cancel_rearm... |
2505 |
|
22df02bb3 workqueue: define... |
2506 |
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) |
1f1f642e2 make cancel_xxx_w... |
2507 |
return 0; |
6e84d644b make cancel_rearm... |
2508 2509 2510 2511 2512 |
/* * The queueing is in progress, or it is already queued. Try to * steal it from ->worklist without clearing WORK_STRUCT_PENDING. */ |
7a22ad757 workqueue: carry ... |
2513 2514 |
gcwq = get_work_gcwq(work); if (!gcwq) |
6e84d644b make cancel_rearm... |
2515 |
return ret; |
8b03ae3cd workqueue: introd... |
2516 |
spin_lock_irq(&gcwq->lock); |
6e84d644b make cancel_rearm... |
2517 2518 |
if (!list_empty(&work->entry)) { /* |
7a22ad757 workqueue: carry ... |
2519 |
* This work is queued, but perhaps we locked the wrong gcwq. |
6e84d644b make cancel_rearm... |
2520 2521 2522 2523 |
* In that case we must see the new value after rmb(), see * insert_work()->wmb(). */ smp_rmb(); |
7a22ad757 workqueue: carry ... |
2524 |
if (gcwq == get_work_gcwq(work)) { |
dc186ad74 workqueue: Add de... |
2525 |
debug_work_deactivate(work); |
6e84d644b make cancel_rearm... |
2526 |
list_del_init(&work->entry); |
7a22ad757 workqueue: carry ... |
2527 |
cwq_dec_nr_in_flight(get_work_cwq(work), |
8a2e8e5de workqueue: fix cw... |
2528 2529 |
get_work_color(work), *work_data_bits(work) & WORK_STRUCT_DELAYED); |
6e84d644b make cancel_rearm... |
2530 2531 2532 |
ret = 1; } } |
8b03ae3cd workqueue: introd... |
2533 |
spin_unlock_irq(&gcwq->lock); |
6e84d644b make cancel_rearm... |
2534 2535 2536 |
return ret; } |
401a8d048 workqueue: cleanu... |
2537 |
static bool __cancel_work_timer(struct work_struct *work, |
1f1f642e2 make cancel_xxx_w... |
2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 |
struct timer_list* timer) { int ret; do { ret = (timer && likely(del_timer(timer))); if (!ret) ret = try_to_grab_pending(work); wait_on_work(work); } while (unlikely(ret < 0)); |
7a22ad757 workqueue: carry ... |
2548 |
clear_work_data(work); |
1f1f642e2 make cancel_xxx_w... |
2549 2550 |
return ret; } |
6e84d644b make cancel_rearm... |
2551 |
/** |
401a8d048 workqueue: cleanu... |
2552 2553 |
* cancel_work_sync - cancel a work and wait for it to finish * @work: the work to cancel |
6e84d644b make cancel_rearm... |
2554 |
* |
401a8d048 workqueue: cleanu... |
2555 2556 2557 2558 |
* Cancel @work and wait for its execution to finish. This function * can be used even if the work re-queues itself or migrates to * another workqueue. On return from this function, @work is * guaranteed to be not pending or executing on any CPU. |
1f1f642e2 make cancel_xxx_w... |
2559 |
* |
401a8d048 workqueue: cleanu... |
2560 2561 |
* cancel_work_sync(&delayed_work->work) must not be used for * delayed_work's. Use cancel_delayed_work_sync() instead. |
6e84d644b make cancel_rearm... |
2562 |
* |
401a8d048 workqueue: cleanu... |
2563 |
* The caller must ensure that the workqueue on which @work was last |
6e84d644b make cancel_rearm... |
2564 |
* queued can't be destroyed before this function returns. |
401a8d048 workqueue: cleanu... |
2565 2566 2567 |
* * RETURNS: * %true if @work was pending, %false otherwise. |
6e84d644b make cancel_rearm... |
2568 |
*/ |
401a8d048 workqueue: cleanu... |
2569 |
bool cancel_work_sync(struct work_struct *work) |
6e84d644b make cancel_rearm... |
2570 |
{ |
1f1f642e2 make cancel_xxx_w... |
2571 |
return __cancel_work_timer(work, NULL); |
b89deed32 implement flush_w... |
2572 |
} |
28e53bddf unify flush_work/... |
2573 |
EXPORT_SYMBOL_GPL(cancel_work_sync); |
b89deed32 implement flush_w... |
2574 |
|
6e84d644b make cancel_rearm... |
2575 |
/** |
401a8d048 workqueue: cleanu... |
2576 2577 |
* flush_delayed_work - wait for a dwork to finish executing the last queueing * @dwork: the delayed work to flush |
6e84d644b make cancel_rearm... |
2578 |
* |
401a8d048 workqueue: cleanu... |
2579 2580 2581 |
* Delayed timer is cancelled and the pending work is queued for * immediate execution. Like flush_work(), this function only * considers the last queueing instance of @dwork. |
1f1f642e2 make cancel_xxx_w... |
2582 |
* |
401a8d048 workqueue: cleanu... |
2583 2584 2585 |
* RETURNS: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. |
6e84d644b make cancel_rearm... |
2586 |
*/ |
401a8d048 workqueue: cleanu... |
2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 |
bool flush_delayed_work(struct delayed_work *dwork) { if (del_timer_sync(&dwork->timer)) __queue_work(raw_smp_processor_id(), get_work_cwq(&dwork->work)->wq, &dwork->work); return flush_work(&dwork->work); } EXPORT_SYMBOL(flush_delayed_work); /** |
09383498c workqueue: implem... |
2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 |
* flush_delayed_work_sync - wait for a dwork to finish * @dwork: the delayed work to flush * * Delayed timer is cancelled and the pending work is queued for * execution immediately. Other than timer handling, its behavior * is identical to flush_work_sync(). * * RETURNS: * %true if flush_work_sync() waited for the work to finish execution, * %false if it was already idle. */ bool flush_delayed_work_sync(struct delayed_work *dwork) { if (del_timer_sync(&dwork->timer)) __queue_work(raw_smp_processor_id(), get_work_cwq(&dwork->work)->wq, &dwork->work); return flush_work_sync(&dwork->work); } EXPORT_SYMBOL(flush_delayed_work_sync); /** |
401a8d048 workqueue: cleanu... |
2618 2619 2620 2621 2622 2623 2624 2625 2626 |
* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish * @dwork: the delayed work cancel * * This is cancel_work_sync() for delayed works. * * RETURNS: * %true if @dwork was pending, %false otherwise. */ bool cancel_delayed_work_sync(struct delayed_work *dwork) |
6e84d644b make cancel_rearm... |
2627 |
{ |
1f1f642e2 make cancel_xxx_w... |
2628 |
return __cancel_work_timer(&dwork->work, &dwork->timer); |
6e84d644b make cancel_rearm... |
2629 |
} |
f5a421a45 rename cancel_rea... |
2630 |
EXPORT_SYMBOL(cancel_delayed_work_sync); |
1da177e4c Linux-2.6.12-rc2 |
2631 |
|
0fcb78c22 [PATCH] Add DocBo... |
2632 2633 2634 2635 |
/** * schedule_work - put work task in global workqueue * @work: job to be done * |
5b0f437df workqueues: Impro... |
2636 2637 2638 2639 2640 2641 |
* Returns zero if @work was already on the kernel-global workqueue and * non-zero otherwise. * * This puts a job in the kernel-global workqueue if it was not already * queued and leaves it in the same position on the kernel-global * workqueue otherwise. |
0fcb78c22 [PATCH] Add DocBo... |
2642 |
*/ |
7ad5b3a50 kernel: remove fa... |
2643 |
int schedule_work(struct work_struct *work) |
1da177e4c Linux-2.6.12-rc2 |
2644 |
{ |
d320c0383 workqueue: s/__cr... |
2645 |
return queue_work(system_wq, work); |
1da177e4c Linux-2.6.12-rc2 |
2646 |
} |
ae90dd5db Move workqueue ex... |
2647 |
EXPORT_SYMBOL(schedule_work); |
1da177e4c Linux-2.6.12-rc2 |
2648 |
|
c1a220e7a pm: introduce new... |
2649 2650 2651 2652 2653 2654 2655 2656 2657 |
/* * schedule_work_on - put work task on a specific cpu * @cpu: cpu to put the work task on * @work: job to be done * * This puts a job on a specific cpu */ int schedule_work_on(int cpu, struct work_struct *work) { |
d320c0383 workqueue: s/__cr... |
2658 |
return queue_work_on(cpu, system_wq, work); |
c1a220e7a pm: introduce new... |
2659 2660 |
} EXPORT_SYMBOL(schedule_work_on); |
0fcb78c22 [PATCH] Add DocBo... |
2661 2662 |
/** * schedule_delayed_work - put work task in global workqueue after delay |
52bad64d9 WorkStruct: Separ... |
2663 2664 |
* @dwork: job to be done * @delay: number of jiffies to wait or 0 for immediate execution |
0fcb78c22 [PATCH] Add DocBo... |
2665 2666 2667 2668 |
* * After waiting for a given time this puts a job in the kernel-global * workqueue. */ |
7ad5b3a50 kernel: remove fa... |
2669 |
int schedule_delayed_work(struct delayed_work *dwork, |
82f67cd9f [PATCH] Add debug... |
2670 |
unsigned long delay) |
1da177e4c Linux-2.6.12-rc2 |
2671 |
{ |
d320c0383 workqueue: s/__cr... |
2672 |
return queue_delayed_work(system_wq, dwork, delay); |
1da177e4c Linux-2.6.12-rc2 |
2673 |
} |
ae90dd5db Move workqueue ex... |
2674 |
EXPORT_SYMBOL(schedule_delayed_work); |
1da177e4c Linux-2.6.12-rc2 |
2675 |
|
0fcb78c22 [PATCH] Add DocBo... |
2676 2677 2678 |
/** * schedule_delayed_work_on - queue work in global workqueue on CPU after delay * @cpu: cpu to use |
52bad64d9 WorkStruct: Separ... |
2679 |
* @dwork: job to be done |
0fcb78c22 [PATCH] Add DocBo... |
2680 2681 2682 2683 2684 |
* @delay: number of jiffies to wait * * After waiting for a given time this puts a job in the kernel-global * workqueue on the specified CPU. */ |
1da177e4c Linux-2.6.12-rc2 |
2685 |
int schedule_delayed_work_on(int cpu, |
52bad64d9 WorkStruct: Separ... |
2686 |
struct delayed_work *dwork, unsigned long delay) |
1da177e4c Linux-2.6.12-rc2 |
2687 |
{ |
d320c0383 workqueue: s/__cr... |
2688 |
return queue_delayed_work_on(cpu, system_wq, dwork, delay); |
1da177e4c Linux-2.6.12-rc2 |
2689 |
} |
ae90dd5db Move workqueue ex... |
2690 |
EXPORT_SYMBOL(schedule_delayed_work_on); |
1da177e4c Linux-2.6.12-rc2 |
2691 |
|
b61367732 [PATCH] schedule_... |
2692 |
/** |
31ddd871f workqueue: Clarif... |
2693 |
* schedule_on_each_cpu - execute a function synchronously on each online CPU |
b61367732 [PATCH] schedule_... |
2694 |
* @func: the function to call |
b61367732 [PATCH] schedule_... |
2695 |
* |
31ddd871f workqueue: Clarif... |
2696 2697 |
* schedule_on_each_cpu() executes @func on each online CPU using the * system workqueue and blocks until all CPUs have completed. |
b61367732 [PATCH] schedule_... |
2698 |
* schedule_on_each_cpu() is very slow. |
31ddd871f workqueue: Clarif... |
2699 2700 2701 |
* * RETURNS: * 0 on success, -errno on failure. |
b61367732 [PATCH] schedule_... |
2702 |
*/ |
65f27f384 WorkStruct: Pass ... |
2703 |
int schedule_on_each_cpu(work_func_t func) |
15316ba81 [PATCH] add sched... |
2704 2705 |
{ int cpu; |
38f515680 workqueue: add mi... |
2706 |
struct work_struct __percpu *works; |
15316ba81 [PATCH] add sched... |
2707 |
|
b61367732 [PATCH] schedule_... |
2708 2709 |
works = alloc_percpu(struct work_struct); if (!works) |
15316ba81 [PATCH] add sched... |
2710 |
return -ENOMEM; |
b61367732 [PATCH] schedule_... |
2711 |
|
939818009 workqueue: fix ra... |
2712 |
get_online_cpus(); |
15316ba81 [PATCH] add sched... |
2713 |
for_each_online_cpu(cpu) { |
9bfb18392 [PATCH] workqueue... |
2714 2715 2716 |
struct work_struct *work = per_cpu_ptr(works, cpu); INIT_WORK(work, func); |
b71ab8c20 workqueue: increa... |
2717 |
schedule_work_on(cpu, work); |
65a644643 HWPOISON: Allow s... |
2718 |
} |
939818009 workqueue: fix ra... |
2719 2720 2721 |
for_each_online_cpu(cpu) flush_work(per_cpu_ptr(works, cpu)); |
95402b382 cpu-hotplug: repl... |
2722 |
put_online_cpus(); |
b61367732 [PATCH] schedule_... |
2723 |
free_percpu(works); |
15316ba81 [PATCH] add sched... |
2724 2725 |
return 0; } |
eef6a7d5c workqueue: warn a... |
2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 |
/** * flush_scheduled_work - ensure that any scheduled work has run to completion. * * Forces execution of the kernel-global workqueue and blocks until its * completion. * * Think twice before calling this function! It's very easy to get into * trouble if you don't take great care. Either of the following situations * will lead to deadlock: * * One of the work items currently on the workqueue needs to acquire * a lock held by your code or its caller. * * Your code is running in the context of a work routine. * * They will be detected by lockdep when they occur, but the first might not * occur very often. It depends on what work items are on the workqueue and * what locks they need, which you have no control over. * * In most situations flushing the entire workqueue is overkill; you merely * need to know that a particular work item isn't queued and isn't running. * In such cases you should use cancel_delayed_work_sync() or * cancel_work_sync() instead. */ |
1da177e4c Linux-2.6.12-rc2 |
2750 2751 |
void flush_scheduled_work(void) { |
d320c0383 workqueue: s/__cr... |
2752 |
flush_workqueue(system_wq); |
1da177e4c Linux-2.6.12-rc2 |
2753 |
} |
ae90dd5db Move workqueue ex... |
2754 |
EXPORT_SYMBOL(flush_scheduled_work); |
1da177e4c Linux-2.6.12-rc2 |
2755 2756 |
/** |
1fa44ecad [SCSI] add execut... |
2757 2758 |
* execute_in_process_context - reliably execute the routine with user context * @fn: the function to execute |
1fa44ecad [SCSI] add execut... |
2759 2760 2761 2762 2763 2764 2765 2766 2767 |
* @ew: guaranteed storage for the execute work structure (must * be available when the work executes) * * Executes the function immediately if process context is available, * otherwise schedules the function for delayed execution. * * Returns: 0 - function was executed * 1 - function was scheduled for execution */ |
65f27f384 WorkStruct: Pass ... |
2768 |
int execute_in_process_context(work_func_t fn, struct execute_work *ew) |
1fa44ecad [SCSI] add execut... |
2769 2770 |
{ if (!in_interrupt()) { |
65f27f384 WorkStruct: Pass ... |
2771 |
fn(&ew->work); |
1fa44ecad [SCSI] add execut... |
2772 2773 |
return 0; } |
65f27f384 WorkStruct: Pass ... |
2774 |
INIT_WORK(&ew->work, fn); |
1fa44ecad [SCSI] add execut... |
2775 2776 2777 2778 2779 |
schedule_work(&ew->work); return 1; } EXPORT_SYMBOL_GPL(execute_in_process_context); |
1da177e4c Linux-2.6.12-rc2 |
2780 2781 |
int keventd_up(void) { |
d320c0383 workqueue: s/__cr... |
2782 |
return system_wq != NULL; |
1da177e4c Linux-2.6.12-rc2 |
2783 |
} |
bdbc5dd7d workqueue: prepar... |
2784 |
static int alloc_cwqs(struct workqueue_struct *wq) |
0f900049c workqueue: update... |
2785 |
{ |
65a644643 HWPOISON: Allow s... |
2786 |
/* |
0f900049c workqueue: update... |
2787 2788 2789 |
* cwqs are forced aligned according to WORK_STRUCT_FLAG_BITS. * Make sure that the alignment isn't lower than that of * unsigned long long. |
65a644643 HWPOISON: Allow s... |
2790 |
*/ |
0f900049c workqueue: update... |
2791 2792 2793 |
const size_t size = sizeof(struct cpu_workqueue_struct); const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS, __alignof__(unsigned long long)); |
931ac77ef workqueue: fix bu... |
2794 2795 2796 2797 2798 |
#ifdef CONFIG_SMP bool percpu = !(wq->flags & WQ_UNBOUND); #else bool percpu = false; #endif |
65a644643 HWPOISON: Allow s... |
2799 |
|
931ac77ef workqueue: fix bu... |
2800 |
if (percpu) |
f34217977 workqueue: implem... |
2801 |
wq->cpu_wq.pcpu = __alloc_percpu(size, align); |
931ac77ef workqueue: fix bu... |
2802 |
else { |
f34217977 workqueue: implem... |
2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 |
void *ptr; /* * Allocate enough room to align cwq and put an extra * pointer at the end pointing back to the originally * allocated pointer which will be used for free. */ ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL); if (ptr) { wq->cpu_wq.single = PTR_ALIGN(ptr, align); *(void **)(wq->cpu_wq.single + 1) = ptr; } |
bdbc5dd7d workqueue: prepar... |
2815 |
} |
f34217977 workqueue: implem... |
2816 |
|
0415b00d1 percpu: Always al... |
2817 |
/* just in case, make sure it's actually aligned */ |
bdbc5dd7d workqueue: prepar... |
2818 2819 |
BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align)); return wq->cpu_wq.v ? 0 : -ENOMEM; |
0f900049c workqueue: update... |
2820 |
} |
bdbc5dd7d workqueue: prepar... |
2821 |
static void free_cwqs(struct workqueue_struct *wq) |
0f900049c workqueue: update... |
2822 |
{ |
931ac77ef workqueue: fix bu... |
2823 2824 2825 2826 2827 2828 2829 |
#ifdef CONFIG_SMP bool percpu = !(wq->flags & WQ_UNBOUND); #else bool percpu = false; #endif if (percpu) |
f34217977 workqueue: implem... |
2830 2831 2832 |
free_percpu(wq->cpu_wq.pcpu); else if (wq->cpu_wq.single) { /* the pointer to free is stored right after the cwq */ |
bdbc5dd7d workqueue: prepar... |
2833 |
kfree(*(void **)(wq->cpu_wq.single + 1)); |
f34217977 workqueue: implem... |
2834 |
} |
0f900049c workqueue: update... |
2835 |
} |
f34217977 workqueue: implem... |
2836 2837 |
static int wq_clamp_max_active(int max_active, unsigned int flags, const char *name) |
b71ab8c20 workqueue: increa... |
2838 |
{ |
f34217977 workqueue: implem... |
2839 2840 2841 |
int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE; if (max_active < 1 || max_active > lim) |
b71ab8c20 workqueue: increa... |
2842 2843 2844 |
printk(KERN_WARNING "workqueue: max_active %d requested for %s " "is out of range, clamping between %d and %d ", |
f34217977 workqueue: implem... |
2845 |
max_active, name, 1, lim); |
b71ab8c20 workqueue: increa... |
2846 |
|
f34217977 workqueue: implem... |
2847 |
return clamp_val(max_active, 1, lim); |
b71ab8c20 workqueue: increa... |
2848 |
} |
d320c0383 workqueue: s/__cr... |
2849 2850 2851 2852 2853 |
struct workqueue_struct *__alloc_workqueue_key(const char *name, unsigned int flags, int max_active, struct lock_class_key *key, const char *lock_name) |
1da177e4c Linux-2.6.12-rc2 |
2854 |
{ |
1da177e4c Linux-2.6.12-rc2 |
2855 |
struct workqueue_struct *wq; |
c34056a3f workqueue: introd... |
2856 |
unsigned int cpu; |
1da177e4c Linux-2.6.12-rc2 |
2857 |
|
f34217977 workqueue: implem... |
2858 |
/* |
6370a6ad3 workqueue: add an... |
2859 2860 2861 2862 2863 2864 2865 |
* Workqueues which may be used during memory reclaim should * have a rescuer to guarantee forward progress. */ if (flags & WQ_MEM_RECLAIM) flags |= WQ_RESCUER; /* |
f34217977 workqueue: implem... |
2866 2867 2868 2869 2870 |
* Unbound workqueues aren't concurrency managed and should be * dispatched to workers immediately. */ if (flags & WQ_UNBOUND) flags |= WQ_HIGHPRI; |
d320c0383 workqueue: s/__cr... |
2871 |
max_active = max_active ?: WQ_DFL_ACTIVE; |
f34217977 workqueue: implem... |
2872 |
max_active = wq_clamp_max_active(max_active, flags, name); |
1e19ffc63 workqueue: implem... |
2873 |
|
3af24433e workqueue: don't ... |
2874 2875 |
wq = kzalloc(sizeof(*wq), GFP_KERNEL); if (!wq) |
4690c4ab5 workqueue: misc/c... |
2876 |
goto err; |
3af24433e workqueue: don't ... |
2877 |
|
97e37d7b9 workqueue: merge ... |
2878 |
wq->flags = flags; |
a0a1a5fd4 workqueue: reimpl... |
2879 |
wq->saved_max_active = max_active; |
73f53c4aa workqueue: reimpl... |
2880 2881 2882 2883 |
mutex_init(&wq->flush_mutex); atomic_set(&wq->nr_cwqs_to_flush, 0); INIT_LIST_HEAD(&wq->flusher_queue); INIT_LIST_HEAD(&wq->flusher_overflow); |
502ca9d81 workqueue: make s... |
2884 |
|
3af24433e workqueue: don't ... |
2885 |
wq->name = name; |
eb13ba873 lockdep: fix work... |
2886 |
lockdep_init_map(&wq->lockdep_map, lock_name, key, 0); |
cce1a1656 workqueue: introd... |
2887 |
INIT_LIST_HEAD(&wq->list); |
3af24433e workqueue: don't ... |
2888 |
|
bdbc5dd7d workqueue: prepar... |
2889 2890 |
if (alloc_cwqs(wq) < 0) goto err; |
f34217977 workqueue: implem... |
2891 |
for_each_cwq_cpu(cpu, wq) { |
1537663f5 workqueue: kill c... |
2892 |
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
8b03ae3cd workqueue: introd... |
2893 |
struct global_cwq *gcwq = get_gcwq(cpu); |
1537663f5 workqueue: kill c... |
2894 |
|
0f900049c workqueue: update... |
2895 |
BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK); |
8b03ae3cd workqueue: introd... |
2896 |
cwq->gcwq = gcwq; |
c34056a3f workqueue: introd... |
2897 |
cwq->wq = wq; |
73f53c4aa workqueue: reimpl... |
2898 |
cwq->flush_color = -1; |
1e19ffc63 workqueue: implem... |
2899 |
cwq->max_active = max_active; |
1e19ffc63 workqueue: implem... |
2900 |
INIT_LIST_HEAD(&cwq->delayed_works); |
e22bee782 workqueue: implem... |
2901 |
} |
1537663f5 workqueue: kill c... |
2902 |
|
e22bee782 workqueue: implem... |
2903 2904 |
if (flags & WQ_RESCUER) { struct worker *rescuer; |
f2e005aaf workqueue: fix ma... |
2905 |
if (!alloc_mayday_mask(&wq->mayday_mask, GFP_KERNEL)) |
e22bee782 workqueue: implem... |
2906 2907 2908 2909 2910 2911 2912 2913 2914 |
goto err; wq->rescuer = rescuer = alloc_worker(); if (!rescuer) goto err; rescuer->task = kthread_create(rescuer_thread, wq, "%s", name); if (IS_ERR(rescuer->task)) goto err; |
e22bee782 workqueue: implem... |
2915 2916 |
rescuer->task->flags |= PF_THREAD_BOUND; wake_up_process(rescuer->task); |
3af24433e workqueue: don't ... |
2917 |
} |
a0a1a5fd4 workqueue: reimpl... |
2918 2919 2920 2921 2922 |
/* * workqueue_lock protects global freeze state and workqueues * list. Grab it, set max_active accordingly and add the new * workqueue to workqueues list. */ |
1537663f5 workqueue: kill c... |
2923 |
spin_lock(&workqueue_lock); |
a0a1a5fd4 workqueue: reimpl... |
2924 |
|
58a69cb47 workqueue, freeze... |
2925 |
if (workqueue_freezing && wq->flags & WQ_FREEZABLE) |
f34217977 workqueue: implem... |
2926 |
for_each_cwq_cpu(cpu, wq) |
a0a1a5fd4 workqueue: reimpl... |
2927 |
get_cwq(cpu, wq)->max_active = 0; |
1537663f5 workqueue: kill c... |
2928 |
list_add(&wq->list, &workqueues); |
a0a1a5fd4 workqueue: reimpl... |
2929 |
|
1537663f5 workqueue: kill c... |
2930 |
spin_unlock(&workqueue_lock); |
3af24433e workqueue: don't ... |
2931 |
return wq; |
4690c4ab5 workqueue: misc/c... |
2932 2933 |
err: if (wq) { |
bdbc5dd7d workqueue: prepar... |
2934 |
free_cwqs(wq); |
f2e005aaf workqueue: fix ma... |
2935 |
free_mayday_mask(wq->mayday_mask); |
e22bee782 workqueue: implem... |
2936 |
kfree(wq->rescuer); |
4690c4ab5 workqueue: misc/c... |
2937 2938 2939 |
kfree(wq); } return NULL; |
3af24433e workqueue: don't ... |
2940 |
} |
d320c0383 workqueue: s/__cr... |
2941 |
EXPORT_SYMBOL_GPL(__alloc_workqueue_key); |
1da177e4c Linux-2.6.12-rc2 |
2942 |
|
3af24433e workqueue: don't ... |
2943 2944 2945 2946 2947 2948 2949 2950 |
/** * destroy_workqueue - safely terminate a workqueue * @wq: target workqueue * * Safely destroy a workqueue. All work currently pending will be done first. */ void destroy_workqueue(struct workqueue_struct *wq) { |
c8e55f360 workqueue: implem... |
2951 |
unsigned int cpu; |
3af24433e workqueue: don't ... |
2952 |
|
9c5a2ba70 workqueue: separa... |
2953 2954 |
/* drain it before proceeding with destruction */ drain_workqueue(wq); |
c8efcc258 workqueue: allow ... |
2955 |
|
a0a1a5fd4 workqueue: reimpl... |
2956 2957 2958 2959 |
/* * wq list is used to freeze wq, remove from list after * flushing is complete in case freeze races us. */ |
95402b382 cpu-hotplug: repl... |
2960 |
spin_lock(&workqueue_lock); |
b1f4ec172 workqueue: introd... |
2961 |
list_del(&wq->list); |
95402b382 cpu-hotplug: repl... |
2962 |
spin_unlock(&workqueue_lock); |
3af24433e workqueue: don't ... |
2963 |
|
e22bee782 workqueue: implem... |
2964 |
/* sanity check */ |
f34217977 workqueue: implem... |
2965 |
for_each_cwq_cpu(cpu, wq) { |
73f53c4aa workqueue: reimpl... |
2966 2967 |
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); int i; |
73f53c4aa workqueue: reimpl... |
2968 2969 |
for (i = 0; i < WORK_NR_COLORS; i++) BUG_ON(cwq->nr_in_flight[i]); |
1e19ffc63 workqueue: implem... |
2970 2971 |
BUG_ON(cwq->nr_active); BUG_ON(!list_empty(&cwq->delayed_works)); |
73f53c4aa workqueue: reimpl... |
2972 |
} |
9b41ea728 [PATCH] workqueue... |
2973 |
|
e22bee782 workqueue: implem... |
2974 2975 |
if (wq->flags & WQ_RESCUER) { kthread_stop(wq->rescuer->task); |
f2e005aaf workqueue: fix ma... |
2976 |
free_mayday_mask(wq->mayday_mask); |
8d9df9f08 workqueue: free r... |
2977 |
kfree(wq->rescuer); |
e22bee782 workqueue: implem... |
2978 |
} |
bdbc5dd7d workqueue: prepar... |
2979 |
free_cwqs(wq); |
3af24433e workqueue: don't ... |
2980 2981 2982 |
kfree(wq); } EXPORT_SYMBOL_GPL(destroy_workqueue); |
dcd989cb7 workqueue: implem... |
2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 |
/** * workqueue_set_max_active - adjust max_active of a workqueue * @wq: target workqueue * @max_active: new max_active value. * * Set max_active of @wq to @max_active. * * CONTEXT: * Don't call from IRQ context. */ void workqueue_set_max_active(struct workqueue_struct *wq, int max_active) { unsigned int cpu; |
f34217977 workqueue: implem... |
2996 |
max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); |
dcd989cb7 workqueue: implem... |
2997 2998 2999 3000 |
spin_lock(&workqueue_lock); wq->saved_max_active = max_active; |
f34217977 workqueue: implem... |
3001 |
for_each_cwq_cpu(cpu, wq) { |
dcd989cb7 workqueue: implem... |
3002 3003 3004 |
struct global_cwq *gcwq = get_gcwq(cpu); spin_lock_irq(&gcwq->lock); |
58a69cb47 workqueue, freeze... |
3005 |
if (!(wq->flags & WQ_FREEZABLE) || |
dcd989cb7 workqueue: implem... |
3006 3007 |
!(gcwq->flags & GCWQ_FREEZING)) get_cwq(gcwq->cpu, wq)->max_active = max_active; |
9bfb18392 [PATCH] workqueue... |
3008 |
|
dcd989cb7 workqueue: implem... |
3009 |
spin_unlock_irq(&gcwq->lock); |
65a644643 HWPOISON: Allow s... |
3010 |
} |
939818009 workqueue: fix ra... |
3011 |
|
dcd989cb7 workqueue: implem... |
3012 |
spin_unlock(&workqueue_lock); |
15316ba81 [PATCH] add sched... |
3013 |
} |
dcd989cb7 workqueue: implem... |
3014 |
EXPORT_SYMBOL_GPL(workqueue_set_max_active); |
15316ba81 [PATCH] add sched... |
3015 |
|
eef6a7d5c workqueue: warn a... |
3016 |
/** |
dcd989cb7 workqueue: implem... |
3017 3018 3019 |
* workqueue_congested - test whether a workqueue is congested * @cpu: CPU in question * @wq: target workqueue |
eef6a7d5c workqueue: warn a... |
3020 |
* |
dcd989cb7 workqueue: implem... |
3021 3022 3023 |
* Test whether @wq's cpu workqueue for @cpu is congested. There is * no synchronization around this function and the test result is * unreliable and only useful as advisory hints or for debugging. |
eef6a7d5c workqueue: warn a... |
3024 |
* |
dcd989cb7 workqueue: implem... |
3025 3026 |
* RETURNS: * %true if congested, %false otherwise. |
eef6a7d5c workqueue: warn a... |
3027 |
*/ |
dcd989cb7 workqueue: implem... |
3028 |
bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq) |
1da177e4c Linux-2.6.12-rc2 |
3029 |
{ |
dcd989cb7 workqueue: implem... |
3030 3031 3032 |
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); return !list_empty(&cwq->delayed_works); |
1da177e4c Linux-2.6.12-rc2 |
3033 |
} |
dcd989cb7 workqueue: implem... |
3034 |
EXPORT_SYMBOL_GPL(workqueue_congested); |
1da177e4c Linux-2.6.12-rc2 |
3035 3036 |
/** |
dcd989cb7 workqueue: implem... |
3037 3038 |
* work_cpu - return the last known associated cpu for @work * @work: the work of interest |
1fa44ecad [SCSI] add execut... |
3039 |
* |
dcd989cb7 workqueue: implem... |
3040 |
* RETURNS: |
bdbc5dd7d workqueue: prepar... |
3041 |
* CPU number if @work was ever queued. WORK_CPU_NONE otherwise. |
1fa44ecad [SCSI] add execut... |
3042 |
*/ |
dcd989cb7 workqueue: implem... |
3043 |
unsigned int work_cpu(struct work_struct *work) |
1fa44ecad [SCSI] add execut... |
3044 |
{ |
dcd989cb7 workqueue: implem... |
3045 |
struct global_cwq *gcwq = get_work_gcwq(work); |
1fa44ecad [SCSI] add execut... |
3046 |
|
bdbc5dd7d workqueue: prepar... |
3047 |
return gcwq ? gcwq->cpu : WORK_CPU_NONE; |
1fa44ecad [SCSI] add execut... |
3048 |
} |
dcd989cb7 workqueue: implem... |
3049 |
EXPORT_SYMBOL_GPL(work_cpu); |
1fa44ecad [SCSI] add execut... |
3050 |
|
dcd989cb7 workqueue: implem... |
3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 |
/** * work_busy - test whether a work is currently pending or running * @work: the work to be tested * * Test whether @work is currently pending or running. There is no * synchronization around this function and the test result is * unreliable and only useful as advisory hints or for debugging. * Especially for reentrant wqs, the pending state might hide the * running state. * * RETURNS: * OR'd bitmask of WORK_BUSY_* bits. */ unsigned int work_busy(struct work_struct *work) |
1da177e4c Linux-2.6.12-rc2 |
3065 |
{ |
dcd989cb7 workqueue: implem... |
3066 3067 3068 |
struct global_cwq *gcwq = get_work_gcwq(work); unsigned long flags; unsigned int ret = 0; |
1da177e4c Linux-2.6.12-rc2 |
3069 |
|
dcd989cb7 workqueue: implem... |
3070 3071 |
if (!gcwq) return false; |
1da177e4c Linux-2.6.12-rc2 |
3072 |
|
dcd989cb7 workqueue: implem... |
3073 |
spin_lock_irqsave(&gcwq->lock, flags); |
1da177e4c Linux-2.6.12-rc2 |
3074 |
|
dcd989cb7 workqueue: implem... |
3075 3076 3077 3078 |
if (work_pending(work)) ret |= WORK_BUSY_PENDING; if (find_worker_executing_work(gcwq, work)) ret |= WORK_BUSY_RUNNING; |
1da177e4c Linux-2.6.12-rc2 |
3079 |
|
dcd989cb7 workqueue: implem... |
3080 |
spin_unlock_irqrestore(&gcwq->lock, flags); |
1da177e4c Linux-2.6.12-rc2 |
3081 |
|
dcd989cb7 workqueue: implem... |
3082 |
return ret; |
1da177e4c Linux-2.6.12-rc2 |
3083 |
} |
dcd989cb7 workqueue: implem... |
3084 |
EXPORT_SYMBOL_GPL(work_busy); |
1da177e4c Linux-2.6.12-rc2 |
3085 |
|
db7bccf45 workqueue: reimpl... |
3086 3087 3088 |
/* * CPU hotplug. * |
e22bee782 workqueue: implem... |
3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 |
* There are two challenges in supporting CPU hotplug. Firstly, there * are a lot of assumptions on strong associations among work, cwq and * gcwq which make migrating pending and scheduled works very * difficult to implement without impacting hot paths. Secondly, * gcwqs serve mix of short, long and very long running works making * blocked draining impractical. * * This is solved by allowing a gcwq to be detached from CPU, running * it with unbound (rogue) workers and allowing it to be reattached * later if the cpu comes back online. A separate thread is created * to govern a gcwq in such state and is called the trustee of the * gcwq. |
db7bccf45 workqueue: reimpl... |
3101 3102 3103 3104 3105 3106 3107 |
* * Trustee states and their descriptions. * * START Command state used on startup. On CPU_DOWN_PREPARE, a * new trustee is started with this state. * * IN_CHARGE Once started, trustee will enter this state after |
e22bee782 workqueue: implem... |
3108 3109 3110 3111 3112 3113 |
* assuming the manager role and making all existing * workers rogue. DOWN_PREPARE waits for trustee to * enter this state. After reaching IN_CHARGE, trustee * tries to execute the pending worklist until it's empty * and the state is set to BUTCHER, or the state is set * to RELEASE. |
db7bccf45 workqueue: reimpl... |
3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 |
* * BUTCHER Command state which is set by the cpu callback after * the cpu has went down. Once this state is set trustee * knows that there will be no new works on the worklist * and once the worklist is empty it can proceed to * killing idle workers. * * RELEASE Command state which is set by the cpu callback if the * cpu down has been canceled or it has come online * again. After recognizing this state, trustee stops |
e22bee782 workqueue: implem... |
3124 3125 3126 |
* trying to drain or butcher and clears ROGUE, rebinds * all remaining workers back to the cpu and releases * manager role. |
db7bccf45 workqueue: reimpl... |
3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 |
* * DONE Trustee will enter this state after BUTCHER or RELEASE * is complete. * * trustee CPU draining * took over down complete * START -----------> IN_CHARGE -----------> BUTCHER -----------> DONE * | | ^ * | CPU is back online v return workers | * ----------------> RELEASE -------------- */ |
1da177e4c Linux-2.6.12-rc2 |
3138 |
|
db7bccf45 workqueue: reimpl... |
3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 |
/** * trustee_wait_event_timeout - timed event wait for trustee * @cond: condition to wait for * @timeout: timeout in jiffies * * wait_event_timeout() for trustee to use. Handles locking and * checks for RELEASE request. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. To be used by trustee. * * RETURNS: * Positive indicating left time if @cond is satisfied, 0 if timed * out, -1 if canceled. */ #define trustee_wait_event_timeout(cond, timeout) ({ \ long __ret = (timeout); \ while (!((cond) || (gcwq->trustee_state == TRUSTEE_RELEASE)) && \ __ret) { \ spin_unlock_irq(&gcwq->lock); \ __wait_event_timeout(gcwq->trustee_wait, (cond) || \ (gcwq->trustee_state == TRUSTEE_RELEASE), \ __ret); \ spin_lock_irq(&gcwq->lock); \ } \ gcwq->trustee_state == TRUSTEE_RELEASE ? -1 : (__ret); \ }) |
3af24433e workqueue: don't ... |
3167 |
|
db7bccf45 workqueue: reimpl... |
3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 |
/** * trustee_wait_event - event wait for trustee * @cond: condition to wait for * * wait_event() for trustee to use. Automatically handles locking and * checks for CANCEL request. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. To be used by trustee. * * RETURNS: * 0 if @cond is satisfied, -1 if canceled. */ #define trustee_wait_event(cond) ({ \ long __ret1; \ __ret1 = trustee_wait_event_timeout(cond, MAX_SCHEDULE_TIMEOUT);\ __ret1 < 0 ? -1 : 0; \ }) |
1da177e4c Linux-2.6.12-rc2 |
3187 |
|
db7bccf45 workqueue: reimpl... |
3188 |
static int __cpuinit trustee_thread(void *__gcwq) |
3af24433e workqueue: don't ... |
3189 |
{ |
db7bccf45 workqueue: reimpl... |
3190 3191 |
struct global_cwq *gcwq = __gcwq; struct worker *worker; |
e22bee782 workqueue: implem... |
3192 |
struct work_struct *work; |
db7bccf45 workqueue: reimpl... |
3193 |
struct hlist_node *pos; |
e22bee782 workqueue: implem... |
3194 |
long rc; |
db7bccf45 workqueue: reimpl... |
3195 |
int i; |
3af24433e workqueue: don't ... |
3196 |
|
db7bccf45 workqueue: reimpl... |
3197 3198 3199 |
BUG_ON(gcwq->cpu != smp_processor_id()); spin_lock_irq(&gcwq->lock); |
3af24433e workqueue: don't ... |
3200 |
/* |
e22bee782 workqueue: implem... |
3201 3202 3203 |
* Claim the manager position and make all workers rogue. * Trustee must be bound to the target cpu and can't be * cancelled. |
3af24433e workqueue: don't ... |
3204 |
*/ |
db7bccf45 workqueue: reimpl... |
3205 |
BUG_ON(gcwq->cpu != smp_processor_id()); |
e22bee782 workqueue: implem... |
3206 3207 |
rc = trustee_wait_event(!(gcwq->flags & GCWQ_MANAGING_WORKERS)); BUG_ON(rc < 0); |
3af24433e workqueue: don't ... |
3208 |
|
e22bee782 workqueue: implem... |
3209 |
gcwq->flags |= GCWQ_MANAGING_WORKERS; |
e1d8aa9f1 tracing: add a ne... |
3210 |
|
db7bccf45 workqueue: reimpl... |
3211 |
list_for_each_entry(worker, &gcwq->idle_list, entry) |
cb4447669 workqueue: use wo... |
3212 |
worker->flags |= WORKER_ROGUE; |
3af24433e workqueue: don't ... |
3213 |
|
db7bccf45 workqueue: reimpl... |
3214 |
for_each_busy_worker(worker, i, pos, gcwq) |
cb4447669 workqueue: use wo... |
3215 |
worker->flags |= WORKER_ROGUE; |
06ba38a9a workqueues: shift... |
3216 |
|
db7bccf45 workqueue: reimpl... |
3217 |
/* |
e22bee782 workqueue: implem... |
3218 3219 3220 3221 3222 3223 3224 3225 |
* Call schedule() so that we cross rq->lock and thus can * guarantee sched callbacks see the rogue flag. This is * necessary as scheduler callbacks may be invoked from other * cpus. */ spin_unlock_irq(&gcwq->lock); schedule(); spin_lock_irq(&gcwq->lock); |
06ba38a9a workqueues: shift... |
3226 |
|
e22bee782 workqueue: implem... |
3227 |
/* |
cb4447669 workqueue: use wo... |
3228 3229 3230 3231 |
* Sched callbacks are disabled now. Zap nr_running. After * this, nr_running stays zero and need_more_worker() and * keep_working() are always true as long as the worklist is * not empty. |
e22bee782 workqueue: implem... |
3232 |
*/ |
cb4447669 workqueue: use wo... |
3233 |
atomic_set(get_gcwq_nr_running(gcwq->cpu), 0); |
1da177e4c Linux-2.6.12-rc2 |
3234 |
|
e22bee782 workqueue: implem... |
3235 3236 3237 |
spin_unlock_irq(&gcwq->lock); del_timer_sync(&gcwq->idle_timer); spin_lock_irq(&gcwq->lock); |
3af24433e workqueue: don't ... |
3238 |
|
e22bee782 workqueue: implem... |
3239 |
/* |
db7bccf45 workqueue: reimpl... |
3240 3241 3242 3243 3244 3245 3246 |
* We're now in charge. Notify and proceed to drain. We need * to keep the gcwq running during the whole CPU down * procedure as other cpu hotunplug callbacks may need to * flush currently running tasks. */ gcwq->trustee_state = TRUSTEE_IN_CHARGE; wake_up_all(&gcwq->trustee_wait); |
3af24433e workqueue: don't ... |
3247 |
|
db7bccf45 workqueue: reimpl... |
3248 3249 3250 |
/* * The original cpu is in the process of dying and may go away * anytime now. When that happens, we and all workers would |
e22bee782 workqueue: implem... |
3251 3252 3253 3254 |
* be migrated to other cpus. Try draining any left work. We * want to get it over with ASAP - spam rescuers, wake up as * many idlers as necessary and create new ones till the * worklist is empty. Note that if the gcwq is frozen, there |
58a69cb47 workqueue, freeze... |
3255 |
* may be frozen works in freezable cwqs. Don't declare |
e22bee782 workqueue: implem... |
3256 |
* completion while frozen. |
db7bccf45 workqueue: reimpl... |
3257 3258 3259 3260 |
*/ while (gcwq->nr_workers != gcwq->nr_idle || gcwq->flags & GCWQ_FREEZING || gcwq->trustee_state == TRUSTEE_IN_CHARGE) { |
e22bee782 workqueue: implem... |
3261 3262 3263 3264 3265 3266 |
int nr_works = 0; list_for_each_entry(work, &gcwq->worklist, entry) { send_mayday(work); nr_works++; } |
3af24433e workqueue: don't ... |
3267 |
|
e22bee782 workqueue: implem... |
3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 |
list_for_each_entry(worker, &gcwq->idle_list, entry) { if (!nr_works--) break; wake_up_process(worker->task); } if (need_to_create_worker(gcwq)) { spin_unlock_irq(&gcwq->lock); worker = create_worker(gcwq, false); spin_lock_irq(&gcwq->lock); if (worker) { |
cb4447669 workqueue: use wo... |
3279 |
worker->flags |= WORKER_ROGUE; |
e22bee782 workqueue: implem... |
3280 3281 |
start_worker(worker); } |
1da177e4c Linux-2.6.12-rc2 |
3282 |
} |
3af24433e workqueue: don't ... |
3283 |
|
db7bccf45 workqueue: reimpl... |
3284 3285 3286 |
/* give a breather */ if (trustee_wait_event_timeout(false, TRUSTEE_COOLDOWN) < 0) break; |
3af24433e workqueue: don't ... |
3287 |
} |
1da177e4c Linux-2.6.12-rc2 |
3288 |
|
14441960e simplify cleanup_... |
3289 |
/* |
e22bee782 workqueue: implem... |
3290 3291 3292 |
* Either all works have been scheduled and cpu is down, or * cpu down has already been canceled. Wait for and butcher * all workers till we're canceled. |
14441960e simplify cleanup_... |
3293 |
*/ |
e22bee782 workqueue: implem... |
3294 3295 3296 3297 3298 3299 |
do { rc = trustee_wait_event(!list_empty(&gcwq->idle_list)); while (!list_empty(&gcwq->idle_list)) destroy_worker(list_first_entry(&gcwq->idle_list, struct worker, entry)); } while (gcwq->nr_workers && rc >= 0); |
4e6045f13 workqueue: debug ... |
3300 |
|
14441960e simplify cleanup_... |
3301 |
/* |
e22bee782 workqueue: implem... |
3302 3303 3304 3305 3306 |
* At this point, either draining has completed and no worker * is left, or cpu down has been canceled or the cpu is being * brought back up. There shouldn't be any idle one left. * Tell the remaining busy ones to rebind once it finishes the * currently scheduled works by scheduling the rebind_work. |
14441960e simplify cleanup_... |
3307 |
*/ |
e22bee782 workqueue: implem... |
3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 |
WARN_ON(!list_empty(&gcwq->idle_list)); for_each_busy_worker(worker, i, pos, gcwq) { struct work_struct *rebind_work = &worker->rebind_work; /* * Rebind_work may race with future cpu hotplug * operations. Use a separate flag to mark that * rebinding is scheduled. */ |
cb4447669 workqueue: use wo... |
3318 3319 |
worker->flags |= WORKER_REBIND; worker->flags &= ~WORKER_ROGUE; |
e22bee782 workqueue: implem... |
3320 3321 3322 3323 3324 3325 3326 |
/* queue rebind_work, wq doesn't matter, use the default one */ if (test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(rebind_work))) continue; debug_work_activate(rebind_work); |
d320c0383 workqueue: s/__cr... |
3327 |
insert_work(get_cwq(gcwq->cpu, system_wq), rebind_work, |
e22bee782 workqueue: implem... |
3328 3329 3330 3331 3332 3333 |
worker->scheduled.next, work_color_to_flags(WORK_NO_COLOR)); } /* relinquish manager role */ gcwq->flags &= ~GCWQ_MANAGING_WORKERS; |
db7bccf45 workqueue: reimpl... |
3334 3335 3336 3337 3338 3339 |
/* notify completion */ gcwq->trustee = NULL; gcwq->trustee_state = TRUSTEE_DONE; wake_up_all(&gcwq->trustee_wait); spin_unlock_irq(&gcwq->lock); return 0; |
3af24433e workqueue: don't ... |
3340 3341 3342 |
} /** |
db7bccf45 workqueue: reimpl... |
3343 3344 3345 |
* wait_trustee_state - wait for trustee to enter the specified state * @gcwq: gcwq the trustee of interest belongs to * @state: target state to wait for |
3af24433e workqueue: don't ... |
3346 |
* |
db7bccf45 workqueue: reimpl... |
3347 3348 3349 3350 3351 |
* Wait for the trustee to reach @state. DONE is already matched. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times. To be used by cpu_callback. |
3af24433e workqueue: don't ... |
3352 |
*/ |
db7bccf45 workqueue: reimpl... |
3353 |
static void __cpuinit wait_trustee_state(struct global_cwq *gcwq, int state) |
06bd6ebff workqueue: annota... |
3354 3355 |
__releases(&gcwq->lock) __acquires(&gcwq->lock) |
3af24433e workqueue: don't ... |
3356 |
{ |
db7bccf45 workqueue: reimpl... |
3357 3358 3359 3360 3361 3362 3363 3364 |
if (!(gcwq->trustee_state == state || gcwq->trustee_state == TRUSTEE_DONE)) { spin_unlock_irq(&gcwq->lock); __wait_event(gcwq->trustee_wait, gcwq->trustee_state == state || gcwq->trustee_state == TRUSTEE_DONE); spin_lock_irq(&gcwq->lock); } |
3af24433e workqueue: don't ... |
3365 |
} |
3af24433e workqueue: don't ... |
3366 3367 3368 3369 3370 3371 |
static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; |
db7bccf45 workqueue: reimpl... |
3372 3373 |
struct global_cwq *gcwq = get_gcwq(cpu); struct task_struct *new_trustee = NULL; |
e22bee782 workqueue: implem... |
3374 |
struct worker *uninitialized_var(new_worker); |
db7bccf45 workqueue: reimpl... |
3375 |
unsigned long flags; |
3af24433e workqueue: don't ... |
3376 |
|
8bb784428 Add suspend-relat... |
3377 |
action &= ~CPU_TASKS_FROZEN; |
3af24433e workqueue: don't ... |
3378 |
switch (action) { |
db7bccf45 workqueue: reimpl... |
3379 3380 3381 3382 3383 3384 3385 |
case CPU_DOWN_PREPARE: new_trustee = kthread_create(trustee_thread, gcwq, "workqueue_trustee/%d ", cpu); if (IS_ERR(new_trustee)) return notifier_from_errno(PTR_ERR(new_trustee)); kthread_bind(new_trustee, cpu); |
e22bee782 workqueue: implem... |
3386 |
/* fall through */ |
3af24433e workqueue: don't ... |
3387 |
case CPU_UP_PREPARE: |
e22bee782 workqueue: implem... |
3388 3389 3390 3391 3392 3393 |
BUG_ON(gcwq->first_idle); new_worker = create_worker(gcwq, false); if (!new_worker) { if (new_trustee) kthread_stop(new_trustee); return NOTIFY_BAD; |
3af24433e workqueue: don't ... |
3394 |
} |
1da177e4c Linux-2.6.12-rc2 |
3395 |
} |
db7bccf45 workqueue: reimpl... |
3396 3397 |
/* some are called w/ irq disabled, don't disturb irq status */ spin_lock_irqsave(&gcwq->lock, flags); |
3af24433e workqueue: don't ... |
3398 |
|
00dfcaf74 workqueues: shrin... |
3399 |
switch (action) { |
db7bccf45 workqueue: reimpl... |
3400 3401 3402 3403 3404 3405 3406 |
case CPU_DOWN_PREPARE: /* initialize trustee and tell it to acquire the gcwq */ BUG_ON(gcwq->trustee || gcwq->trustee_state != TRUSTEE_DONE); gcwq->trustee = new_trustee; gcwq->trustee_state = TRUSTEE_START; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE); |
e22bee782 workqueue: implem... |
3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 |
/* fall through */ case CPU_UP_PREPARE: BUG_ON(gcwq->first_idle); gcwq->first_idle = new_worker; break; case CPU_DYING: /* * Before this, the trustee and all workers except for * the ones which are still executing works from * before the last CPU down must be on the cpu. After * this, they'll all be diasporas. */ gcwq->flags |= GCWQ_DISASSOCIATED; |
db7bccf45 workqueue: reimpl... |
3421 |
break; |
3da1c84c0 workqueues: make ... |
3422 |
case CPU_POST_DEAD: |
db7bccf45 workqueue: reimpl... |
3423 |
gcwq->trustee_state = TRUSTEE_BUTCHER; |
e22bee782 workqueue: implem... |
3424 3425 3426 3427 |
/* fall through */ case CPU_UP_CANCELED: destroy_worker(gcwq->first_idle); gcwq->first_idle = NULL; |
db7bccf45 workqueue: reimpl... |
3428 3429 3430 3431 |
break; case CPU_DOWN_FAILED: case CPU_ONLINE: |
e22bee782 workqueue: implem... |
3432 |
gcwq->flags &= ~GCWQ_DISASSOCIATED; |
db7bccf45 workqueue: reimpl... |
3433 3434 3435 3436 |
if (gcwq->trustee_state != TRUSTEE_DONE) { gcwq->trustee_state = TRUSTEE_RELEASE; wake_up_process(gcwq->trustee); wait_trustee_state(gcwq, TRUSTEE_DONE); |
3af24433e workqueue: don't ... |
3437 |
} |
db7bccf45 workqueue: reimpl... |
3438 |
|
e22bee782 workqueue: implem... |
3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 |
/* * Trustee is done and there might be no worker left. * Put the first_idle in and request a real manager to * take a look. */ spin_unlock_irq(&gcwq->lock); kthread_bind(gcwq->first_idle->task, cpu); spin_lock_irq(&gcwq->lock); gcwq->flags |= GCWQ_MANAGE_WORKERS; start_worker(gcwq->first_idle); gcwq->first_idle = NULL; |
db7bccf45 workqueue: reimpl... |
3450 |
break; |
00dfcaf74 workqueues: shrin... |
3451 |
} |
db7bccf45 workqueue: reimpl... |
3452 |
spin_unlock_irqrestore(&gcwq->lock, flags); |
1537663f5 workqueue: kill c... |
3453 |
return notifier_from_errno(0); |
1da177e4c Linux-2.6.12-rc2 |
3454 |
} |
1da177e4c Linux-2.6.12-rc2 |
3455 |
|
2d3854a37 cpumask: introduc... |
3456 |
#ifdef CONFIG_SMP |
8ccad40df work_on_cpu: Use ... |
3457 |
|
2d3854a37 cpumask: introduc... |
3458 |
struct work_for_cpu { |
6b44003e5 work_on_cpu(): re... |
3459 |
struct completion completion; |
2d3854a37 cpumask: introduc... |
3460 3461 3462 3463 |
long (*fn)(void *); void *arg; long ret; }; |
6b44003e5 work_on_cpu(): re... |
3464 |
static int do_work_for_cpu(void *_wfc) |
2d3854a37 cpumask: introduc... |
3465 |
{ |
6b44003e5 work_on_cpu(): re... |
3466 |
struct work_for_cpu *wfc = _wfc; |
2d3854a37 cpumask: introduc... |
3467 |
wfc->ret = wfc->fn(wfc->arg); |
6b44003e5 work_on_cpu(): re... |
3468 3469 |
complete(&wfc->completion); return 0; |
2d3854a37 cpumask: introduc... |
3470 3471 3472 3473 3474 3475 3476 3477 |
} /** * work_on_cpu - run a function in user context on a particular cpu * @cpu: the cpu to run on * @fn: the function to run * @arg: the function arg * |
31ad90812 work_on_cpu: don'... |
3478 3479 |
* This will return the value @fn returns. * It is up to the caller to ensure that the cpu doesn't go offline. |
6b44003e5 work_on_cpu(): re... |
3480 |
* The caller must not hold any locks which would prevent @fn from completing. |
2d3854a37 cpumask: introduc... |
3481 3482 3483 |
*/ long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg) { |
6b44003e5 work_on_cpu(): re... |
3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 |
struct task_struct *sub_thread; struct work_for_cpu wfc = { .completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion), .fn = fn, .arg = arg, }; sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu"); if (IS_ERR(sub_thread)) return PTR_ERR(sub_thread); kthread_bind(sub_thread, cpu); wake_up_process(sub_thread); wait_for_completion(&wfc.completion); |
2d3854a37 cpumask: introduc... |
3497 3498 3499 3500 |
return wfc.ret; } EXPORT_SYMBOL_GPL(work_on_cpu); #endif /* CONFIG_SMP */ |
a0a1a5fd4 workqueue: reimpl... |
3501 3502 3503 3504 3505 |
#ifdef CONFIG_FREEZER /** * freeze_workqueues_begin - begin freezing workqueues * |
58a69cb47 workqueue, freeze... |
3506 3507 3508 |
* Start freezing workqueues. After this function returns, all freezable * workqueues will queue new works to their frozen_works list instead of * gcwq->worklist. |
a0a1a5fd4 workqueue: reimpl... |
3509 3510 |
* * CONTEXT: |
8b03ae3cd workqueue: introd... |
3511 |
* Grabs and releases workqueue_lock and gcwq->lock's. |
a0a1a5fd4 workqueue: reimpl... |
3512 3513 3514 |
*/ void freeze_workqueues_begin(void) { |
a0a1a5fd4 workqueue: reimpl... |
3515 3516 3517 3518 3519 3520 |
unsigned int cpu; spin_lock(&workqueue_lock); BUG_ON(workqueue_freezing); workqueue_freezing = true; |
f34217977 workqueue: implem... |
3521 |
for_each_gcwq_cpu(cpu) { |
8b03ae3cd workqueue: introd... |
3522 |
struct global_cwq *gcwq = get_gcwq(cpu); |
bdbc5dd7d workqueue: prepar... |
3523 |
struct workqueue_struct *wq; |
8b03ae3cd workqueue: introd... |
3524 3525 |
spin_lock_irq(&gcwq->lock); |
db7bccf45 workqueue: reimpl... |
3526 3527 |
BUG_ON(gcwq->flags & GCWQ_FREEZING); gcwq->flags |= GCWQ_FREEZING; |
a0a1a5fd4 workqueue: reimpl... |
3528 3529 |
list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
58a69cb47 workqueue, freeze... |
3530 |
if (cwq && wq->flags & WQ_FREEZABLE) |
a0a1a5fd4 workqueue: reimpl... |
3531 |
cwq->max_active = 0; |
a0a1a5fd4 workqueue: reimpl... |
3532 |
} |
8b03ae3cd workqueue: introd... |
3533 3534 |
spin_unlock_irq(&gcwq->lock); |
a0a1a5fd4 workqueue: reimpl... |
3535 3536 3537 3538 3539 3540 |
} spin_unlock(&workqueue_lock); } /** |
58a69cb47 workqueue, freeze... |
3541 |
* freeze_workqueues_busy - are freezable workqueues still busy? |
a0a1a5fd4 workqueue: reimpl... |
3542 3543 3544 3545 3546 3547 3548 3549 |
* * Check whether freezing is complete. This function must be called * between freeze_workqueues_begin() and thaw_workqueues(). * * CONTEXT: * Grabs and releases workqueue_lock. * * RETURNS: |
58a69cb47 workqueue, freeze... |
3550 3551 |
* %true if some freezable workqueues are still busy. %false if freezing * is complete. |
a0a1a5fd4 workqueue: reimpl... |
3552 3553 3554 |
*/ bool freeze_workqueues_busy(void) { |
a0a1a5fd4 workqueue: reimpl... |
3555 3556 3557 3558 3559 3560 |
unsigned int cpu; bool busy = false; spin_lock(&workqueue_lock); BUG_ON(!workqueue_freezing); |
f34217977 workqueue: implem... |
3561 |
for_each_gcwq_cpu(cpu) { |
bdbc5dd7d workqueue: prepar... |
3562 |
struct workqueue_struct *wq; |
a0a1a5fd4 workqueue: reimpl... |
3563 3564 3565 3566 3567 3568 |
/* * nr_active is monotonically decreasing. It's safe * to peek without lock. */ list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
58a69cb47 workqueue, freeze... |
3569 |
if (!cwq || !(wq->flags & WQ_FREEZABLE)) |
a0a1a5fd4 workqueue: reimpl... |
3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 |
continue; BUG_ON(cwq->nr_active < 0); if (cwq->nr_active) { busy = true; goto out_unlock; } } } out_unlock: spin_unlock(&workqueue_lock); return busy; } /** * thaw_workqueues - thaw workqueues * * Thaw workqueues. Normal queueing is restored and all collected |
7e11629d0 workqueue: use sh... |
3588 |
* frozen works are transferred to their respective gcwq worklists. |
a0a1a5fd4 workqueue: reimpl... |
3589 3590 |
* * CONTEXT: |
8b03ae3cd workqueue: introd... |
3591 |
* Grabs and releases workqueue_lock and gcwq->lock's. |
a0a1a5fd4 workqueue: reimpl... |
3592 3593 3594 |
*/ void thaw_workqueues(void) { |
a0a1a5fd4 workqueue: reimpl... |
3595 3596 3597 3598 3599 3600 |
unsigned int cpu; spin_lock(&workqueue_lock); if (!workqueue_freezing) goto out_unlock; |
f34217977 workqueue: implem... |
3601 |
for_each_gcwq_cpu(cpu) { |
8b03ae3cd workqueue: introd... |
3602 |
struct global_cwq *gcwq = get_gcwq(cpu); |
bdbc5dd7d workqueue: prepar... |
3603 |
struct workqueue_struct *wq; |
8b03ae3cd workqueue: introd... |
3604 3605 |
spin_lock_irq(&gcwq->lock); |
db7bccf45 workqueue: reimpl... |
3606 3607 |
BUG_ON(!(gcwq->flags & GCWQ_FREEZING)); gcwq->flags &= ~GCWQ_FREEZING; |
a0a1a5fd4 workqueue: reimpl... |
3608 3609 |
list_for_each_entry(wq, &workqueues, list) { struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); |
58a69cb47 workqueue, freeze... |
3610 |
if (!cwq || !(wq->flags & WQ_FREEZABLE)) |
a0a1a5fd4 workqueue: reimpl... |
3611 |
continue; |
a0a1a5fd4 workqueue: reimpl... |
3612 3613 3614 3615 3616 3617 |
/* restore max_active and repopulate worklist */ cwq->max_active = wq->saved_max_active; while (!list_empty(&cwq->delayed_works) && cwq->nr_active < cwq->max_active) cwq_activate_first_delayed(cwq); |
a0a1a5fd4 workqueue: reimpl... |
3618 |
} |
8b03ae3cd workqueue: introd... |
3619 |
|
e22bee782 workqueue: implem... |
3620 |
wake_up_worker(gcwq); |
8b03ae3cd workqueue: introd... |
3621 |
spin_unlock_irq(&gcwq->lock); |
a0a1a5fd4 workqueue: reimpl... |
3622 3623 3624 3625 3626 3627 3628 |
} workqueue_freezing = false; out_unlock: spin_unlock(&workqueue_lock); } #endif /* CONFIG_FREEZER */ |
6ee0578b4 workqueue: mark i... |
3629 |
static int __init init_workqueues(void) |
1da177e4c Linux-2.6.12-rc2 |
3630 |
{ |
c34056a3f workqueue: introd... |
3631 |
unsigned int cpu; |
c8e55f360 workqueue: implem... |
3632 |
int i; |
c34056a3f workqueue: introd... |
3633 |
|
f6500947a workqueue: workqu... |
3634 |
cpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE); |
8b03ae3cd workqueue: introd... |
3635 3636 |
/* initialize gcwqs */ |
f34217977 workqueue: implem... |
3637 |
for_each_gcwq_cpu(cpu) { |
8b03ae3cd workqueue: introd... |
3638 3639 3640 |
struct global_cwq *gcwq = get_gcwq(cpu); spin_lock_init(&gcwq->lock); |
7e11629d0 workqueue: use sh... |
3641 |
INIT_LIST_HEAD(&gcwq->worklist); |
8b03ae3cd workqueue: introd... |
3642 |
gcwq->cpu = cpu; |
477a3c33d workqueue: fix GC... |
3643 |
gcwq->flags |= GCWQ_DISASSOCIATED; |
8b03ae3cd workqueue: introd... |
3644 |
|
c8e55f360 workqueue: implem... |
3645 3646 3647 |
INIT_LIST_HEAD(&gcwq->idle_list); for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++) INIT_HLIST_HEAD(&gcwq->busy_hash[i]); |
e22bee782 workqueue: implem... |
3648 3649 3650 |
init_timer_deferrable(&gcwq->idle_timer); gcwq->idle_timer.function = idle_worker_timeout; gcwq->idle_timer.data = (unsigned long)gcwq; |
e7577c50f cpumask: convert ... |
3651 |
|
e22bee782 workqueue: implem... |
3652 3653 |
setup_timer(&gcwq->mayday_timer, gcwq_mayday_timeout, (unsigned long)gcwq); |
8b03ae3cd workqueue: introd... |
3654 |
ida_init(&gcwq->worker_ida); |
db7bccf45 workqueue: reimpl... |
3655 3656 3657 |
gcwq->trustee_state = TRUSTEE_DONE; init_waitqueue_head(&gcwq->trustee_wait); |
8b03ae3cd workqueue: introd... |
3658 |
} |
e22bee782 workqueue: implem... |
3659 |
/* create the initial worker */ |
f34217977 workqueue: implem... |
3660 |
for_each_online_gcwq_cpu(cpu) { |
e22bee782 workqueue: implem... |
3661 3662 |
struct global_cwq *gcwq = get_gcwq(cpu); struct worker *worker; |
477a3c33d workqueue: fix GC... |
3663 3664 |
if (cpu != WORK_CPU_UNBOUND) gcwq->flags &= ~GCWQ_DISASSOCIATED; |
e22bee782 workqueue: implem... |
3665 3666 3667 3668 3669 3670 |
worker = create_worker(gcwq, true); BUG_ON(!worker); spin_lock_irq(&gcwq->lock); start_worker(worker); spin_unlock_irq(&gcwq->lock); } |
d320c0383 workqueue: s/__cr... |
3671 3672 3673 |
system_wq = alloc_workqueue("events", 0, 0); system_long_wq = alloc_workqueue("events_long", 0, 0); system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0); |
f34217977 workqueue: implem... |
3674 3675 |
system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE); |
24d51add7 workqueue: fix bu... |
3676 3677 |
system_freezable_wq = alloc_workqueue("events_freezable", WQ_FREEZABLE, 0); |
e5cba24e3 workqueue: check ... |
3678 |
BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq || |
24d51add7 workqueue: fix bu... |
3679 |
!system_unbound_wq || !system_freezable_wq); |
6ee0578b4 workqueue: mark i... |
3680 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
3681 |
} |
6ee0578b4 workqueue: mark i... |
3682 |
early_initcall(init_workqueues); |