Blame view
kernel/sched/wait.c
17.7 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 |
/* * Generic waiting primitives. * |
6d49e352a propagate name ch... |
4 |
* (C) 2004 Nadia Yvette Chambers, Oracle |
1da177e4c Linux-2.6.12-rc2 |
5 |
*/ |
1da177e4c Linux-2.6.12-rc2 |
6 |
#include <linux/init.h> |
9984de1a5 kernel: Map most ... |
7 |
#include <linux/export.h> |
1da177e4c Linux-2.6.12-rc2 |
8 9 10 11 |
#include <linux/sched.h> #include <linux/mm.h> #include <linux/wait.h> #include <linux/hash.h> |
cb6538e74 sched/wait: Fix a... |
12 |
#include <linux/kthread.h> |
1da177e4c Linux-2.6.12-rc2 |
13 |
|
f07fdec50 lockdep/waitqueue... |
14 |
void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *key) |
21d71f513 [PATCH] uninline ... |
15 16 |
{ spin_lock_init(&q->lock); |
f07fdec50 lockdep/waitqueue... |
17 |
lockdep_set_class_and_name(&q->lock, key, name); |
21d71f513 [PATCH] uninline ... |
18 19 |
INIT_LIST_HEAD(&q->task_list); } |
eb4542b98 [PATCH] lockdep: ... |
20 |
|
2fc391112 locking, sched: G... |
21 |
EXPORT_SYMBOL(__init_waitqueue_head); |
eb4542b98 [PATCH] lockdep: ... |
22 |
|
7ad5b3a50 kernel: remove fa... |
23 |
void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
24 25 26 27 28 29 30 31 32 |
{ unsigned long flags; wait->flags &= ~WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); __add_wait_queue(q, wait); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(add_wait_queue); |
7ad5b3a50 kernel: remove fa... |
33 |
void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
34 35 36 37 38 39 40 41 42 |
{ unsigned long flags; wait->flags |= WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); __add_wait_queue_tail(q, wait); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(add_wait_queue_exclusive); |
7ad5b3a50 kernel: remove fa... |
43 |
void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
44 45 46 47 48 49 50 51 52 53 54 |
{ unsigned long flags; spin_lock_irqsave(&q->lock, flags); __remove_wait_queue(q, wait); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(remove_wait_queue); /* |
b4145872f sched: Move wait ... |
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
* The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve * number) then we wake all the non-exclusive tasks and one exclusive task. * * There are circumstances in which we can try to wake a task which has already * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns * zero in this (rare) case, and we handle it by continuing to scan the queue. */ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, int nr_exclusive, int wake_flags, void *key) { wait_queue_t *curr, *next; list_for_each_entry_safe(curr, next, &q->task_list, task_list) { unsigned flags = curr->flags; if (curr->func(curr, mode, wake_flags, key) && (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) break; } } /** * __wake_up - wake up threads blocked on a waitqueue. * @q: the waitqueue * @mode: which threads * @nr_exclusive: how many wake-one or wake-many threads to wake up * @key: is directly passed to the wakeup function * * It may be assumed that this function implies a write memory barrier before * changing the task state if and only if any tasks are woken up. */ void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr_exclusive, void *key) { unsigned long flags; spin_lock_irqsave(&q->lock, flags); __wake_up_common(q, mode, nr_exclusive, 0, key); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(__wake_up); /* * Same as __wake_up but called with the spinlock in wait_queue_head_t held. */ void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr) { __wake_up_common(q, mode, nr, 0, NULL); } EXPORT_SYMBOL_GPL(__wake_up_locked); void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key) { __wake_up_common(q, mode, 1, 0, key); } EXPORT_SYMBOL_GPL(__wake_up_locked_key); /** * __wake_up_sync_key - wake up threads blocked on a waitqueue. * @q: the waitqueue * @mode: which threads * @nr_exclusive: how many wake-one or wake-many threads to wake up * @key: opaque value to be passed to wakeup targets * * The sync wakeup differs that the waker knows that it will schedule * away soon, so while the target thread will be woken up, it will not * be migrated to another CPU - ie. the two threads are 'synchronized' * with each other. This can prevent needless bouncing between CPUs. * * On UP it can prevent extra preemption. * * It may be assumed that this function implies a write memory barrier before * changing the task state if and only if any tasks are woken up. */ void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr_exclusive, void *key) { unsigned long flags; int wake_flags = 1; /* XXX WF_SYNC */ if (unlikely(!q)) return; if (unlikely(nr_exclusive != 1)) wake_flags = 0; spin_lock_irqsave(&q->lock, flags); __wake_up_common(q, mode, nr_exclusive, wake_flags, key); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL_GPL(__wake_up_sync_key); /* * __wake_up_sync - see __wake_up_sync_key() */ void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) { __wake_up_sync_key(q, mode, nr_exclusive, NULL); } EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ /* |
1da177e4c Linux-2.6.12-rc2 |
158 159 160 161 162 163 164 165 166 |
* Note: we use "set_current_state()" _after_ the wait-queue add, * because we need a memory barrier there on SMP, so that any * wake-function that tests for the wait-queue being active * will be guaranteed to see waitqueue addition _or_ subsequent * tests in this thread will see the wakeup having taken place. * * The spin_unlock() itself is semi-permeable and only protects * one way (it only protects stuff inside the critical region and * stops them from bleeding out - it would still allow subsequent |
59c51591a Fix occurrences o... |
167 |
* loads to move into the critical region). |
1da177e4c Linux-2.6.12-rc2 |
168 |
*/ |
7ad5b3a50 kernel: remove fa... |
169 |
void |
1da177e4c Linux-2.6.12-rc2 |
170 171 172 173 174 175 176 177 |
prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state) { unsigned long flags; wait->flags &= ~WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); if (list_empty(&wait->task_list)) __add_wait_queue(q, wait); |
a25d644fc wait: kill is_syn... |
178 |
set_current_state(state); |
1da177e4c Linux-2.6.12-rc2 |
179 180 181 |
spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(prepare_to_wait); |
7ad5b3a50 kernel: remove fa... |
182 |
void |
1da177e4c Linux-2.6.12-rc2 |
183 184 185 186 187 188 189 190 |
prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state) { unsigned long flags; wait->flags |= WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); if (list_empty(&wait->task_list)) __add_wait_queue_tail(q, wait); |
a25d644fc wait: kill is_syn... |
191 |
set_current_state(state); |
1da177e4c Linux-2.6.12-rc2 |
192 193 194 |
spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(prepare_to_wait_exclusive); |
c2d816443 sched/wait: Intro... |
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 |
long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state) { unsigned long flags; if (signal_pending_state(state, current)) return -ERESTARTSYS; wait->private = current; wait->func = autoremove_wake_function; spin_lock_irqsave(&q->lock, flags); if (list_empty(&wait->task_list)) { if (wait->flags & WQ_FLAG_EXCLUSIVE) __add_wait_queue_tail(q, wait); else __add_wait_queue(q, wait); } set_current_state(state); spin_unlock_irqrestore(&q->lock, flags); return 0; } EXPORT_SYMBOL(prepare_to_wait_event); |
ee2f154a5 docbook: add more... |
218 |
/** |
777c6c5f1 wait: prevent exc... |
219 220 221 222 223 224 225 226 |
* finish_wait - clean up after waiting in a queue * @q: waitqueue waited on * @wait: wait descriptor * * Sets current thread back to running state and removes * the wait descriptor from the given waitqueue if still * queued. */ |
7ad5b3a50 kernel: remove fa... |
227 |
void finish_wait(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 |
{ unsigned long flags; __set_current_state(TASK_RUNNING); /* * We can check for list emptiness outside the lock * IFF: * - we use the "careful" check that verifies both * the next and prev pointers, so that there cannot * be any half-pending updates in progress on other * CPU's that we haven't seen yet (and that might * still change the stack area. * and * - all other users take the lock (ie we can only * have _one_ other CPU that looks at or modifies * the list). */ if (!list_empty_careful(&wait->task_list)) { spin_lock_irqsave(&q->lock, flags); list_del_init(&wait->task_list); spin_unlock_irqrestore(&q->lock, flags); } } EXPORT_SYMBOL(finish_wait); |
ee2f154a5 docbook: add more... |
252 |
/** |
777c6c5f1 wait: prevent exc... |
253 254 255 |
* abort_exclusive_wait - abort exclusive waiting in a queue * @q: waitqueue waited on * @wait: wait descriptor |
ee2f154a5 docbook: add more... |
256 |
* @mode: runstate of the waiter to be woken |
777c6c5f1 wait: prevent exc... |
257 258 259 260 261 262 263 264 265 266 |
* @key: key to identify a wait bit queue or %NULL * * Sets current thread back to running state and removes * the wait descriptor from the given waitqueue if still * queued. * * Wakes up the next waiter if the caller is concurrently * woken up through the queue. * * This prevents waiter starvation where an exclusive waiter |
25985edce Fix common misspe... |
267 |
* aborts and is woken up concurrently and no one wakes up |
777c6c5f1 wait: prevent exc... |
268 269 270 271 272 273 274 275 276 277 278 279 |
* the next waiter. */ void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, unsigned int mode, void *key) { unsigned long flags; __set_current_state(TASK_RUNNING); spin_lock_irqsave(&q->lock, flags); if (!list_empty(&wait->task_list)) list_del_init(&wait->task_list); else if (waitqueue_active(q)) |
78ddb08fe wait: don't use _... |
280 |
__wake_up_locked_key(q, mode, key); |
777c6c5f1 wait: prevent exc... |
281 282 283 |
spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(abort_exclusive_wait); |
1da177e4c Linux-2.6.12-rc2 |
284 285 286 287 288 289 290 291 292 |
int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) { int ret = default_wake_function(wait, mode, sync, key); if (ret) list_del_init(&wait->task_list); return ret; } EXPORT_SYMBOL(autoremove_wake_function); |
cb6538e74 sched/wait: Fix a... |
293 294 295 296 |
static inline bool is_kthread_should_stop(void) { return (current->flags & PF_KTHREAD) && kthread_should_stop(); } |
61ada528d sched/wait: Provi... |
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 |
/* * DEFINE_WAIT_FUNC(wait, woken_wake_func); * * add_wait_queue(&wq, &wait); * for (;;) { * if (condition) * break; * * p->state = mode; condition = true; * smp_mb(); // A smp_wmb(); // C * if (!wait->flags & WQ_FLAG_WOKEN) wait->flags |= WQ_FLAG_WOKEN; * schedule() try_to_wake_up(); * p->state = TASK_RUNNING; ~~~~~~~~~~~~~~~~~~ * wait->flags &= ~WQ_FLAG_WOKEN; condition = true; * smp_mb() // B smp_wmb(); // C * wait->flags |= WQ_FLAG_WOKEN; * } * remove_wait_queue(&wq, &wait); * */ long wait_woken(wait_queue_t *wait, unsigned mode, long timeout) { set_current_state(mode); /* A */ /* * The above implies an smp_mb(), which matches with the smp_wmb() from * woken_wake_function() such that if we observe WQ_FLAG_WOKEN we must * also observe all state before the wakeup. */ |
cb6538e74 sched/wait: Fix a... |
326 |
if (!(wait->flags & WQ_FLAG_WOKEN) && !is_kthread_should_stop()) |
61ada528d sched/wait: Provi... |
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 |
timeout = schedule_timeout(timeout); __set_current_state(TASK_RUNNING); /* * The below implies an smp_mb(), it too pairs with the smp_wmb() from * woken_wake_function() such that we must either observe the wait * condition being true _OR_ WQ_FLAG_WOKEN such that we will not miss * an event. */ set_mb(wait->flags, wait->flags & ~WQ_FLAG_WOKEN); /* B */ return timeout; } EXPORT_SYMBOL(wait_woken); int woken_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) { /* * Although this function is called under waitqueue lock, LOCK * doesn't imply write barrier and the users expects write * barrier semantics on wakeup functions. The following * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up() * and is paired with set_mb() in wait_woken(). */ smp_wmb(); /* C */ wait->flags |= WQ_FLAG_WOKEN; return default_wake_function(wait, mode, sync, key); } EXPORT_SYMBOL(woken_wake_function); |
1da177e4c Linux-2.6.12-rc2 |
357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 |
int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue *wait_bit = container_of(wait, struct wait_bit_queue, wait); if (wait_bit->key.flags != key->flags || wait_bit->key.bit_nr != key->bit_nr || test_bit(key->bit_nr, key->flags)) return 0; else return autoremove_wake_function(wait, mode, sync, key); } EXPORT_SYMBOL(wake_bit_function); /* * To allow interruptible waiting and asynchronous (i.e. nonblocking) * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are * permitted return codes. Nonzero return codes halt waiting and return. */ |
7ad5b3a50 kernel: remove fa... |
377 |
int __sched |
1da177e4c Linux-2.6.12-rc2 |
378 |
__wait_on_bit(wait_queue_head_t *wq, struct wait_bit_queue *q, |
c1221321b sched: Allow wait... |
379 |
wait_bit_action_f *action, unsigned mode) |
1da177e4c Linux-2.6.12-rc2 |
380 381 382 383 384 385 |
{ int ret = 0; do { prepare_to_wait(wq, &q->wait, mode); if (test_bit(q->key.bit_nr, q->key.flags)) |
c1221321b sched: Allow wait... |
386 |
ret = (*action)(&q->key); |
1da177e4c Linux-2.6.12-rc2 |
387 388 389 390 391 |
} while (test_bit(q->key.bit_nr, q->key.flags) && !ret); finish_wait(wq, &q->wait); return ret; } EXPORT_SYMBOL(__wait_on_bit); |
7ad5b3a50 kernel: remove fa... |
392 |
int __sched out_of_line_wait_on_bit(void *word, int bit, |
c1221321b sched: Allow wait... |
393 |
wait_bit_action_f *action, unsigned mode) |
1da177e4c Linux-2.6.12-rc2 |
394 395 396 397 398 399 400 |
{ wait_queue_head_t *wq = bit_waitqueue(word, bit); DEFINE_WAIT_BIT(wait, word, bit); return __wait_on_bit(wq, &wait, action, mode); } EXPORT_SYMBOL(out_of_line_wait_on_bit); |
cbbce8220 SCHED: add some "... |
401 402 403 404 405 406 407 408 409 410 411 |
int __sched out_of_line_wait_on_bit_timeout( void *word, int bit, wait_bit_action_f *action, unsigned mode, unsigned long timeout) { wait_queue_head_t *wq = bit_waitqueue(word, bit); DEFINE_WAIT_BIT(wait, word, bit); wait.key.timeout = jiffies + timeout; return __wait_on_bit(wq, &wait, action, mode); } EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout); |
7ad5b3a50 kernel: remove fa... |
412 |
int __sched |
1da177e4c Linux-2.6.12-rc2 |
413 |
__wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q, |
c1221321b sched: Allow wait... |
414 |
wait_bit_action_f *action, unsigned mode) |
1da177e4c Linux-2.6.12-rc2 |
415 |
{ |
1da177e4c Linux-2.6.12-rc2 |
416 |
do { |
777c6c5f1 wait: prevent exc... |
417 |
int ret; |
1da177e4c Linux-2.6.12-rc2 |
418 |
prepare_to_wait_exclusive(wq, &q->wait, mode); |
777c6c5f1 wait: prevent exc... |
419 420 |
if (!test_bit(q->key.bit_nr, q->key.flags)) continue; |
c1221321b sched: Allow wait... |
421 |
ret = action(&q->key); |
777c6c5f1 wait: prevent exc... |
422 423 424 425 |
if (!ret) continue; abort_exclusive_wait(wq, &q->wait, mode, &q->key); return ret; |
1da177e4c Linux-2.6.12-rc2 |
426 427 |
} while (test_and_set_bit(q->key.bit_nr, q->key.flags)); finish_wait(wq, &q->wait); |
777c6c5f1 wait: prevent exc... |
428 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
429 430 |
} EXPORT_SYMBOL(__wait_on_bit_lock); |
7ad5b3a50 kernel: remove fa... |
431 |
int __sched out_of_line_wait_on_bit_lock(void *word, int bit, |
c1221321b sched: Allow wait... |
432 |
wait_bit_action_f *action, unsigned mode) |
1da177e4c Linux-2.6.12-rc2 |
433 434 435 436 437 438 439 |
{ wait_queue_head_t *wq = bit_waitqueue(word, bit); DEFINE_WAIT_BIT(wait, word, bit); return __wait_on_bit_lock(wq, &wait, action, mode); } EXPORT_SYMBOL(out_of_line_wait_on_bit_lock); |
7ad5b3a50 kernel: remove fa... |
440 |
void __wake_up_bit(wait_queue_head_t *wq, void *word, int bit) |
1da177e4c Linux-2.6.12-rc2 |
441 442 443 |
{ struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); if (waitqueue_active(wq)) |
e64d66c8e wait: Use TASK_NO... |
444 |
__wake_up(wq, TASK_NORMAL, 1, &key); |
1da177e4c Linux-2.6.12-rc2 |
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 |
} EXPORT_SYMBOL(__wake_up_bit); /** * wake_up_bit - wake up a waiter on a bit * @word: the word being waited on, a kernel virtual address * @bit: the bit of the word being waited on * * There is a standard hashed waitqueue table for generic use. This * is the part of the hashtable's accessor API that wakes up waiters * on a bit. For instance, if one were to have waiters on a bitflag, * one would call wake_up_bit() after clearing the bit. * * In order for this to function properly, as it uses waitqueue_active() * internally, some kind of memory barrier must be done prior to calling |
4e857c58e arch: Mass conver... |
460 |
* this. Typically, this will be smp_mb__after_atomic(), but in some |
1da177e4c Linux-2.6.12-rc2 |
461 462 463 464 |
* cases where bitflags are manipulated non-atomically under a lock, one * may need to use a less regular barrier, such fs/inode.c's smp_mb(), * because spin_unlock() does not guarantee a memory barrier. */ |
7ad5b3a50 kernel: remove fa... |
465 |
void wake_up_bit(void *word, int bit) |
1da177e4c Linux-2.6.12-rc2 |
466 467 468 469 |
{ __wake_up_bit(bit_waitqueue(word, bit), word, bit); } EXPORT_SYMBOL(wake_up_bit); |
7ad5b3a50 kernel: remove fa... |
470 |
wait_queue_head_t *bit_waitqueue(void *word, int bit) |
1da177e4c Linux-2.6.12-rc2 |
471 472 473 474 475 476 477 478 |
{ const int shift = BITS_PER_LONG == 32 ? 5 : 6; const struct zone *zone = page_zone(virt_to_page(word)); unsigned long val = (unsigned long)word << shift | bit; return &zone->wait_table[hash_long(val, zone->wait_table_bits)]; } EXPORT_SYMBOL(bit_waitqueue); |
cb65537ee Add wait_on_atomi... |
479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 |
/* * Manipulate the atomic_t address to produce a better bit waitqueue table hash * index (we're keying off bit -1, but that would produce a horrible hash * value). */ static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p) { if (BITS_PER_LONG == 64) { unsigned long q = (unsigned long)p; return bit_waitqueue((void *)(q & ~1), q & 1); } return bit_waitqueue(p, 0); } static int wake_atomic_t_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue *wait_bit = container_of(wait, struct wait_bit_queue, wait); atomic_t *val = key->flags; if (wait_bit->key.flags != key->flags || wait_bit->key.bit_nr != key->bit_nr || atomic_read(val) != 0) return 0; return autoremove_wake_function(wait, mode, sync, key); } /* * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting, * the actions of __wait_on_atomic_t() are permitted return codes. Nonzero * return codes halt waiting and return. */ static __sched int __wait_on_atomic_t(wait_queue_head_t *wq, struct wait_bit_queue *q, int (*action)(atomic_t *), unsigned mode) { atomic_t *val; int ret = 0; do { prepare_to_wait(wq, &q->wait, mode); val = q->key.flags; if (atomic_read(val) == 0) |
42577ca8c Fix __wait_on_ato... |
525 526 |
break; ret = (*action)(val); |
cb65537ee Add wait_on_atomi... |
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 |
} while (!ret && atomic_read(val) != 0); finish_wait(wq, &q->wait); return ret; } #define DEFINE_WAIT_ATOMIC_T(name, p) \ struct wait_bit_queue name = { \ .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \ .wait = { \ .private = current, \ .func = wake_atomic_t_function, \ .task_list = \ LIST_HEAD_INIT((name).wait.task_list), \ }, \ } __sched int out_of_line_wait_on_atomic_t(atomic_t *p, int (*action)(atomic_t *), unsigned mode) { wait_queue_head_t *wq = atomic_t_waitqueue(p); DEFINE_WAIT_ATOMIC_T(wait, p); return __wait_on_atomic_t(wq, &wait, action, mode); } EXPORT_SYMBOL(out_of_line_wait_on_atomic_t); /** * wake_up_atomic_t - Wake up a waiter on a atomic_t |
2203547f8 kernel: fix new k... |
555 |
* @p: The atomic_t being waited on, a kernel virtual address |
cb65537ee Add wait_on_atomi... |
556 557 558 559 560 561 562 563 564 565 566 |
* * Wake up anyone waiting for the atomic_t to go to zero. * * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t * check is done by the waiter's wake function, not the by the waker itself). */ void wake_up_atomic_t(atomic_t *p) { __wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR); } EXPORT_SYMBOL(wake_up_atomic_t); |
743162013 sched: Remove pro... |
567 |
|
c1221321b sched: Allow wait... |
568 |
__sched int bit_wait(struct wait_bit_key *word) |
743162013 sched: Remove pro... |
569 570 571 572 573 574 575 |
{ if (signal_pending_state(current->state, current)) return 1; schedule(); return 0; } EXPORT_SYMBOL(bit_wait); |
c1221321b sched: Allow wait... |
576 |
__sched int bit_wait_io(struct wait_bit_key *word) |
743162013 sched: Remove pro... |
577 578 579 580 581 582 583 |
{ if (signal_pending_state(current->state, current)) return 1; io_schedule(); return 0; } EXPORT_SYMBOL(bit_wait_io); |
cbbce8220 SCHED: add some "... |
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 |
__sched int bit_wait_timeout(struct wait_bit_key *word) { unsigned long now = ACCESS_ONCE(jiffies); if (signal_pending_state(current->state, current)) return 1; if (time_after_eq(now, word->timeout)) return -EAGAIN; schedule_timeout(word->timeout - now); return 0; } EXPORT_SYMBOL_GPL(bit_wait_timeout); __sched int bit_wait_io_timeout(struct wait_bit_key *word) { unsigned long now = ACCESS_ONCE(jiffies); if (signal_pending_state(current->state, current)) return 1; if (time_after_eq(now, word->timeout)) return -EAGAIN; io_schedule_timeout(word->timeout - now); return 0; } EXPORT_SYMBOL_GPL(bit_wait_io_timeout); |