Blame view
kernel/wait.c
10.7 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 |
/* * Generic waiting primitives. * |
6d49e352a propagate name ch... |
4 |
* (C) 2004 Nadia Yvette Chambers, Oracle |
1da177e4c Linux-2.6.12-rc2 |
5 |
*/ |
1da177e4c Linux-2.6.12-rc2 |
6 |
#include <linux/init.h> |
9984de1a5 kernel: Map most ... |
7 |
#include <linux/export.h> |
1da177e4c Linux-2.6.12-rc2 |
8 9 10 11 |
#include <linux/sched.h> #include <linux/mm.h> #include <linux/wait.h> #include <linux/hash.h> |
f07fdec50 lockdep/waitqueue... |
12 |
void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct lock_class_key *key) |
21d71f513 [PATCH] uninline ... |
13 14 |
{ spin_lock_init(&q->lock); |
f07fdec50 lockdep/waitqueue... |
15 |
lockdep_set_class_and_name(&q->lock, key, name); |
21d71f513 [PATCH] uninline ... |
16 17 |
INIT_LIST_HEAD(&q->task_list); } |
eb4542b98 [PATCH] lockdep: ... |
18 |
|
2fc391112 locking, sched: G... |
19 |
EXPORT_SYMBOL(__init_waitqueue_head); |
eb4542b98 [PATCH] lockdep: ... |
20 |
|
7ad5b3a50 kernel: remove fa... |
21 |
void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
22 23 24 25 26 27 28 29 30 |
{ unsigned long flags; wait->flags &= ~WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); __add_wait_queue(q, wait); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(add_wait_queue); |
7ad5b3a50 kernel: remove fa... |
31 |
void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
32 33 34 35 36 37 38 39 40 |
{ unsigned long flags; wait->flags |= WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); __add_wait_queue_tail(q, wait); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(add_wait_queue_exclusive); |
7ad5b3a50 kernel: remove fa... |
41 |
void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
{ unsigned long flags; spin_lock_irqsave(&q->lock, flags); __remove_wait_queue(q, wait); spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(remove_wait_queue); /* * Note: we use "set_current_state()" _after_ the wait-queue add, * because we need a memory barrier there on SMP, so that any * wake-function that tests for the wait-queue being active * will be guaranteed to see waitqueue addition _or_ subsequent * tests in this thread will see the wakeup having taken place. * * The spin_unlock() itself is semi-permeable and only protects * one way (it only protects stuff inside the critical region and * stops them from bleeding out - it would still allow subsequent |
59c51591a Fix occurrences o... |
62 |
* loads to move into the critical region). |
1da177e4c Linux-2.6.12-rc2 |
63 |
*/ |
7ad5b3a50 kernel: remove fa... |
64 |
void |
1da177e4c Linux-2.6.12-rc2 |
65 66 67 68 69 70 71 72 |
prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state) { unsigned long flags; wait->flags &= ~WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); if (list_empty(&wait->task_list)) __add_wait_queue(q, wait); |
a25d644fc wait: kill is_syn... |
73 |
set_current_state(state); |
1da177e4c Linux-2.6.12-rc2 |
74 75 76 |
spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(prepare_to_wait); |
7ad5b3a50 kernel: remove fa... |
77 |
void |
1da177e4c Linux-2.6.12-rc2 |
78 79 80 81 82 83 84 85 |
prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state) { unsigned long flags; wait->flags |= WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); if (list_empty(&wait->task_list)) __add_wait_queue_tail(q, wait); |
a25d644fc wait: kill is_syn... |
86 |
set_current_state(state); |
1da177e4c Linux-2.6.12-rc2 |
87 88 89 |
spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(prepare_to_wait_exclusive); |
ee2f154a5 docbook: add more... |
90 |
/** |
777c6c5f1 wait: prevent exc... |
91 92 93 94 95 96 97 98 |
* finish_wait - clean up after waiting in a queue * @q: waitqueue waited on * @wait: wait descriptor * * Sets current thread back to running state and removes * the wait descriptor from the given waitqueue if still * queued. */ |
7ad5b3a50 kernel: remove fa... |
99 |
void finish_wait(wait_queue_head_t *q, wait_queue_t *wait) |
1da177e4c Linux-2.6.12-rc2 |
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
{ unsigned long flags; __set_current_state(TASK_RUNNING); /* * We can check for list emptiness outside the lock * IFF: * - we use the "careful" check that verifies both * the next and prev pointers, so that there cannot * be any half-pending updates in progress on other * CPU's that we haven't seen yet (and that might * still change the stack area. * and * - all other users take the lock (ie we can only * have _one_ other CPU that looks at or modifies * the list). */ if (!list_empty_careful(&wait->task_list)) { spin_lock_irqsave(&q->lock, flags); list_del_init(&wait->task_list); spin_unlock_irqrestore(&q->lock, flags); } } EXPORT_SYMBOL(finish_wait); |
ee2f154a5 docbook: add more... |
124 |
/** |
777c6c5f1 wait: prevent exc... |
125 126 127 |
* abort_exclusive_wait - abort exclusive waiting in a queue * @q: waitqueue waited on * @wait: wait descriptor |
ee2f154a5 docbook: add more... |
128 |
* @mode: runstate of the waiter to be woken |
777c6c5f1 wait: prevent exc... |
129 130 131 132 133 134 135 136 137 138 |
* @key: key to identify a wait bit queue or %NULL * * Sets current thread back to running state and removes * the wait descriptor from the given waitqueue if still * queued. * * Wakes up the next waiter if the caller is concurrently * woken up through the queue. * * This prevents waiter starvation where an exclusive waiter |
25985edce Fix common misspe... |
139 |
* aborts and is woken up concurrently and no one wakes up |
777c6c5f1 wait: prevent exc... |
140 141 142 143 144 145 146 147 148 149 150 151 |
* the next waiter. */ void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, unsigned int mode, void *key) { unsigned long flags; __set_current_state(TASK_RUNNING); spin_lock_irqsave(&q->lock, flags); if (!list_empty(&wait->task_list)) list_del_init(&wait->task_list); else if (waitqueue_active(q)) |
78ddb08fe wait: don't use _... |
152 |
__wake_up_locked_key(q, mode, key); |
777c6c5f1 wait: prevent exc... |
153 154 155 |
spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(abort_exclusive_wait); |
1da177e4c Linux-2.6.12-rc2 |
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 |
int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) { int ret = default_wake_function(wait, mode, sync, key); if (ret) list_del_init(&wait->task_list); return ret; } EXPORT_SYMBOL(autoremove_wake_function); int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue *wait_bit = container_of(wait, struct wait_bit_queue, wait); if (wait_bit->key.flags != key->flags || wait_bit->key.bit_nr != key->bit_nr || test_bit(key->bit_nr, key->flags)) return 0; else return autoremove_wake_function(wait, mode, sync, key); } EXPORT_SYMBOL(wake_bit_function); /* * To allow interruptible waiting and asynchronous (i.e. nonblocking) * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are * permitted return codes. Nonzero return codes halt waiting and return. */ |
7ad5b3a50 kernel: remove fa... |
186 |
int __sched |
1da177e4c Linux-2.6.12-rc2 |
187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
__wait_on_bit(wait_queue_head_t *wq, struct wait_bit_queue *q, int (*action)(void *), unsigned mode) { int ret = 0; do { prepare_to_wait(wq, &q->wait, mode); if (test_bit(q->key.bit_nr, q->key.flags)) ret = (*action)(q->key.flags); } while (test_bit(q->key.bit_nr, q->key.flags) && !ret); finish_wait(wq, &q->wait); return ret; } EXPORT_SYMBOL(__wait_on_bit); |
7ad5b3a50 kernel: remove fa... |
201 |
int __sched out_of_line_wait_on_bit(void *word, int bit, |
1da177e4c Linux-2.6.12-rc2 |
202 203 204 205 206 207 208 209 |
int (*action)(void *), unsigned mode) { wait_queue_head_t *wq = bit_waitqueue(word, bit); DEFINE_WAIT_BIT(wait, word, bit); return __wait_on_bit(wq, &wait, action, mode); } EXPORT_SYMBOL(out_of_line_wait_on_bit); |
7ad5b3a50 kernel: remove fa... |
210 |
int __sched |
1da177e4c Linux-2.6.12-rc2 |
211 212 213 |
__wait_on_bit_lock(wait_queue_head_t *wq, struct wait_bit_queue *q, int (*action)(void *), unsigned mode) { |
1da177e4c Linux-2.6.12-rc2 |
214 |
do { |
777c6c5f1 wait: prevent exc... |
215 |
int ret; |
1da177e4c Linux-2.6.12-rc2 |
216 |
prepare_to_wait_exclusive(wq, &q->wait, mode); |
777c6c5f1 wait: prevent exc... |
217 218 219 220 221 222 223 |
if (!test_bit(q->key.bit_nr, q->key.flags)) continue; ret = action(q->key.flags); if (!ret) continue; abort_exclusive_wait(wq, &q->wait, mode, &q->key); return ret; |
1da177e4c Linux-2.6.12-rc2 |
224 225 |
} while (test_and_set_bit(q->key.bit_nr, q->key.flags)); finish_wait(wq, &q->wait); |
777c6c5f1 wait: prevent exc... |
226 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
227 228 |
} EXPORT_SYMBOL(__wait_on_bit_lock); |
7ad5b3a50 kernel: remove fa... |
229 |
int __sched out_of_line_wait_on_bit_lock(void *word, int bit, |
1da177e4c Linux-2.6.12-rc2 |
230 231 232 233 234 235 236 237 |
int (*action)(void *), unsigned mode) { wait_queue_head_t *wq = bit_waitqueue(word, bit); DEFINE_WAIT_BIT(wait, word, bit); return __wait_on_bit_lock(wq, &wait, action, mode); } EXPORT_SYMBOL(out_of_line_wait_on_bit_lock); |
7ad5b3a50 kernel: remove fa... |
238 |
void __wake_up_bit(wait_queue_head_t *wq, void *word, int bit) |
1da177e4c Linux-2.6.12-rc2 |
239 240 241 |
{ struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); if (waitqueue_active(wq)) |
e64d66c8e wait: Use TASK_NO... |
242 |
__wake_up(wq, TASK_NORMAL, 1, &key); |
1da177e4c Linux-2.6.12-rc2 |
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 |
} EXPORT_SYMBOL(__wake_up_bit); /** * wake_up_bit - wake up a waiter on a bit * @word: the word being waited on, a kernel virtual address * @bit: the bit of the word being waited on * * There is a standard hashed waitqueue table for generic use. This * is the part of the hashtable's accessor API that wakes up waiters * on a bit. For instance, if one were to have waiters on a bitflag, * one would call wake_up_bit() after clearing the bit. * * In order for this to function properly, as it uses waitqueue_active() * internally, some kind of memory barrier must be done prior to calling * this. Typically, this will be smp_mb__after_clear_bit(), but in some * cases where bitflags are manipulated non-atomically under a lock, one * may need to use a less regular barrier, such fs/inode.c's smp_mb(), * because spin_unlock() does not guarantee a memory barrier. */ |
7ad5b3a50 kernel: remove fa... |
263 |
void wake_up_bit(void *word, int bit) |
1da177e4c Linux-2.6.12-rc2 |
264 265 266 267 |
{ __wake_up_bit(bit_waitqueue(word, bit), word, bit); } EXPORT_SYMBOL(wake_up_bit); |
7ad5b3a50 kernel: remove fa... |
268 |
wait_queue_head_t *bit_waitqueue(void *word, int bit) |
1da177e4c Linux-2.6.12-rc2 |
269 270 271 272 273 274 275 276 |
{ const int shift = BITS_PER_LONG == 32 ? 5 : 6; const struct zone *zone = page_zone(virt_to_page(word)); unsigned long val = (unsigned long)word << shift | bit; return &zone->wait_table[hash_long(val, zone->wait_table_bits)]; } EXPORT_SYMBOL(bit_waitqueue); |
cb65537ee Add wait_on_atomi... |
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 |
/* * Manipulate the atomic_t address to produce a better bit waitqueue table hash * index (we're keying off bit -1, but that would produce a horrible hash * value). */ static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p) { if (BITS_PER_LONG == 64) { unsigned long q = (unsigned long)p; return bit_waitqueue((void *)(q & ~1), q & 1); } return bit_waitqueue(p, 0); } static int wake_atomic_t_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue *wait_bit = container_of(wait, struct wait_bit_queue, wait); atomic_t *val = key->flags; if (wait_bit->key.flags != key->flags || wait_bit->key.bit_nr != key->bit_nr || atomic_read(val) != 0) return 0; return autoremove_wake_function(wait, mode, sync, key); } /* * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting, * the actions of __wait_on_atomic_t() are permitted return codes. Nonzero * return codes halt waiting and return. */ static __sched int __wait_on_atomic_t(wait_queue_head_t *wq, struct wait_bit_queue *q, int (*action)(atomic_t *), unsigned mode) { atomic_t *val; int ret = 0; do { prepare_to_wait(wq, &q->wait, mode); val = q->key.flags; if (atomic_read(val) == 0) |
42577ca8c Fix __wait_on_ato... |
323 324 |
break; ret = (*action)(val); |
cb65537ee Add wait_on_atomi... |
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 |
} while (!ret && atomic_read(val) != 0); finish_wait(wq, &q->wait); return ret; } #define DEFINE_WAIT_ATOMIC_T(name, p) \ struct wait_bit_queue name = { \ .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \ .wait = { \ .private = current, \ .func = wake_atomic_t_function, \ .task_list = \ LIST_HEAD_INIT((name).wait.task_list), \ }, \ } __sched int out_of_line_wait_on_atomic_t(atomic_t *p, int (*action)(atomic_t *), unsigned mode) { wait_queue_head_t *wq = atomic_t_waitqueue(p); DEFINE_WAIT_ATOMIC_T(wait, p); return __wait_on_atomic_t(wq, &wait, action, mode); } EXPORT_SYMBOL(out_of_line_wait_on_atomic_t); /** * wake_up_atomic_t - Wake up a waiter on a atomic_t * @word: The word being waited on, a kernel virtual address * @bit: The bit of the word being waited on * * Wake up anyone waiting for the atomic_t to go to zero. * * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t * check is done by the waiter's wake function, not the by the waker itself). */ void wake_up_atomic_t(atomic_t *p) { __wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR); } EXPORT_SYMBOL(wake_up_atomic_t); |