Blame view
kernel/spinlock.c
8.96 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 |
/* * Copyright (2004) Linus Torvalds * * Author: Zwane Mwaikambo <zwane@fsmlabs.com> * |
fb1c8f93d [PATCH] spinlock ... |
6 7 8 9 |
* Copyright (2004, 2005) Ingo Molnar * * This file contains the spinlock/rwlock implementations for the * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) |
0cb91a229 [PATCH] i386: Acc... |
10 11 12 13 14 |
* * Note that some architectures have special knowledge about the * stack frames of these functions in their profile_pc. If you * change anything significant here that could change the stack * frame contact the architecture maintainers. |
1da177e4c Linux-2.6.12-rc2 |
15 |
*/ |
1da177e4c Linux-2.6.12-rc2 |
16 17 18 19 |
#include <linux/linkage.h> #include <linux/preempt.h> #include <linux/spinlock.h> #include <linux/interrupt.h> |
8a25d5deb [PATCH] lockdep: ... |
20 |
#include <linux/debug_locks.h> |
1da177e4c Linux-2.6.12-rc2 |
21 |
#include <linux/module.h> |
8e13c7b77 locking: Reduce i... |
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
/* * If lockdep is enabled then we use the non-preemption spin-ops * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are * not re-enabled during lock-acquire (which the preempt-spin-ops do): */ #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) /* * The __lock_function inlines are taken from * include/linux/spinlock_api_smp.h */ #else /* * We build the __lock_function inlines here. They are too large for * inlining all over the place, but here is only one user per function * which embedds them into the calling _lock_function below. * * This could be a long-held lock. We both prepare to spin for a long * time (making _this_ CPU preemptable if possible), and we also signal * towards that other CPU that it should break the lock ASAP. */ #define BUILD_LOCK_OPS(op, locktype) \ void __lockfunc __##op##_lock(locktype##_t *lock) \ { \ for (;;) { \ preempt_disable(); \ if (likely(_raw_##op##_trylock(lock))) \ break; \ preempt_enable(); \ \ if (!(lock)->break_lock) \ (lock)->break_lock = 1; \ while (!op##_can_lock(lock) && (lock)->break_lock) \ _raw_##op##_relax(&lock->raw_lock); \ } \ (lock)->break_lock = 0; \ } \ \ unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock) \ { \ unsigned long flags; \ \ for (;;) { \ preempt_disable(); \ local_irq_save(flags); \ if (likely(_raw_##op##_trylock(lock))) \ break; \ local_irq_restore(flags); \ preempt_enable(); \ \ if (!(lock)->break_lock) \ (lock)->break_lock = 1; \ while (!op##_can_lock(lock) && (lock)->break_lock) \ _raw_##op##_relax(&lock->raw_lock); \ } \ (lock)->break_lock = 0; \ return flags; \ } \ \ void __lockfunc __##op##_lock_irq(locktype##_t *lock) \ { \ _##op##_lock_irqsave(lock); \ } \ \ void __lockfunc __##op##_lock_bh(locktype##_t *lock) \ { \ unsigned long flags; \ \ /* */ \ /* Careful: we must exclude softirqs too, hence the */ \ /* irq-disabling. We use the generic preemption-aware */ \ /* function: */ \ /**/ \ flags = _##op##_lock_irqsave(lock); \ local_bh_disable(); \ local_irq_restore(flags); \ } \ /* * Build preemption-friendly versions of the following * lock-spinning functions: * * __[spin|read|write]_lock() * __[spin|read|write]_lock_irq() * __[spin|read|write]_lock_irqsave() * __[spin|read|write]_lock_bh() */ BUILD_LOCK_OPS(spin, spinlock); BUILD_LOCK_OPS(read, rwlock); BUILD_LOCK_OPS(write, rwlock); #endif #ifdef CONFIG_DEBUG_LOCK_ALLOC void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) { preempt_disable(); spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); } EXPORT_SYMBOL(_spin_lock_nested); unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass) { unsigned long flags; local_irq_save(flags); preempt_disable(); spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock, _raw_spin_lock_flags, &flags); return flags; } EXPORT_SYMBOL(_spin_lock_irqsave_nested); void __lockfunc _spin_lock_nest_lock(spinlock_t *lock, struct lockdep_map *nest_lock) { preempt_disable(); spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); } EXPORT_SYMBOL(_spin_lock_nest_lock); #endif |
6beb00092 locking: Make inl... |
148 |
#ifndef CONFIG_INLINE_SPIN_TRYLOCK |
1da177e4c Linux-2.6.12-rc2 |
149 150 |
int __lockfunc _spin_trylock(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
151 |
return __spin_trylock(lock); |
1da177e4c Linux-2.6.12-rc2 |
152 153 |
} EXPORT_SYMBOL(_spin_trylock); |
892a7c67c locking: Allow ar... |
154 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
155 |
|
6beb00092 locking: Make inl... |
156 |
#ifndef CONFIG_INLINE_READ_TRYLOCK |
1da177e4c Linux-2.6.12-rc2 |
157 158 |
int __lockfunc _read_trylock(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
159 |
return __read_trylock(lock); |
1da177e4c Linux-2.6.12-rc2 |
160 161 |
} EXPORT_SYMBOL(_read_trylock); |
892a7c67c locking: Allow ar... |
162 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
163 |
|
6beb00092 locking: Make inl... |
164 |
#ifndef CONFIG_INLINE_WRITE_TRYLOCK |
1da177e4c Linux-2.6.12-rc2 |
165 166 |
int __lockfunc _write_trylock(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
167 |
return __write_trylock(lock); |
1da177e4c Linux-2.6.12-rc2 |
168 169 |
} EXPORT_SYMBOL(_write_trylock); |
892a7c67c locking: Allow ar... |
170 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
171 |
|
6beb00092 locking: Make inl... |
172 |
#ifndef CONFIG_INLINE_READ_LOCK |
1da177e4c Linux-2.6.12-rc2 |
173 174 |
void __lockfunc _read_lock(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
175 |
__read_lock(lock); |
1da177e4c Linux-2.6.12-rc2 |
176 177 |
} EXPORT_SYMBOL(_read_lock); |
892a7c67c locking: Allow ar... |
178 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
179 |
|
6beb00092 locking: Make inl... |
180 |
#ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE |
1da177e4c Linux-2.6.12-rc2 |
181 182 |
unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
183 |
return __spin_lock_irqsave(lock); |
1da177e4c Linux-2.6.12-rc2 |
184 185 |
} EXPORT_SYMBOL(_spin_lock_irqsave); |
892a7c67c locking: Allow ar... |
186 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
187 |
|
6beb00092 locking: Make inl... |
188 |
#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ |
1da177e4c Linux-2.6.12-rc2 |
189 190 |
void __lockfunc _spin_lock_irq(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
191 |
__spin_lock_irq(lock); |
1da177e4c Linux-2.6.12-rc2 |
192 193 |
} EXPORT_SYMBOL(_spin_lock_irq); |
892a7c67c locking: Allow ar... |
194 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
195 |
|
6beb00092 locking: Make inl... |
196 |
#ifndef CONFIG_INLINE_SPIN_LOCK_BH |
1da177e4c Linux-2.6.12-rc2 |
197 198 |
void __lockfunc _spin_lock_bh(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
199 |
__spin_lock_bh(lock); |
1da177e4c Linux-2.6.12-rc2 |
200 201 |
} EXPORT_SYMBOL(_spin_lock_bh); |
892a7c67c locking: Allow ar... |
202 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
203 |
|
6beb00092 locking: Make inl... |
204 |
#ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE |
1da177e4c Linux-2.6.12-rc2 |
205 206 |
unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
207 |
return __read_lock_irqsave(lock); |
1da177e4c Linux-2.6.12-rc2 |
208 209 |
} EXPORT_SYMBOL(_read_lock_irqsave); |
892a7c67c locking: Allow ar... |
210 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
211 |
|
6beb00092 locking: Make inl... |
212 |
#ifndef CONFIG_INLINE_READ_LOCK_IRQ |
1da177e4c Linux-2.6.12-rc2 |
213 214 |
void __lockfunc _read_lock_irq(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
215 |
__read_lock_irq(lock); |
1da177e4c Linux-2.6.12-rc2 |
216 217 |
} EXPORT_SYMBOL(_read_lock_irq); |
892a7c67c locking: Allow ar... |
218 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
219 |
|
6beb00092 locking: Make inl... |
220 |
#ifndef CONFIG_INLINE_READ_LOCK_BH |
1da177e4c Linux-2.6.12-rc2 |
221 222 |
void __lockfunc _read_lock_bh(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
223 |
__read_lock_bh(lock); |
1da177e4c Linux-2.6.12-rc2 |
224 225 |
} EXPORT_SYMBOL(_read_lock_bh); |
892a7c67c locking: Allow ar... |
226 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
227 |
|
6beb00092 locking: Make inl... |
228 |
#ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE |
1da177e4c Linux-2.6.12-rc2 |
229 230 |
unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
231 |
return __write_lock_irqsave(lock); |
1da177e4c Linux-2.6.12-rc2 |
232 233 |
} EXPORT_SYMBOL(_write_lock_irqsave); |
892a7c67c locking: Allow ar... |
234 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
235 |
|
6beb00092 locking: Make inl... |
236 |
#ifndef CONFIG_INLINE_WRITE_LOCK_IRQ |
1da177e4c Linux-2.6.12-rc2 |
237 238 |
void __lockfunc _write_lock_irq(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
239 |
__write_lock_irq(lock); |
1da177e4c Linux-2.6.12-rc2 |
240 241 |
} EXPORT_SYMBOL(_write_lock_irq); |
892a7c67c locking: Allow ar... |
242 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
243 |
|
6beb00092 locking: Make inl... |
244 |
#ifndef CONFIG_INLINE_WRITE_LOCK_BH |
1da177e4c Linux-2.6.12-rc2 |
245 246 |
void __lockfunc _write_lock_bh(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
247 |
__write_lock_bh(lock); |
1da177e4c Linux-2.6.12-rc2 |
248 249 |
} EXPORT_SYMBOL(_write_lock_bh); |
892a7c67c locking: Allow ar... |
250 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
251 |
|
6beb00092 locking: Make inl... |
252 |
#ifndef CONFIG_INLINE_SPIN_LOCK |
1da177e4c Linux-2.6.12-rc2 |
253 254 |
void __lockfunc _spin_lock(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
255 |
__spin_lock(lock); |
1da177e4c Linux-2.6.12-rc2 |
256 |
} |
1da177e4c Linux-2.6.12-rc2 |
257 |
EXPORT_SYMBOL(_spin_lock); |
892a7c67c locking: Allow ar... |
258 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
259 |
|
6beb00092 locking: Make inl... |
260 |
#ifndef CONFIG_INLINE_WRITE_LOCK |
1da177e4c Linux-2.6.12-rc2 |
261 262 |
void __lockfunc _write_lock(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
263 |
__write_lock(lock); |
1da177e4c Linux-2.6.12-rc2 |
264 |
} |
1da177e4c Linux-2.6.12-rc2 |
265 |
EXPORT_SYMBOL(_write_lock); |
892a7c67c locking: Allow ar... |
266 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
267 |
|
6beb00092 locking: Make inl... |
268 |
#ifndef CONFIG_INLINE_SPIN_UNLOCK |
1da177e4c Linux-2.6.12-rc2 |
269 270 |
void __lockfunc _spin_unlock(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
271 |
__spin_unlock(lock); |
1da177e4c Linux-2.6.12-rc2 |
272 273 |
} EXPORT_SYMBOL(_spin_unlock); |
892a7c67c locking: Allow ar... |
274 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
275 |
|
6beb00092 locking: Make inl... |
276 |
#ifndef CONFIG_INLINE_WRITE_UNLOCK |
1da177e4c Linux-2.6.12-rc2 |
277 278 |
void __lockfunc _write_unlock(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
279 |
__write_unlock(lock); |
1da177e4c Linux-2.6.12-rc2 |
280 281 |
} EXPORT_SYMBOL(_write_unlock); |
892a7c67c locking: Allow ar... |
282 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
283 |
|
6beb00092 locking: Make inl... |
284 |
#ifndef CONFIG_INLINE_READ_UNLOCK |
1da177e4c Linux-2.6.12-rc2 |
285 286 |
void __lockfunc _read_unlock(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
287 |
__read_unlock(lock); |
1da177e4c Linux-2.6.12-rc2 |
288 289 |
} EXPORT_SYMBOL(_read_unlock); |
892a7c67c locking: Allow ar... |
290 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
291 |
|
6beb00092 locking: Make inl... |
292 |
#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE |
1da177e4c Linux-2.6.12-rc2 |
293 294 |
void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) { |
69d0ee737 locking: Move spi... |
295 |
__spin_unlock_irqrestore(lock, flags); |
1da177e4c Linux-2.6.12-rc2 |
296 297 |
} EXPORT_SYMBOL(_spin_unlock_irqrestore); |
892a7c67c locking: Allow ar... |
298 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
299 |
|
6beb00092 locking: Make inl... |
300 |
#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ |
1da177e4c Linux-2.6.12-rc2 |
301 302 |
void __lockfunc _spin_unlock_irq(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
303 |
__spin_unlock_irq(lock); |
1da177e4c Linux-2.6.12-rc2 |
304 305 |
} EXPORT_SYMBOL(_spin_unlock_irq); |
892a7c67c locking: Allow ar... |
306 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
307 |
|
6beb00092 locking: Make inl... |
308 |
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH |
1da177e4c Linux-2.6.12-rc2 |
309 310 |
void __lockfunc _spin_unlock_bh(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
311 |
__spin_unlock_bh(lock); |
1da177e4c Linux-2.6.12-rc2 |
312 313 |
} EXPORT_SYMBOL(_spin_unlock_bh); |
892a7c67c locking: Allow ar... |
314 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
315 |
|
6beb00092 locking: Make inl... |
316 |
#ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE |
1da177e4c Linux-2.6.12-rc2 |
317 318 |
void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { |
69d0ee737 locking: Move spi... |
319 |
__read_unlock_irqrestore(lock, flags); |
1da177e4c Linux-2.6.12-rc2 |
320 321 |
} EXPORT_SYMBOL(_read_unlock_irqrestore); |
892a7c67c locking: Allow ar... |
322 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
323 |
|
6beb00092 locking: Make inl... |
324 |
#ifndef CONFIG_INLINE_READ_UNLOCK_IRQ |
1da177e4c Linux-2.6.12-rc2 |
325 326 |
void __lockfunc _read_unlock_irq(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
327 |
__read_unlock_irq(lock); |
1da177e4c Linux-2.6.12-rc2 |
328 329 |
} EXPORT_SYMBOL(_read_unlock_irq); |
892a7c67c locking: Allow ar... |
330 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
331 |
|
6beb00092 locking: Make inl... |
332 |
#ifndef CONFIG_INLINE_READ_UNLOCK_BH |
1da177e4c Linux-2.6.12-rc2 |
333 334 |
void __lockfunc _read_unlock_bh(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
335 |
__read_unlock_bh(lock); |
1da177e4c Linux-2.6.12-rc2 |
336 337 |
} EXPORT_SYMBOL(_read_unlock_bh); |
892a7c67c locking: Allow ar... |
338 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
339 |
|
6beb00092 locking: Make inl... |
340 |
#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE |
1da177e4c Linux-2.6.12-rc2 |
341 342 |
void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { |
69d0ee737 locking: Move spi... |
343 |
__write_unlock_irqrestore(lock, flags); |
1da177e4c Linux-2.6.12-rc2 |
344 345 |
} EXPORT_SYMBOL(_write_unlock_irqrestore); |
892a7c67c locking: Allow ar... |
346 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
347 |
|
6beb00092 locking: Make inl... |
348 |
#ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ |
1da177e4c Linux-2.6.12-rc2 |
349 350 |
void __lockfunc _write_unlock_irq(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
351 |
__write_unlock_irq(lock); |
1da177e4c Linux-2.6.12-rc2 |
352 353 |
} EXPORT_SYMBOL(_write_unlock_irq); |
892a7c67c locking: Allow ar... |
354 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
355 |
|
6beb00092 locking: Make inl... |
356 |
#ifndef CONFIG_INLINE_WRITE_UNLOCK_BH |
1da177e4c Linux-2.6.12-rc2 |
357 358 |
void __lockfunc _write_unlock_bh(rwlock_t *lock) { |
69d0ee737 locking: Move spi... |
359 |
__write_unlock_bh(lock); |
1da177e4c Linux-2.6.12-rc2 |
360 361 |
} EXPORT_SYMBOL(_write_unlock_bh); |
892a7c67c locking: Allow ar... |
362 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
363 |
|
6beb00092 locking: Make inl... |
364 |
#ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH |
1da177e4c Linux-2.6.12-rc2 |
365 366 |
int __lockfunc _spin_trylock_bh(spinlock_t *lock) { |
69d0ee737 locking: Move spi... |
367 |
return __spin_trylock_bh(lock); |
1da177e4c Linux-2.6.12-rc2 |
368 369 |
} EXPORT_SYMBOL(_spin_trylock_bh); |
892a7c67c locking: Allow ar... |
370 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
371 |
|
0764d23cf ftrace: lockdep n... |
372 |
notrace int in_lock_functions(unsigned long addr) |
1da177e4c Linux-2.6.12-rc2 |
373 374 375 376 377 378 379 380 |
{ /* Linker adds these: start and end of __lockfunc functions */ extern char __lock_text_start[], __lock_text_end[]; return addr >= (unsigned long)__lock_text_start && addr < (unsigned long)__lock_text_end; } EXPORT_SYMBOL(in_lock_functions); |