Blame view
kernel/locking/rwsem.c
44.5 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
c4e05116a [PATCH] lockdep: ... |
2 3 4 5 |
/* kernel/rwsem.c: R/W semaphores, public implementation * * Written by David Howells (dhowells@redhat.com). * Derived from asm-i386/semaphore.h |
5dec94d49 locking/rwsem: Me... |
6 7 8 9 10 11 12 |
* * Writer lock-stealing by Alex Shi <alex.shi@intel.com> * and Michel Lespinasse <walken@google.com> * * Optimistic spinning by Tim Chen <tim.c.chen@intel.com> * and Davidlohr Bueso <davidlohr@hp.com>. Based on mutexes. * |
4f23dbc1e locking/rwsem: Im... |
13 14 15 |
* Rwsem count bit fields re-definition and rwsem rearchitecture by * Waiman Long <longman@redhat.com> and * Peter Zijlstra <peterz@infradead.org>. |
c4e05116a [PATCH] lockdep: ... |
16 17 18 19 |
*/ #include <linux/types.h> #include <linux/kernel.h> |
c7af77b58 sched: mark rwsem... |
20 |
#include <linux/sched.h> |
5dec94d49 locking/rwsem: Me... |
21 22 |
#include <linux/sched/rt.h> #include <linux/sched/task.h> |
b17b01533 sched/headers: Pr... |
23 |
#include <linux/sched/debug.h> |
5dec94d49 locking/rwsem: Me... |
24 25 |
#include <linux/sched/wake_q.h> #include <linux/sched/signal.h> |
7d43f1ce9 locking/rwsem: En... |
26 |
#include <linux/sched/clock.h> |
9984de1a5 kernel: Map most ... |
27 |
#include <linux/export.h> |
c4e05116a [PATCH] lockdep: ... |
28 |
#include <linux/rwsem.h> |
60063497a atomic: use <linu... |
29 |
#include <linux/atomic.h> |
c4e05116a [PATCH] lockdep: ... |
30 |
|
42254105d locking/rwsem: Ad... |
31 |
#ifndef CONFIG_PREEMPT_RT |
5dec94d49 locking/rwsem: Me... |
32 33 34 |
#include "lock_events.h" /* |
617f3ef95 locking/rwsem: Re... |
35 |
* The least significant 2 bits of the owner value has the following |
5dec94d49 locking/rwsem: Me... |
36 |
* meanings when set. |
02f1082b0 locking/rwsem: Cl... |
37 |
* - Bit 0: RWSEM_READER_OWNED - The rwsem is owned by readers |
617f3ef95 locking/rwsem: Re... |
38 |
* - Bit 1: RWSEM_NONSPINNABLE - Cannot spin on a reader-owned lock |
5dec94d49 locking/rwsem: Me... |
39 |
* |
617f3ef95 locking/rwsem: Re... |
40 41 |
* When the rwsem is reader-owned and a spinning writer has timed out, * the nonspinnable bit will be set to disable optimistic spinning. |
7d43f1ce9 locking/rwsem: En... |
42 |
|
5dec94d49 locking/rwsem: Me... |
43 44 45 46 |
* When a writer acquires a rwsem, it puts its task_struct pointer * into the owner field. It is cleared after an unlock. * * When a reader acquires a rwsem, it will also puts its task_struct |
7d43f1ce9 locking/rwsem: En... |
47 48 49 50 |
* pointer into the owner field with the RWSEM_READER_OWNED bit set. * On unlock, the owner field will largely be left untouched. So * for a free or reader-owned rwsem, the owner value may contain * information about the last reader that acquires the rwsem. |
5dec94d49 locking/rwsem: Me... |
51 52 53 54 55 |
* * That information may be helpful in debugging cases where the system * seems to hang on a reader owned rwsem especially if only one reader * is involved. Ideally we would like to track all the readers that own * a rwsem, but the overhead is simply too big. |
5cfd92e12 locking/rwsem: Ad... |
56 |
* |
617f3ef95 locking/rwsem: Re... |
57 58 59 60 61 |
* A fast path reader optimistic lock stealing is supported when the rwsem * is previously owned by a writer and the following conditions are met: * - OSQ is empty * - rwsem is not currently writer owned * - the handoff isn't set. |
5dec94d49 locking/rwsem: Me... |
62 63 |
*/ #define RWSEM_READER_OWNED (1UL << 0) |
617f3ef95 locking/rwsem: Re... |
64 |
#define RWSEM_NONSPINNABLE (1UL << 1) |
02f1082b0 locking/rwsem: Cl... |
65 |
#define RWSEM_OWNER_FLAGS_MASK (RWSEM_READER_OWNED | RWSEM_NONSPINNABLE) |
5dec94d49 locking/rwsem: Me... |
66 67 68 69 |
#ifdef CONFIG_DEBUG_RWSEMS # define DEBUG_RWSEMS_WARN_ON(c, sem) do { \ if (!debug_locks_silent && \ |
fce45cd41 locking/rwsem: Ch... |
70 71 |
WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, magic = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty ",\ |
5dec94d49 locking/rwsem: Me... |
72 |
#c, atomic_long_read(&(sem)->count), \ |
fce45cd41 locking/rwsem: Ch... |
73 |
(unsigned long) sem->magic, \ |
94a9717b3 locking/rwsem: Ma... |
74 |
atomic_long_read(&(sem)->owner), (long)current, \ |
5dec94d49 locking/rwsem: Me... |
75 76 77 78 79 80 81 82 |
list_empty(&(sem)->wait_list) ? "" : "not ")) \ debug_locks_off(); \ } while (0) #else # define DEBUG_RWSEMS_WARN_ON(c, sem) #endif /* |
a15ea1a35 locking/rwsem: Gu... |
83 |
* On 64-bit architectures, the bit definitions of the count are: |
5dec94d49 locking/rwsem: Me... |
84 |
* |
a15ea1a35 locking/rwsem: Gu... |
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 |
* Bit 0 - writer locked bit * Bit 1 - waiters present bit * Bit 2 - lock handoff bit * Bits 3-7 - reserved * Bits 8-62 - 55-bit reader count * Bit 63 - read fail bit * * On 32-bit architectures, the bit definitions of the count are: * * Bit 0 - writer locked bit * Bit 1 - waiters present bit * Bit 2 - lock handoff bit * Bits 3-7 - reserved * Bits 8-30 - 23-bit reader count * Bit 31 - read fail bit * * It is not likely that the most significant bit (read fail bit) will ever * be set. This guard bit is still checked anyway in the down_read() fastpath * just in case we need to use up more of the reader bits for other purpose * in the future. |
5dec94d49 locking/rwsem: Me... |
105 106 107 |
* * atomic_long_fetch_add() is used to obtain reader lock, whereas * atomic_long_cmpxchg() will be used to obtain writer lock. |
4f23dbc1e locking/rwsem: Im... |
108 109 |
* * There are three places where the lock handoff bit may be set or cleared. |
76723ed1f locking/rwsem: Ma... |
110 111 112 |
* 1) rwsem_mark_wake() for readers -- set, clear * 2) rwsem_try_write_lock() for writers -- set, clear * 3) rwsem_del_waiter() -- clear |
4f23dbc1e locking/rwsem: Im... |
113 114 115 116 |
* * For all the above cases, wait_lock will be held. A writer must also * be the first one in the wait_list to be eligible for setting the handoff * bit. So concurrent setting/clearing of handoff bit is not possible. |
5dec94d49 locking/rwsem: Me... |
117 118 119 |
*/ #define RWSEM_WRITER_LOCKED (1UL << 0) #define RWSEM_FLAG_WAITERS (1UL << 1) |
4f23dbc1e locking/rwsem: Im... |
120 |
#define RWSEM_FLAG_HANDOFF (1UL << 2) |
a15ea1a35 locking/rwsem: Gu... |
121 |
#define RWSEM_FLAG_READFAIL (1UL << (BITS_PER_LONG - 1)) |
4f23dbc1e locking/rwsem: Im... |
122 |
|
5dec94d49 locking/rwsem: Me... |
123 124 125 126 127 |
#define RWSEM_READER_SHIFT 8 #define RWSEM_READER_BIAS (1UL << RWSEM_READER_SHIFT) #define RWSEM_READER_MASK (~(RWSEM_READER_BIAS - 1)) #define RWSEM_WRITER_MASK RWSEM_WRITER_LOCKED #define RWSEM_LOCK_MASK (RWSEM_WRITER_MASK|RWSEM_READER_MASK) |
4f23dbc1e locking/rwsem: Im... |
128 |
#define RWSEM_READ_FAILED_MASK (RWSEM_WRITER_MASK|RWSEM_FLAG_WAITERS|\ |
a15ea1a35 locking/rwsem: Gu... |
129 |
RWSEM_FLAG_HANDOFF|RWSEM_FLAG_READFAIL) |
5dec94d49 locking/rwsem: Me... |
130 131 132 133 134 135 136 137 138 139 |
/* * All writes to owner are protected by WRITE_ONCE() to make sure that * store tearing can't happen as optimistic spinners may read and use * the owner value concurrently without lock. Read from owner, however, * may not need READ_ONCE() as long as the pointer value is only used * for comparison and isn't being dereferenced. */ static inline void rwsem_set_owner(struct rw_semaphore *sem) { |
94a9717b3 locking/rwsem: Ma... |
140 |
atomic_long_set(&sem->owner, (long)current); |
5dec94d49 locking/rwsem: Me... |
141 142 143 144 |
} static inline void rwsem_clear_owner(struct rw_semaphore *sem) { |
94a9717b3 locking/rwsem: Ma... |
145 146 147 148 149 150 151 152 153 |
atomic_long_set(&sem->owner, 0); } /* * Test the flags in the owner field. */ static inline bool rwsem_test_oflags(struct rw_semaphore *sem, long flags) { return atomic_long_read(&sem->owner) & flags; |
5dec94d49 locking/rwsem: Me... |
154 155 156 157 158 159 160 161 162 |
} /* * The task_struct pointer of the last owning reader will be left in * the owner field. * * Note that the owner value just indicates the task has owned the rwsem * previously, it may not be the real owner or one of the real owners * anymore when that field is examined, so take it with a grain of salt. |
5cfd92e12 locking/rwsem: Ad... |
163 164 |
* * The reader non-spinnable bit is preserved. |
5dec94d49 locking/rwsem: Me... |
165 166 167 168 |
*/ static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, struct task_struct *owner) { |
5cfd92e12 locking/rwsem: Ad... |
169 |
unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED | |
617f3ef95 locking/rwsem: Re... |
170 |
(atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE); |
5dec94d49 locking/rwsem: Me... |
171 |
|
94a9717b3 locking/rwsem: Ma... |
172 |
atomic_long_set(&sem->owner, val); |
5dec94d49 locking/rwsem: Me... |
173 174 175 176 177 178 179 180 |
} static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) { __rwsem_set_reader_owned(sem, current); } /* |
94a9717b3 locking/rwsem: Ma... |
181 |
* Return true if the rwsem is owned by a reader. |
5dec94d49 locking/rwsem: Me... |
182 |
*/ |
94a9717b3 locking/rwsem: Ma... |
183 |
static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
184 |
{ |
94a9717b3 locking/rwsem: Ma... |
185 186 187 188 189 190 191 192 193 194 |
#ifdef CONFIG_DEBUG_RWSEMS /* * Check the count to see if it is write-locked. */ long count = atomic_long_read(&sem->count); if (count & RWSEM_WRITER_MASK) return false; #endif return rwsem_test_oflags(sem, RWSEM_READER_OWNED); |
5dec94d49 locking/rwsem: Me... |
195 196 197 198 199 200 201 202 203 204 205 |
} #ifdef CONFIG_DEBUG_RWSEMS /* * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there * is a task pointer in owner of a reader-owned rwsem, it will be the * real owner or one of the real owners. The only exception is when the * unlock is done by up_read_non_owner(). */ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) { |
94a9717b3 locking/rwsem: Ma... |
206 207 208 209 210 211 212 |
unsigned long val = atomic_long_read(&sem->owner); while ((val & ~RWSEM_OWNER_FLAGS_MASK) == (unsigned long)current) { if (atomic_long_try_cmpxchg(&sem->owner, &val, val & RWSEM_OWNER_FLAGS_MASK)) return; } |
5dec94d49 locking/rwsem: Me... |
213 214 215 216 217 218 219 220 |
} #else static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) { } #endif /* |
7d43f1ce9 locking/rwsem: En... |
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 |
* Set the RWSEM_NONSPINNABLE bits if the RWSEM_READER_OWNED flag * remains set. Otherwise, the operation will be aborted. */ static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) { unsigned long owner = atomic_long_read(&sem->owner); do { if (!(owner & RWSEM_READER_OWNED)) break; if (owner & RWSEM_NONSPINNABLE) break; } while (!atomic_long_try_cmpxchg(&sem->owner, &owner, owner | RWSEM_NONSPINNABLE)); } |
c8fe8b056 locking/rwsem: Pa... |
236 |
static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) |
a15ea1a35 locking/rwsem: Gu... |
237 |
{ |
c8fe8b056 locking/rwsem: Pa... |
238 |
*cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count); |
3379116a0 locking/rwsem: Be... |
239 |
|
c8fe8b056 locking/rwsem: Pa... |
240 |
if (WARN_ON_ONCE(*cntp < 0)) |
a15ea1a35 locking/rwsem: Gu... |
241 |
rwsem_set_nonspinnable(sem); |
3379116a0 locking/rwsem: Be... |
242 |
|
c8fe8b056 locking/rwsem: Pa... |
243 |
if (!(*cntp & RWSEM_READ_FAILED_MASK)) { |
3379116a0 locking/rwsem: Be... |
244 245 246 247 248 |
rwsem_set_reader_owned(sem); return true; } return false; |
a15ea1a35 locking/rwsem: Gu... |
249 |
} |
285c61aed locking/rwsem: In... |
250 251 252 253 254 255 256 257 258 259 260 |
static inline bool rwsem_write_trylock(struct rw_semaphore *sem) { long tmp = RWSEM_UNLOCKED_VALUE; if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) { rwsem_set_owner(sem); return true; } return false; } |
7d43f1ce9 locking/rwsem: En... |
261 |
/* |
94a9717b3 locking/rwsem: Ma... |
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 |
* Return just the real task structure pointer of the owner */ static inline struct task_struct *rwsem_owner(struct rw_semaphore *sem) { return (struct task_struct *) (atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK); } /* * Return the real task structure pointer of the owner and the embedded * flags in the owner. pflags must be non-NULL. */ static inline struct task_struct * rwsem_owner_flags(struct rw_semaphore *sem, unsigned long *pflags) { unsigned long owner = atomic_long_read(&sem->owner); *pflags = owner & RWSEM_OWNER_FLAGS_MASK; return (struct task_struct *)(owner & ~RWSEM_OWNER_FLAGS_MASK); } /* |
5dec94d49 locking/rwsem: Me... |
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 |
* Guide to the rw_semaphore's count field. * * When the RWSEM_WRITER_LOCKED bit in count is set, the lock is owned * by a writer. * * The lock is owned by readers when * (1) the RWSEM_WRITER_LOCKED isn't set in count, * (2) some of the reader bits are set in count, and * (3) the owner field has RWSEM_READ_OWNED bit set. * * Having some reader bits set is not enough to guarantee a readers owned * lock as the readers may be in the process of backing out from the count * and a writer has just released the lock. So another writer may steal * the lock immediately after that. */ /* * Initialize an rwsem: */ void __init_rwsem(struct rw_semaphore *sem, const char *name, struct lock_class_key *key) { #ifdef CONFIG_DEBUG_LOCK_ALLOC /* * Make sure we are not reinitializing a held semaphore: */ debug_check_no_locks_freed((void *)sem, sizeof(*sem)); |
de8f5e4f2 lockdep: Introduc... |
311 |
lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); |
5dec94d49 locking/rwsem: Me... |
312 |
#endif |
fce45cd41 locking/rwsem: Ch... |
313 314 315 |
#ifdef CONFIG_DEBUG_RWSEMS sem->magic = sem; #endif |
5dec94d49 locking/rwsem: Me... |
316 317 318 |
atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); raw_spin_lock_init(&sem->wait_lock); INIT_LIST_HEAD(&sem->wait_list); |
94a9717b3 locking/rwsem: Ma... |
319 |
atomic_long_set(&sem->owner, 0L); |
5dec94d49 locking/rwsem: Me... |
320 321 322 323 |
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER osq_lock_init(&sem->osq); #endif } |
5dec94d49 locking/rwsem: Me... |
324 325 326 327 328 329 330 331 332 333 334 |
EXPORT_SYMBOL(__init_rwsem); enum rwsem_waiter_type { RWSEM_WAITING_FOR_WRITE, RWSEM_WAITING_FOR_READ }; struct rwsem_waiter { struct list_head list; struct task_struct *task; enum rwsem_waiter_type type; |
4f23dbc1e locking/rwsem: Im... |
335 |
unsigned long timeout; |
76723ed1f locking/rwsem: Ma... |
336 |
bool handoff_set; |
5dec94d49 locking/rwsem: Me... |
337 |
}; |
4f23dbc1e locking/rwsem: Im... |
338 339 |
#define rwsem_first_waiter(sem) \ list_first_entry(&sem->wait_list, struct rwsem_waiter, list) |
5dec94d49 locking/rwsem: Me... |
340 341 342 343 344 345 |
enum rwsem_wake_type { RWSEM_WAKE_ANY, /* Wake whatever's at head of wait list */ RWSEM_WAKE_READERS, /* Wake readers only */ RWSEM_WAKE_READ_OWNED /* Waker thread holds the read lock */ }; |
4f23dbc1e locking/rwsem: Im... |
346 347 348 349 350 351 |
/* * The typical HZ value is either 250 or 1000. So set the minimum waiting * time to at least 4ms or 1 jiffy (if it is higher than 4ms) in the wait * queue before initiating the handoff protocol. */ #define RWSEM_WAIT_TIMEOUT DIV_ROUND_UP(HZ, 250) |
5dec94d49 locking/rwsem: Me... |
352 |
/* |
d3681e269 locking/rwsem: Wa... |
353 354 355 356 357 358 |
* Magic number to batch-wakeup waiting readers, even when writers are * also present in the queue. This both limits the amount of work the * waking thread must do and also prevents any potential counter overflow, * however unlikely. */ #define MAX_READERS_WAKEUP 0x100 |
76723ed1f locking/rwsem: Ma... |
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 |
static inline void rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) { lockdep_assert_held(&sem->wait_lock); list_add_tail(&waiter->list, &sem->wait_list); /* caller will set RWSEM_FLAG_WAITERS */ } /* * Remove a waiter from the wait_list and clear flags. * * Both rwsem_mark_wake() and rwsem_try_write_lock() contain a full 'copy' of * this function. Modify with care. */ static inline void rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) { lockdep_assert_held(&sem->wait_lock); list_del(&waiter->list); if (likely(!list_empty(&sem->wait_list))) return; atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count); } |
d3681e269 locking/rwsem: Wa... |
383 |
/* |
5dec94d49 locking/rwsem: Me... |
384 385 386 387 388 389 390 391 392 393 |
* handle the lock release when processes blocked on it that can now run * - if we come here from up_xxxx(), then the RWSEM_FLAG_WAITERS bit must * have been set. * - there must be someone on the queue * - the wait_lock must be held by the caller * - tasks are marked for wakeup, the caller must later invoke wake_up_q() * to actually wakeup the blocked task(s) and drop the reference count, * preferably when the wait_lock is released * - woken process blocks are discarded from the list after having task zeroed * - writers are only marked woken if downgrading is false |
76723ed1f locking/rwsem: Ma... |
394 395 |
* * Implies rwsem_del_waiter() for all woken readers. |
5dec94d49 locking/rwsem: Me... |
396 |
*/ |
6cef7ff6e locking/rwsem: Co... |
397 398 399 |
static void rwsem_mark_wake(struct rw_semaphore *sem, enum rwsem_wake_type wake_type, struct wake_q_head *wake_q) |
5dec94d49 locking/rwsem: Me... |
400 401 402 403 |
{ struct rwsem_waiter *waiter, *tmp; long oldcount, woken = 0, adjustment = 0; struct list_head wlist; |
4f23dbc1e locking/rwsem: Im... |
404 |
lockdep_assert_held(&sem->wait_lock); |
5dec94d49 locking/rwsem: Me... |
405 406 407 408 |
/* * Take a peek at the queue head waiter such that we can determine * the wakeup(s) to perform. */ |
4f23dbc1e locking/rwsem: Im... |
409 |
waiter = rwsem_first_waiter(sem); |
5dec94d49 locking/rwsem: Me... |
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 |
if (waiter->type == RWSEM_WAITING_FOR_WRITE) { if (wake_type == RWSEM_WAKE_ANY) { /* * Mark writer at the front of the queue for wakeup. * Until the task is actually later awoken later by * the caller, other writers are able to steal it. * Readers, on the other hand, will block as they * will notice the queued writer. */ wake_q_add(wake_q, waiter->task); lockevent_inc(rwsem_wake_writer); } return; } /* |
a15ea1a35 locking/rwsem: Gu... |
428 429 430 431 432 433 |
* No reader wakeup if there are too many of them already. */ if (unlikely(atomic_long_read(&sem->count) < 0)) return; /* |
5dec94d49 locking/rwsem: Me... |
434 435 436 437 438 |
* Writers might steal the lock before we grant it to the next reader. * We prefer to do the first reader grant before counting readers * so we can bail out early if a writer stole the lock. */ if (wake_type != RWSEM_WAKE_READ_OWNED) { |
5cfd92e12 locking/rwsem: Ad... |
439 |
struct task_struct *owner; |
5dec94d49 locking/rwsem: Me... |
440 441 442 |
adjustment = RWSEM_READER_BIAS; oldcount = atomic_long_fetch_add(adjustment, &sem->count); if (unlikely(oldcount & RWSEM_WRITER_MASK)) { |
4f23dbc1e locking/rwsem: Im... |
443 444 445 446 447 |
/* * When we've been waiting "too" long (for writers * to give up the lock), request a HANDOFF to * force the issue. */ |
d10e819d1 locking/rwsem: Al... |
448 449 450 451 452 453 |
if (time_after(jiffies, waiter->timeout)) { if (!(oldcount & RWSEM_FLAG_HANDOFF)) { adjustment -= RWSEM_FLAG_HANDOFF; lockevent_inc(rwsem_rlock_handoff); } waiter->handoff_set = true; |
4f23dbc1e locking/rwsem: Im... |
454 455 456 |
} atomic_long_add(-adjustment, &sem->count); |
5dec94d49 locking/rwsem: Me... |
457 458 459 460 461 |
return; } /* * Set it to reader-owned to give spinners an early * indication that readers now have the lock. |
5cfd92e12 locking/rwsem: Ad... |
462 463 |
* The reader nonspinnable bit seen at slowpath entry of * the reader is copied over. |
5dec94d49 locking/rwsem: Me... |
464 |
*/ |
5cfd92e12 locking/rwsem: Ad... |
465 |
owner = waiter->task; |
5cfd92e12 locking/rwsem: Ad... |
466 |
__rwsem_set_reader_owned(sem, owner); |
5dec94d49 locking/rwsem: Me... |
467 468 469 |
} /* |
d3681e269 locking/rwsem: Wa... |
470 471 |
* Grant up to MAX_READERS_WAKEUP read locks to all the readers in the * queue. We know that the woken will be at least 1 as we accounted |
5dec94d49 locking/rwsem: Me... |
472 473 474 |
* for above. Note we increment the 'active part' of the count by the * number of readers before waking any processes up. * |
d3681e269 locking/rwsem: Wa... |
475 476 477 478 479 480 |
* This is an adaptation of the phase-fair R/W locks where at the * reader phase (first waiter is a reader), all readers are eligible * to acquire the lock at the same time irrespective of their order * in the queue. The writers acquire the lock according to their * order in the queue. * |
5dec94d49 locking/rwsem: Me... |
481 482 483 484 485 486 487 488 489 490 491 |
* We have to do wakeup in 2 passes to prevent the possibility that * the reader count may be decremented before it is incremented. It * is because the to-be-woken waiter may not have slept yet. So it * may see waiter->task got cleared, finish its critical section and * do an unlock before the reader count increment. * * 1) Collect the read-waiters in a separate list, count them and * fully increment the reader count in rwsem. * 2) For each waiters in the new list, clear waiter->task and * put them into wake_q to be woken up later. */ |
d3681e269 locking/rwsem: Wa... |
492 493 |
INIT_LIST_HEAD(&wlist); list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { |
5dec94d49 locking/rwsem: Me... |
494 |
if (waiter->type == RWSEM_WAITING_FOR_WRITE) |
d3681e269 locking/rwsem: Wa... |
495 |
continue; |
5dec94d49 locking/rwsem: Me... |
496 497 |
woken++; |
d3681e269 locking/rwsem: Wa... |
498 499 500 501 502 503 504 |
list_move_tail(&waiter->list, &wlist); /* * Limit # of readers that can be woken up per wakeup call. */ if (woken >= MAX_READERS_WAKEUP) break; |
5dec94d49 locking/rwsem: Me... |
505 |
} |
5dec94d49 locking/rwsem: Me... |
506 507 508 |
adjustment = woken * RWSEM_READER_BIAS - adjustment; lockevent_cond_inc(rwsem_wake_reader, woken); |
76723ed1f locking/rwsem: Ma... |
509 510 |
oldcount = atomic_long_read(&sem->count); |
5dec94d49 locking/rwsem: Me... |
511 |
if (list_empty(&sem->wait_list)) { |
76723ed1f locking/rwsem: Ma... |
512 513 514 515 |
/* * Combined with list_move_tail() above, this implies * rwsem_del_waiter(). */ |
5dec94d49 locking/rwsem: Me... |
516 |
adjustment -= RWSEM_FLAG_WAITERS; |
76723ed1f locking/rwsem: Ma... |
517 518 519 520 521 522 523 524 525 |
if (oldcount & RWSEM_FLAG_HANDOFF) adjustment -= RWSEM_FLAG_HANDOFF; } else if (woken) { /* * When we've woken a reader, we no longer need to force * writers to give up the lock and we can clear HANDOFF. */ if (oldcount & RWSEM_FLAG_HANDOFF) adjustment -= RWSEM_FLAG_HANDOFF; |
5dec94d49 locking/rwsem: Me... |
526 527 528 529 530 531 532 533 534 535 536 537 538 539 |
} if (adjustment) atomic_long_add(adjustment, &sem->count); /* 2nd pass */ list_for_each_entry_safe(waiter, tmp, &wlist, list) { struct task_struct *tsk; tsk = waiter->task; get_task_struct(tsk); /* * Ensure calling get_task_struct() before setting the reader |
6cef7ff6e locking/rwsem: Co... |
540 |
* waiter to nil such that rwsem_down_read_slowpath() cannot |
5dec94d49 locking/rwsem: Me... |
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 |
* race with do_exit() by always holding a reference count * to the task to wakeup. */ smp_store_release(&waiter->task, NULL); /* * Ensure issuing the wakeup (either by us or someone else) * after setting the reader waiter to nil. */ wake_q_add_safe(wake_q, tsk); } } /* * This function must be called with the sem->wait_lock held to prevent * race conditions between checking the rwsem wait list and setting the * sem->count accordingly. |
4f23dbc1e locking/rwsem: Im... |
557 |
* |
76723ed1f locking/rwsem: Ma... |
558 |
* Implies rwsem_del_waiter() on success. |
5dec94d49 locking/rwsem: Me... |
559 |
*/ |
00f3c5a3d locking/rwsem: Al... |
560 |
static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, |
76723ed1f locking/rwsem: Ma... |
561 |
struct rwsem_waiter *waiter) |
5dec94d49 locking/rwsem: Me... |
562 |
{ |
d10e819d1 locking/rwsem: Al... |
563 |
struct rwsem_waiter *first = rwsem_first_waiter(sem); |
00f3c5a3d locking/rwsem: Al... |
564 |
long count, new; |
5dec94d49 locking/rwsem: Me... |
565 |
|
4f23dbc1e locking/rwsem: Im... |
566 |
lockdep_assert_held(&sem->wait_lock); |
5dec94d49 locking/rwsem: Me... |
567 |
|
00f3c5a3d locking/rwsem: Al... |
568 |
count = atomic_long_read(&sem->count); |
4f23dbc1e locking/rwsem: Im... |
569 570 |
do { bool has_handoff = !!(count & RWSEM_FLAG_HANDOFF); |
5dec94d49 locking/rwsem: Me... |
571 |
|
76723ed1f locking/rwsem: Ma... |
572 |
if (has_handoff) { |
d10e819d1 locking/rwsem: Al... |
573 574 575 576 577 578 |
/* * Honor handoff bit and yield only when the first * waiter is the one that set it. Otherwisee, we * still try to acquire the rwsem. */ if (first->handoff_set && (waiter != first)) |
76723ed1f locking/rwsem: Ma... |
579 |
return false; |
d10e819d1 locking/rwsem: Al... |
580 581 582 583 584 585 |
/* * First waiter can inherit a previously set handoff * bit and spin on rwsem if lock acquisition fails. */ if (waiter == first) waiter->handoff_set = true; |
76723ed1f locking/rwsem: Ma... |
586 |
} |
5dec94d49 locking/rwsem: Me... |
587 |
|
4f23dbc1e locking/rwsem: Im... |
588 589 590 |
new = count; if (count & RWSEM_LOCK_MASK) { |
76723ed1f locking/rwsem: Ma... |
591 592 |
if (has_handoff || (!rt_task(waiter->task) && !time_after(jiffies, waiter->timeout))) |
4f23dbc1e locking/rwsem: Im... |
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 |
return false; new |= RWSEM_FLAG_HANDOFF; } else { new |= RWSEM_WRITER_LOCKED; new &= ~RWSEM_FLAG_HANDOFF; if (list_is_singular(&sem->wait_list)) new &= ~RWSEM_FLAG_WAITERS; } } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new)); /* * We have either acquired the lock with handoff bit cleared or * set the handoff bit. */ |
76723ed1f locking/rwsem: Ma... |
609 610 611 |
if (new & RWSEM_FLAG_HANDOFF) { waiter->handoff_set = true; lockevent_inc(rwsem_wlock_handoff); |
4f23dbc1e locking/rwsem: Im... |
612 |
return false; |
76723ed1f locking/rwsem: Ma... |
613 |
} |
4f23dbc1e locking/rwsem: Im... |
614 |
|
76723ed1f locking/rwsem: Ma... |
615 616 617 618 619 |
/* * Have rwsem_try_write_lock() fully imply rwsem_del_waiter() on * success. */ list_del(&waiter->list); |
4f23dbc1e locking/rwsem: Im... |
620 621 |
rwsem_set_owner(sem); return true; |
5dec94d49 locking/rwsem: Me... |
622 |
} |
562d350a8 locking/rwsem: Di... |
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 |
/* * The rwsem_spin_on_owner() function returns the following 4 values * depending on the lock owner state. * OWNER_NULL : owner is currently NULL * OWNER_WRITER: when owner changes and is a writer * OWNER_READER: when owner changes and the new owner may be a reader. * OWNER_NONSPINNABLE: * when optimistic spinning has to stop because either the * owner stops running, is unknown, or its timeslice has * been used up. */ enum owner_state { OWNER_NULL = 1 << 0, OWNER_WRITER = 1 << 1, OWNER_READER = 1 << 2, OWNER_NONSPINNABLE = 1 << 3, }; |
5dec94d49 locking/rwsem: Me... |
640 641 642 643 644 645 646 |
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER /* * Try to acquire write lock before the writer has been put on wait queue. */ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) { long count = atomic_long_read(&sem->count); |
4f23dbc1e locking/rwsem: Im... |
647 |
while (!(count & (RWSEM_LOCK_MASK|RWSEM_FLAG_HANDOFF))) { |
5dec94d49 locking/rwsem: Me... |
648 |
if (atomic_long_try_cmpxchg_acquire(&sem->count, &count, |
4f23dbc1e locking/rwsem: Im... |
649 |
count | RWSEM_WRITER_LOCKED)) { |
5dec94d49 locking/rwsem: Me... |
650 |
rwsem_set_owner(sem); |
617f3ef95 locking/rwsem: Re... |
651 |
lockevent_inc(rwsem_opt_lock); |
5dec94d49 locking/rwsem: Me... |
652 653 654 655 656 657 658 659 660 661 662 663 664 665 |
return true; } } return false; } static inline bool owner_on_cpu(struct task_struct *owner) { /* * As lock holder preemption issue, we both skip spinning if * task is not on cpu or its cpu is preempted */ return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner)); } |
617f3ef95 locking/rwsem: Re... |
666 |
static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
667 668 |
{ struct task_struct *owner; |
94a9717b3 locking/rwsem: Ma... |
669 |
unsigned long flags; |
5dec94d49 locking/rwsem: Me... |
670 |
bool ret = true; |
cf69482d6 locking/rwsem: En... |
671 672 |
if (need_resched()) { lockevent_inc(rwsem_opt_fail); |
5dec94d49 locking/rwsem: Me... |
673 |
return false; |
cf69482d6 locking/rwsem: En... |
674 |
} |
5dec94d49 locking/rwsem: Me... |
675 |
|
cf69482d6 locking/rwsem: En... |
676 |
preempt_disable(); |
5dec94d49 locking/rwsem: Me... |
677 |
rcu_read_lock(); |
94a9717b3 locking/rwsem: Ma... |
678 |
owner = rwsem_owner_flags(sem, &flags); |
781343005 locking/rwsem: Do... |
679 680 681 |
/* * Don't check the read-owner as the entry may be stale. */ |
617f3ef95 locking/rwsem: Re... |
682 |
if ((flags & RWSEM_NONSPINNABLE) || |
781343005 locking/rwsem: Do... |
683 |
(owner && !(flags & RWSEM_READER_OWNED) && !owner_on_cpu(owner))) |
94a9717b3 locking/rwsem: Ma... |
684 |
ret = false; |
5dec94d49 locking/rwsem: Me... |
685 |
rcu_read_unlock(); |
cf69482d6 locking/rwsem: En... |
686 687 688 |
preempt_enable(); lockevent_cond_inc(rwsem_opt_fail, !ret); |
5dec94d49 locking/rwsem: Me... |
689 690 |
return ret; } |
7d43f1ce9 locking/rwsem: En... |
691 |
#define OWNER_SPINNABLE (OWNER_NULL | OWNER_WRITER | OWNER_READER) |
3f6d517a3 locking/rwsem: Ma... |
692 |
|
94a9717b3 locking/rwsem: Ma... |
693 |
static inline enum owner_state |
617f3ef95 locking/rwsem: Re... |
694 |
rwsem_owner_state(struct task_struct *owner, unsigned long flags) |
5dec94d49 locking/rwsem: Me... |
695 |
{ |
617f3ef95 locking/rwsem: Re... |
696 |
if (flags & RWSEM_NONSPINNABLE) |
3f6d517a3 locking/rwsem: Ma... |
697 |
return OWNER_NONSPINNABLE; |
94a9717b3 locking/rwsem: Ma... |
698 |
if (flags & RWSEM_READER_OWNED) |
3f6d517a3 locking/rwsem: Ma... |
699 |
return OWNER_READER; |
94a9717b3 locking/rwsem: Ma... |
700 |
return owner ? OWNER_WRITER : OWNER_NULL; |
3f6d517a3 locking/rwsem: Ma... |
701 |
} |
7d43f1ce9 locking/rwsem: En... |
702 |
static noinline enum owner_state |
617f3ef95 locking/rwsem: Re... |
703 |
rwsem_spin_on_owner(struct rw_semaphore *sem) |
3f6d517a3 locking/rwsem: Ma... |
704 |
{ |
94a9717b3 locking/rwsem: Ma... |
705 706 707 |
struct task_struct *new, *owner; unsigned long flags, new_flags; enum owner_state state; |
3f6d517a3 locking/rwsem: Ma... |
708 |
|
94a9717b3 locking/rwsem: Ma... |
709 |
owner = rwsem_owner_flags(sem, &flags); |
617f3ef95 locking/rwsem: Re... |
710 |
state = rwsem_owner_state(owner, flags); |
3f6d517a3 locking/rwsem: Ma... |
711 712 |
if (state != OWNER_WRITER) return state; |
5dec94d49 locking/rwsem: Me... |
713 714 |
rcu_read_lock(); |
3f6d517a3 locking/rwsem: Ma... |
715 |
for (;;) { |
91d2a812d locking/rwsem: Ma... |
716 717 718 719 720 721 |
/* * When a waiting writer set the handoff flag, it may spin * on the owner as well. Once that writer acquires the lock, * we can spin on it. So we don't need to quit even when the * handoff bit is set. */ |
94a9717b3 locking/rwsem: Ma... |
722 723 |
new = rwsem_owner_flags(sem, &new_flags); if ((new != owner) || (new_flags != flags)) { |
617f3ef95 locking/rwsem: Re... |
724 |
state = rwsem_owner_state(new, new_flags); |
3f6d517a3 locking/rwsem: Ma... |
725 726 |
break; } |
5dec94d49 locking/rwsem: Me... |
727 728 729 730 731 732 733 |
/* * Ensure we emit the owner->on_cpu, dereference _after_ * checking sem->owner still matches owner, if that fails, * owner might point to free()d memory, if it still matches, * the rcu_read_lock() ensures the memory stays valid. */ barrier(); |
5dec94d49 locking/rwsem: Me... |
734 |
if (need_resched() || !owner_on_cpu(owner)) { |
3f6d517a3 locking/rwsem: Ma... |
735 736 |
state = OWNER_NONSPINNABLE; break; |
5dec94d49 locking/rwsem: Me... |
737 738 739 740 741 |
} cpu_relax(); } rcu_read_unlock(); |
3f6d517a3 locking/rwsem: Ma... |
742 |
return state; |
5dec94d49 locking/rwsem: Me... |
743 |
} |
7d43f1ce9 locking/rwsem: En... |
744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 |
/* * Calculate reader-owned rwsem spinning threshold for writer * * The more readers own the rwsem, the longer it will take for them to * wind down and free the rwsem. So the empirical formula used to * determine the actual spinning time limit here is: * * Spinning threshold = (10 + nr_readers/2)us * * The limit is capped to a maximum of 25us (30 readers). This is just * a heuristic and is subjected to change in the future. */ static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem) { long count = atomic_long_read(&sem->count); int readers = count >> RWSEM_READER_SHIFT; u64 delta; if (readers > 30) readers = 30; delta = (20 + readers) * NSEC_PER_USEC / 2; return sched_clock() + delta; } |
617f3ef95 locking/rwsem: Re... |
768 |
static bool rwsem_optimistic_spin(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
769 770 |
{ bool taken = false; |
990fa7384 locking/rwsem: Mo... |
771 |
int prev_owner_state = OWNER_NULL; |
7d43f1ce9 locking/rwsem: En... |
772 773 |
int loop = 0; u64 rspin_threshold = 0; |
5dec94d49 locking/rwsem: Me... |
774 775 776 777 |
preempt_disable(); /* sem->wait_lock should not be held when doing optimistic spinning */ |
5dec94d49 locking/rwsem: Me... |
778 779 780 781 782 783 784 |
if (!osq_lock(&sem->osq)) goto done; /* * Optimistically spin on the owner field and attempt to acquire the * lock whenever the owner changes. Spinning will be stopped when: * 1) the owning writer isn't running; or |
7d43f1ce9 locking/rwsem: En... |
785 |
* 2) readers own the lock and spinning time has exceeded limit. |
5dec94d49 locking/rwsem: Me... |
786 |
*/ |
990fa7384 locking/rwsem: Mo... |
787 |
for (;;) { |
7d43f1ce9 locking/rwsem: En... |
788 |
enum owner_state owner_state; |
990fa7384 locking/rwsem: Mo... |
789 |
|
617f3ef95 locking/rwsem: Re... |
790 |
owner_state = rwsem_spin_on_owner(sem); |
990fa7384 locking/rwsem: Mo... |
791 792 |
if (!(owner_state & OWNER_SPINNABLE)) break; |
5dec94d49 locking/rwsem: Me... |
793 794 795 |
/* * Try to acquire the lock */ |
617f3ef95 locking/rwsem: Re... |
796 |
taken = rwsem_try_write_lock_unqueued(sem); |
cf69482d6 locking/rwsem: En... |
797 798 |
if (taken) |
5dec94d49 locking/rwsem: Me... |
799 |
break; |
5dec94d49 locking/rwsem: Me... |
800 801 |
/* |
7d43f1ce9 locking/rwsem: En... |
802 803 |
* Time-based reader-owned rwsem optimistic spinning */ |
617f3ef95 locking/rwsem: Re... |
804 |
if (owner_state == OWNER_READER) { |
7d43f1ce9 locking/rwsem: En... |
805 806 807 808 809 810 811 812 |
/* * Re-initialize rspin_threshold every time when * the owner state changes from non-reader to reader. * This allows a writer to steal the lock in between * 2 reader phases and have the threshold reset at * the beginning of the 2nd reader phase. */ if (prev_owner_state != OWNER_READER) { |
617f3ef95 locking/rwsem: Re... |
813 |
if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE)) |
7d43f1ce9 locking/rwsem: En... |
814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 |
break; rspin_threshold = rwsem_rspin_threshold(sem); loop = 0; } /* * Check time threshold once every 16 iterations to * avoid calling sched_clock() too frequently so * as to reduce the average latency between the times * when the lock becomes free and when the spinner * is ready to do a trylock. */ else if (!(++loop & 0xf) && (sched_clock() > rspin_threshold)) { rwsem_set_nonspinnable(sem); lockevent_inc(rwsem_opt_nospin); break; } } /* |
990fa7384 locking/rwsem: Mo... |
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 |
* An RT task cannot do optimistic spinning if it cannot * be sure the lock holder is running or live-lock may * happen if the current task and the lock holder happen * to run in the same CPU. However, aborting optimistic * spinning while a NULL owner is detected may miss some * opportunity where spinning can continue without causing * problem. * * There are 2 possible cases where an RT task may be able * to continue spinning. * * 1) The lock owner is in the process of releasing the * lock, sem->owner is cleared but the lock has not * been released yet. * 2) The lock was free and owner cleared, but another * task just comes in and acquire the lock before * we try to get it. The new owner may be a spinnable * writer. * |
e2db7592b locking: Fix typo... |
853 |
* To take advantage of two scenarios listed above, the RT |
990fa7384 locking/rwsem: Mo... |
854 855 856 857 858 859 860 861 862 |
* task is made to retry one more time to see if it can * acquire the lock or continue spinning on the new owning * writer. Of course, if the time lag is long enough or the * new owner is not a writer or spinnable, the RT task will * quit spinning. * * If the owner is a writer, the need_resched() check is * done inside rwsem_spin_on_owner(). If the owner is not * a writer, need_resched() check needs to be done here. |
5dec94d49 locking/rwsem: Me... |
863 |
*/ |
990fa7384 locking/rwsem: Mo... |
864 865 866 867 868 869 870 871 |
if (owner_state != OWNER_WRITER) { if (need_resched()) break; if (rt_task(current) && (prev_owner_state != OWNER_WRITER)) break; } prev_owner_state = owner_state; |
5dec94d49 locking/rwsem: Me... |
872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 |
/* * The cpu_relax() call is a compiler barrier which forces * everything in this loop to be re-loaded. We don't need * memory barriers as we'll eventually observe the right * values at the cost of a few extra spins. */ cpu_relax(); } osq_unlock(&sem->osq); done: preempt_enable(); lockevent_cond_inc(rwsem_opt_fail, !taken); return taken; } |
7d43f1ce9 locking/rwsem: En... |
887 888 |
/* |
617f3ef95 locking/rwsem: Re... |
889 |
* Clear the owner's RWSEM_NONSPINNABLE bit if it is set. This should |
7d43f1ce9 locking/rwsem: En... |
890 |
* only be called when the reader count reaches 0. |
5cfd92e12 locking/rwsem: Ad... |
891 |
*/ |
617f3ef95 locking/rwsem: Re... |
892 |
static inline void clear_nonspinnable(struct rw_semaphore *sem) |
5cfd92e12 locking/rwsem: Ad... |
893 |
{ |
617f3ef95 locking/rwsem: Re... |
894 895 |
if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE)) atomic_long_andnot(RWSEM_NONSPINNABLE, &sem->owner); |
1a728dff8 locking/rwsem: En... |
896 |
} |
5dec94d49 locking/rwsem: Me... |
897 |
#else |
617f3ef95 locking/rwsem: Re... |
898 |
static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) |
cf69482d6 locking/rwsem: En... |
899 900 901 |
{ return false; } |
617f3ef95 locking/rwsem: Re... |
902 |
static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
903 904 905 |
{ return false; } |
7d43f1ce9 locking/rwsem: En... |
906 |
|
617f3ef95 locking/rwsem: Re... |
907 |
static inline void clear_nonspinnable(struct rw_semaphore *sem) { } |
1a728dff8 locking/rwsem: En... |
908 |
|
562d350a8 locking/rwsem: Di... |
909 |
static inline enum owner_state |
617f3ef95 locking/rwsem: Re... |
910 |
rwsem_spin_on_owner(struct rw_semaphore *sem) |
91d2a812d locking/rwsem: Ma... |
911 |
{ |
562d350a8 locking/rwsem: Di... |
912 |
return OWNER_NONSPINNABLE; |
91d2a812d locking/rwsem: Ma... |
913 |
} |
5dec94d49 locking/rwsem: Me... |
914 915 916 917 918 |
#endif /* * Wait for the read lock to be granted */ |
6cef7ff6e locking/rwsem: Co... |
919 |
static struct rw_semaphore __sched * |
2f064a59a sched: Change tas... |
920 |
rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int state) |
5dec94d49 locking/rwsem: Me... |
921 |
{ |
617f3ef95 locking/rwsem: Re... |
922 |
long adjustment = -RWSEM_READER_BIAS; |
2f06f7029 locking/rwsem: Pr... |
923 |
long rcnt = (count >> RWSEM_READER_SHIFT); |
5dec94d49 locking/rwsem: Me... |
924 925 |
struct rwsem_waiter waiter; DEFINE_WAKE_Q(wake_q); |
a15ea1a35 locking/rwsem: Gu... |
926 |
bool wake = false; |
5dec94d49 locking/rwsem: Me... |
927 |
|
5cfd92e12 locking/rwsem: Ad... |
928 |
/* |
2f06f7029 locking/rwsem: Pr... |
929 |
* To prevent a constant stream of readers from starving a sleeping |
617f3ef95 locking/rwsem: Re... |
930 931 |
* waiter, don't attempt optimistic lock stealing if the lock is * currently owned by readers. |
2f06f7029 locking/rwsem: Pr... |
932 |
*/ |
617f3ef95 locking/rwsem: Re... |
933 934 |
if ((atomic_long_read(&sem->owner) & RWSEM_READER_OWNED) && (rcnt > 1) && !(count & RWSEM_WRITER_LOCKED)) |
2f06f7029 locking/rwsem: Pr... |
935 936 937 |
goto queue; /* |
617f3ef95 locking/rwsem: Re... |
938 |
* Reader optimistic lock stealing. |
1a728dff8 locking/rwsem: En... |
939 |
*/ |
617f3ef95 locking/rwsem: Re... |
940 |
if (!(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { |
1a728dff8 locking/rwsem: En... |
941 942 |
rwsem_set_reader_owned(sem); lockevent_inc(rwsem_rlock_steal); |
1a728dff8 locking/rwsem: En... |
943 |
|
cf69482d6 locking/rwsem: En... |
944 |
/* |
617f3ef95 locking/rwsem: Re... |
945 946 |
* Wake up other readers in the wait queue if it is * the first reader. |
cf69482d6 locking/rwsem: En... |
947 |
*/ |
617f3ef95 locking/rwsem: Re... |
948 |
if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) { |
cf69482d6 locking/rwsem: En... |
949 950 951 952 953 954 955 956 957 958 959 |
raw_spin_lock_irq(&sem->wait_lock); if (!list_empty(&sem->wait_list)) rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); raw_spin_unlock_irq(&sem->wait_lock); wake_up_q(&wake_q); } return sem; } queue: |
5dec94d49 locking/rwsem: Me... |
960 961 |
waiter.task = current; waiter.type = RWSEM_WAITING_FOR_READ; |
4f23dbc1e locking/rwsem: Im... |
962 |
waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT; |
d10e819d1 locking/rwsem: Al... |
963 |
waiter.handoff_set = false; |
5dec94d49 locking/rwsem: Me... |
964 965 966 967 968 |
raw_spin_lock_irq(&sem->wait_lock); if (list_empty(&sem->wait_list)) { /* * In case the wait queue is empty and the lock isn't owned |
4f23dbc1e locking/rwsem: Im... |
969 970 971 |
* by a writer or has the handoff bit set, this reader can * exit the slowpath and return immediately as its * RWSEM_READER_BIAS has already been set in the count. |
5dec94d49 locking/rwsem: Me... |
972 |
*/ |
617f3ef95 locking/rwsem: Re... |
973 |
if (!(atomic_long_read(&sem->count) & |
4f23dbc1e locking/rwsem: Im... |
974 |
(RWSEM_WRITER_MASK | RWSEM_FLAG_HANDOFF))) { |
e1b98fa31 locking/rwsem: Ad... |
975 976 |
/* Provide lock ACQUIRE */ smp_acquire__after_ctrl_dep(); |
5dec94d49 locking/rwsem: Me... |
977 978 979 980 981 982 983 |
raw_spin_unlock_irq(&sem->wait_lock); rwsem_set_reader_owned(sem); lockevent_inc(rwsem_rlock_fast); return sem; } adjustment += RWSEM_FLAG_WAITERS; } |
76723ed1f locking/rwsem: Ma... |
984 |
rwsem_add_waiter(sem, &waiter); |
5dec94d49 locking/rwsem: Me... |
985 986 |
/* we're now waiting on the lock, but no longer actively locking */ |
617f3ef95 locking/rwsem: Re... |
987 |
count = atomic_long_add_return(adjustment, &sem->count); |
5dec94d49 locking/rwsem: Me... |
988 989 990 991 992 993 994 |
/* * If there are no active locks, wake the front queued process(es). * * If there are no writers and we are first in the queue, * wake our own waiter to join the existing active readers ! */ |
7d43f1ce9 locking/rwsem: En... |
995 |
if (!(count & RWSEM_LOCK_MASK)) { |
617f3ef95 locking/rwsem: Re... |
996 |
clear_nonspinnable(sem); |
7d43f1ce9 locking/rwsem: En... |
997 998 999 1000 |
wake = true; } if (wake || (!(count & RWSEM_WRITER_MASK) && (adjustment & RWSEM_FLAG_WAITERS))) |
6cef7ff6e locking/rwsem: Co... |
1001 |
rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); |
5dec94d49 locking/rwsem: Me... |
1002 1003 1004 1005 1006 |
raw_spin_unlock_irq(&sem->wait_lock); wake_up_q(&wake_q); /* wait to be given the lock */ |
6ffddfb9e locking/rwsem: Ad... |
1007 |
for (;;) { |
5dec94d49 locking/rwsem: Me... |
1008 |
set_current_state(state); |
99143f82a lcoking/rwsem: Ad... |
1009 |
if (!smp_load_acquire(&waiter.task)) { |
6ffddfb9e locking/rwsem: Ad... |
1010 |
/* Matches rwsem_mark_wake()'s smp_store_release(). */ |
5dec94d49 locking/rwsem: Me... |
1011 |
break; |
99143f82a lcoking/rwsem: Ad... |
1012 |
} |
5dec94d49 locking/rwsem: Me... |
1013 1014 1015 1016 1017 |
if (signal_pending_state(state, current)) { raw_spin_lock_irq(&sem->wait_lock); if (waiter.task) goto out_nolock; raw_spin_unlock_irq(&sem->wait_lock); |
6ffddfb9e locking/rwsem: Ad... |
1018 |
/* Ordered by sem->wait_lock against rwsem_mark_wake(). */ |
5dec94d49 locking/rwsem: Me... |
1019 1020 1021 1022 1023 1024 1025 1026 1027 |
break; } schedule(); lockevent_inc(rwsem_sleep_reader); } __set_current_state(TASK_RUNNING); lockevent_inc(rwsem_rlock); return sem; |
6ffddfb9e locking/rwsem: Ad... |
1028 |
|
5dec94d49 locking/rwsem: Me... |
1029 |
out_nolock: |
76723ed1f locking/rwsem: Ma... |
1030 |
rwsem_del_waiter(sem, &waiter); |
5dec94d49 locking/rwsem: Me... |
1031 1032 1033 1034 1035 |
raw_spin_unlock_irq(&sem->wait_lock); __set_current_state(TASK_RUNNING); lockevent_inc(rwsem_rlock_fail); return ERR_PTR(-EINTR); } |
5dec94d49 locking/rwsem: Me... |
1036 1037 1038 |
/* * Wait until we successfully acquire the write lock */ |
6cef7ff6e locking/rwsem: Co... |
1039 1040 |
static struct rw_semaphore * rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) |
5dec94d49 locking/rwsem: Me... |
1041 1042 |
{ long count; |
5dec94d49 locking/rwsem: Me... |
1043 |
struct rwsem_waiter waiter; |
5dec94d49 locking/rwsem: Me... |
1044 1045 1046 |
DEFINE_WAKE_Q(wake_q); /* do optimistic spinning and steal lock if possible */ |
617f3ef95 locking/rwsem: Re... |
1047 |
if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) { |
6ffddfb9e locking/rwsem: Ad... |
1048 |
/* rwsem_optimistic_spin() implies ACQUIRE on success */ |
5dec94d49 locking/rwsem: Me... |
1049 |
return sem; |
6ffddfb9e locking/rwsem: Ad... |
1050 |
} |
5dec94d49 locking/rwsem: Me... |
1051 1052 1053 1054 1055 1056 1057 |
/* * Optimistic spinning failed, proceed to the slowpath * and block until we can acquire the sem. */ waiter.task = current; waiter.type = RWSEM_WAITING_FOR_WRITE; |
4f23dbc1e locking/rwsem: Im... |
1058 |
waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT; |
76723ed1f locking/rwsem: Ma... |
1059 |
waiter.handoff_set = false; |
5dec94d49 locking/rwsem: Me... |
1060 1061 |
raw_spin_lock_irq(&sem->wait_lock); |
76723ed1f locking/rwsem: Ma... |
1062 |
rwsem_add_waiter(sem, &waiter); |
5dec94d49 locking/rwsem: Me... |
1063 1064 |
/* we're now waiting on the lock */ |
76723ed1f locking/rwsem: Ma... |
1065 |
if (rwsem_first_waiter(sem) != &waiter) { |
5dec94d49 locking/rwsem: Me... |
1066 1067 1068 |
count = atomic_long_read(&sem->count); /* |
4f23dbc1e locking/rwsem: Im... |
1069 |
* If there were already threads queued before us and: |
c034f48e9 kernel: delete re... |
1070 |
* 1) there are no active locks, wake the front |
4f23dbc1e locking/rwsem: Im... |
1071 1072 1073 1074 |
* queued process(es) as the handoff bit might be set. * 2) there are no active writers and some readers, the lock * must be read owned; so we try to wake any read lock * waiters that were queued ahead of us. |
5dec94d49 locking/rwsem: Me... |
1075 |
*/ |
4f23dbc1e locking/rwsem: Im... |
1076 1077 |
if (count & RWSEM_WRITER_MASK) goto wait; |
5dec94d49 locking/rwsem: Me... |
1078 |
|
4f23dbc1e locking/rwsem: Im... |
1079 1080 1081 |
rwsem_mark_wake(sem, (count & RWSEM_READER_MASK) ? RWSEM_WAKE_READERS : RWSEM_WAKE_ANY, &wake_q); |
5dec94d49 locking/rwsem: Me... |
1082 |
|
00f3c5a3d locking/rwsem: Al... |
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 |
if (!wake_q_empty(&wake_q)) { /* * We want to minimize wait_lock hold time especially * when a large number of readers are to be woken up. */ raw_spin_unlock_irq(&sem->wait_lock); wake_up_q(&wake_q); wake_q_init(&wake_q); /* Used again, reinit */ raw_spin_lock_irq(&sem->wait_lock); } |
5dec94d49 locking/rwsem: Me... |
1093 |
} else { |
00f3c5a3d locking/rwsem: Al... |
1094 |
atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count); |
5dec94d49 locking/rwsem: Me... |
1095 |
} |
4f23dbc1e locking/rwsem: Im... |
1096 |
wait: |
5dec94d49 locking/rwsem: Me... |
1097 1098 |
/* wait until we successfully acquire the lock */ set_current_state(state); |
6ffddfb9e locking/rwsem: Ad... |
1099 |
for (;;) { |
76723ed1f locking/rwsem: Ma... |
1100 |
if (rwsem_try_write_lock(sem, &waiter)) { |
6ffddfb9e locking/rwsem: Ad... |
1101 |
/* rwsem_try_write_lock() implies ACQUIRE on success */ |
5dec94d49 locking/rwsem: Me... |
1102 |
break; |
6ffddfb9e locking/rwsem: Ad... |
1103 |
} |
4f23dbc1e locking/rwsem: Im... |
1104 |
|
5dec94d49 locking/rwsem: Me... |
1105 |
raw_spin_unlock_irq(&sem->wait_lock); |
76723ed1f locking/rwsem: Ma... |
1106 1107 |
if (signal_pending_state(state, current)) goto out_nolock; |
91d2a812d locking/rwsem: Ma... |
1108 1109 1110 1111 1112 1113 1114 1115 |
/* * After setting the handoff bit and failing to acquire * the lock, attempt to spin on owner to accelerate lock * transfer. If the previous owner is a on-cpu writer and it * has just released the lock, OWNER_NULL will be returned. * In this case, we attempt to acquire the lock again * without sleeping. */ |
76723ed1f locking/rwsem: Ma... |
1116 |
if (waiter.handoff_set) { |
562d350a8 locking/rwsem: Di... |
1117 1118 1119 1120 1121 1122 1123 1124 1125 |
enum owner_state owner_state; preempt_disable(); owner_state = rwsem_spin_on_owner(sem); preempt_enable(); if (owner_state == OWNER_NULL) goto trylock_again; } |
91d2a812d locking/rwsem: Ma... |
1126 |
|
76723ed1f locking/rwsem: Ma... |
1127 1128 1129 |
schedule(); lockevent_inc(rwsem_sleep_writer); set_current_state(state); |
91d2a812d locking/rwsem: Ma... |
1130 |
trylock_again: |
5dec94d49 locking/rwsem: Me... |
1131 1132 1133 |
raw_spin_lock_irq(&sem->wait_lock); } __set_current_state(TASK_RUNNING); |
5dec94d49 locking/rwsem: Me... |
1134 1135 |
raw_spin_unlock_irq(&sem->wait_lock); lockevent_inc(rwsem_wlock); |
76723ed1f locking/rwsem: Ma... |
1136 |
return sem; |
5dec94d49 locking/rwsem: Me... |
1137 1138 1139 1140 |
out_nolock: __set_current_state(TASK_RUNNING); raw_spin_lock_irq(&sem->wait_lock); |
76723ed1f locking/rwsem: Ma... |
1141 1142 |
rwsem_del_waiter(sem, &waiter); if (!list_empty(&sem->wait_list)) |
6cef7ff6e locking/rwsem: Co... |
1143 |
rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); |
5dec94d49 locking/rwsem: Me... |
1144 1145 1146 |
raw_spin_unlock_irq(&sem->wait_lock); wake_up_q(&wake_q); lockevent_inc(rwsem_wlock_fail); |
5dec94d49 locking/rwsem: Me... |
1147 1148 |
return ERR_PTR(-EINTR); } |
5dec94d49 locking/rwsem: Me... |
1149 1150 1151 1152 |
/* * handle waking up a waiter on the semaphore * - up_read/up_write has decremented the active part of count if we come here */ |
d4e5076c3 locking/rwsem: Re... |
1153 |
static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
1154 1155 1156 1157 1158 1159 1160 |
{ unsigned long flags; DEFINE_WAKE_Q(wake_q); raw_spin_lock_irqsave(&sem->wait_lock, flags); if (!list_empty(&sem->wait_list)) |
6cef7ff6e locking/rwsem: Co... |
1161 |
rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); |
5dec94d49 locking/rwsem: Me... |
1162 1163 1164 1165 1166 1167 |
raw_spin_unlock_irqrestore(&sem->wait_lock, flags); wake_up_q(&wake_q); return sem; } |
5dec94d49 locking/rwsem: Me... |
1168 1169 1170 1171 1172 1173 |
/* * downgrade a write lock into a read lock * - caller incremented waiting part of count and discovered it still negative * - just wake up any readers at the front of the queue */ |
6cef7ff6e locking/rwsem: Co... |
1174 |
static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
1175 1176 1177 1178 1179 1180 1181 |
{ unsigned long flags; DEFINE_WAKE_Q(wake_q); raw_spin_lock_irqsave(&sem->wait_lock, flags); if (!list_empty(&sem->wait_list)) |
6cef7ff6e locking/rwsem: Co... |
1182 |
rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); |
5dec94d49 locking/rwsem: Me... |
1183 1184 1185 1186 1187 1188 |
raw_spin_unlock_irqrestore(&sem->wait_lock, flags); wake_up_q(&wake_q); return sem; } |
5dec94d49 locking/rwsem: Me... |
1189 1190 1191 1192 |
/* * lock for reading */ |
c995e638c locking/rwsem: Fo... |
1193 |
static inline int __down_read_common(struct rw_semaphore *sem, int state) |
5dec94d49 locking/rwsem: Me... |
1194 |
{ |
c8fe8b056 locking/rwsem: Pa... |
1195 1196 1197 1198 |
long count; if (!rwsem_read_trylock(sem, &count)) { if (IS_ERR(rwsem_down_read_slowpath(sem, count, state))) |
c995e638c locking/rwsem: Fo... |
1199 |
return -EINTR; |
94a9717b3 locking/rwsem: Ma... |
1200 |
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); |
5dec94d49 locking/rwsem: Me... |
1201 |
} |
c995e638c locking/rwsem: Fo... |
1202 1203 1204 1205 1206 1207 |
return 0; } static inline void __down_read(struct rw_semaphore *sem) { __down_read_common(sem, TASK_UNINTERRUPTIBLE); |
5dec94d49 locking/rwsem: Me... |
1208 |
} |
31784cff7 rwsem: Implement ... |
1209 1210 |
static inline int __down_read_interruptible(struct rw_semaphore *sem) { |
c995e638c locking/rwsem: Fo... |
1211 |
return __down_read_common(sem, TASK_INTERRUPTIBLE); |
31784cff7 rwsem: Implement ... |
1212 |
} |
5dec94d49 locking/rwsem: Me... |
1213 1214 |
static inline int __down_read_killable(struct rw_semaphore *sem) { |
c995e638c locking/rwsem: Fo... |
1215 |
return __down_read_common(sem, TASK_KILLABLE); |
5dec94d49 locking/rwsem: Me... |
1216 1217 1218 1219 |
} static inline int __down_read_trylock(struct rw_semaphore *sem) { |
fce45cd41 locking/rwsem: Ch... |
1220 1221 1222 |
long tmp; DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); |
5dec94d49 locking/rwsem: Me... |
1223 1224 1225 |
/* * Optimize for the case when the rwsem is not locked at all. */ |
fce45cd41 locking/rwsem: Ch... |
1226 |
tmp = RWSEM_UNLOCKED_VALUE; |
5dec94d49 locking/rwsem: Me... |
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 |
do { if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, tmp + RWSEM_READER_BIAS)) { rwsem_set_reader_owned(sem); return 1; } } while (!(tmp & RWSEM_READ_FAILED_MASK)); return 0; } /* * lock for writing */ |
c995e638c locking/rwsem: Fo... |
1240 |
static inline int __down_write_common(struct rw_semaphore *sem, int state) |
5dec94d49 locking/rwsem: Me... |
1241 |
{ |
285c61aed locking/rwsem: In... |
1242 |
if (unlikely(!rwsem_write_trylock(sem))) { |
c995e638c locking/rwsem: Fo... |
1243 |
if (IS_ERR(rwsem_down_write_slowpath(sem, state))) |
5dec94d49 locking/rwsem: Me... |
1244 |
return -EINTR; |
6cef7ff6e locking/rwsem: Co... |
1245 |
} |
285c61aed locking/rwsem: In... |
1246 |
|
5dec94d49 locking/rwsem: Me... |
1247 1248 |
return 0; } |
c995e638c locking/rwsem: Fo... |
1249 1250 1251 1252 1253 1254 1255 1256 1257 |
static inline void __down_write(struct rw_semaphore *sem) { __down_write_common(sem, TASK_UNINTERRUPTIBLE); } static inline int __down_write_killable(struct rw_semaphore *sem) { return __down_write_common(sem, TASK_KILLABLE); } |
5dec94d49 locking/rwsem: Me... |
1258 1259 |
static inline int __down_write_trylock(struct rw_semaphore *sem) { |
fce45cd41 locking/rwsem: Ch... |
1260 |
DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); |
285c61aed locking/rwsem: In... |
1261 |
return rwsem_write_trylock(sem); |
5dec94d49 locking/rwsem: Me... |
1262 1263 1264 1265 1266 |
} /* * unlock after reading */ |
7f26482a8 locking/percpu-rw... |
1267 |
static inline void __up_read(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
1268 1269 |
{ long tmp; |
fce45cd41 locking/rwsem: Ch... |
1270 |
DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); |
94a9717b3 locking/rwsem: Ma... |
1271 |
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); |
fce45cd41 locking/rwsem: Ch... |
1272 |
|
5dec94d49 locking/rwsem: Me... |
1273 1274 |
rwsem_clear_reader_owned(sem); tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); |
a15ea1a35 locking/rwsem: Gu... |
1275 |
DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); |
6cef7ff6e locking/rwsem: Co... |
1276 |
if (unlikely((tmp & (RWSEM_LOCK_MASK|RWSEM_FLAG_WAITERS)) == |
7d43f1ce9 locking/rwsem: En... |
1277 |
RWSEM_FLAG_WAITERS)) { |
617f3ef95 locking/rwsem: Re... |
1278 |
clear_nonspinnable(sem); |
d4e5076c3 locking/rwsem: Re... |
1279 |
rwsem_wake(sem); |
7d43f1ce9 locking/rwsem: En... |
1280 |
} |
5dec94d49 locking/rwsem: Me... |
1281 1282 1283 1284 1285 |
} /* * unlock after writing */ |
7f26482a8 locking/percpu-rw... |
1286 |
static inline void __up_write(struct rw_semaphore *sem) |
5dec94d49 locking/rwsem: Me... |
1287 |
{ |
6cef7ff6e locking/rwsem: Co... |
1288 |
long tmp; |
fce45cd41 locking/rwsem: Ch... |
1289 |
DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); |
02f1082b0 locking/rwsem: Cl... |
1290 1291 1292 1293 |
/* * sem->owner may differ from current if the ownership is transferred * to an anonymous writer by setting the RWSEM_NONSPINNABLE bits. */ |
94a9717b3 locking/rwsem: Ma... |
1294 1295 |
DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) && !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem); |
fce45cd41 locking/rwsem: Ch... |
1296 |
|
5dec94d49 locking/rwsem: Me... |
1297 |
rwsem_clear_owner(sem); |
6cef7ff6e locking/rwsem: Co... |
1298 1299 |
tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); if (unlikely(tmp & RWSEM_FLAG_WAITERS)) |
d4e5076c3 locking/rwsem: Re... |
1300 |
rwsem_wake(sem); |
5dec94d49 locking/rwsem: Me... |
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 |
} /* * downgrade write lock to read lock */ static inline void __downgrade_write(struct rw_semaphore *sem) { long tmp; /* * When downgrading from exclusive to shared ownership, * anything inside the write-locked region cannot leak * into the read side. In contrast, anything in the * read-locked region is ok to be re-ordered into the * write side. As such, rely on RELEASE semantics. */ |
94a9717b3 locking/rwsem: Ma... |
1317 |
DEBUG_RWSEMS_WARN_ON(rwsem_owner(sem) != current, sem); |
5dec94d49 locking/rwsem: Me... |
1318 1319 1320 1321 1322 1323 |
tmp = atomic_long_fetch_add_release( -RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count); rwsem_set_reader_owned(sem); if (tmp & RWSEM_FLAG_WAITERS) rwsem_downgrade_wake(sem); } |
4fc828e24 locking/rwsem: Su... |
1324 |
|
42254105d locking/rwsem: Ad... |
1325 |
#else /* !CONFIG_PREEMPT_RT */ |
e17ba59b7 locking/rtmutex: ... |
1326 |
#define RT_MUTEX_BUILD_MUTEX |
42254105d locking/rwsem: Ad... |
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 |
#include "rtmutex.c" #define rwbase_set_and_save_current_state(state) \ set_current_state(state) #define rwbase_restore_current_state() \ __set_current_state(TASK_RUNNING) #define rwbase_rtmutex_lock_state(rtm, state) \ __rt_mutex_lock(rtm, state) #define rwbase_rtmutex_slowlock_locked(rtm, state) \ |
add461325 locking/rtmutex: ... |
1339 |
__rt_mutex_slowlock_locked(rtm, NULL, state) |
42254105d locking/rwsem: Ad... |
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 |
#define rwbase_rtmutex_unlock(rtm) \ __rt_mutex_unlock(rtm) #define rwbase_rtmutex_trylock(rtm) \ __rt_mutex_trylock(rtm) #define rwbase_signal_pending_state(state, current) \ signal_pending_state(state, current) #define rwbase_schedule() \ schedule() #include "rwbase_rt.c" |
15eb7c888 locking/rwsem: Ad... |
1354 |
void __init_rwsem(struct rw_semaphore *sem, const char *name, |
42254105d locking/rwsem: Ad... |
1355 1356 |
struct lock_class_key *key) { |
15eb7c888 locking/rwsem: Ad... |
1357 1358 1359 |
init_rwbase_rt(&(sem)->rwbase); #ifdef CONFIG_DEBUG_LOCK_ALLOC |
42254105d locking/rwsem: Ad... |
1360 1361 |
debug_check_no_locks_freed((void *)sem, sizeof(*sem)); lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); |
42254105d locking/rwsem: Ad... |
1362 |
#endif |
15eb7c888 locking/rwsem: Ad... |
1363 1364 |
} EXPORT_SYMBOL(__init_rwsem); |
42254105d locking/rwsem: Ad... |
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 |
static inline void __down_read(struct rw_semaphore *sem) { rwbase_read_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); } static inline int __down_read_interruptible(struct rw_semaphore *sem) { return rwbase_read_lock(&sem->rwbase, TASK_INTERRUPTIBLE); } static inline int __down_read_killable(struct rw_semaphore *sem) { return rwbase_read_lock(&sem->rwbase, TASK_KILLABLE); } static inline int __down_read_trylock(struct rw_semaphore *sem) { return rwbase_read_trylock(&sem->rwbase); } static inline void __up_read(struct rw_semaphore *sem) { rwbase_read_unlock(&sem->rwbase, TASK_NORMAL); } static inline void __sched __down_write(struct rw_semaphore *sem) { rwbase_write_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); } static inline int __sched __down_write_killable(struct rw_semaphore *sem) { return rwbase_write_lock(&sem->rwbase, TASK_KILLABLE); } static inline int __down_write_trylock(struct rw_semaphore *sem) { return rwbase_write_trylock(&sem->rwbase); } static inline void __up_write(struct rw_semaphore *sem) { rwbase_write_unlock(&sem->rwbase); } static inline void __downgrade_write(struct rw_semaphore *sem) { rwbase_write_downgrade(&sem->rwbase); } /* Debug stubs for the common API */ #define DEBUG_RWSEMS_WARN_ON(c, sem) static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, struct task_struct *owner) { } static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem) { int count = atomic_read(&sem->rwbase.readers); return count < 0 && count != READER_BIAS; } #endif /* CONFIG_PREEMPT_RT */ |
c4e05116a [PATCH] lockdep: ... |
1432 1433 1434 |
/* * lock for reading */ |
c7af77b58 sched: mark rwsem... |
1435 |
void __sched down_read(struct rw_semaphore *sem) |
c4e05116a [PATCH] lockdep: ... |
1436 1437 1438 |
{ might_sleep(); rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); |
4fe87745a lockstat: hook in... |
1439 |
LOCK_CONTENDED(sem, __down_read_trylock, __down_read); |
c4e05116a [PATCH] lockdep: ... |
1440 |
} |
c4e05116a [PATCH] lockdep: ... |
1441 |
EXPORT_SYMBOL(down_read); |
31784cff7 rwsem: Implement ... |
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 |
int __sched down_read_interruptible(struct rw_semaphore *sem) { might_sleep(); rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_interruptible)) { rwsem_release(&sem->dep_map, _RET_IP_); return -EINTR; } return 0; } EXPORT_SYMBOL(down_read_interruptible); |
76f8507f7 locking/rwsem: Ad... |
1455 1456 1457 1458 1459 1460 |
int __sched down_read_killable(struct rw_semaphore *sem) { might_sleep(); rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { |
5facae4f3 locking/lockdep: ... |
1461 |
rwsem_release(&sem->dep_map, _RET_IP_); |
76f8507f7 locking/rwsem: Ad... |
1462 1463 |
return -EINTR; } |
76f8507f7 locking/rwsem: Ad... |
1464 1465 |
return 0; } |
76f8507f7 locking/rwsem: Ad... |
1466 |
EXPORT_SYMBOL(down_read_killable); |
c4e05116a [PATCH] lockdep: ... |
1467 1468 1469 1470 1471 1472 |
/* * trylock for reading -- returns 1 if successful, 0 if contention */ int down_read_trylock(struct rw_semaphore *sem) { int ret = __down_read_trylock(sem); |
c7580c1e8 locking/rwsem: Mo... |
1473 |
if (ret == 1) |
c4e05116a [PATCH] lockdep: ... |
1474 1475 1476 |
rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); return ret; } |
c4e05116a [PATCH] lockdep: ... |
1477 1478 1479 1480 1481 |
EXPORT_SYMBOL(down_read_trylock); /* * lock for writing */ |
c7af77b58 sched: mark rwsem... |
1482 |
void __sched down_write(struct rw_semaphore *sem) |
c4e05116a [PATCH] lockdep: ... |
1483 1484 1485 |
{ might_sleep(); rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); |
4fe87745a lockstat: hook in... |
1486 |
LOCK_CONTENDED(sem, __down_write_trylock, __down_write); |
c4e05116a [PATCH] lockdep: ... |
1487 |
} |
c4e05116a [PATCH] lockdep: ... |
1488 1489 1490 |
EXPORT_SYMBOL(down_write); /* |
916633a40 locking/rwsem: Pr... |
1491 1492 1493 1494 1495 1496 |
* lock for writing */ int __sched down_write_killable(struct rw_semaphore *sem) { might_sleep(); rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); |
6cef7ff6e locking/rwsem: Co... |
1497 1498 |
if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) { |
5facae4f3 locking/lockdep: ... |
1499 |
rwsem_release(&sem->dep_map, _RET_IP_); |
916633a40 locking/rwsem: Pr... |
1500 1501 |
return -EINTR; } |
916633a40 locking/rwsem: Pr... |
1502 1503 |
return 0; } |
916633a40 locking/rwsem: Pr... |
1504 1505 1506 |
EXPORT_SYMBOL(down_write_killable); /* |
c4e05116a [PATCH] lockdep: ... |
1507 1508 1509 1510 1511 |
* trylock for writing -- returns 1 if successful, 0 if contention */ int down_write_trylock(struct rw_semaphore *sem) { int ret = __down_write_trylock(sem); |
c7580c1e8 locking/rwsem: Mo... |
1512 |
if (ret == 1) |
428e6ce02 Lockdep treats do... |
1513 |
rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_); |
4fc828e24 locking/rwsem: Su... |
1514 |
|
c4e05116a [PATCH] lockdep: ... |
1515 1516 |
return ret; } |
c4e05116a [PATCH] lockdep: ... |
1517 1518 1519 1520 1521 1522 1523 |
EXPORT_SYMBOL(down_write_trylock); /* * release a read lock */ void up_read(struct rw_semaphore *sem) { |
5facae4f3 locking/lockdep: ... |
1524 |
rwsem_release(&sem->dep_map, _RET_IP_); |
c4e05116a [PATCH] lockdep: ... |
1525 1526 |
__up_read(sem); } |
c4e05116a [PATCH] lockdep: ... |
1527 1528 1529 1530 1531 1532 1533 |
EXPORT_SYMBOL(up_read); /* * release a write lock */ void up_write(struct rw_semaphore *sem) { |
5facae4f3 locking/lockdep: ... |
1534 |
rwsem_release(&sem->dep_map, _RET_IP_); |
c4e05116a [PATCH] lockdep: ... |
1535 1536 |
__up_write(sem); } |
c4e05116a [PATCH] lockdep: ... |
1537 1538 1539 1540 1541 1542 1543 |
EXPORT_SYMBOL(up_write); /* * downgrade write lock to read lock */ void downgrade_write(struct rw_semaphore *sem) { |
6419c4af7 locking/lockdep: ... |
1544 |
lock_downgrade(&sem->dep_map, _RET_IP_); |
c4e05116a [PATCH] lockdep: ... |
1545 1546 |
__downgrade_write(sem); } |
c4e05116a [PATCH] lockdep: ... |
1547 |
EXPORT_SYMBOL(downgrade_write); |
4ea2176df [PATCH] lockdep: ... |
1548 1549 1550 1551 1552 1553 1554 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC void down_read_nested(struct rw_semaphore *sem, int subclass) { might_sleep(); rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); |
4fe87745a lockstat: hook in... |
1555 |
LOCK_CONTENDED(sem, __down_read_trylock, __down_read); |
4ea2176df [PATCH] lockdep: ... |
1556 |
} |
4ea2176df [PATCH] lockdep: ... |
1557 |
EXPORT_SYMBOL(down_read_nested); |
0f9368b5b rwsem: Implement ... |
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 |
int down_read_killable_nested(struct rw_semaphore *sem, int subclass) { might_sleep(); rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { rwsem_release(&sem->dep_map, _RET_IP_); return -EINTR; } return 0; } EXPORT_SYMBOL(down_read_killable_nested); |
1b963c81b lockdep, rwsem: p... |
1571 1572 1573 1574 |
void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest) { might_sleep(); rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_); |
1b963c81b lockdep, rwsem: p... |
1575 1576 |
LOCK_CONTENDED(sem, __down_write_trylock, __down_write); } |
1b963c81b lockdep, rwsem: p... |
1577 |
EXPORT_SYMBOL(_down_write_nest_lock); |
84759c6d1 Revert "rw_semaph... |
1578 1579 1580 |
void down_read_non_owner(struct rw_semaphore *sem) { might_sleep(); |
84759c6d1 Revert "rw_semaph... |
1581 |
__down_read(sem); |
925b9cd1b locking/rwsem: Ma... |
1582 |
__rwsem_set_reader_owned(sem, NULL); |
84759c6d1 Revert "rw_semaph... |
1583 |
} |
84759c6d1 Revert "rw_semaph... |
1584 |
EXPORT_SYMBOL(down_read_non_owner); |
4ea2176df [PATCH] lockdep: ... |
1585 1586 1587 1588 |
void down_write_nested(struct rw_semaphore *sem, int subclass) { might_sleep(); rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); |
4fe87745a lockstat: hook in... |
1589 |
LOCK_CONTENDED(sem, __down_write_trylock, __down_write); |
4ea2176df [PATCH] lockdep: ... |
1590 |
} |
4ea2176df [PATCH] lockdep: ... |
1591 |
EXPORT_SYMBOL(down_write_nested); |
887bddfa9 add down_write_ki... |
1592 1593 1594 1595 |
int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass) { might_sleep(); rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); |
6cef7ff6e locking/rwsem: Co... |
1596 1597 |
if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) { |
5facae4f3 locking/lockdep: ... |
1598 |
rwsem_release(&sem->dep_map, _RET_IP_); |
887bddfa9 add down_write_ki... |
1599 1600 |
return -EINTR; } |
887bddfa9 add down_write_ki... |
1601 1602 |
return 0; } |
887bddfa9 add down_write_ki... |
1603 |
EXPORT_SYMBOL(down_write_killable_nested); |
84759c6d1 Revert "rw_semaph... |
1604 1605 |
void up_read_non_owner(struct rw_semaphore *sem) { |
94a9717b3 locking/rwsem: Ma... |
1606 |
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); |
84759c6d1 Revert "rw_semaph... |
1607 1608 |
__up_read(sem); } |
84759c6d1 Revert "rw_semaph... |
1609 |
EXPORT_SYMBOL(up_read_non_owner); |
4ea2176df [PATCH] lockdep: ... |
1610 |
#endif |