Blame view
arch/x86/include/asm/rwsem.h
5.91 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 |
/* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+ * * Written by David Howells (dhowells@redhat.com). * |
99122a3fe x86: remove more ... |
5 |
* Derived from asm-x86/semaphore.h |
1da177e4c Linux-2.6.12-rc2 |
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
* * * The MSW of the count is the negated number of active writers and waiting * lockers, and the LSW is the total number of active locks * * The lock count is initialized to 0 (no active and no waiting lockers). * * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an * uncontended lock. This can be determined because XADD returns the old value. * Readers increment by 1 and see a positive value when uncontended, negative * if there are writers (and maybe) readers waiting (in which case it goes to * sleep). * * The value of WAITING_BIAS supports up to 32766 waiting processes. This can * be extended to 65534 by manually checking the whole MSW rather than relying * on the S flag. * * The value of ACTIVE_BIAS supports up to 65535 active processes. * * This should be totally fair - if anything is waiting, a process that wants a * lock will go to the back of the queue. When the currently active lock is * released, if there's a writer at the front of the queue, then that and only * that will be woken up; if there's a bunch of consequtive readers at the * front, then they'll all be woken up, but no other readers will be. */ |
1965aae3c x86: Fix ASM_X86_... |
31 32 |
#ifndef _ASM_X86_RWSEM_H #define _ASM_X86_RWSEM_H |
1da177e4c Linux-2.6.12-rc2 |
33 34 35 36 37 38 |
#ifndef _LINUX_RWSEM_H #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" #endif #ifdef __KERNEL__ |
1838ef1d7 x86-64, rwsem: 64... |
39 |
#include <asm/asm.h> |
1da177e4c Linux-2.6.12-rc2 |
40 |
|
1da177e4c Linux-2.6.12-rc2 |
41 |
/* |
1838ef1d7 x86-64, rwsem: 64... |
42 43 44 |
* The bias values and the counter type limits the number of * potential readers/writers to 32767 for 32 bits and 2147483647 * for 64 bits. |
1da177e4c Linux-2.6.12-rc2 |
45 |
*/ |
6e5609a97 include/asm-x86/r... |
46 |
|
1838ef1d7 x86-64, rwsem: 64... |
47 48 49 50 51 52 53 54 55 |
#ifdef CONFIG_X86_64 # define RWSEM_ACTIVE_MASK 0xffffffffL #else # define RWSEM_ACTIVE_MASK 0x0000ffffL #endif #define RWSEM_UNLOCKED_VALUE 0x00000000L #define RWSEM_ACTIVE_BIAS 0x00000001L #define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) |
1da177e4c Linux-2.6.12-rc2 |
56 57 |
#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) |
6e5609a97 include/asm-x86/r... |
58 |
|
1da177e4c Linux-2.6.12-rc2 |
59 60 61 62 63 |
/* * lock for reading */ static inline void __down_read(struct rw_semaphore *sem) { |
6e5609a97 include/asm-x86/r... |
64 65 |
asm volatile("# beginning down_read \t" |
1838ef1d7 x86-64, rwsem: 64... |
66 67 |
LOCK_PREFIX _ASM_INC "(%1) \t" |
b4bcb4c28 x86, rwsem: Minor... |
68 |
/* adds 0x00000001 */ |
6e5609a97 include/asm-x86/r... |
69 70 71 72 73 74 75 76 77 78 79 |
" jns 1f " " call call_rwsem_down_read_failed " "1: \t" "# ending down_read \t" : "+m" (sem->count) : "a" (sem) : "memory", "cc"); |
1da177e4c Linux-2.6.12-rc2 |
80 81 82 83 84 85 86 |
} /* * trylock for reading -- returns 1 if successful, 0 if contention */ static inline int __down_read_trylock(struct rw_semaphore *sem) { |
bde11efbc x86: Cleanup rwse... |
87 |
long result, tmp; |
6e5609a97 include/asm-x86/r... |
88 89 |
asm volatile("# beginning __down_read_trylock \t" |
59c33fa77 x86-32: clean up ... |
90 91 |
" mov %0,%1 \t" |
6e5609a97 include/asm-x86/r... |
92 93 |
"1: \t" |
59c33fa77 x86-32: clean up ... |
94 95 96 97 |
" mov %1,%2 \t" " add %3,%2 \t" |
6e5609a97 include/asm-x86/r... |
98 99 |
" jle 2f \t" |
59c33fa77 x86-32: clean up ... |
100 101 |
LOCK_PREFIX " cmpxchg %2,%0 \t" |
6e5609a97 include/asm-x86/r... |
102 103 104 105 106 107 108 109 110 111 |
" jnz 1b \t" "2: \t" "# ending __down_read_trylock \t" : "+m" (sem->count), "=&a" (result), "=&r" (tmp) : "i" (RWSEM_ACTIVE_READ_BIAS) : "memory", "cc"); return result >= 0 ? 1 : 0; |
1da177e4c Linux-2.6.12-rc2 |
112 113 114 115 116 |
} /* * lock for writing */ |
4ea2176df [PATCH] lockdep: ... |
117 |
static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) |
1da177e4c Linux-2.6.12-rc2 |
118 |
{ |
bde11efbc x86: Cleanup rwse... |
119 |
long tmp; |
6e5609a97 include/asm-x86/r... |
120 121 |
asm volatile("# beginning down_write \t" |
59c33fa77 x86-32: clean up ... |
122 123 |
LOCK_PREFIX " xadd %1,(%2) \t" |
b4bcb4c28 x86, rwsem: Minor... |
124 |
/* adds 0xffff0001, returns the old value */ |
59c33fa77 x86-32: clean up ... |
125 126 |
" test %1,%1 \t" |
6e5609a97 include/asm-x86/r... |
127 128 129 130 131 132 133 134 135 |
/* was the count 0 before? */ " jz 1f " " call call_rwsem_down_write_failed " "1: " "# ending down_write" : "+m" (sem->count), "=d" (tmp) |
b4bcb4c28 x86, rwsem: Minor... |
136 |
: "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS) |
6e5609a97 include/asm-x86/r... |
137 |
: "memory", "cc"); |
1da177e4c Linux-2.6.12-rc2 |
138 |
} |
4ea2176df [PATCH] lockdep: ... |
139 140 141 142 |
static inline void __down_write(struct rw_semaphore *sem) { __down_write_nested(sem, 0); } |
1da177e4c Linux-2.6.12-rc2 |
143 144 145 146 147 |
/* * trylock for writing -- returns 1 if successful, 0 if contention */ static inline int __down_write_trylock(struct rw_semaphore *sem) { |
bde11efbc x86: Cleanup rwse... |
148 149 |
long ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE, RWSEM_ACTIVE_WRITE_BIAS); |
1da177e4c Linux-2.6.12-rc2 |
150 151 152 153 154 155 156 157 158 159 |
if (ret == RWSEM_UNLOCKED_VALUE) return 1; return 0; } /* * unlock after reading */ static inline void __up_read(struct rw_semaphore *sem) { |
bde11efbc x86: Cleanup rwse... |
160 |
long tmp; |
6e5609a97 include/asm-x86/r... |
161 162 |
asm volatile("# beginning __up_read \t" |
59c33fa77 x86-32: clean up ... |
163 164 |
LOCK_PREFIX " xadd %1,(%2) \t" |
6e5609a97 include/asm-x86/r... |
165 166 167 |
/* subtracts 1, returns the old value */ " jns 1f \t" |
b4bcb4c28 x86, rwsem: Minor... |
168 169 |
" call call_rwsem_wake " /* expects old value in %edx */ |
6e5609a97 include/asm-x86/r... |
170 171 172 173 174 |
"1: " "# ending __up_read " : "+m" (sem->count), "=d" (tmp) |
b4bcb4c28 x86, rwsem: Minor... |
175 |
: "a" (sem), "1" (-RWSEM_ACTIVE_READ_BIAS) |
6e5609a97 include/asm-x86/r... |
176 |
: "memory", "cc"); |
1da177e4c Linux-2.6.12-rc2 |
177 178 179 180 181 182 183 |
} /* * unlock after writing */ static inline void __up_write(struct rw_semaphore *sem) { |
bde11efbc x86: Cleanup rwse... |
184 |
long tmp; |
6e5609a97 include/asm-x86/r... |
185 186 |
asm volatile("# beginning __up_write \t" |
59c33fa77 x86-32: clean up ... |
187 188 |
LOCK_PREFIX " xadd %1,(%2) \t" |
a751bd858 x86, rwsem: Stay ... |
189 190 191 |
/* subtracts 0xffff0001, returns the old value */ " jns 1f \t" |
b4bcb4c28 x86, rwsem: Minor... |
192 193 |
" call call_rwsem_wake " /* expects old value in %edx */ |
6e5609a97 include/asm-x86/r... |
194 195 196 197 |
"1: \t" "# ending __up_write " |
59c33fa77 x86-32: clean up ... |
198 199 200 |
: "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS) : "memory", "cc"); |
1da177e4c Linux-2.6.12-rc2 |
201 202 203 204 205 206 207 |
} /* * downgrade write lock to read lock */ static inline void __downgrade_write(struct rw_semaphore *sem) { |
6e5609a97 include/asm-x86/r... |
208 209 |
asm volatile("# beginning __downgrade_write \t" |
1838ef1d7 x86-64, rwsem: 64... |
210 211 |
LOCK_PREFIX _ASM_ADD "%2,(%1) \t" |
0d1622d7f x86-64, rwsem: Av... |
212 213 214 215 |
/* * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386) * 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64) */ |
6e5609a97 include/asm-x86/r... |
216 217 218 219 220 221 222 223 224 |
" jns 1f \t" " call call_rwsem_downgrade_wake " "1: \t" "# ending __downgrade_write " : "+m" (sem->count) |
0d1622d7f x86-64, rwsem: Av... |
225 |
: "a" (sem), "er" (-RWSEM_WAITING_BIAS) |
6e5609a97 include/asm-x86/r... |
226 |
: "memory", "cc"); |
1da177e4c Linux-2.6.12-rc2 |
227 228 229 230 231 |
} /* * implement atomic add functionality */ |
bde11efbc x86: Cleanup rwse... |
232 |
static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) |
1da177e4c Linux-2.6.12-rc2 |
233 |
{ |
1838ef1d7 x86-64, rwsem: 64... |
234 |
asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0" |
6e5609a97 include/asm-x86/r... |
235 |
: "+m" (sem->count) |
1838ef1d7 x86-64, rwsem: 64... |
236 |
: "er" (delta)); |
1da177e4c Linux-2.6.12-rc2 |
237 238 239 240 241 |
} /* * implement exchange and add functionality */ |
bde11efbc x86: Cleanup rwse... |
242 |
static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) |
1da177e4c Linux-2.6.12-rc2 |
243 |
{ |
8b8bc2f73 x86: Use xadd hel... |
244 |
return delta + xadd(&sem->count, delta); |
1da177e4c Linux-2.6.12-rc2 |
245 246 247 |
} #endif /* __KERNEL__ */ |
1965aae3c x86: Fix ASM_X86_... |
248 |
#endif /* _ASM_X86_RWSEM_H */ |