Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit c8de2fa4dc2778ae3605925c127b3deac54b2b3a

Authored by Linus Torvalds
2 parents 0f47c9423c b5f541810e
Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga

Merge branch 'rwsem-optimizations' 

Merge rwsem optimizations from Michel Lespinasse:
 "These patches extend Alex Shi's work (which added write lock stealing
  on the rwsem slow path) in order to provide rwsem write lock stealing
  on the fast path (that is, without taking the rwsem's wait_lock).

  I have unfortunately been unable to push this through -next before due
  to Ingo Molnar / David Howells / Peter Zijlstra being busy with other
  things.  However, this has gotten some attention from Rik van Riel and
  Davidlohr Bueso who both commented that they felt this was ready for
  v3.10, and Ingo Molnar has said that he was OK with me pushing
  directly to you.  So, here goes :)

  Davidlohr got the following test results from pgbench running on a
  quad-core laptop:

    | db_size | clients  |  tps-vanilla   |   tps-rwsem  |
    +---------+----------+----------------+--------------+
    | 160 MB   |       1 |           5803 |         6906 | + 19.0%
    | 160 MB   |       2 |          13092 |        15931 |
    | 160 MB   |       4 |          29412 |        33021 |
    | 160 MB   |       8 |          32448 |        34626 |
    | 160 MB   |      16 |          32758 |        33098 |
    | 160 MB   |      20 |          26940 |        31343 | + 16.3%
    | 160 MB   |      30 |          25147 |        28961 |
    | 160 MB   |      40 |          25484 |        26902 |
    | 160 MB   |      50 |          24528 |        25760 |
    ------------------------------------------------------
    | 1.6 GB   |       1 |           5733 |         7729 | + 34.8%
    | 1.6 GB   |       2 |           9411 |        19009 | + 101.9%
    | 1.6 GB   |       4 |          31818 |        33185 |
    | 1.6 GB   |       8 |          33700 |        34550 |
    | 1.6 GB   |      16 |          32751 |        33079 |
    | 1.6 GB   |      20 |          30919 |        31494 |
    | 1.6 GB   |      30 |          28540 |        28535 |
    | 1.6 GB   |      40 |          26380 |        27054 |
    | 1.6 GB   |      50 |          25241 |        25591 |
    ------------------------------------------------------
    | 7.6 GB   |       1 |           5779 |         6224 |
    | 7.6 GB   |       2 |          10897 |        13611 | + 24.9%
    | 7.6 GB   |       4 |          32683 |        33108 |
    | 7.6 GB   |       8 |          33968 |        34712 |
    | 7.6 GB   |      16 |          32287 |        32895 |
    | 7.6 GB   |      20 |          27770 |        31689 | + 14.1%
    | 7.6 GB   |      30 |          26739 |        29003 |
    | 7.6 GB   |      40 |          24901 |        26683 |
    | 7.6 GB   |      50 |          17115 |        25925 | + 51.5%
    ------------------------------------------------------

  (Davidlohr also has one additional patch which further improves
  throughput, though I will ask him to send it directly to you as I have
  suggested some minor changes)."

* emailed patches from Michel Lespinasse <walken@google.com>:
  rwsem: no need for explicit signed longs
  x86 rwsem: avoid taking slow path when stealing write lock
  rwsem: do not block readers at head of queue if other readers are active
  rwsem: implement support for write lock stealing on the fastpath
  rwsem: simplify __rwsem_do_wake
  rwsem: skip initial trylock in rwsem_down_write_failed
  rwsem: avoid taking wait_lock in rwsem_down_write_failed
  rwsem: use cmpxchg for trying to steal write lock
  rwsem: more agressive lock stealing in rwsem_down_write_failed
  rwsem: simplify rwsem_down_write_failed
  rwsem: simplify rwsem_down_read_failed
  rwsem: move rwsem_down_failed_common code into rwsem_down_{read,write}_failed
  rwsem: shorter spinlocked section in rwsem_down_failed_common()
  rwsem: make the waiter type an enumeration rather than a bitmask

Showing 3 changed files Side-by-side Diff

arch/x86/include/asm/rwsem.h
Diff comments   View file @ c8de2fa
... ... @@ -105,8 +105,8 @@
105 105 asm volatile("# beginning down_write\n\t"
106 106 LOCK_PREFIX " xadd %1,(%2)\n\t"
107 107 /* adds 0xffff0001, returns the old value */
108   - " test %1,%1\n\t"
109   - /* was the count 0 before? */
  108 + " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
  109 + /* was the active mask 0 before? */
110 110 " jz 1f\n"
111 111 " call call_rwsem_down_write_failed\n"
112 112 "1:\n"
... ... @@ -126,11 +126,25 @@
126 126 */
127 127 static inline int __down_write_trylock(struct rw_semaphore *sem)
128 128 {
129   - long ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
130   - RWSEM_ACTIVE_WRITE_BIAS);
131   - if (ret == RWSEM_UNLOCKED_VALUE)
132   - return 1;
133   - return 0;
  129 + long result, tmp;
  130 + asm volatile("# beginning __down_write_trylock\n\t"
  131 + " mov %0,%1\n\t"
  132 + "1:\n\t"
  133 + " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
  134 + /* was the active mask 0 before? */
  135 + " jnz 2f\n\t"
  136 + " mov %1,%2\n\t"
  137 + " add %3,%2\n\t"
  138 + LOCK_PREFIX " cmpxchg %2,%0\n\t"
  139 + " jnz 1b\n\t"
  140 + "2:\n\t"
  141 + " sete %b1\n\t"
  142 + " movzbl %b1, %k1\n\t"
  143 + "# ending __down_write_trylock\n\t"
  144 + : "+m" (sem->count), "=&a" (result), "=&r" (tmp)
  145 + : "er" (RWSEM_ACTIVE_WRITE_BIAS)
  146 + : "memory", "cc");
  147 + return result;
134 148 }
135 149  
136 150 /*
lib/rwsem-spinlock.c
Diff comments   View file @ c8de2fa
... ... @@ -9,12 +9,15 @@
9 9 #include <linux/sched.h>
10 10 #include <linux/export.h>
11 11  
  12 +enum rwsem_waiter_type {
  13 + RWSEM_WAITING_FOR_WRITE,
  14 + RWSEM_WAITING_FOR_READ
  15 +};
  16 +
12 17 struct rwsem_waiter {
13 18 struct list_head list;
14 19 struct task_struct *task;
15   - unsigned int flags;
16   -#define RWSEM_WAITING_FOR_READ 0x00000001
17   -#define RWSEM_WAITING_FOR_WRITE 0x00000002
  20 + enum rwsem_waiter_type type;
18 21 };
19 22  
20 23 int rwsem_is_locked(struct rw_semaphore *sem)
21 24  
22 25  
... ... @@ -67,26 +70,17 @@
67 70  
68 71 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
69 72  
70   - if (!wakewrite) {
71   - if (waiter->flags & RWSEM_WAITING_FOR_WRITE)
72   - goto out;
73   - goto dont_wake_writers;
74   - }
75   -
76   - /*
77   - * as we support write lock stealing, we can't set sem->activity
78   - * to -1 here to indicate we get the lock. Instead, we wake it up
79   - * to let it go get it again.
80   - */
81   - if (waiter->flags & RWSEM_WAITING_FOR_WRITE) {
82   - wake_up_process(waiter->task);
  73 + if (waiter->type == RWSEM_WAITING_FOR_WRITE) {
  74 + if (wakewrite)
  75 + /* Wake up a writer. Note that we do not grant it the
  76 + * lock - it will have to acquire it when it runs. */
  77 + wake_up_process(waiter->task);
83 78 goto out;
84 79 }
85 80  
86 81 /* grant an infinite number of read locks to the front of the queue */
87   - dont_wake_writers:
88 82 woken = 0;
89   - while (waiter->flags & RWSEM_WAITING_FOR_READ) {
  83 + do {
90 84 struct list_head *next = waiter->list.next;
91 85  
92 86 list_del(&waiter->list);
93 87  
... ... @@ -96,10 +90,10 @@
96 90 wake_up_process(tsk);
97 91 put_task_struct(tsk);
98 92 woken++;
99   - if (list_empty(&sem->wait_list))
  93 + if (next == &sem->wait_list)
100 94 break;
101 95 waiter = list_entry(next, struct rwsem_waiter, list);
102   - }
  96 + } while (waiter->type != RWSEM_WAITING_FOR_WRITE);
103 97  
104 98 sem->activity += woken;
105 99  
... ... @@ -144,7 +138,7 @@
144 138  
145 139 /* set up my own style of waitqueue */
146 140 waiter.task = tsk;
147   - waiter.flags = RWSEM_WAITING_FOR_READ;
  141 + waiter.type = RWSEM_WAITING_FOR_READ;
148 142 get_task_struct(tsk);
149 143  
150 144 list_add_tail(&waiter.list, &sem->wait_list);
... ... @@ -201,7 +195,7 @@
201 195 /* set up my own style of waitqueue */
202 196 tsk = current;
203 197 waiter.task = tsk;
204   - waiter.flags = RWSEM_WAITING_FOR_WRITE;
  198 + waiter.type = RWSEM_WAITING_FOR_WRITE;
205 199 list_add_tail(&waiter.list, &sem->wait_list);
206 200  
207 201 /* wait for someone to release the lock */
... ... @@ -4,6 +4,7 @@
4 4 * Derived from arch/i386/kernel/semaphore.c
5 5 *
6 6 * Writer lock-stealing by Alex Shi <alex.shi@intel.com>
  7 + * and Michel Lespinasse <walken@google.com>
7 8 */
8 9 #include <linux/rwsem.h>
9 10 #include <linux/sched.h>
10 11  
11 12  
... ... @@ -30,21 +31,22 @@
30 31  
31 32 EXPORT_SYMBOL(__init_rwsem);
32 33  
  34 +enum rwsem_waiter_type {
  35 + RWSEM_WAITING_FOR_WRITE,
  36 + RWSEM_WAITING_FOR_READ
  37 +};
  38 +
33 39 struct rwsem_waiter {
34 40 struct list_head list;
35 41 struct task_struct *task;
36   - unsigned int flags;
37   -#define RWSEM_WAITING_FOR_READ 0x00000001
38   -#define RWSEM_WAITING_FOR_WRITE 0x00000002
  42 + enum rwsem_waiter_type type;
39 43 };
40 44  
41   -/* Wake types for __rwsem_do_wake(). Note that RWSEM_WAKE_NO_ACTIVE and
42   - * RWSEM_WAKE_READ_OWNED imply that the spinlock must have been kept held
43   - * since the rwsem value was observed.
44   - */
45   -#define RWSEM_WAKE_ANY 0 /* Wake whatever's at head of wait list */
46   -#define RWSEM_WAKE_NO_ACTIVE 1 /* rwsem was observed with no active thread */
47   -#define RWSEM_WAKE_READ_OWNED 2 /* rwsem was observed to be read owned */
  45 +enum rwsem_wake_type {
  46 + RWSEM_WAKE_ANY, /* Wake whatever's at head of wait list */
  47 + RWSEM_WAKE_READERS, /* Wake readers only */
  48 + RWSEM_WAKE_READ_OWNED /* Waker thread holds the read lock */
  49 +};
48 50  
49 51 /*
50 52 * handle the lock release when processes blocked on it that can now run
51 53  
52 54  
53 55  
54 56  
55 57  
... ... @@ -57,46 +59,43 @@
57 59 * - writers are only woken if downgrading is false
58 60 */
59 61 static struct rw_semaphore *
60   -__rwsem_do_wake(struct rw_semaphore *sem, int wake_type)
  62 +__rwsem_do_wake(struct rw_semaphore *sem, enum rwsem_wake_type wake_type)
61 63 {
62 64 struct rwsem_waiter *waiter;
63 65 struct task_struct *tsk;
64 66 struct list_head *next;
65   - signed long woken, loop, adjustment;
  67 + long oldcount, woken, loop, adjustment;
66 68  
67 69 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
68   - if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE))
69   - goto readers_only;
70   -
71   - if (wake_type == RWSEM_WAKE_READ_OWNED)
72   - /* Another active reader was observed, so wakeup is not
73   - * likely to succeed. Save the atomic op.
74   - */
  70 + if (waiter->type == RWSEM_WAITING_FOR_WRITE) {
  71 + if (wake_type == RWSEM_WAKE_ANY)
  72 + /* Wake writer at the front of the queue, but do not
  73 + * grant it the lock yet as we want other writers
  74 + * to be able to steal it. Readers, on the other hand,
  75 + * will block as they will notice the queued writer.
  76 + */
  77 + wake_up_process(waiter->task);
75 78 goto out;
  79 + }
76 80  
77   - /* Wake up the writing waiter and let the task grab the sem: */
78   - wake_up_process(waiter->task);
79   - goto out;
80   -
81   - readers_only:
82   - /* If we come here from up_xxxx(), another thread might have reached
83   - * rwsem_down_failed_common() before we acquired the spinlock and
84   - * woken up a waiter, making it now active. We prefer to check for
85   - * this first in order to not spend too much time with the spinlock
86   - * held if we're not going to be able to wake up readers in the end.
87   - *
88   - * Note that we do not need to update the rwsem count: any writer
89   - * trying to acquire rwsem will run rwsem_down_write_failed() due
90   - * to the waiting threads and block trying to acquire the spinlock.
91   - *
92   - * We use a dummy atomic update in order to acquire the cache line
93   - * exclusively since we expect to succeed and run the final rwsem
94   - * count adjustment pretty soon.
  81 + /* Writers might steal the lock before we grant it to the next reader.
  82 + * We prefer to do the first reader grant before counting readers
  83 + * so we can bail out early if a writer stole the lock.
95 84 */
96   - if (wake_type == RWSEM_WAKE_ANY &&
97   - rwsem_atomic_update(0, sem) < RWSEM_WAITING_BIAS)
98   - /* Someone grabbed the sem for write already */
99   - goto out;
  85 + adjustment = 0;
  86 + if (wake_type != RWSEM_WAKE_READ_OWNED) {
  87 + adjustment = RWSEM_ACTIVE_READ_BIAS;
  88 + try_reader_grant:
  89 + oldcount = rwsem_atomic_update(adjustment, sem) - adjustment;
  90 + if (unlikely(oldcount < RWSEM_WAITING_BIAS)) {
  91 + /* A writer stole the lock. Undo our reader grant. */
  92 + if (rwsem_atomic_update(-adjustment, sem) &
  93 + RWSEM_ACTIVE_MASK)
  94 + goto out;
  95 + /* Last active locker left. Retry waking readers. */
  96 + goto try_reader_grant;
  97 + }
  98 + }
100 99  
101 100 /* Grant an infinite number of read locks to the readers at the front
102 101 * of the queue. Note we increment the 'active part' of the count by
103 102  
104 103  
105 104  
... ... @@ -112,17 +111,19 @@
112 111 waiter = list_entry(waiter->list.next,
113 112 struct rwsem_waiter, list);
114 113  
115   - } while (waiter->flags & RWSEM_WAITING_FOR_READ);
  114 + } while (waiter->type != RWSEM_WAITING_FOR_WRITE);
116 115  
117   - adjustment = woken * RWSEM_ACTIVE_READ_BIAS;
118   - if (waiter->flags & RWSEM_WAITING_FOR_READ)
  116 + adjustment = woken * RWSEM_ACTIVE_READ_BIAS - adjustment;
  117 + if (waiter->type != RWSEM_WAITING_FOR_WRITE)
119 118 /* hit end of list above */
120 119 adjustment -= RWSEM_WAITING_BIAS;
121 120  
122   - rwsem_atomic_add(adjustment, sem);
  121 + if (adjustment)
  122 + rwsem_atomic_add(adjustment, sem);
123 123  
124 124 next = sem->wait_list.next;
125   - for (loop = woken; loop > 0; loop--) {
  125 + loop = woken;
  126 + do {
126 127 waiter = list_entry(next, struct rwsem_waiter, list);
127 128 next = waiter->list.next;
128 129 tsk = waiter->task;
... ... @@ -130,7 +131,7 @@
130 131 waiter->task = NULL;
131 132 wake_up_process(tsk);
132 133 put_task_struct(tsk);
133   - }
  134 + } while (--loop);
134 135  
135 136 sem->wait_list.next = next;
136 137 next->prev = &sem->wait_list;
137 138  
138 139  
139 140  
140 141  
141 142  
142 143  
143 144  
144 145  
... ... @@ -139,60 +140,21 @@
139 140 return sem;
140 141 }
141 142  
142   -/* Try to get write sem, caller holds sem->wait_lock: */
143   -static int try_get_writer_sem(struct rw_semaphore *sem,
144   - struct rwsem_waiter *waiter)
145   -{
146   - struct rwsem_waiter *fwaiter;
147   - long oldcount, adjustment;
148   -
149   - /* only steal when first waiter is writing */
150   - fwaiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
151   - if (!(fwaiter->flags & RWSEM_WAITING_FOR_WRITE))
152   - return 0;
153   -
154   - adjustment = RWSEM_ACTIVE_WRITE_BIAS;
155   - /* Only one waiter in the queue: */
156   - if (fwaiter == waiter && waiter->list.next == &sem->wait_list)
157   - adjustment -= RWSEM_WAITING_BIAS;
158   -
159   -try_again_write:
160   - oldcount = rwsem_atomic_update(adjustment, sem) - adjustment;
161   - if (!(oldcount & RWSEM_ACTIVE_MASK)) {
162   - /* No active lock: */
163   - struct task_struct *tsk = waiter->task;
164   -
165   - list_del(&waiter->list);
166   - smp_mb();
167   - put_task_struct(tsk);
168   - tsk->state = TASK_RUNNING;
169   - return 1;
170   - }
171   - /* some one grabbed the sem already */
172   - if (rwsem_atomic_update(-adjustment, sem) & RWSEM_ACTIVE_MASK)
173   - return 0;
174   - goto try_again_write;
175   -}
176   -
177 143 /*
178   - * wait for a lock to be granted
  144 + * wait for the read lock to be granted
179 145 */
180   -static struct rw_semaphore __sched *
181   -rwsem_down_failed_common(struct rw_semaphore *sem,
182   - unsigned int flags, signed long adjustment)
  146 +struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
183 147 {
  148 + long count, adjustment = -RWSEM_ACTIVE_READ_BIAS;
184 149 struct rwsem_waiter waiter;
185 150 struct task_struct *tsk = current;
186   - signed long count;
187 151  
188   - set_task_state(tsk, TASK_UNINTERRUPTIBLE);
189   -
190 152 /* set up my own style of waitqueue */
191   - raw_spin_lock_irq(&sem->wait_lock);
192 153 waiter.task = tsk;
193   - waiter.flags = flags;
  154 + waiter.type = RWSEM_WAITING_FOR_READ;
194 155 get_task_struct(tsk);
195 156  
  157 + raw_spin_lock_irq(&sem->wait_lock);
196 158 if (list_empty(&sem->wait_list))
197 159 adjustment += RWSEM_WAITING_BIAS;
198 160 list_add_tail(&waiter.list, &sem->wait_list);
199 161  
200 162  
201 163  
202 164  
... ... @@ -200,35 +162,24 @@
200 162 /* we're now waiting on the lock, but no longer actively locking */
201 163 count = rwsem_atomic_update(adjustment, sem);
202 164  
203   - /* If there are no active locks, wake the front queued process(es) up.
  165 + /* If there are no active locks, wake the front queued process(es).
204 166 *
205   - * Alternatively, if we're called from a failed down_write(), there
206   - * were already threads queued before us and there are no active
207   - * writers, the lock must be read owned; so we try to wake any read
208   - * locks that were queued ahead of us. */
209   - if (count == RWSEM_WAITING_BIAS)
210   - sem = __rwsem_do_wake(sem, RWSEM_WAKE_NO_ACTIVE);
211   - else if (count > RWSEM_WAITING_BIAS &&
212   - adjustment == -RWSEM_ACTIVE_WRITE_BIAS)
213   - sem = __rwsem_do_wake(sem, RWSEM_WAKE_READ_OWNED);
  167 + * If there are no writers and we are first in the queue,
  168 + * wake our own waiter to join the existing active readers !
  169 + */
  170 + if (count == RWSEM_WAITING_BIAS ||
  171 + (count > RWSEM_WAITING_BIAS &&
  172 + adjustment != -RWSEM_ACTIVE_READ_BIAS))
  173 + sem = __rwsem_do_wake(sem, RWSEM_WAKE_ANY);
214 174  
215 175 raw_spin_unlock_irq(&sem->wait_lock);
216 176  
217 177 /* wait to be given the lock */
218   - for (;;) {
  178 + while (true) {
  179 + set_task_state(tsk, TASK_UNINTERRUPTIBLE);
219 180 if (!waiter.task)
220 181 break;
221   -
222   - raw_spin_lock_irq(&sem->wait_lock);
223   - /* Try to get the writer sem, may steal from the head writer: */
224   - if (flags == RWSEM_WAITING_FOR_WRITE)
225   - if (try_get_writer_sem(sem, &waiter)) {
226   - raw_spin_unlock_irq(&sem->wait_lock);
227   - return sem;
228   - }
229   - raw_spin_unlock_irq(&sem->wait_lock);
230 182 schedule();
231   - set_task_state(tsk, TASK_UNINTERRUPTIBLE);
232 183 }
233 184  
234 185 tsk->state = TASK_RUNNING;
235 186  
236 187  
... ... @@ -237,21 +188,62 @@
237 188 }
238 189  
239 190 /*
240   - * wait for the read lock to be granted
  191 + * wait until we successfully acquire the write lock
241 192 */
242   -struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
243   -{
244   - return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_READ,
245   - -RWSEM_ACTIVE_READ_BIAS);
246   -}
247   -
248   -/*
249   - * wait for the write lock to be granted
250   - */
251 193 struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
252 194 {
253   - return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_WRITE,
254   - -RWSEM_ACTIVE_WRITE_BIAS);
  195 + long count, adjustment = -RWSEM_ACTIVE_WRITE_BIAS;
  196 + struct rwsem_waiter waiter;
  197 + struct task_struct *tsk = current;
  198 +
  199 + /* set up my own style of waitqueue */
  200 + waiter.task = tsk;
  201 + waiter.type = RWSEM_WAITING_FOR_WRITE;
  202 +
  203 + raw_spin_lock_irq(&sem->wait_lock);
  204 + if (list_empty(&sem->wait_list))
  205 + adjustment += RWSEM_WAITING_BIAS;
  206 + list_add_tail(&waiter.list, &sem->wait_list);
  207 +
  208 + /* we're now waiting on the lock, but no longer actively locking */
  209 + count = rwsem_atomic_update(adjustment, sem);
  210 +
  211 + /* If there were already threads queued before us and there are no
  212 + * active writers, the lock must be read owned; so we try to wake
  213 + * any read locks that were queued ahead of us. */
  214 + if (count > RWSEM_WAITING_BIAS &&
  215 + adjustment == -RWSEM_ACTIVE_WRITE_BIAS)
  216 + sem = __rwsem_do_wake(sem, RWSEM_WAKE_READERS);
  217 +
  218 + /* wait until we successfully acquire the lock */
  219 + set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  220 + while (true) {
  221 + if (!(count & RWSEM_ACTIVE_MASK)) {
  222 + /* Try acquiring the write lock. */
  223 + count = RWSEM_ACTIVE_WRITE_BIAS;
  224 + if (!list_is_singular(&sem->wait_list))
  225 + count += RWSEM_WAITING_BIAS;
  226 + if (cmpxchg(&sem->count, RWSEM_WAITING_BIAS, count) ==
  227 + RWSEM_WAITING_BIAS)
  228 + break;
  229 + }
  230 +
  231 + raw_spin_unlock_irq(&sem->wait_lock);
  232 +
  233 + /* Block until there are no active lockers. */
  234 + do {
  235 + schedule();
  236 + set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  237 + } while ((count = sem->count) & RWSEM_ACTIVE_MASK);
  238 +
  239 + raw_spin_lock_irq(&sem->wait_lock);
  240 + }
  241 +
  242 + list_del(&waiter.list);
  243 + raw_spin_unlock_irq(&sem->wait_lock);
  244 + tsk->state = TASK_RUNNING;
  245 +
  246 + return sem;
255 247 }
256 248  
257 249 /*