Blame view

kernel/rtmutex.c 26.4 KB
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
1
2
3
4
5
6
7
8
9
  /*
   * RT-Mutexes: simple blocking mutual exclusion locks with PI support
   *
   * started by Ingo Molnar and Thomas Gleixner.
   *
   *  Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
   *  Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
   *  Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
   *  Copyright (C) 2006 Esben Nielsen
d07fe82c2   Steven Rostedt   [PATCH] reference...
10
11
   *
   *  See Documentation/rt-mutex-design.txt for details.
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
12
13
   */
  #include <linux/spinlock.h>
9984de1a5   Paul Gortmaker   kernel: Map most ...
14
  #include <linux/export.h>
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
15
16
17
18
  #include <linux/sched.h>
  #include <linux/timer.h>
  
  #include "rtmutex_common.h"
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
19
20
21
  /*
   * lock->owner state tracking:
   *
8161239a8   Lai Jiangshan   rtmutex: Simplify...
22
23
   * lock->owner holds the task_struct pointer of the owner. Bit 0
   * is used to keep track of the "lock has waiters" state.
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
24
   *
8161239a8   Lai Jiangshan   rtmutex: Simplify...
25
26
27
28
29
30
   * owner	bit0
   * NULL		0	lock is free (fast acquire possible)
   * NULL		1	lock is free and has waiters and the top waiter
   *				is going to take the lock*
   * taskpointer	0	lock is held (fast release possible)
   * taskpointer	1	lock is held and has waiters**
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
31
32
   *
   * The fast atomic compare exchange based acquire and release is only
8161239a8   Lai Jiangshan   rtmutex: Simplify...
33
34
35
36
37
38
   * possible when bit 0 of lock->owner is 0.
   *
   * (*) It also can be a transitional state when grabbing the lock
   * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
   * we need to set the bit0 before looking at the lock, and the owner may be
   * NULL in this small time, hence this can be a transitional state.
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
39
   *
8161239a8   Lai Jiangshan   rtmutex: Simplify...
40
41
42
43
   * (**) There is a small time when bit 0 is set but there are no
   * waiters. This can happen when grabbing the lock in the slow path.
   * To prevent a cmpxchg of the owner releasing the lock, we need to
   * set this bit before looking at the lock.
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
44
   */
bd197234b   Thomas Gleixner   Revert "futex_req...
45
  static void
8161239a8   Lai Jiangshan   rtmutex: Simplify...
46
  rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
47
  {
8161239a8   Lai Jiangshan   rtmutex: Simplify...
48
  	unsigned long val = (unsigned long)owner;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
  
  	if (rt_mutex_has_waiters(lock))
  		val |= RT_MUTEX_HAS_WAITERS;
  
  	lock->owner = (struct task_struct *)val;
  }
  
  static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
  {
  	lock->owner = (struct task_struct *)
  			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
  }
  
  static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
  {
  	if (!rt_mutex_has_waiters(lock))
  		clear_rt_mutex_waiters(lock);
  }
  
  /*
bd197234b   Thomas Gleixner   Revert "futex_req...
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
   * We can speed up the acquire/release, if the architecture
   * supports cmpxchg and if there's no debugging state to be set up
   */
  #if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
  # define rt_mutex_cmpxchg(l,c,n)	(cmpxchg(&l->owner, c, n) == c)
  static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  {
  	unsigned long owner, *p = (unsigned long *) &lock->owner;
  
  	do {
  		owner = *p;
  	} while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
  }
  #else
  # define rt_mutex_cmpxchg(l,c,n)	(0)
  static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  {
  	lock->owner = (struct task_struct *)
  			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
  }
  #endif
  
  /*
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
   * Calculate task priority from the waiter list priority
   *
   * Return task->normal_prio when the waiter list is empty or when
   * the waiter is not allowed to do priority boosting
   */
  int rt_mutex_getprio(struct task_struct *task)
  {
  	if (likely(!task_has_pi_waiters(task)))
  		return task->normal_prio;
  
  	return min(task_top_pi_waiter(task)->pi_list_entry.prio,
  		   task->normal_prio);
  }
  
  /*
   * Adjust the priority of a task, after its pi_waiters got modified.
   *
   * This can be both boosting and unboosting. task->pi_lock must be held.
   */
bd197234b   Thomas Gleixner   Revert "futex_req...
111
  static void __rt_mutex_adjust_prio(struct task_struct *task)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
  {
  	int prio = rt_mutex_getprio(task);
  
  	if (task->prio != prio)
  		rt_mutex_setprio(task, prio);
  }
  
  /*
   * Adjust task priority (undo boosting). Called from the exit path of
   * rt_mutex_slowunlock() and rt_mutex_slowlock().
   *
   * (Note: We do this outside of the protection of lock->wait_lock to
   * allow the lock to be taken while or before we readjust the priority
   * of task. We do not use the spin_xx_mutex() variants here as we are
   * outside of the debug path.)
   */
  static void rt_mutex_adjust_prio(struct task_struct *task)
  {
  	unsigned long flags;
1d6154825   Thomas Gleixner   sched: Convert pi...
131
  	raw_spin_lock_irqsave(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
132
  	__rt_mutex_adjust_prio(task);
1d6154825   Thomas Gleixner   sched: Convert pi...
133
  	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
134
135
136
137
138
139
140
141
142
143
144
145
  }
  
  /*
   * Max number of times we'll walk the boosting chain:
   */
  int max_lock_depth = 1024;
  
  /*
   * Adjust the priority chain. Also used for deadlock detection.
   * Decreases task's usage by one - may thus free the task.
   * Returns 0 or -EDEADLK.
   */
bd197234b   Thomas Gleixner   Revert "futex_req...
146
147
148
149
150
  static int rt_mutex_adjust_prio_chain(struct task_struct *task,
  				      int deadlock_detect,
  				      struct rt_mutex *orig_lock,
  				      struct rt_mutex_waiter *orig_waiter,
  				      struct task_struct *top_task)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
  {
  	struct rt_mutex *lock;
  	struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
  	int detect_deadlock, ret = 0, depth = 0;
  	unsigned long flags;
  
  	detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
  							 deadlock_detect);
  
  	/*
  	 * The (de)boosting is a step by step approach with a lot of
  	 * pitfalls. We want this to be preemptible and we want hold a
  	 * maximum of two locks per step. So we have to check
  	 * carefully whether things change under us.
  	 */
   again:
  	if (++depth > max_lock_depth) {
  		static int prev_max;
  
  		/*
  		 * Print this only once. If the admin changes the limit,
  		 * print a new message when reaching the limit again.
  		 */
  		if (prev_max != max_lock_depth) {
  			prev_max = max_lock_depth;
  			printk(KERN_WARNING "Maximum lock depth %d reached "
  			       "task: %s (%d)
  ", max_lock_depth,
ba25f9dcc   Pavel Emelyanov   Use helpers to ob...
179
  			       top_task->comm, task_pid_nr(top_task));
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
180
181
182
183
184
185
186
187
188
  		}
  		put_task_struct(task);
  
  		return deadlock_detect ? -EDEADLK : 0;
  	}
   retry:
  	/*
  	 * Task can not go away as we did a get_task() before !
  	 */
1d6154825   Thomas Gleixner   sched: Convert pi...
189
  	raw_spin_lock_irqsave(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
190
191
192
193
194
195
196
  
  	waiter = task->pi_blocked_on;
  	/*
  	 * Check whether the end of the boosting chain has been
  	 * reached or the state of the chain has changed while we
  	 * dropped the locks.
  	 */
8161239a8   Lai Jiangshan   rtmutex: Simplify...
197
  	if (!waiter)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
198
  		goto out_unlock_pi;
1a539a872   Thomas Gleixner   rt-mutex: fix cha...
199
200
  	/*
  	 * Check the orig_waiter state. After we dropped the locks,
8161239a8   Lai Jiangshan   rtmutex: Simplify...
201
  	 * the previous owner of the lock might have released the lock.
1a539a872   Thomas Gleixner   rt-mutex: fix cha...
202
  	 */
8161239a8   Lai Jiangshan   rtmutex: Simplify...
203
  	if (orig_waiter && !rt_mutex_owner(orig_lock))
1a539a872   Thomas Gleixner   rt-mutex: fix cha...
204
205
206
207
208
209
210
  		goto out_unlock_pi;
  
  	/*
  	 * Drop out, when the task has no waiters. Note,
  	 * top_waiter can be NULL, when we are in the deboosting
  	 * mode!
  	 */
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
211
212
213
214
215
216
217
218
219
220
221
222
  	if (top_waiter && (!task_has_pi_waiters(task) ||
  			   top_waiter != task_top_pi_waiter(task)))
  		goto out_unlock_pi;
  
  	/*
  	 * When deadlock detection is off then we check, if further
  	 * priority adjustment is necessary.
  	 */
  	if (!detect_deadlock && waiter->list_entry.prio == task->prio)
  		goto out_unlock_pi;
  
  	lock = waiter->lock;
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
223
  	if (!raw_spin_trylock(&lock->wait_lock)) {
1d6154825   Thomas Gleixner   sched: Convert pi...
224
  		raw_spin_unlock_irqrestore(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
225
226
227
228
229
  		cpu_relax();
  		goto retry;
  	}
  
  	/* Deadlock detection */
95e02ca9b   Thomas Gleixner   [PATCH] rtmutex: ...
230
  	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
231
  		debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
232
  		raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
233
234
235
236
237
238
239
240
241
242
243
244
  		ret = deadlock_detect ? -EDEADLK : 0;
  		goto out_unlock_pi;
  	}
  
  	top_waiter = rt_mutex_top_waiter(lock);
  
  	/* Requeue the waiter */
  	plist_del(&waiter->list_entry, &lock->wait_list);
  	waiter->list_entry.prio = task->prio;
  	plist_add(&waiter->list_entry, &lock->wait_list);
  
  	/* Release the task */
1d6154825   Thomas Gleixner   sched: Convert pi...
245
  	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
246
247
248
249
250
251
252
253
254
255
256
  	if (!rt_mutex_owner(lock)) {
  		/*
  		 * If the requeue above changed the top waiter, then we need
  		 * to wake the new top waiter up to try to get the lock.
  		 */
  
  		if (top_waiter != rt_mutex_top_waiter(lock))
  			wake_up_process(rt_mutex_top_waiter(lock)->task);
  		raw_spin_unlock(&lock->wait_lock);
  		goto out_put_task;
  	}
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
257
258
259
260
  	put_task_struct(task);
  
  	/* Grab the next task */
  	task = rt_mutex_owner(lock);
db630637b   Steven Rostedt   [PATCH] clean up ...
261
  	get_task_struct(task);
1d6154825   Thomas Gleixner   sched: Convert pi...
262
  	raw_spin_lock_irqsave(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
  
  	if (waiter == rt_mutex_top_waiter(lock)) {
  		/* Boost the owner */
  		plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
  		waiter->pi_list_entry.prio = waiter->list_entry.prio;
  		plist_add(&waiter->pi_list_entry, &task->pi_waiters);
  		__rt_mutex_adjust_prio(task);
  
  	} else if (top_waiter == waiter) {
  		/* Deboost the owner */
  		plist_del(&waiter->pi_list_entry, &task->pi_waiters);
  		waiter = rt_mutex_top_waiter(lock);
  		waiter->pi_list_entry.prio = waiter->list_entry.prio;
  		plist_add(&waiter->pi_list_entry, &task->pi_waiters);
  		__rt_mutex_adjust_prio(task);
  	}
1d6154825   Thomas Gleixner   sched: Convert pi...
279
  	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
280
281
  
  	top_waiter = rt_mutex_top_waiter(lock);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
282
  	raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
283
284
285
286
287
288
289
  
  	if (!detect_deadlock && waiter != top_waiter)
  		goto out_put_task;
  
  	goto again;
  
   out_unlock_pi:
1d6154825   Thomas Gleixner   sched: Convert pi...
290
  	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
291
292
   out_put_task:
  	put_task_struct(task);
36c8b5868   Ingo Molnar   [PATCH] sched: cl...
293

23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
294
295
296
297
  	return ret;
  }
  
  /*
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
298
299
   * Try to take an rt-mutex
   *
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
300
   * Must be called with lock->wait_lock held.
8161239a8   Lai Jiangshan   rtmutex: Simplify...
301
302
303
304
   *
   * @lock:   the lock to be acquired.
   * @task:   the task which wants to acquire the lock
   * @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
305
   */
8161239a8   Lai Jiangshan   rtmutex: Simplify...
306
307
  static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
  		struct rt_mutex_waiter *waiter)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
  {
  	/*
  	 * We have to be careful here if the atomic speedups are
  	 * enabled, such that, when
  	 *  - no other waiter is on the lock
  	 *  - the lock has been released since we did the cmpxchg
  	 * the lock can be released or taken while we are doing the
  	 * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
  	 *
  	 * The atomic acquire/release aware variant of
  	 * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
  	 * the WAITERS bit, the atomic release / acquire can not
  	 * happen anymore and lock->wait_lock protects us from the
  	 * non-atomic case.
  	 *
  	 * Note, that this might set lock->owner =
  	 * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
  	 * any more. This is fixed up when we take the ownership.
  	 * This is the transitional state explained at the top of this file.
  	 */
  	mark_rt_mutex_waiters(lock);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
329
  	if (rt_mutex_owner(lock))
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
330
  		return 0;
8161239a8   Lai Jiangshan   rtmutex: Simplify...
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
  	/*
  	 * It will get the lock because of one of these conditions:
  	 * 1) there is no waiter
  	 * 2) higher priority than waiters
  	 * 3) it is top waiter
  	 */
  	if (rt_mutex_has_waiters(lock)) {
  		if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
  			if (!waiter || waiter != rt_mutex_top_waiter(lock))
  				return 0;
  		}
  	}
  
  	if (waiter || rt_mutex_has_waiters(lock)) {
  		unsigned long flags;
  		struct rt_mutex_waiter *top;
  
  		raw_spin_lock_irqsave(&task->pi_lock, flags);
  
  		/* remove the queued waiter. */
  		if (waiter) {
  			plist_del(&waiter->list_entry, &lock->wait_list);
  			task->pi_blocked_on = NULL;
  		}
  
  		/*
  		 * We have to enqueue the top waiter(if it exists) into
  		 * task->pi_waiters list.
  		 */
  		if (rt_mutex_has_waiters(lock)) {
  			top = rt_mutex_top_waiter(lock);
  			top->pi_list_entry.prio = top->list_entry.prio;
  			plist_add(&top->pi_list_entry, &task->pi_waiters);
  		}
  		raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  	}
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
367
  	/* We got the lock. */
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
368
  	debug_rt_mutex_lock(lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
369

8161239a8   Lai Jiangshan   rtmutex: Simplify...
370
  	rt_mutex_set_owner(lock, task);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
371

8161239a8   Lai Jiangshan   rtmutex: Simplify...
372
  	rt_mutex_deadlock_account_lock(lock, task);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
373
374
375
376
377
378
379
380
381
382
383
384
385
  
  	return 1;
  }
  
  /*
   * Task blocks on lock.
   *
   * Prepare waiter and propagate pi chain
   *
   * This must be called with lock->wait_lock held.
   */
  static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
  				   struct rt_mutex_waiter *waiter,
8dac456a6   Darren Hart   rt_mutex: add pro...
386
  				   struct task_struct *task,
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
387
  				   int detect_deadlock)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
388
  {
36c8b5868   Ingo Molnar   [PATCH] sched: cl...
389
  	struct task_struct *owner = rt_mutex_owner(lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
390
  	struct rt_mutex_waiter *top_waiter = waiter;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
391
  	unsigned long flags;
db630637b   Steven Rostedt   [PATCH] clean up ...
392
  	int chain_walk = 0, res;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
393

1d6154825   Thomas Gleixner   sched: Convert pi...
394
  	raw_spin_lock_irqsave(&task->pi_lock, flags);
8dac456a6   Darren Hart   rt_mutex: add pro...
395
396
  	__rt_mutex_adjust_prio(task);
  	waiter->task = task;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
397
  	waiter->lock = lock;
8dac456a6   Darren Hart   rt_mutex: add pro...
398
399
  	plist_node_init(&waiter->list_entry, task->prio);
  	plist_node_init(&waiter->pi_list_entry, task->prio);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
400
401
402
403
404
  
  	/* Get the top priority waiter on the lock */
  	if (rt_mutex_has_waiters(lock))
  		top_waiter = rt_mutex_top_waiter(lock);
  	plist_add(&waiter->list_entry, &lock->wait_list);
8dac456a6   Darren Hart   rt_mutex: add pro...
405
  	task->pi_blocked_on = waiter;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
406

1d6154825   Thomas Gleixner   sched: Convert pi...
407
  	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
408

8161239a8   Lai Jiangshan   rtmutex: Simplify...
409
410
  	if (!owner)
  		return 0;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
411
  	if (waiter == rt_mutex_top_waiter(lock)) {
1d6154825   Thomas Gleixner   sched: Convert pi...
412
  		raw_spin_lock_irqsave(&owner->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
413
414
415
416
  		plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
  		plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
  
  		__rt_mutex_adjust_prio(owner);
db630637b   Steven Rostedt   [PATCH] clean up ...
417
418
  		if (owner->pi_blocked_on)
  			chain_walk = 1;
1d6154825   Thomas Gleixner   sched: Convert pi...
419
  		raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
420
  	}
db630637b   Steven Rostedt   [PATCH] clean up ...
421
422
423
424
  	else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
  		chain_walk = 1;
  
  	if (!chain_walk)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
425
  		return 0;
db630637b   Steven Rostedt   [PATCH] clean up ...
426
427
428
429
430
431
  	/*
  	 * The owner can't disappear while holding a lock,
  	 * so the owner struct is protected by wait_lock.
  	 * Gets dropped in rt_mutex_adjust_prio_chain()!
  	 */
  	get_task_struct(owner);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
432
  	raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
433

95e02ca9b   Thomas Gleixner   [PATCH] rtmutex: ...
434
  	res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
8dac456a6   Darren Hart   rt_mutex: add pro...
435
  					 task);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
436

d209d74d5   Thomas Gleixner   rtmutes: Convert ...
437
  	raw_spin_lock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
438
439
440
441
442
443
444
  
  	return res;
  }
  
  /*
   * Wake up the next waiter on the lock.
   *
8161239a8   Lai Jiangshan   rtmutex: Simplify...
445
   * Remove the top waiter from the current tasks waiter list and wake it up.
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
446
447
448
449
450
451
   *
   * Called with lock->wait_lock held.
   */
  static void wakeup_next_waiter(struct rt_mutex *lock)
  {
  	struct rt_mutex_waiter *waiter;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
452
  	unsigned long flags;
1d6154825   Thomas Gleixner   sched: Convert pi...
453
  	raw_spin_lock_irqsave(&current->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
454
455
  
  	waiter = rt_mutex_top_waiter(lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
456
457
458
459
460
461
462
463
  
  	/*
  	 * Remove it from current->pi_waiters. We do not adjust a
  	 * possible priority boost right now. We execute wakeup in the
  	 * boosted mode and go back to normal after releasing
  	 * lock->wait_lock.
  	 */
  	plist_del(&waiter->pi_list_entry, &current->pi_waiters);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
464

8161239a8   Lai Jiangshan   rtmutex: Simplify...
465
  	rt_mutex_set_owner(lock, NULL);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
466

1d6154825   Thomas Gleixner   sched: Convert pi...
467
  	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
468

8161239a8   Lai Jiangshan   rtmutex: Simplify...
469
  	wake_up_process(waiter->task);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
470
471
472
  }
  
  /*
8161239a8   Lai Jiangshan   rtmutex: Simplify...
473
   * Remove a waiter from a lock and give up
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
474
   *
8161239a8   Lai Jiangshan   rtmutex: Simplify...
475
476
   * Must be called with lock->wait_lock held and
   * have just failed to try_to_take_rt_mutex().
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
477
   */
bd197234b   Thomas Gleixner   Revert "futex_req...
478
479
  static void remove_waiter(struct rt_mutex *lock,
  			  struct rt_mutex_waiter *waiter)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
480
481
  {
  	int first = (waiter == rt_mutex_top_waiter(lock));
36c8b5868   Ingo Molnar   [PATCH] sched: cl...
482
  	struct task_struct *owner = rt_mutex_owner(lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
483
  	unsigned long flags;
db630637b   Steven Rostedt   [PATCH] clean up ...
484
  	int chain_walk = 0;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
485

1d6154825   Thomas Gleixner   sched: Convert pi...
486
  	raw_spin_lock_irqsave(&current->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
487
  	plist_del(&waiter->list_entry, &lock->wait_list);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
488
  	current->pi_blocked_on = NULL;
1d6154825   Thomas Gleixner   sched: Convert pi...
489
  	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
490

8161239a8   Lai Jiangshan   rtmutex: Simplify...
491
492
493
494
  	if (!owner)
  		return;
  
  	if (first) {
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
495

1d6154825   Thomas Gleixner   sched: Convert pi...
496
  		raw_spin_lock_irqsave(&owner->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
497
498
499
500
501
502
503
504
505
506
  
  		plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
  
  		if (rt_mutex_has_waiters(lock)) {
  			struct rt_mutex_waiter *next;
  
  			next = rt_mutex_top_waiter(lock);
  			plist_add(&next->pi_list_entry, &owner->pi_waiters);
  		}
  		__rt_mutex_adjust_prio(owner);
db630637b   Steven Rostedt   [PATCH] clean up ...
507
508
  		if (owner->pi_blocked_on)
  			chain_walk = 1;
1d6154825   Thomas Gleixner   sched: Convert pi...
509
  		raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
510
511
512
  	}
  
  	WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
db630637b   Steven Rostedt   [PATCH] clean up ...
513
  	if (!chain_walk)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
514
  		return;
db630637b   Steven Rostedt   [PATCH] clean up ...
515
516
  	/* gets dropped in rt_mutex_adjust_prio_chain()! */
  	get_task_struct(owner);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
517
  	raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
518

9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
519
  	rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
520

d209d74d5   Thomas Gleixner   rtmutes: Convert ...
521
  	raw_spin_lock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
522
523
524
  }
  
  /*
95e02ca9b   Thomas Gleixner   [PATCH] rtmutex: ...
525
526
527
528
529
530
531
532
   * Recheck the pi chain, in case we got a priority setting
   *
   * Called from sched_setscheduler
   */
  void rt_mutex_adjust_pi(struct task_struct *task)
  {
  	struct rt_mutex_waiter *waiter;
  	unsigned long flags;
1d6154825   Thomas Gleixner   sched: Convert pi...
533
  	raw_spin_lock_irqsave(&task->pi_lock, flags);
95e02ca9b   Thomas Gleixner   [PATCH] rtmutex: ...
534
535
536
  
  	waiter = task->pi_blocked_on;
  	if (!waiter || waiter->list_entry.prio == task->prio) {
1d6154825   Thomas Gleixner   sched: Convert pi...
537
  		raw_spin_unlock_irqrestore(&task->pi_lock, flags);
95e02ca9b   Thomas Gleixner   [PATCH] rtmutex: ...
538
539
  		return;
  	}
1d6154825   Thomas Gleixner   sched: Convert pi...
540
  	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
95e02ca9b   Thomas Gleixner   [PATCH] rtmutex: ...
541

db630637b   Steven Rostedt   [PATCH] clean up ...
542
543
  	/* gets dropped in rt_mutex_adjust_prio_chain()! */
  	get_task_struct(task);
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
544
  	rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
95e02ca9b   Thomas Gleixner   [PATCH] rtmutex: ...
545
  }
8dac456a6   Darren Hart   rt_mutex: add pro...
546
547
548
549
550
551
552
  /**
   * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
   * @lock:		 the rt_mutex to take
   * @state:		 the state the task should block in (TASK_INTERRUPTIBLE
   * 			 or TASK_UNINTERRUPTIBLE)
   * @timeout:		 the pre-initialized and started timer, or NULL for none
   * @waiter:		 the pre-initialized rt_mutex_waiter
8dac456a6   Darren Hart   rt_mutex: add pro...
553
554
   *
   * lock->wait_lock must be held by the caller.
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
555
556
   */
  static int __sched
8dac456a6   Darren Hart   rt_mutex: add pro...
557
558
  __rt_mutex_slowlock(struct rt_mutex *lock, int state,
  		    struct hrtimer_sleeper *timeout,
8161239a8   Lai Jiangshan   rtmutex: Simplify...
559
  		    struct rt_mutex_waiter *waiter)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
560
  {
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
561
  	int ret = 0;
5342e269b   Paul E. McKenney   rcu: Permit rt_mu...
562
  	int was_disabled;
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
563

23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
564
565
  	for (;;) {
  		/* Try to acquire the lock: */
8161239a8   Lai Jiangshan   rtmutex: Simplify...
566
  		if (try_to_take_rt_mutex(lock, current, waiter))
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
  			break;
  
  		/*
  		 * TASK_INTERRUPTIBLE checks for signals and
  		 * timeout. Ignored otherwise.
  		 */
  		if (unlikely(state == TASK_INTERRUPTIBLE)) {
  			/* Signal pending? */
  			if (signal_pending(current))
  				ret = -EINTR;
  			if (timeout && !timeout->task)
  				ret = -ETIMEDOUT;
  			if (ret)
  				break;
  		}
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
582
  		raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
583

5342e269b   Paul E. McKenney   rcu: Permit rt_mu...
584
585
586
  		was_disabled = irqs_disabled();
  		if (was_disabled)
  			local_irq_enable();
8dac456a6   Darren Hart   rt_mutex: add pro...
587
  		debug_rt_mutex_print_deadlock(waiter);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
588

8161239a8   Lai Jiangshan   rtmutex: Simplify...
589
  		schedule_rt_mutex(lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
590

5342e269b   Paul E. McKenney   rcu: Permit rt_mu...
591
592
  		if (was_disabled)
  			local_irq_disable();
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
593
  		raw_spin_lock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
594
595
  		set_current_state(state);
  	}
8dac456a6   Darren Hart   rt_mutex: add pro...
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
  	return ret;
  }
  
  /*
   * Slow path lock function:
   */
  static int __sched
  rt_mutex_slowlock(struct rt_mutex *lock, int state,
  		  struct hrtimer_sleeper *timeout,
  		  int detect_deadlock)
  {
  	struct rt_mutex_waiter waiter;
  	int ret = 0;
  
  	debug_rt_mutex_init_waiter(&waiter);
8dac456a6   Darren Hart   rt_mutex: add pro...
611

d209d74d5   Thomas Gleixner   rtmutes: Convert ...
612
  	raw_spin_lock(&lock->wait_lock);
8dac456a6   Darren Hart   rt_mutex: add pro...
613
614
  
  	/* Try to acquire the lock again: */
8161239a8   Lai Jiangshan   rtmutex: Simplify...
615
  	if (try_to_take_rt_mutex(lock, current, NULL)) {
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
616
  		raw_spin_unlock(&lock->wait_lock);
8dac456a6   Darren Hart   rt_mutex: add pro...
617
618
619
620
621
622
623
624
625
626
627
  		return 0;
  	}
  
  	set_current_state(state);
  
  	/* Setup the timer, when timeout != NULL */
  	if (unlikely(timeout)) {
  		hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
  		if (!hrtimer_active(&timeout->timer))
  			timeout->task = NULL;
  	}
8161239a8   Lai Jiangshan   rtmutex: Simplify...
628
629
630
631
  	ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
  
  	if (likely(!ret))
  		ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
8dac456a6   Darren Hart   rt_mutex: add pro...
632

23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
633
  	set_current_state(TASK_RUNNING);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
634
  	if (unlikely(ret))
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
635
  		remove_waiter(lock, &waiter);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
636
637
638
639
640
641
  
  	/*
  	 * try_to_take_rt_mutex() sets the waiter bit
  	 * unconditionally. We might have to fix that up.
  	 */
  	fixup_rt_mutex_waiters(lock);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
642
  	raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
643
644
645
646
  
  	/* Remove pending timer: */
  	if (unlikely(timeout))
  		hrtimer_cancel(&timeout->timer);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
647
648
649
650
651
652
653
654
655
  	debug_rt_mutex_free_waiter(&waiter);
  
  	return ret;
  }
  
  /*
   * Slow path try-lock function:
   */
  static inline int
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
656
  rt_mutex_slowtrylock(struct rt_mutex *lock)
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
657
658
  {
  	int ret = 0;
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
659
  	raw_spin_lock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
660
661
  
  	if (likely(rt_mutex_owner(lock) != current)) {
8161239a8   Lai Jiangshan   rtmutex: Simplify...
662
  		ret = try_to_take_rt_mutex(lock, current, NULL);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
663
664
665
666
667
668
  		/*
  		 * try_to_take_rt_mutex() sets the lock waiters
  		 * bit unconditionally. Clean this up.
  		 */
  		fixup_rt_mutex_waiters(lock);
  	}
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
669
  	raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
670
671
672
673
674
675
676
677
678
679
  
  	return ret;
  }
  
  /*
   * Slow path to release a rt-mutex:
   */
  static void __sched
  rt_mutex_slowunlock(struct rt_mutex *lock)
  {
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
680
  	raw_spin_lock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
681
682
683
684
685
686
687
  
  	debug_rt_mutex_unlock(lock);
  
  	rt_mutex_deadlock_account_unlock(current);
  
  	if (!rt_mutex_has_waiters(lock)) {
  		lock->owner = NULL;
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
688
  		raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
689
690
691
692
  		return;
  	}
  
  	wakeup_next_waiter(lock);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
693
  	raw_spin_unlock(&lock->wait_lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
  
  	/* Undo pi boosting if necessary: */
  	rt_mutex_adjust_prio(current);
  }
  
  /*
   * debug aware fast / slowpath lock,trylock,unlock
   *
   * The atomic acquire/release ops are compiled away, when either the
   * architecture does not support cmpxchg or when debugging is enabled.
   */
  static inline int
  rt_mutex_fastlock(struct rt_mutex *lock, int state,
  		  int detect_deadlock,
  		  int (*slowfn)(struct rt_mutex *lock, int state,
  				struct hrtimer_sleeper *timeout,
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
710
  				int detect_deadlock))
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
711
712
713
714
715
  {
  	if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
  		rt_mutex_deadlock_account_lock(lock, current);
  		return 0;
  	} else
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
716
  		return slowfn(lock, state, NULL, detect_deadlock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
717
718
719
720
721
722
723
  }
  
  static inline int
  rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
  			struct hrtimer_sleeper *timeout, int detect_deadlock,
  			int (*slowfn)(struct rt_mutex *lock, int state,
  				      struct hrtimer_sleeper *timeout,
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
724
  				      int detect_deadlock))
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
725
726
727
728
729
  {
  	if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
  		rt_mutex_deadlock_account_lock(lock, current);
  		return 0;
  	} else
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
730
  		return slowfn(lock, state, timeout, detect_deadlock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
731
732
733
734
  }
  
  static inline int
  rt_mutex_fasttrylock(struct rt_mutex *lock,
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
735
  		     int (*slowfn)(struct rt_mutex *lock))
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
736
737
738
739
740
  {
  	if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
  		rt_mutex_deadlock_account_lock(lock, current);
  		return 1;
  	}
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
741
  	return slowfn(lock);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
  }
  
  static inline void
  rt_mutex_fastunlock(struct rt_mutex *lock,
  		    void (*slowfn)(struct rt_mutex *lock))
  {
  	if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
  		rt_mutex_deadlock_account_unlock(current);
  	else
  		slowfn(lock);
  }
  
  /**
   * rt_mutex_lock - lock a rt_mutex
   *
   * @lock: the rt_mutex to be locked
   */
  void __sched rt_mutex_lock(struct rt_mutex *lock)
  {
  	might_sleep();
  
  	rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
  }
  EXPORT_SYMBOL_GPL(rt_mutex_lock);
  
  /**
   * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
   *
   * @lock: 		the rt_mutex to be locked
   * @detect_deadlock:	deadlock detection on/off
   *
   * Returns:
   *  0 		on success
   * -EINTR 	when interrupted by a signal
   * -EDEADLK	when the lock would deadlock (when deadlock detection is on)
   */
  int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
  						 int detect_deadlock)
  {
  	might_sleep();
  
  	return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
  				 detect_deadlock, rt_mutex_slowlock);
  }
  EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
  
  /**
23b94b967   Luis Henriques   locking, rtmutex....
789
790
791
   * rt_mutex_timed_lock - lock a rt_mutex interruptible
   *			the timeout structure is provided
   *			by the caller
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
792
793
794
795
796
797
798
799
   *
   * @lock: 		the rt_mutex to be locked
   * @timeout:		timeout structure or NULL (no timeout)
   * @detect_deadlock:	deadlock detection on/off
   *
   * Returns:
   *  0 		on success
   * -EINTR 	when interrupted by a signal
3ac49a1c9   Jean Delvare   trivial: fix ETIM...
800
   * -ETIMEDOUT	when the timeout expired
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
   * -EDEADLK	when the lock would deadlock (when deadlock detection is on)
   */
  int
  rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
  		    int detect_deadlock)
  {
  	might_sleep();
  
  	return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
  				       detect_deadlock, rt_mutex_slowlock);
  }
  EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
  
  /**
   * rt_mutex_trylock - try to lock a rt_mutex
   *
   * @lock:	the rt_mutex to be locked
   *
   * Returns 1 on success and 0 on contention
   */
  int __sched rt_mutex_trylock(struct rt_mutex *lock)
  {
  	return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
  }
  EXPORT_SYMBOL_GPL(rt_mutex_trylock);
  
  /**
   * rt_mutex_unlock - unlock a rt_mutex
   *
   * @lock: the rt_mutex to be unlocked
   */
  void __sched rt_mutex_unlock(struct rt_mutex *lock)
  {
  	rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
  }
  EXPORT_SYMBOL_GPL(rt_mutex_unlock);
23b94b967   Luis Henriques   locking, rtmutex....
837
  /**
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
   * rt_mutex_destroy - mark a mutex unusable
   * @lock: the mutex to be destroyed
   *
   * This function marks the mutex uninitialized, and any subsequent
   * use of the mutex is forbidden. The mutex must not be locked when
   * this function is called.
   */
  void rt_mutex_destroy(struct rt_mutex *lock)
  {
  	WARN_ON(rt_mutex_is_locked(lock));
  #ifdef CONFIG_DEBUG_RT_MUTEXES
  	lock->magic = NULL;
  #endif
  }
  
  EXPORT_SYMBOL_GPL(rt_mutex_destroy);
  
  /**
   * __rt_mutex_init - initialize the rt lock
   *
   * @lock: the rt lock to be initialized
   *
   * Initialize the rt lock to unlocked state.
   *
   * Initializing of a locked rt lock is not allowed
   */
  void __rt_mutex_init(struct rt_mutex *lock, const char *name)
  {
  	lock->owner = NULL;
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
867
  	raw_spin_lock_init(&lock->wait_lock);
732375c6a   Dima Zavin   plist: Remove the...
868
  	plist_head_init(&lock->wait_list);
23f78d4a0   Ingo Molnar   [PATCH] pi-futex:...
869
870
871
872
  
  	debug_rt_mutex_init(lock, name);
  }
  EXPORT_SYMBOL_GPL(__rt_mutex_init);
0cdbee992   Ingo Molnar   [PATCH] pi-futex:...
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
  
  /**
   * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
   *				proxy owner
   *
   * @lock: 	the rt_mutex to be locked
   * @proxy_owner:the task to set as owner
   *
   * No locking. Caller has to do serializing itself
   * Special API call for PI-futex support
   */
  void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
  				struct task_struct *proxy_owner)
  {
  	__rt_mutex_init(lock, NULL);
9a11b49a8   Ingo Molnar   [PATCH] lockdep: ...
888
  	debug_rt_mutex_proxy_lock(lock, proxy_owner);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
889
  	rt_mutex_set_owner(lock, proxy_owner);
0cdbee992   Ingo Molnar   [PATCH] pi-futex:...
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
  	rt_mutex_deadlock_account_lock(lock, proxy_owner);
  }
  
  /**
   * rt_mutex_proxy_unlock - release a lock on behalf of owner
   *
   * @lock: 	the rt_mutex to be locked
   *
   * No locking. Caller has to do serializing itself
   * Special API call for PI-futex support
   */
  void rt_mutex_proxy_unlock(struct rt_mutex *lock,
  			   struct task_struct *proxy_owner)
  {
  	debug_rt_mutex_proxy_unlock(lock);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
905
  	rt_mutex_set_owner(lock, NULL);
0cdbee992   Ingo Molnar   [PATCH] pi-futex:...
906
907
908
909
  	rt_mutex_deadlock_account_unlock(proxy_owner);
  }
  
  /**
8dac456a6   Darren Hart   rt_mutex: add pro...
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
   * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
   * @lock:		the rt_mutex to take
   * @waiter:		the pre-initialized rt_mutex_waiter
   * @task:		the task to prepare
   * @detect_deadlock:	perform deadlock detection (1) or not (0)
   *
   * Returns:
   *  0 - task blocked on lock
   *  1 - acquired the lock for task, caller should wake it up
   * <0 - error
   *
   * Special API call for FUTEX_REQUEUE_PI support.
   */
  int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  			      struct rt_mutex_waiter *waiter,
  			      struct task_struct *task, int detect_deadlock)
  {
  	int ret;
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
928
  	raw_spin_lock(&lock->wait_lock);
8dac456a6   Darren Hart   rt_mutex: add pro...
929

8161239a8   Lai Jiangshan   rtmutex: Simplify...
930
  	if (try_to_take_rt_mutex(lock, task, NULL)) {
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
931
  		raw_spin_unlock(&lock->wait_lock);
8dac456a6   Darren Hart   rt_mutex: add pro...
932
933
934
935
  		return 1;
  	}
  
  	ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
936
  	if (ret && !rt_mutex_owner(lock)) {
8dac456a6   Darren Hart   rt_mutex: add pro...
937
938
939
940
941
942
943
944
  		/*
  		 * Reset the return value. We might have
  		 * returned with -EDEADLK and the owner
  		 * released the lock while we were walking the
  		 * pi chain.  Let the waiter sort it out.
  		 */
  		ret = 0;
  	}
8161239a8   Lai Jiangshan   rtmutex: Simplify...
945
946
947
  
  	if (unlikely(ret))
  		remove_waiter(lock, waiter);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
948
  	raw_spin_unlock(&lock->wait_lock);
8dac456a6   Darren Hart   rt_mutex: add pro...
949
950
951
952
953
954
955
  
  	debug_rt_mutex_print_deadlock(waiter);
  
  	return ret;
  }
  
  /**
0cdbee992   Ingo Molnar   [PATCH] pi-futex:...
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
   * rt_mutex_next_owner - return the next owner of the lock
   *
   * @lock: the rt lock query
   *
   * Returns the next owner of the lock or NULL
   *
   * Caller has to serialize against other accessors to the lock
   * itself.
   *
   * Special API call for PI-futex support
   */
  struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
  {
  	if (!rt_mutex_has_waiters(lock))
  		return NULL;
  
  	return rt_mutex_top_waiter(lock)->task;
  }
8dac456a6   Darren Hart   rt_mutex: add pro...
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
  
  /**
   * rt_mutex_finish_proxy_lock() - Complete lock acquisition
   * @lock:		the rt_mutex we were woken on
   * @to:			the timeout, null if none. hrtimer should already have
   * 			been started.
   * @waiter:		the pre-initialized rt_mutex_waiter
   * @detect_deadlock:	perform deadlock detection (1) or not (0)
   *
   * Complete the lock acquisition started our behalf by another thread.
   *
   * Returns:
   *  0 - success
   * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK
   *
   * Special API call for PI-futex requeue support
   */
  int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
  			       struct hrtimer_sleeper *to,
  			       struct rt_mutex_waiter *waiter,
  			       int detect_deadlock)
  {
  	int ret;
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
997
  	raw_spin_lock(&lock->wait_lock);
8dac456a6   Darren Hart   rt_mutex: add pro...
998
999
  
  	set_current_state(TASK_INTERRUPTIBLE);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
1000
  	ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
8dac456a6   Darren Hart   rt_mutex: add pro...
1001
1002
  
  	set_current_state(TASK_RUNNING);
8161239a8   Lai Jiangshan   rtmutex: Simplify...
1003
  	if (unlikely(ret))
8dac456a6   Darren Hart   rt_mutex: add pro...
1004
1005
1006
1007
1008
1009
1010
  		remove_waiter(lock, waiter);
  
  	/*
  	 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
  	 * have to fix that up.
  	 */
  	fixup_rt_mutex_waiters(lock);
d209d74d5   Thomas Gleixner   rtmutes: Convert ...
1011
  	raw_spin_unlock(&lock->wait_lock);
8dac456a6   Darren Hart   rt_mutex: add pro...
1012

8dac456a6   Darren Hart   rt_mutex: add pro...
1013
1014
  	return ret;
  }