FUTEX: Tidy up the code

The recent PRIVATE and REQUEUE_PI changes to the futex code made it hard to read. Tidy it up. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Ingo Molnar <mingo@elte.hu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

FUTEX: Tidy up the code
The recent PRIVATE and REQUEUE_PI changes to the futex code made it hard to read. Tidy it up. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Ingo Molnar <mingo@elte.hu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Thomas Gleixner · Linus Torvalds
1 parent 0746aec3c7
Showing 4 changed files with 74 additions and 85 deletions Side-by-side Diff
kernel/futex.c
kernel/rtmutex-debug.c
kernel/rtmutex.c
kernel/rtmutex_common.h
@@ -121,6 +121,24 @@
 static struct vfsmount *futex_mnt;
  
 /*
+ * Take mm->mmap_sem, when futex is shared
+ */
+static inline void futex_lock_mm(struct rw_semaphore *fshared)
+{
+	if (fshared)
+		down_read(fshared);
+}
+
+/*
+ * Release mm->mmap_sem, when the futex is shared
+ */
+static inline void futex_unlock_mm(struct rw_semaphore *fshared)
+{
+	if (fshared)
+		up_read(fshared);
+}
+
+/*
  * We hash on the keys returned from get_futex_key (see below).
  */
 static struct futex_hash_bucket *hash_futex(union futex_key *key)
  
@@ -287,8 +305,19 @@
 }
 EXPORT_SYMBOL_GPL(drop_futex_key_refs);
  
-static inline int get_futex_value_locked(u32 *dest, u32 __user *from)
+static u32 cmpxchg_futex_value_locked(u32 __user *uaddr, u32 uval, u32 newval)
 {
+	u32 curval;
+
+	pagefault_disable();
+	curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
+	pagefault_enable();
+
+	return curval;
+}
+
+static int get_futex_value_locked(u32 *dest, u32 __user *from)
+{
 	int ret;
  
 	pagefault_disable();
@@ -620,9 +649,7 @@
  
 		newval = FUTEX_WAITERS | new_owner->pid;
  
-		pagefault_disable();
-		curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
-		pagefault_enable();
+		curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
  
 		if (curval == -EFAULT)
 			ret = -EFAULT;
@@ -659,9 +686,7 @@
 	 * There is no waiter, so we unlock the futex. The owner died
 	 * bit has not to be preserved here. We are the owner:
 	 */
-	pagefault_disable();
-	oldval = futex_atomic_cmpxchg_inatomic(uaddr, uval, 0);
-	pagefault_enable();
+	oldval = cmpxchg_futex_value_locked(uaddr, uval, 0);
  
 	if (oldval == -EFAULT)
 		return oldval;
@@ -700,8 +725,7 @@
 	union futex_key key;
 	int ret;
  
-	if (fshared)
-		down_read(fshared);
+	futex_lock_mm(fshared);
  
 	ret = get_futex_key(uaddr, fshared, &key);
 	if (unlikely(ret != 0))
@@ -725,8 +749,7 @@
  
 	spin_unlock(&hb->lock);
 out:
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
 	return ret;
 }
  
@@ -746,8 +769,7 @@
 	int ret, op_ret, attempt = 0;
  
 retryfull:
-	if (fshared)
-		down_read(fshared);
+	futex_lock_mm(fshared);
  
 	ret = get_futex_key(uaddr1, fshared, &key1);
 	if (unlikely(ret != 0))
@@ -793,7 +815,7 @@
 		 */
 		if (attempt++) {
 			ret = futex_handle_fault((unsigned long)uaddr2,
-						fshared, attempt);
+						 fshared, attempt);
 			if (ret)
 				goto out;
 			goto retry;
@@ -803,8 +825,7 @@
 		 * If we would have faulted, release mmap_sem,
 		 * fault it in and start all over again.
 		 */
-		if (fshared)
-			up_read(fshared);
+		futex_unlock_mm(fshared);
  
 		ret = get_user(dummy, uaddr2);
 		if (ret)
@@ -841,8 +862,8 @@
 	if (hb1 != hb2)
 		spin_unlock(&hb2->lock);
 out:
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
+
 	return ret;
 }
  
@@ -861,8 +882,7 @@
 	int ret, drop_count = 0;
  
  retry:
-	if (fshared)
-		down_read(fshared);
+	futex_lock_mm(fshared);
  
 	ret = get_futex_key(uaddr1, fshared, &key1);
 	if (unlikely(ret != 0))
@@ -890,8 +910,7 @@
 			 * If we would have faulted, release mmap_sem, fault
 			 * it in and start all over again.
 			 */
-			if (fshared)
-				up_read(fshared);
+			futex_unlock_mm(fshared);
  
 			ret = get_user(curval, uaddr1);
  
@@ -944,8 +963,7 @@
 		drop_futex_key_refs(&key1);
  
 out:
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
 	return ret;
 }
  
@@ -1113,10 +1131,7 @@
 	while (!ret) {
 		newval = (uval & FUTEX_OWNER_DIED) | newtid;
  
-		pagefault_disable();
-		curval = futex_atomic_cmpxchg_inatomic(uaddr,
-						       uval, newval);
-		pagefault_enable();
+		curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
  
 		if (curval == -EFAULT)
 			ret = -EFAULT;
@@ -1134,6 +1149,7 @@
 #define ARG3_SHARED  1
  
 static long futex_wait_restart(struct restart_block *restart);
+
 static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
 		      u32 val, ktime_t *abs_time)
 {
@@ -1148,8 +1164,7 @@
  
 	q.pi_state = NULL;
  retry:
-	if (fshared)
-		down_read(fshared);
+	futex_lock_mm(fshared);
  
 	ret = get_futex_key(uaddr, fshared, &q.key);
 	if (unlikely(ret != 0))
@@ -1186,8 +1201,7 @@
 		 * If we would have faulted, release mmap_sem, fault it in and
 		 * start all over again.
 		 */
-		if (fshared)
-			up_read(fshared);
+		futex_unlock_mm(fshared);
  
 		ret = get_user(uval, uaddr);
  
@@ -1206,8 +1220,7 @@
 	 * Now the futex is queued and we have checked the data, we
 	 * don't want to hold mmap_sem while we sleep.
 	 */
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
  
 	/*
 	 * There might have been scheduling since the queue_me(), as we
@@ -1285,8 +1298,7 @@
 	queue_unlock(&q, hb);
  
  out_release_sem:
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
 	return ret;
 }
  
@@ -1333,8 +1345,7 @@
  
 	q.pi_state = NULL;
  retry:
-	if (fshared)
-		down_read(fshared);
+	futex_lock_mm(fshared);
  
 	ret = get_futex_key(uaddr, fshared, &q.key);
 	if (unlikely(ret != 0))
@@ -1353,9 +1364,7 @@
 	 */
 	newval = current->pid;
  
-	pagefault_disable();
-	curval = futex_atomic_cmpxchg_inatomic(uaddr, 0, newval);
-	pagefault_enable();
+	curval = cmpxchg_futex_value_locked(uaddr, 0, newval);
  
 	if (unlikely(curval == -EFAULT))
 		goto uaddr_faulted;
@@ -1398,9 +1407,7 @@
 		lock_taken = 1;
 	}
  
-	pagefault_disable();
-	curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
-	pagefault_enable();
+	curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
  
 	if (unlikely(curval == -EFAULT))
 		goto uaddr_faulted;
@@ -1428,8 +1435,7 @@
 			 * exit to complete.
 			 */
 			queue_unlock(&q, hb);
-			if (fshared)
-				up_read(fshared);
+			futex_unlock_mm(fshared);
 			cond_resched();
 			goto retry;
  
@@ -1465,8 +1471,7 @@
 	 * Now the futex is queued and we have checked the data, we
 	 * don't want to hold mmap_sem while we sleep.
 	 */
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
  
 	WARN_ON(!q.pi_state);
 	/*
@@ -1480,8 +1485,7 @@
 		ret = ret ? 0 : -EWOULDBLOCK;
 	}
  
-	if (fshared)
-		down_read(fshared);
+	futex_lock_mm(fshared);
 	spin_lock(q.lock_ptr);
  
 	if (!ret) {
@@ -1518,8 +1522,7 @@
  
 	/* Unqueue and drop the lock */
 	unqueue_me_pi(&q);
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
  
 	return ret != -EINTR ? ret : -ERESTARTNOINTR;
  
@@ -1527,8 +1530,7 @@
 	queue_unlock(&q, hb);
  
  out_release_sem:
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
 	return ret;
  
  uaddr_faulted:
@@ -1550,8 +1552,7 @@
 		goto retry_unlocked;
 	}
  
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
  
 	ret = get_user(uval, uaddr);
 	if (!ret && (uval != -EFAULT))
@@ -1585,8 +1586,7 @@
 	/*
 	 * First take all the futex related locks:
 	 */
-	if (fshared)
-		down_read(fshared);
+	futex_lock_mm(fshared);
  
 	ret = get_futex_key(uaddr, fshared, &key);
 	if (unlikely(ret != 0))
  
@@ -1601,12 +1601,10 @@
 	 * again. If it succeeds then we can return without waking
 	 * anyone else up:
 	 */
-	if (!(uval & FUTEX_OWNER_DIED)) {
-		pagefault_disable();
-		uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0);
-		pagefault_enable();
-	}
+	if (!(uval & FUTEX_OWNER_DIED))
+		uval = cmpxchg_futex_value_locked(uaddr, current->pid, 0);
  
+
 	if (unlikely(uval == -EFAULT))
 		goto pi_faulted;
 	/*
@@ -1647,8 +1645,7 @@
 out_unlock:
 	spin_unlock(&hb->lock);
 out:
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
  
 	return ret;
  
@@ -1671,8 +1668,7 @@
 		goto retry_unlocked;
 	}
  
-	if (fshared)
-		up_read(fshared);
+	futex_unlock_mm(fshared);
  
 	ret = get_user(uval, uaddr);
 	if (!ret && (uval != -EFAULT))
@@ -1729,8 +1725,8 @@
  
 	if (printk_timed_ratelimit(&printk_interval, 60 * 60 * 1000)) {
 		printk(KERN_WARNING "Process `%s' used FUTEX_FD, which "
-		    	"will be removed from the kernel in June 2007\n",
-			current->comm);
+		       "will be removed from the kernel in June 2007\n",
+		       current->comm);
 	}
  
 	ret = -EINVAL;
  
@@ -1908,10 +1904,8 @@
 		 * Wake robust non-PI futexes here. The wakeup of
 		 * PI futexes happens in exit_pi_state():
 		 */
-		if (!pi) {
-			if (uval & FUTEX_WAITERS)
+		if (!pi && (uval & FUTEX_WAITERS))
 				futex_wake(uaddr, &curr->mm->mmap_sem, 1);
-		}
 	}
 	return 0;
 }
@@ -29,12 +29,6 @@
  
 #include "rtmutex_common.h"
  
-#ifdef CONFIG_DEBUG_RT_MUTEXES
-# include "rtmutex-debug.h"
-#else
-# include "rtmutex.h"
-#endif
-
 # define TRACE_WARN_ON(x)			WARN_ON(x)
 # define TRACE_BUG_ON(x)			BUG_ON(x)
  
@@ -17,12 +17,6 @@
  
 #include "rtmutex_common.h"
  
-#ifdef CONFIG_DEBUG_RT_MUTEXES
-# include "rtmutex-debug.h"
-#else
-# include "rtmutex.h"
-#endif
-
 /*
  * lock->owner state tracking:
  *
@@ -103,7 +103,7 @@
  
 static inline struct task_struct *rt_mutex_real_owner(struct rt_mutex *lock)
 {
- 	return (struct task_struct *)
+	return (struct task_struct *)
 		((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
 }
  
@@ -120,5 +120,12 @@
 				       struct task_struct *proxy_owner);
 extern void rt_mutex_proxy_unlock(struct rt_mutex *lock,
 				  struct task_struct *proxy_owner);
+
+#ifdef CONFIG_DEBUG_RT_MUTEXES
+# include "rtmutex-debug.h"
+#else
+# include "rtmutex.h"
+#endif
+
 #endif
...	...	@@ -121,6 +121,24 @@
121	121	static struct vfsmount *futex_mnt;
122	122
123	123	/*
	124	+ * Take mm->mmap_sem, when futex is shared
	125	+ */
	126	+static inline void futex_lock_mm(struct rw_semaphore *fshared)
	127	+{
	128	+ if (fshared)
	129	+ down_read(fshared);
	130	+}
	131	+
	132	+/*
	133	+ * Release mm->mmap_sem, when the futex is shared
	134	+ */
	135	+static inline void futex_unlock_mm(struct rw_semaphore *fshared)
	136	+{
	137	+ if (fshared)
	138	+ up_read(fshared);
	139	+}
	140	+
	141	+/*
124	142	* We hash on the keys returned from get_futex_key (see below).
125	143	*/
126	144	static struct futex_hash_bucket hash_futex(union futex_key key)
127	145
...	...	@@ -287,8 +305,19 @@
287	305	}
288	306	EXPORT_SYMBOL_GPL(drop_futex_key_refs);
289	307
290		-static inline int get_futex_value_locked(u32 dest, u32 __user from)
	308	+static u32 cmpxchg_futex_value_locked(u32 __user *uaddr, u32 uval, u32 newval)
291	309	{
	310	+ u32 curval;
	311	+
	312	+ pagefault_disable();
	313	+ curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
	314	+ pagefault_enable();
	315	+
	316	+ return curval;
	317	+}
	318	+
	319	+static int get_futex_value_locked(u32 dest, u32 __user from)
	320	+{
292	321	int ret;
293	322
294	323	pagefault_disable();
...	...	@@ -620,9 +649,7 @@
620	649
621	650	newval = FUTEX_WAITERS \| new_owner->pid;
622	651
623		- pagefault_disable();
624		- curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
625		- pagefault_enable();
	652	+ curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
626	653
627	654	if (curval == -EFAULT)
628	655	ret = -EFAULT;
...	...	@@ -659,9 +686,7 @@
659	686	* There is no waiter, so we unlock the futex. The owner died
660	687	* bit has not to be preserved here. We are the owner:
661	688	*/
662		- pagefault_disable();
663		- oldval = futex_atomic_cmpxchg_inatomic(uaddr, uval, 0);
664		- pagefault_enable();
	689	+ oldval = cmpxchg_futex_value_locked(uaddr, uval, 0);
665	690
666	691	if (oldval == -EFAULT)
667	692	return oldval;
...	...	@@ -700,8 +725,7 @@
700	725	union futex_key key;
701	726	int ret;
702	727
703		- if (fshared)
704		- down_read(fshared);
	728	+ futex_lock_mm(fshared);
705	729
706	730	ret = get_futex_key(uaddr, fshared, &key);
707	731	if (unlikely(ret != 0))
...	...	@@ -725,8 +749,7 @@
725	749
726	750	spin_unlock(&hb->lock);
727	751	out:
728		- if (fshared)
729		- up_read(fshared);
	752	+ futex_unlock_mm(fshared);
730	753	return ret;
731	754	}
732	755
...	...	@@ -746,8 +769,7 @@
746	769	int ret, op_ret, attempt = 0;
747	770
748	771	retryfull:
749		- if (fshared)
750		- down_read(fshared);
	772	+ futex_lock_mm(fshared);
751	773
752	774	ret = get_futex_key(uaddr1, fshared, &key1);
753	775	if (unlikely(ret != 0))
...	...	@@ -793,7 +815,7 @@
793	815	*/
794	816	if (attempt++) {
795	817	ret = futex_handle_fault((unsigned long)uaddr2,
796		- fshared, attempt);
	818	+ fshared, attempt);
797	819	if (ret)
798	820	goto out;
799	821	goto retry;
...	...	@@ -803,8 +825,7 @@
803	825	* If we would have faulted, release mmap_sem,
804	826	* fault it in and start all over again.
805	827	*/
806		- if (fshared)
807		- up_read(fshared);
	828	+ futex_unlock_mm(fshared);
808	829
809	830	ret = get_user(dummy, uaddr2);
810	831	if (ret)
...	...	@@ -841,8 +862,8 @@
841	862	if (hb1 != hb2)
842	863	spin_unlock(&hb2->lock);
843	864	out:
844		- if (fshared)
845		- up_read(fshared);
	865	+ futex_unlock_mm(fshared);
	866	+
846	867	return ret;
847	868	}
848	869
...	...	@@ -861,8 +882,7 @@
861	882	int ret, drop_count = 0;
862	883
863	884	retry:
864		- if (fshared)
865		- down_read(fshared);
	885	+ futex_lock_mm(fshared);
866	886
867	887	ret = get_futex_key(uaddr1, fshared, &key1);
868	888	if (unlikely(ret != 0))
...	...	@@ -890,8 +910,7 @@
890	910	* If we would have faulted, release mmap_sem, fault
891	911	* it in and start all over again.
892	912	*/
893		- if (fshared)
894		- up_read(fshared);
	913	+ futex_unlock_mm(fshared);
895	914
896	915	ret = get_user(curval, uaddr1);
897	916
...	...	@@ -944,8 +963,7 @@
944	963	drop_futex_key_refs(&key1);
945	964
946	965	out:
947		- if (fshared)
948		- up_read(fshared);
	966	+ futex_unlock_mm(fshared);
949	967	return ret;
950	968	}
951	969
...	...	@@ -1113,10 +1131,7 @@
1113	1131	while (!ret) {
1114	1132	newval = (uval & FUTEX_OWNER_DIED) \| newtid;
1115	1133
1116		- pagefault_disable();
1117		- curval = futex_atomic_cmpxchg_inatomic(uaddr,
1118		- uval, newval);
1119		- pagefault_enable();
	1134	+ curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
1120	1135
1121	1136	if (curval == -EFAULT)
1122	1137	ret = -EFAULT;
...	...	@@ -1134,6 +1149,7 @@
1134	1149	#define ARG3_SHARED 1
1135	1150
1136	1151	static long futex_wait_restart(struct restart_block *restart);
	1152	+
1137	1153	static int futex_wait(u32 __user uaddr, struct rw_semaphore fshared,
1138	1154	u32 val, ktime_t *abs_time)
1139	1155	{
...	...	@@ -1148,8 +1164,7 @@
1148	1164
1149	1165	q.pi_state = NULL;
1150	1166	retry:
1151		- if (fshared)
1152		- down_read(fshared);
	1167	+ futex_lock_mm(fshared);
1153	1168
1154	1169	ret = get_futex_key(uaddr, fshared, &q.key);
1155	1170	if (unlikely(ret != 0))
...	...	@@ -1186,8 +1201,7 @@
1186	1201	* If we would have faulted, release mmap_sem, fault it in and
1187	1202	* start all over again.
1188	1203	*/
1189		- if (fshared)
1190		- up_read(fshared);
	1204	+ futex_unlock_mm(fshared);
1191	1205
1192	1206	ret = get_user(uval, uaddr);
1193	1207
...	...	@@ -1206,8 +1220,7 @@
1206	1220	* Now the futex is queued and we have checked the data, we
1207	1221	* don't want to hold mmap_sem while we sleep.
1208	1222	*/
1209		- if (fshared)
1210		- up_read(fshared);
	1223	+ futex_unlock_mm(fshared);
1211	1224
1212	1225	/*
1213	1226	* There might have been scheduling since the queue_me(), as we
...	...	@@ -1285,8 +1298,7 @@
1285	1298	queue_unlock(&q, hb);
1286	1299
1287	1300	out_release_sem:
1288		- if (fshared)
1289		- up_read(fshared);
	1301	+ futex_unlock_mm(fshared);
1290	1302	return ret;
1291	1303	}
1292	1304
...	...	@@ -1333,8 +1345,7 @@
1333	1345
1334	1346	q.pi_state = NULL;
1335	1347	retry:
1336		- if (fshared)
1337		- down_read(fshared);
	1348	+ futex_lock_mm(fshared);
1338	1349
1339	1350	ret = get_futex_key(uaddr, fshared, &q.key);
1340	1351	if (unlikely(ret != 0))
...	...	@@ -1353,9 +1364,7 @@
1353	1364	*/
1354	1365	newval = current->pid;
1355	1366
1356		- pagefault_disable();
1357		- curval = futex_atomic_cmpxchg_inatomic(uaddr, 0, newval);
1358		- pagefault_enable();
	1367	+ curval = cmpxchg_futex_value_locked(uaddr, 0, newval);
1359	1368
1360	1369	if (unlikely(curval == -EFAULT))
1361	1370	goto uaddr_faulted;
...	...	@@ -1398,9 +1407,7 @@
1398	1407	lock_taken = 1;
1399	1408	}
1400	1409
1401		- pagefault_disable();
1402		- curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
1403		- pagefault_enable();
	1410	+ curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
1404	1411
1405	1412	if (unlikely(curval == -EFAULT))
1406	1413	goto uaddr_faulted;
...	...	@@ -1428,8 +1435,7 @@
1428	1435	* exit to complete.
1429	1436	*/
1430	1437	queue_unlock(&q, hb);
1431		- if (fshared)
1432		- up_read(fshared);
	1438	+ futex_unlock_mm(fshared);
1433	1439	cond_resched();
1434	1440	goto retry;
1435	1441
...	...	@@ -1465,8 +1471,7 @@
1465	1471	* Now the futex is queued and we have checked the data, we
1466	1472	* don't want to hold mmap_sem while we sleep.
1467	1473	*/
1468		- if (fshared)
1469		- up_read(fshared);
	1474	+ futex_unlock_mm(fshared);
1470	1475
1471	1476	WARN_ON(!q.pi_state);
1472	1477	/*
...	...	@@ -1480,8 +1485,7 @@
1480	1485	ret = ret ? 0 : -EWOULDBLOCK;
1481	1486	}
1482	1487
1483		- if (fshared)
1484		- down_read(fshared);
	1488	+ futex_lock_mm(fshared);
1485	1489	spin_lock(q.lock_ptr);
1486	1490
1487	1491	if (!ret) {
...	...	@@ -1518,8 +1522,7 @@
1518	1522
1519	1523	/* Unqueue and drop the lock */
1520	1524	unqueue_me_pi(&q);
1521		- if (fshared)
1522		- up_read(fshared);
	1525	+ futex_unlock_mm(fshared);
1523	1526
1524	1527	return ret != -EINTR ? ret : -ERESTARTNOINTR;
1525	1528
...	...	@@ -1527,8 +1530,7 @@
1527	1530	queue_unlock(&q, hb);
1528	1531
1529	1532	out_release_sem:
1530		- if (fshared)
1531		- up_read(fshared);
	1533	+ futex_unlock_mm(fshared);
1532	1534	return ret;
1533	1535
1534	1536	uaddr_faulted:
...	...	@@ -1550,8 +1552,7 @@
1550	1552	goto retry_unlocked;
1551	1553	}
1552	1554
1553		- if (fshared)
1554		- up_read(fshared);
	1555	+ futex_unlock_mm(fshared);
1555	1556
1556	1557	ret = get_user(uval, uaddr);
1557	1558	if (!ret && (uval != -EFAULT))
...	...	@@ -1585,8 +1586,7 @@
1585	1586	/*
1586	1587	* First take all the futex related locks:
1587	1588	*/
1588		- if (fshared)
1589		- down_read(fshared);
	1589	+ futex_lock_mm(fshared);
1590	1590
1591	1591	ret = get_futex_key(uaddr, fshared, &key);
1592	1592	if (unlikely(ret != 0))
1593	1593
...	...	@@ -1601,12 +1601,10 @@
1601	1601	* again. If it succeeds then we can return without waking
1602	1602	* anyone else up:
1603	1603	*/
1604		- if (!(uval & FUTEX_OWNER_DIED)) {
1605		- pagefault_disable();
1606		- uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0);
1607		- pagefault_enable();
1608		- }
	1604	+ if (!(uval & FUTEX_OWNER_DIED))
	1605	+ uval = cmpxchg_futex_value_locked(uaddr, current->pid, 0);
1609	1606
	1607	+
1610	1608	if (unlikely(uval == -EFAULT))
1611	1609	goto pi_faulted;
1612	1610	/*
...	...	@@ -1647,8 +1645,7 @@
1647	1645	out_unlock:
1648	1646	spin_unlock(&hb->lock);
1649	1647	out:
1650		- if (fshared)
1651		- up_read(fshared);
	1648	+ futex_unlock_mm(fshared);
1652	1649
1653	1650	return ret;
1654	1651
...	...	@@ -1671,8 +1668,7 @@
1671	1668	goto retry_unlocked;
1672	1669	}
1673	1670
1674		- if (fshared)
1675		- up_read(fshared);
	1671	+ futex_unlock_mm(fshared);
1676	1672
1677	1673	ret = get_user(uval, uaddr);
1678	1674	if (!ret && (uval != -EFAULT))
...	...	@@ -1729,8 +1725,8 @@
1729	1725
1730	1726	if (printk_timed_ratelimit(&printk_interval, 60 * 60 * 1000)) {
1731	1727	printk(KERN_WARNING "Process `%s' used FUTEX_FD, which "
1732		- "will be removed from the kernel in June 2007\n",
1733		- current->comm);
	1728	+ "will be removed from the kernel in June 2007\n",
	1729	+ current->comm);
1734	1730	}
1735	1731
1736	1732	ret = -EINVAL;
1737	1733
...	...	@@ -1908,10 +1904,8 @@
1908	1904	* Wake robust non-PI futexes here. The wakeup of
1909	1905	* PI futexes happens in exit_pi_state():
1910	1906	*/
1911		- if (!pi) {
1912		- if (uval & FUTEX_WAITERS)
	1907	+ if (!pi && (uval & FUTEX_WAITERS))
1913	1908	futex_wake(uaddr, &curr->mm->mmap_sem, 1);
1914		- }
1915	1909	}
1916	1910	return 0;
1917	1911	}
...	...	@@ -29,12 +29,6 @@
29	29
30	30	#include "rtmutex_common.h"
31	31
32		-#ifdef CONFIG_DEBUG_RT_MUTEXES
33		-# include "rtmutex-debug.h"
34		-#else
35		-# include "rtmutex.h"
36		-#endif
37		-
38	32	# define TRACE_WARN_ON(x) WARN_ON(x)
39	33	# define TRACE_BUG_ON(x) BUG_ON(x)
40	34
...	...	@@ -17,12 +17,6 @@
17	17
18	18	#include "rtmutex_common.h"
19	19
20		-#ifdef CONFIG_DEBUG_RT_MUTEXES
21		-# include "rtmutex-debug.h"
22		-#else
23		-# include "rtmutex.h"
24		-#endif
25		-
26	20	/*
27	21	* lock->owner state tracking:
28	22	*
...	...	@@ -103,7 +103,7 @@
103	103
104	104	static inline struct task_struct rt_mutex_real_owner(struct rt_mutex lock)
105	105	{
106		- return (struct task_struct *)
	106	+ return (struct task_struct *)
107	107	((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
108	108	}
109	109
...	...	@@ -120,5 +120,12 @@
120	120	struct task_struct *proxy_owner);
121	121	extern void rt_mutex_proxy_unlock(struct rt_mutex *lock,
122	122	struct task_struct *proxy_owner);
	123	+
	124	+#ifdef CONFIG_DEBUG_RT_MUTEXES
	125	+# include "rtmutex-debug.h"
	126	+#else
	127	+# include "rtmutex.h"
	128	+#endif
	129	+
123	130	#endif