Merge branch 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull core/locking changes from Ingo Molnar: "Main changes: - another mutex optimization, from Davidlohr Bueso - improved lglock lockdep tracking, from Michel Lespinasse - [ assorted smaller updates, improvements, cleanups. ]" * 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: generic-ipi/locking: Fix misleading smp_call_function_any() description hung_task debugging: Print more info when reporting the problem mutex: Avoid label warning when !CONFIG_MUTEX_SPIN_ON_OWNER mutex: Do not unnecessarily deal with waiters mutex: Fix/document access-once assumption in mutex_can_spin_on_owner() lglock: Update lockdep annotations to report recursive local locks lockdep: Introduce lock_acquire_exclusive()/shared() helper macros

Merge branch 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull core/locking changes from Ingo Molnar: "Main changes: - another mutex optimization, from Davidlohr Bueso - improved lglock lockdep tracking, from Michel Lespinasse - [ assorted smaller updates, improvements, cleanups. ]" * 'core-locking-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: generic-ipi/locking: Fix misleading smp_call_function_any() description hung_task debugging: Print more info when reporting the problem mutex: Avoid label warning when !CONFIG_MUTEX_SPIN_ON_OWNER mutex: Do not unnecessarily deal with waiters mutex: Fix/document access-once assumption in mutex_can_spin_on_owner() lglock: Update lockdep annotations to report recursive local locks lockdep: Introduce lock_acquire_exclusive()/shared() helper macros
Linus Torvalds
2 parents b854e4de0b 15e71911fc
Showing 5 changed files Side-by-side Diff
include/linux/lockdep.h
kernel/hung_task.c
kernel/lglock.c
kernel/mutex.c
kernel/smp.c
@@ -365,7 +365,7 @@
  
 #define lockdep_recursing(tsk)	((tsk)->lockdep_recursion)
  
-#else /* !LOCKDEP */
+#else /* !CONFIG_LOCKDEP */
  
 static inline void lockdep_off(void)
 {
  
  
  
  
  
  
  
@@ -479,82 +479,36 @@
  * on the per lock-class debug mode:
  */
  
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-# ifdef CONFIG_PROVE_LOCKING
-#  define spin_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
-#  define spin_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 2, n, i)
-# else
-#  define spin_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
-#  define spin_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 1, NULL, i)
-# endif
-# define spin_release(l, n, i)			lock_release(l, n, i)
+#ifdef CONFIG_PROVE_LOCKING
+ #define lock_acquire_exclusive(l, s, t, n, i)		lock_acquire(l, s, t, 0, 2, n, i)
+ #define lock_acquire_shared(l, s, t, n, i)		lock_acquire(l, s, t, 1, 2, n, i)
+ #define lock_acquire_shared_recursive(l, s, t, n, i)	lock_acquire(l, s, t, 2, 2, n, i)
 #else
-# define spin_acquire(l, s, t, i)		do { } while (0)
-# define spin_release(l, n, i)			do { } while (0)
+ #define lock_acquire_exclusive(l, s, t, n, i)		lock_acquire(l, s, t, 0, 1, n, i)
+ #define lock_acquire_shared(l, s, t, n, i)		lock_acquire(l, s, t, 1, 1, n, i)
+ #define lock_acquire_shared_recursive(l, s, t, n, i)	lock_acquire(l, s, t, 2, 1, n, i)
 #endif
  
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-# ifdef CONFIG_PROVE_LOCKING
-#  define rwlock_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
-#  define rwlock_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 2, 2, NULL, i)
-# else
-#  define rwlock_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
-#  define rwlock_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 2, 1, NULL, i)
-# endif
-# define rwlock_release(l, n, i)		lock_release(l, n, i)
-#else
-# define rwlock_acquire(l, s, t, i)		do { } while (0)
-# define rwlock_acquire_read(l, s, t, i)	do { } while (0)
-# define rwlock_release(l, n, i)		do { } while (0)
-#endif
+#define spin_acquire(l, s, t, i)		lock_acquire_exclusive(l, s, t, NULL, i)
+#define spin_acquire_nest(l, s, t, n, i)	lock_acquire_exclusive(l, s, t, n, i)
+#define spin_release(l, n, i)			lock_release(l, n, i)
  
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-# ifdef CONFIG_PROVE_LOCKING
-#  define mutex_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
-#  define mutex_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 2, n, i)
-# else
-#  define mutex_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
-#  define mutex_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 1, n, i)
-# endif
-# define mutex_release(l, n, i)			lock_release(l, n, i)
-#else
-# define mutex_acquire(l, s, t, i)		do { } while (0)
-# define mutex_acquire_nest(l, s, t, n, i)	do { } while (0)
-# define mutex_release(l, n, i)			do { } while (0)
-#endif
+#define rwlock_acquire(l, s, t, i)		lock_acquire_exclusive(l, s, t, NULL, i)
+#define rwlock_acquire_read(l, s, t, i)		lock_acquire_shared_recursive(l, s, t, NULL, i)
+#define rwlock_release(l, n, i)			lock_release(l, n, i)
  
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-# ifdef CONFIG_PROVE_LOCKING
-#  define rwsem_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
-#  define rwsem_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 2, n, i)
-#  define rwsem_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 1, 2, NULL, i)
-# else
-#  define rwsem_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
-#  define rwsem_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 1, n, i)
-#  define rwsem_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 1, 1, NULL, i)
-# endif
+#define mutex_acquire(l, s, t, i)		lock_acquire_exclusive(l, s, t, NULL, i)
+#define mutex_acquire_nest(l, s, t, n, i)	lock_acquire_exclusive(l, s, t, n, i)
+#define mutex_release(l, n, i)			lock_release(l, n, i)
+
+#define rwsem_acquire(l, s, t, i)		lock_acquire_exclusive(l, s, t, NULL, i)
+#define rwsem_acquire_nest(l, s, t, n, i)	lock_acquire_exclusive(l, s, t, n, i)
+#define rwsem_acquire_read(l, s, t, i)		lock_acquire_shared(l, s, t, NULL, i)
 # define rwsem_release(l, n, i)			lock_release(l, n, i)
-#else
-# define rwsem_acquire(l, s, t, i)		do { } while (0)
-# define rwsem_acquire_nest(l, s, t, n, i)	do { } while (0)
-# define rwsem_acquire_read(l, s, t, i)		do { } while (0)
-# define rwsem_release(l, n, i)			do { } while (0)
-#endif
  
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-# ifdef CONFIG_PROVE_LOCKING
-#  define lock_map_acquire(l)		lock_acquire(l, 0, 0, 0, 2, NULL, _THIS_IP_)
-#  define lock_map_acquire_read(l)	lock_acquire(l, 0, 0, 2, 2, NULL, _THIS_IP_)
-# else
-#  define lock_map_acquire(l)		lock_acquire(l, 0, 0, 0, 1, NULL, _THIS_IP_)
-#  define lock_map_acquire_read(l)	lock_acquire(l, 0, 0, 2, 1, NULL, _THIS_IP_)
-# endif
+#define lock_map_acquire(l)			lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_)
+#define lock_map_acquire_read(l)		lock_acquire_shared_recursive(l, 0, 0, NULL, _THIS_IP_)
 # define lock_map_release(l)			lock_release(l, 1, _THIS_IP_)
-#else
-# define lock_map_acquire(l)			do { } while (0)
-# define lock_map_acquire_read(l)		do { } while (0)
-# define lock_map_release(l)			do { } while (0)
-#endif
  
 #ifdef CONFIG_PROVE_LOCKING
 # define might_lock(lock) 						\
@@ -15,6 +15,7 @@
 #include <linux/lockdep.h>
 #include <linux/export.h>
 #include <linux/sysctl.h>
+#include <linux/utsname.h>
  
 /*
  * The number of tasks checked:
@@ -99,10 +100,14 @@
 	 * Ok, the task did not get scheduled for more than 2 minutes,
 	 * complain:
 	 */
-	printk(KERN_ERR "INFO: task %s:%d blocked for more than "
-			"%ld seconds.\n", t->comm, t->pid, timeout);
-	printk(KERN_ERR "\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
-			" disables this message.\n");
+	pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
+		t->comm, t->pid, timeout);
+	pr_err("      %s %s %.*s\n",
+		print_tainted(), init_utsname()->release,
+		(int)strcspn(init_utsname()->version, " "),
+		init_utsname()->version);
+	pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
+		" disables this message.\n");
 	sched_show_task(t);
 	debug_show_held_locks(t);
  
@@ -21,7 +21,7 @@
 	arch_spinlock_t *lock;
  
 	preempt_disable();
-	rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
+	lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
 	lock = this_cpu_ptr(lg->lock);
 	arch_spin_lock(lock);
 }
@@ -31,7 +31,7 @@
 {
 	arch_spinlock_t *lock;
  
-	rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+	lock_release(&lg->lock_dep_map, 1, _RET_IP_);
 	lock = this_cpu_ptr(lg->lock);
 	arch_spin_unlock(lock);
 	preempt_enable();
@@ -43,7 +43,7 @@
 	arch_spinlock_t *lock;
  
 	preempt_disable();
-	rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
+	lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
 	lock = per_cpu_ptr(lg->lock, cpu);
 	arch_spin_lock(lock);
 }
@@ -53,7 +53,7 @@
 {
 	arch_spinlock_t *lock;
  
-	rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+	lock_release(&lg->lock_dep_map, 1, _RET_IP_);
 	lock = per_cpu_ptr(lg->lock, cpu);
 	arch_spin_unlock(lock);
 	preempt_enable();
@@ -65,7 +65,7 @@
 	int i;
  
 	preempt_disable();
-	rwlock_acquire(&lg->lock_dep_map, 0, 0, _RET_IP_);
+	lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
 	for_each_possible_cpu(i) {
 		arch_spinlock_t *lock;
 		lock = per_cpu_ptr(lg->lock, i);
@@ -78,7 +78,7 @@
 {
 	int i;
  
-	rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
+	lock_release(&lg->lock_dep_map, 1, _RET_IP_);
 	for_each_possible_cpu(i) {
 		arch_spinlock_t *lock;
 		lock = per_cpu_ptr(lg->lock, i);
@@ -209,11 +209,13 @@
  */
 static inline int mutex_can_spin_on_owner(struct mutex *lock)
 {
+	struct task_struct *owner;
 	int retval = 1;
  
 	rcu_read_lock();
-	if (lock->owner)
-		retval = lock->owner->on_cpu;
+	owner = ACCESS_ONCE(lock->owner);
+	if (owner)
+		retval = owner->on_cpu;
 	rcu_read_unlock();
 	/*
 	 * if lock->owner is not set, the mutex owner may have just acquired
@@ -461,7 +463,7 @@
 			 * performed the optimistic spinning cannot be done.
 			 */
 			if (ACCESS_ONCE(ww->ctx))
-				break;
+				goto slowpath;
 		}
  
 		/*
@@ -472,7 +474,7 @@
 		owner = ACCESS_ONCE(lock->owner);
 		if (owner && !mutex_spin_on_owner(lock, owner)) {
 			mspin_unlock(MLOCK(lock), &node);
-			break;
+			goto slowpath;
 		}
  
 		if ((atomic_read(&lock->count) == 1) &&
@@ -499,7 +501,7 @@
 		 * the owner complete.
 		 */
 		if (!owner && (need_resched() || rt_task(task)))
-			break;
+			goto slowpath;
  
 		/*
 		 * The cpu_relax() call is a compiler barrier which forces
@@ -513,6 +515,10 @@
 #endif
 	spin_lock_mutex(&lock->wait_lock, flags);
  
+	/* once more, can we acquire the lock? */
+	if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
+		goto skip_wait;
+
 	debug_mutex_lock_common(lock, &waiter);
 	debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
  
@@ -520,9 +526,6 @@
 	list_add_tail(&waiter.list, &lock->wait_list);
 	waiter.task = task;
  
-	if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, -1) == 1))
-		goto done;
-
 	lock_contended(&lock->dep_map, ip);
  
 	for (;;) {
@@ -536,7 +539,7 @@
 		 * other waiters:
 		 */
 		if (MUTEX_SHOW_NO_WAITER(lock) &&
-		   (atomic_xchg(&lock->count, -1) == 1))
+		    (atomic_xchg(&lock->count, -1) == 1))
 			break;
  
 		/*
  
  
  
  
@@ -561,24 +564,25 @@
 		schedule_preempt_disabled();
 		spin_lock_mutex(&lock->wait_lock, flags);
 	}
+	mutex_remove_waiter(lock, &waiter, current_thread_info());
+	/* set it to 0 if there are no waiters left: */
+	if (likely(list_empty(&lock->wait_list)))
+		atomic_set(&lock->count, 0);
+	debug_mutex_free_waiter(&waiter);
  
-done:
+skip_wait:
+	/* got the lock - cleanup and rejoice! */
 	lock_acquired(&lock->dep_map, ip);
-	/* got the lock - rejoice! */
-	mutex_remove_waiter(lock, &waiter, current_thread_info());
 	mutex_set_owner(lock);
  
 	if (!__builtin_constant_p(ww_ctx == NULL)) {
-		struct ww_mutex *ww = container_of(lock,
-						      struct ww_mutex,
-						      base);
+		struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
 		struct mutex_waiter *cur;
  
 		/*
 		 * This branch gets optimized out for the common case,
 		 * and is only important for ww_mutex_lock.
 		 */
-
 		ww_mutex_lock_acquired(ww, ww_ctx);
 		ww->ctx = ww_ctx;
  
  
  
@@ -592,15 +596,8 @@
 		}
 	}
  
-	/* set it to 0 if there are no waiters left: */
-	if (likely(list_empty(&lock->wait_list)))
-		atomic_set(&lock->count, 0);
-
 	spin_unlock_mutex(&lock->wait_lock, flags);
-
-	debug_mutex_free_waiter(&waiter);
 	preempt_enable();
-
 	return 0;
  
 err:
@@ -278,8 +278,6 @@
  * @wait: If true, wait until function has completed.
  *
  * Returns 0 on success, else a negative status code (if no cpus were online).
- * Note that @wait will be implicitly turned on in case of allocation failures,
- * since we fall back to on-stack allocation.
  *
  * Selection preference:
  *	1) current cpu if in @mask
...	...	@@ -365,7 +365,7 @@
365	365
366	366	#define lockdep_recursing(tsk) ((tsk)->lockdep_recursion)
367	367
368		-#else /* !LOCKDEP */
	368	+#else /* !CONFIG_LOCKDEP */
369	369
370	370	static inline void lockdep_off(void)
371	371	{
372	372
373	373
374	374
375	375
376	376
377	377
378	378
...	...	@@ -479,82 +479,36 @@
479	479	* on the per lock-class debug mode:
480	480	*/
481	481
482		-#ifdef CONFIG_DEBUG_LOCK_ALLOC
483		-# ifdef CONFIG_PROVE_LOCKING
484		-# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
485		-# define spin_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 2, n, i)
486		-# else
487		-# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
488		-# define spin_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 1, NULL, i)
489		-# endif
490		-# define spin_release(l, n, i) lock_release(l, n, i)
	482	+#ifdef CONFIG_PROVE_LOCKING
	483	+ #define lock_acquire_exclusive(l, s, t, n, i) lock_acquire(l, s, t, 0, 2, n, i)
	484	+ #define lock_acquire_shared(l, s, t, n, i) lock_acquire(l, s, t, 1, 2, n, i)
	485	+ #define lock_acquire_shared_recursive(l, s, t, n, i) lock_acquire(l, s, t, 2, 2, n, i)
491	486	#else
492		-# define spin_acquire(l, s, t, i) do { } while (0)
493		-# define spin_release(l, n, i) do { } while (0)
	487	+ #define lock_acquire_exclusive(l, s, t, n, i) lock_acquire(l, s, t, 0, 1, n, i)
	488	+ #define lock_acquire_shared(l, s, t, n, i) lock_acquire(l, s, t, 1, 1, n, i)
	489	+ #define lock_acquire_shared_recursive(l, s, t, n, i) lock_acquire(l, s, t, 2, 1, n, i)
494	490	#endif
495	491
496		-#ifdef CONFIG_DEBUG_LOCK_ALLOC
497		-# ifdef CONFIG_PROVE_LOCKING
498		-# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
499		-# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 2, NULL, i)
500		-# else
501		-# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
502		-# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 1, NULL, i)
503		-# endif
504		-# define rwlock_release(l, n, i) lock_release(l, n, i)
505		-#else
506		-# define rwlock_acquire(l, s, t, i) do { } while (0)
507		-# define rwlock_acquire_read(l, s, t, i) do { } while (0)
508		-# define rwlock_release(l, n, i) do { } while (0)
509		-#endif
	492	+#define spin_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
	493	+#define spin_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i)
	494	+#define spin_release(l, n, i) lock_release(l, n, i)
510	495
511		-#ifdef CONFIG_DEBUG_LOCK_ALLOC
512		-# ifdef CONFIG_PROVE_LOCKING
513		-# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
514		-# define mutex_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 2, n, i)
515		-# else
516		-# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
517		-# define mutex_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 1, n, i)
518		-# endif
519		-# define mutex_release(l, n, i) lock_release(l, n, i)
520		-#else
521		-# define mutex_acquire(l, s, t, i) do { } while (0)
522		-# define mutex_acquire_nest(l, s, t, n, i) do { } while (0)
523		-# define mutex_release(l, n, i) do { } while (0)
524		-#endif
	496	+#define rwlock_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
	497	+#define rwlock_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i)
	498	+#define rwlock_release(l, n, i) lock_release(l, n, i)
525	499
526		-#ifdef CONFIG_DEBUG_LOCK_ALLOC
527		-# ifdef CONFIG_PROVE_LOCKING
528		-# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
529		-# define rwsem_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 2, n, i)
530		-# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 2, NULL, i)
531		-# else
532		-# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
533		-# define rwsem_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 1, n, i)
534		-# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 1, NULL, i)
535		-# endif
	500	+#define mutex_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
	501	+#define mutex_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i)
	502	+#define mutex_release(l, n, i) lock_release(l, n, i)
	503	+
	504	+#define rwsem_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
	505	+#define rwsem_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i)
	506	+#define rwsem_acquire_read(l, s, t, i) lock_acquire_shared(l, s, t, NULL, i)
536	507	# define rwsem_release(l, n, i) lock_release(l, n, i)
537		-#else
538		-# define rwsem_acquire(l, s, t, i) do { } while (0)
539		-# define rwsem_acquire_nest(l, s, t, n, i) do { } while (0)
540		-# define rwsem_acquire_read(l, s, t, i) do { } while (0)
541		-# define rwsem_release(l, n, i) do { } while (0)
542		-#endif
543	508
544		-#ifdef CONFIG_DEBUG_LOCK_ALLOC
545		-# ifdef CONFIG_PROVE_LOCKING
546		-# define lock_map_acquire(l) lock_acquire(l, 0, 0, 0, 2, NULL, _THIS_IP_)
547		-# define lock_map_acquire_read(l) lock_acquire(l, 0, 0, 2, 2, NULL, _THIS_IP_)
548		-# else
549		-# define lock_map_acquire(l) lock_acquire(l, 0, 0, 0, 1, NULL, _THIS_IP_)
550		-# define lock_map_acquire_read(l) lock_acquire(l, 0, 0, 2, 1, NULL, _THIS_IP_)
551		-# endif
	509	+#define lock_map_acquire(l) lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_)
	510	+#define lock_map_acquire_read(l) lock_acquire_shared_recursive(l, 0, 0, NULL, _THIS_IP_)
552	511	# define lock_map_release(l) lock_release(l, 1, _THIS_IP_)
553		-#else
554		-# define lock_map_acquire(l) do { } while (0)
555		-# define lock_map_acquire_read(l) do { } while (0)
556		-# define lock_map_release(l) do { } while (0)
557		-#endif
558	512
559	513	#ifdef CONFIG_PROVE_LOCKING
560	514	# define might_lock(lock) \
...	...	@@ -15,6 +15,7 @@
15	15	#include <linux/lockdep.h>
16	16	#include <linux/export.h>
17	17	#include <linux/sysctl.h>
	18	+#include <linux/utsname.h>
18	19
19	20	/*
20	21	* The number of tasks checked:
...	...	@@ -99,10 +100,14 @@
99	100	* Ok, the task did not get scheduled for more than 2 minutes,
100	101	* complain:
101	102	*/
102		- printk(KERN_ERR "INFO: task %s:%d blocked for more than "
103		- "%ld seconds.\n", t->comm, t->pid, timeout);
104		- printk(KERN_ERR "\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
105		- " disables this message.\n");
	103	+ pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
	104	+ t->comm, t->pid, timeout);
	105	+ pr_err(" %s %s %.*s\n",
	106	+ print_tainted(), init_utsname()->release,
	107	+ (int)strcspn(init_utsname()->version, " "),
	108	+ init_utsname()->version);
	109	+ pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
	110	+ " disables this message.\n");
106	111	sched_show_task(t);
107	112	debug_show_held_locks(t);
108	113
...	...	@@ -21,7 +21,7 @@
21	21	arch_spinlock_t *lock;
22	22
23	23	preempt_disable();
24		- rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
	24	+ lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
25	25	lock = this_cpu_ptr(lg->lock);
26	26	arch_spin_lock(lock);
27	27	}
...	...	@@ -31,7 +31,7 @@
31	31	{
32	32	arch_spinlock_t *lock;
33	33
34		- rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
	34	+ lock_release(&lg->lock_dep_map, 1, _RET_IP_);
35	35	lock = this_cpu_ptr(lg->lock);
36	36	arch_spin_unlock(lock);
37	37	preempt_enable();
...	...	@@ -43,7 +43,7 @@
43	43	arch_spinlock_t *lock;
44	44
45	45	preempt_disable();
46		- rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_);
	46	+ lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
47	47	lock = per_cpu_ptr(lg->lock, cpu);
48	48	arch_spin_lock(lock);
49	49	}
...	...	@@ -53,7 +53,7 @@
53	53	{
54	54	arch_spinlock_t *lock;
55	55
56		- rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
	56	+ lock_release(&lg->lock_dep_map, 1, _RET_IP_);
57	57	lock = per_cpu_ptr(lg->lock, cpu);
58	58	arch_spin_unlock(lock);
59	59	preempt_enable();
...	...	@@ -65,7 +65,7 @@
65	65	int i;
66	66
67	67	preempt_disable();
68		- rwlock_acquire(&lg->lock_dep_map, 0, 0, _RET_IP_);
	68	+ lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_);
69	69	for_each_possible_cpu(i) {
70	70	arch_spinlock_t *lock;
71	71	lock = per_cpu_ptr(lg->lock, i);
...	...	@@ -78,7 +78,7 @@
78	78	{
79	79	int i;
80	80
81		- rwlock_release(&lg->lock_dep_map, 1, _RET_IP_);
	81	+ lock_release(&lg->lock_dep_map, 1, _RET_IP_);
82	82	for_each_possible_cpu(i) {
83	83	arch_spinlock_t *lock;
84	84	lock = per_cpu_ptr(lg->lock, i);
...	...	@@ -209,11 +209,13 @@
209	209	*/
210	210	static inline int mutex_can_spin_on_owner(struct mutex *lock)
211	211	{
	212	+ struct task_struct *owner;
212	213	int retval = 1;
213	214
214	215	rcu_read_lock();
215		- if (lock->owner)
216		- retval = lock->owner->on_cpu;
	216	+ owner = ACCESS_ONCE(lock->owner);
	217	+ if (owner)
	218	+ retval = owner->on_cpu;
217	219	rcu_read_unlock();
218	220	/*
219	221	* if lock->owner is not set, the mutex owner may have just acquired
...	...	@@ -461,7 +463,7 @@
461	463	* performed the optimistic spinning cannot be done.
462	464	*/
463	465	if (ACCESS_ONCE(ww->ctx))
464		- break;
	466	+ goto slowpath;
465	467	}
466	468
467	469	/*
...	...	@@ -472,7 +474,7 @@
472	474	owner = ACCESS_ONCE(lock->owner);
473	475	if (owner && !mutex_spin_on_owner(lock, owner)) {
474	476	mspin_unlock(MLOCK(lock), &node);
475		- break;
	477	+ goto slowpath;
476	478	}
477	479
478	480	if ((atomic_read(&lock->count) == 1) &&
...	...	@@ -499,7 +501,7 @@
499	501	* the owner complete.
500	502	*/
501	503	if (!owner && (need_resched() \|\| rt_task(task)))
502		- break;
	504	+ goto slowpath;
503	505
504	506	/*
505	507	* The cpu_relax() call is a compiler barrier which forces
...	...	@@ -513,6 +515,10 @@
513	515	#endif
514	516	spin_lock_mutex(&lock->wait_lock, flags);
515	517
	518	+ /* once more, can we acquire the lock? */
	519	+ if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
	520	+ goto skip_wait;
	521	+
516	522	debug_mutex_lock_common(lock, &waiter);
517	523	debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
518	524
...	...	@@ -520,9 +526,6 @@
520	526	list_add_tail(&waiter.list, &lock->wait_list);
521	527	waiter.task = task;
522	528
523		- if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, -1) == 1))
524		- goto done;
525		-
526	529	lock_contended(&lock->dep_map, ip);
527	530
528	531	for (;;) {
...	...	@@ -536,7 +539,7 @@
536	539	* other waiters:
537	540	*/
538	541	if (MUTEX_SHOW_NO_WAITER(lock) &&
539		- (atomic_xchg(&lock->count, -1) == 1))
	542	+ (atomic_xchg(&lock->count, -1) == 1))
540	543	break;
541	544
542	545	/*
543	546
544	547
545	548
546	549
...	...	@@ -561,24 +564,25 @@
561	564	schedule_preempt_disabled();
562	565	spin_lock_mutex(&lock->wait_lock, flags);
563	566	}
	567	+ mutex_remove_waiter(lock, &waiter, current_thread_info());
	568	+ /* set it to 0 if there are no waiters left: */
	569	+ if (likely(list_empty(&lock->wait_list)))
	570	+ atomic_set(&lock->count, 0);
	571	+ debug_mutex_free_waiter(&waiter);
564	572
565		-done:
	573	+skip_wait:
	574	+ /* got the lock - cleanup and rejoice! */
566	575	lock_acquired(&lock->dep_map, ip);
567		- /* got the lock - rejoice! */
568		- mutex_remove_waiter(lock, &waiter, current_thread_info());
569	576	mutex_set_owner(lock);
570	577
571	578	if (!__builtin_constant_p(ww_ctx == NULL)) {
572		- struct ww_mutex *ww = container_of(lock,
573		- struct ww_mutex,
574		- base);
	579	+ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
575	580	struct mutex_waiter *cur;
576	581
577	582	/*
578	583	* This branch gets optimized out for the common case,
579	584	* and is only important for ww_mutex_lock.
580	585	*/
581		-
582	586	ww_mutex_lock_acquired(ww, ww_ctx);
583	587	ww->ctx = ww_ctx;
584	588
585	589
586	590
...	...	@@ -592,15 +596,8 @@
592	596	}
593	597	}
594	598
595		- /* set it to 0 if there are no waiters left: */
596		- if (likely(list_empty(&lock->wait_list)))
597		- atomic_set(&lock->count, 0);
598		-
599	599	spin_unlock_mutex(&lock->wait_lock, flags);
600		-
601		- debug_mutex_free_waiter(&waiter);
602	600	preempt_enable();
603		-
604	601	return 0;
605	602
606	603	err:
...	...	@@ -278,8 +278,6 @@
278	278	* @wait: If true, wait until function has completed.
279	279	*
280	280	* Returns 0 on success, else a negative status code (if no cpus were online).
281		- * Note that @wait will be implicitly turned on in case of allocation failures,
282		- * since we fall back to on-stack allocation.
283	281	*
284	282	* Selection preference:
285	283	* 1) current cpu if in @mask