Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulm…

…ck/linux-2.6-rcu into core/urgent

Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulm…
…ck/linux-2.6-rcu into core/urgent
Ingo Molnar
2 parents e6625fa48e a841796f11
Showing 5 changed files Side-by-side Diff
include/linux/sched.h
kernel/rcutree_plugin.h
kernel/sched.c
kernel/signal.c
kernel/softirq.c
@@ -1254,6 +1254,9 @@
 #ifdef CONFIG_PREEMPT_RCU
 	int rcu_read_lock_nesting;
 	char rcu_read_unlock_special;
+#if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU)
+	int rcu_boosted;
+#endif /* #if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU) */
 	struct list_head rcu_node_entry;
 #endif /* #ifdef CONFIG_PREEMPT_RCU */
 #ifdef CONFIG_TREE_PREEMPT_RCU
@@ -68,6 +68,7 @@
 DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
 static struct rcu_state *rcu_state = &rcu_preempt_state;
  
+static void rcu_read_unlock_special(struct task_struct *t);
 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
  
 /*
@@ -147,7 +148,7 @@
 	struct rcu_data *rdp;
 	struct rcu_node *rnp;
  
-	if (t->rcu_read_lock_nesting &&
+	if (t->rcu_read_lock_nesting > 0 &&
 	    (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
  
 		/* Possibly blocking in an RCU read-side critical section. */
@@ -190,6 +191,14 @@
 				rnp->gp_tasks = &t->rcu_node_entry;
 		}
 		raw_spin_unlock_irqrestore(&rnp->lock, flags);
+	} else if (t->rcu_read_lock_nesting < 0 &&
+		   t->rcu_read_unlock_special) {
+
+		/*
+		 * Complete exit from RCU read-side critical section on
+		 * behalf of preempted instance of __rcu_read_unlock().
+		 */
+		rcu_read_unlock_special(t);
 	}
  
 	/*
@@ -284,7 +293,7 @@
  * notify RCU core processing or task having blocked during the RCU
  * read-side critical section.
  */
-static void rcu_read_unlock_special(struct task_struct *t)
+static noinline void rcu_read_unlock_special(struct task_struct *t)
 {
 	int empty;
 	int empty_exp;
@@ -309,7 +318,7 @@
 	}
  
 	/* Hardware IRQ handlers cannot block. */
-	if (in_irq()) {
+	if (in_irq() || in_serving_softirq()) {
 		local_irq_restore(flags);
 		return;
 	}
@@ -342,6 +351,11 @@
 #ifdef CONFIG_RCU_BOOST
 		if (&t->rcu_node_entry == rnp->boost_tasks)
 			rnp->boost_tasks = np;
+		/* Snapshot and clear ->rcu_boosted with rcu_node lock held. */
+		if (t->rcu_boosted) {
+			special |= RCU_READ_UNLOCK_BOOSTED;
+			t->rcu_boosted = 0;
+		}
 #endif /* #ifdef CONFIG_RCU_BOOST */
 		t->rcu_blocked_node = NULL;
  
@@ -358,7 +372,6 @@
 #ifdef CONFIG_RCU_BOOST
 		/* Unboost if we were boosted. */
 		if (special & RCU_READ_UNLOCK_BOOSTED) {
-			t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BOOSTED;
 			rt_mutex_unlock(t->rcu_boost_mutex);
 			t->rcu_boost_mutex = NULL;
 		}
  
@@ -387,13 +400,22 @@
 	struct task_struct *t = current;
  
 	barrier();  /* needed if we ever invoke rcu_read_unlock in rcutree.c */
-	--t->rcu_read_lock_nesting;
-	barrier();  /* decrement before load of ->rcu_read_unlock_special */
-	if (t->rcu_read_lock_nesting == 0 &&
-	    unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
-		rcu_read_unlock_special(t);
+	if (t->rcu_read_lock_nesting != 1)
+		--t->rcu_read_lock_nesting;
+	else {
+		t->rcu_read_lock_nesting = INT_MIN;
+		barrier();  /* assign before ->rcu_read_unlock_special load */
+		if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
+			rcu_read_unlock_special(t);
+		barrier();  /* ->rcu_read_unlock_special load before assign */
+		t->rcu_read_lock_nesting = 0;
+	}
 #ifdef CONFIG_PROVE_LOCKING
-	WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
+	{
+		int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
+
+		WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
+	}
 #endif /* #ifdef CONFIG_PROVE_LOCKING */
 }
 EXPORT_SYMBOL_GPL(__rcu_read_unlock);
@@ -589,7 +611,8 @@
 		rcu_preempt_qs(cpu);
 		return;
 	}
-	if (per_cpu(rcu_preempt_data, cpu).qs_pending)
+	if (t->rcu_read_lock_nesting > 0 &&
+	    per_cpu(rcu_preempt_data, cpu).qs_pending)
 		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
 }
  
  
  
@@ -695,9 +718,12 @@
  
 	raw_spin_lock_irqsave(&rnp->lock, flags);
 	for (;;) {
-		if (!sync_rcu_preempt_exp_done(rnp))
+		if (!sync_rcu_preempt_exp_done(rnp)) {
+			raw_spin_unlock_irqrestore(&rnp->lock, flags);
 			break;
+		}
 		if (rnp->parent == NULL) {
+			raw_spin_unlock_irqrestore(&rnp->lock, flags);
 			wake_up(&sync_rcu_preempt_exp_wq);
 			break;
 		}
@@ -707,7 +733,6 @@
 		raw_spin_lock(&rnp->lock); /* irqs already disabled */
 		rnp->expmask &= ~mask;
 	}
-	raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
  
 /*
@@ -1174,7 +1199,7 @@
 	t = container_of(tb, struct task_struct, rcu_node_entry);
 	rt_mutex_init_proxy_locked(&mtx, t);
 	t->rcu_boost_mutex = &mtx;
-	t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
+	t->rcu_boosted = 1;
 	raw_spin_unlock_irqrestore(&rnp->lock, flags);
 	rt_mutex_lock(&mtx);  /* Side effect: boosts task t's priority. */
 	rt_mutex_unlock(&mtx);  /* Keep lockdep happy. */
@@ -2544,14 +2544,10 @@
 }
  
 #ifdef CONFIG_SMP
-static void sched_ttwu_pending(void)
+static void sched_ttwu_do_pending(struct task_struct *list)
 {
 	struct rq *rq = this_rq();
-	struct task_struct *list = xchg(&rq->wake_list, NULL);
  
-	if (!list)
-		return;
-
 	raw_spin_lock(&rq->lock);
  
 	while (list) {
  
@@ -2563,9 +2559,45 @@
 	raw_spin_unlock(&rq->lock);
 }
  
+#ifdef CONFIG_HOTPLUG_CPU
+
+static void sched_ttwu_pending(void)
+{
+	struct rq *rq = this_rq();
+	struct task_struct *list = xchg(&rq->wake_list, NULL);
+
+	if (!list)
+		return;
+
+	sched_ttwu_do_pending(list);
+}
+
+#endif /* CONFIG_HOTPLUG_CPU */
+
 void scheduler_ipi(void)
 {
-	sched_ttwu_pending();
+	struct rq *rq = this_rq();
+	struct task_struct *list = xchg(&rq->wake_list, NULL);
+
+	if (!list)
+		return;
+
+	/*
+	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
+	 * traditionally all their work was done from the interrupt return
+	 * path. Now that we actually do some work, we need to make sure
+	 * we do call them.
+	 *
+	 * Some archs already do call them, luckily irq_enter/exit nest
+	 * properly.
+	 *
+	 * Arguably we should visit all archs and update all handlers,
+	 * however a fair share of IPIs are still resched only so this would
+	 * somewhat pessimize the simple resched case.
+	 */
+	irq_enter();
+	sched_ttwu_do_pending(list);
+	irq_exit();
 }
  
 static void ttwu_queue_remote(struct task_struct *p, int cpu)
@@ -1178,18 +1178,25 @@
 {
 	struct sighand_struct *sighand;
  
-	rcu_read_lock();
 	for (;;) {
+		local_irq_save(*flags);
+		rcu_read_lock();
 		sighand = rcu_dereference(tsk->sighand);
-		if (unlikely(sighand == NULL))
+		if (unlikely(sighand == NULL)) {
+			rcu_read_unlock();
+			local_irq_restore(*flags);
 			break;
+		}
  
-		spin_lock_irqsave(&sighand->siglock, *flags);
-		if (likely(sighand == tsk->sighand))
+		spin_lock(&sighand->siglock);
+		if (likely(sighand == tsk->sighand)) {
+			rcu_read_unlock();
 			break;
-		spin_unlock_irqrestore(&sighand->siglock, *flags);
+		}
+		spin_unlock(&sighand->siglock);
+		rcu_read_unlock();
+		local_irq_restore(*flags);
 	}
-	rcu_read_unlock();
  
 	return sighand;
 }
@@ -315,16 +315,24 @@
 {
 	if (!force_irqthreads)
 		__do_softirq();
-	else
+	else {
+		__local_bh_disable((unsigned long)__builtin_return_address(0),
+				SOFTIRQ_OFFSET);
 		wakeup_softirqd();
+		__local_bh_enable(SOFTIRQ_OFFSET);
+	}
 }
 #else
 static inline void invoke_softirq(void)
 {
 	if (!force_irqthreads)
 		do_softirq();
-	else
+	else {
+		__local_bh_disable((unsigned long)__builtin_return_address(0),
+				SOFTIRQ_OFFSET);
 		wakeup_softirqd();
+		__local_bh_enable(SOFTIRQ_OFFSET);
+	}
 }
 #endif
...	...	@@ -1254,6 +1254,9 @@
1254	1254	#ifdef CONFIG_PREEMPT_RCU
1255	1255	int rcu_read_lock_nesting;
1256	1256	char rcu_read_unlock_special;
	1257	+#if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU)
	1258	+ int rcu_boosted;
	1259	+#endif /* #if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU) */
1257	1260	struct list_head rcu_node_entry;
1258	1261	#endif /* #ifdef CONFIG_PREEMPT_RCU */
1259	1262	#ifdef CONFIG_TREE_PREEMPT_RCU
...	...	@@ -68,6 +68,7 @@
68	68	DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
69	69	static struct rcu_state *rcu_state = &rcu_preempt_state;
70	70
	71	+static void rcu_read_unlock_special(struct task_struct *t);
71	72	static int rcu_preempted_readers_exp(struct rcu_node *rnp);
72	73
73	74	/*
...	...	@@ -147,7 +148,7 @@
147	148	struct rcu_data *rdp;
148	149	struct rcu_node *rnp;
149	150
150		- if (t->rcu_read_lock_nesting &&
	151	+ if (t->rcu_read_lock_nesting > 0 &&
151	152	(t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
152	153
153	154	/* Possibly blocking in an RCU read-side critical section. */
...	...	@@ -190,6 +191,14 @@
190	191	rnp->gp_tasks = &t->rcu_node_entry;
191	192	}
192	193	raw_spin_unlock_irqrestore(&rnp->lock, flags);
	194	+ } else if (t->rcu_read_lock_nesting < 0 &&
	195	+ t->rcu_read_unlock_special) {
	196	+
	197	+ /*
	198	+ * Complete exit from RCU read-side critical section on
	199	+ * behalf of preempted instance of __rcu_read_unlock().
	200	+ */
	201	+ rcu_read_unlock_special(t);
193	202	}
194	203
195	204	/*
...	...	@@ -284,7 +293,7 @@
284	293	* notify RCU core processing or task having blocked during the RCU
285	294	* read-side critical section.
286	295	*/
287		-static void rcu_read_unlock_special(struct task_struct *t)
	296	+static noinline void rcu_read_unlock_special(struct task_struct *t)
288	297	{
289	298	int empty;
290	299	int empty_exp;
...	...	@@ -309,7 +318,7 @@
309	318	}
310	319
311	320	/* Hardware IRQ handlers cannot block. */
312		- if (in_irq()) {
	321	+ if (in_irq() \|\| in_serving_softirq()) {
313	322	local_irq_restore(flags);
314	323	return;
315	324	}
...	...	@@ -342,6 +351,11 @@
342	351	#ifdef CONFIG_RCU_BOOST
343	352	if (&t->rcu_node_entry == rnp->boost_tasks)
344	353	rnp->boost_tasks = np;
	354	+ /* Snapshot and clear ->rcu_boosted with rcu_node lock held. */
	355	+ if (t->rcu_boosted) {
	356	+ special \|= RCU_READ_UNLOCK_BOOSTED;
	357	+ t->rcu_boosted = 0;
	358	+ }
345	359	#endif /* #ifdef CONFIG_RCU_BOOST */
346	360	t->rcu_blocked_node = NULL;
347	361
...	...	@@ -358,7 +372,6 @@
358	372	#ifdef CONFIG_RCU_BOOST
359	373	/* Unboost if we were boosted. */
360	374	if (special & RCU_READ_UNLOCK_BOOSTED) {
361		- t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BOOSTED;
362	375	rt_mutex_unlock(t->rcu_boost_mutex);
363	376	t->rcu_boost_mutex = NULL;
364	377	}
365	378
...	...	@@ -387,13 +400,22 @@
387	400	struct task_struct *t = current;
388	401
389	402	barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */
390		- --t->rcu_read_lock_nesting;
391		- barrier(); /* decrement before load of ->rcu_read_unlock_special */
392		- if (t->rcu_read_lock_nesting == 0 &&
393		- unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
394		- rcu_read_unlock_special(t);
	403	+ if (t->rcu_read_lock_nesting != 1)
	404	+ --t->rcu_read_lock_nesting;
	405	+ else {
	406	+ t->rcu_read_lock_nesting = INT_MIN;
	407	+ barrier(); /* assign before ->rcu_read_unlock_special load */
	408	+ if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
	409	+ rcu_read_unlock_special(t);
	410	+ barrier(); /* ->rcu_read_unlock_special load before assign */
	411	+ t->rcu_read_lock_nesting = 0;
	412	+ }
395	413	#ifdef CONFIG_PROVE_LOCKING
396		- WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
	414	+ {
	415	+ int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
	416	+
	417	+ WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
	418	+ }
397	419	#endif /* #ifdef CONFIG_PROVE_LOCKING */
398	420	}
399	421	EXPORT_SYMBOL_GPL(__rcu_read_unlock);
...	...	@@ -589,7 +611,8 @@
589	611	rcu_preempt_qs(cpu);
590	612	return;
591	613	}
592		- if (per_cpu(rcu_preempt_data, cpu).qs_pending)
	614	+ if (t->rcu_read_lock_nesting > 0 &&
	615	+ per_cpu(rcu_preempt_data, cpu).qs_pending)
593	616	t->rcu_read_unlock_special \|= RCU_READ_UNLOCK_NEED_QS;
594	617	}
595	618
596	619
597	620
...	...	@@ -695,9 +718,12 @@
695	718
696	719	raw_spin_lock_irqsave(&rnp->lock, flags);
697	720	for (;;) {
698		- if (!sync_rcu_preempt_exp_done(rnp))
	721	+ if (!sync_rcu_preempt_exp_done(rnp)) {
	722	+ raw_spin_unlock_irqrestore(&rnp->lock, flags);
699	723	break;
	724	+ }
700	725	if (rnp->parent == NULL) {
	726	+ raw_spin_unlock_irqrestore(&rnp->lock, flags);
701	727	wake_up(&sync_rcu_preempt_exp_wq);
702	728	break;
703	729	}
...	...	@@ -707,7 +733,6 @@
707	733	raw_spin_lock(&rnp->lock); /* irqs already disabled */
708	734	rnp->expmask &= ~mask;
709	735	}
710		- raw_spin_unlock_irqrestore(&rnp->lock, flags);
711	736	}
712	737
713	738	/*
...	...	@@ -1174,7 +1199,7 @@
1174	1199	t = container_of(tb, struct task_struct, rcu_node_entry);
1175	1200	rt_mutex_init_proxy_locked(&mtx, t);
1176	1201	t->rcu_boost_mutex = &mtx;
1177		- t->rcu_read_unlock_special \|= RCU_READ_UNLOCK_BOOSTED;
	1202	+ t->rcu_boosted = 1;
1178	1203	raw_spin_unlock_irqrestore(&rnp->lock, flags);
1179	1204	rt_mutex_lock(&mtx); /* Side effect: boosts task t's priority. */
1180	1205	rt_mutex_unlock(&mtx); /* Keep lockdep happy. */
...	...	@@ -2544,14 +2544,10 @@
2544	2544	}
2545	2545
2546	2546	#ifdef CONFIG_SMP
2547		-static void sched_ttwu_pending(void)
	2547	+static void sched_ttwu_do_pending(struct task_struct *list)
2548	2548	{
2549	2549	struct rq *rq = this_rq();
2550		- struct task_struct *list = xchg(&rq->wake_list, NULL);
2551	2550
2552		- if (!list)
2553		- return;
2554		-
2555	2551	raw_spin_lock(&rq->lock);
2556	2552
2557	2553	while (list) {
2558	2554
...	...	@@ -2563,9 +2559,45 @@
2563	2559	raw_spin_unlock(&rq->lock);
2564	2560	}
2565	2561
	2562	+#ifdef CONFIG_HOTPLUG_CPU
	2563	+
	2564	+static void sched_ttwu_pending(void)
	2565	+{
	2566	+ struct rq *rq = this_rq();
	2567	+ struct task_struct *list = xchg(&rq->wake_list, NULL);
	2568	+
	2569	+ if (!list)
	2570	+ return;
	2571	+
	2572	+ sched_ttwu_do_pending(list);
	2573	+}
	2574	+
	2575	+#endif /* CONFIG_HOTPLUG_CPU */
	2576	+
2566	2577	void scheduler_ipi(void)
2567	2578	{
2568		- sched_ttwu_pending();
	2579	+ struct rq *rq = this_rq();
	2580	+ struct task_struct *list = xchg(&rq->wake_list, NULL);
	2581	+
	2582	+ if (!list)
	2583	+ return;
	2584	+
	2585	+ /*
	2586	+ * Not all reschedule IPI handlers call irq_enter/irq_exit, since
	2587	+ * traditionally all their work was done from the interrupt return
	2588	+ * path. Now that we actually do some work, we need to make sure
	2589	+ * we do call them.
	2590	+ *
	2591	+ * Some archs already do call them, luckily irq_enter/exit nest
	2592	+ * properly.
	2593	+ *
	2594	+ * Arguably we should visit all archs and update all handlers,
	2595	+ * however a fair share of IPIs are still resched only so this would
	2596	+ * somewhat pessimize the simple resched case.
	2597	+ */
	2598	+ irq_enter();
	2599	+ sched_ttwu_do_pending(list);
	2600	+ irq_exit();
2569	2601	}
2570	2602
2571	2603	static void ttwu_queue_remote(struct task_struct *p, int cpu)
...	...	@@ -1178,18 +1178,25 @@
1178	1178	{
1179	1179	struct sighand_struct *sighand;
1180	1180
1181		- rcu_read_lock();
1182	1181	for (;;) {
	1182	+ local_irq_save(*flags);
	1183	+ rcu_read_lock();
1183	1184	sighand = rcu_dereference(tsk->sighand);
1184		- if (unlikely(sighand == NULL))
	1185	+ if (unlikely(sighand == NULL)) {
	1186	+ rcu_read_unlock();
	1187	+ local_irq_restore(*flags);
1185	1188	break;
	1189	+ }
1186	1190
1187		- spin_lock_irqsave(&sighand->siglock, *flags);
1188		- if (likely(sighand == tsk->sighand))
	1191	+ spin_lock(&sighand->siglock);
	1192	+ if (likely(sighand == tsk->sighand)) {
	1193	+ rcu_read_unlock();
1189	1194	break;
1190		- spin_unlock_irqrestore(&sighand->siglock, *flags);
	1195	+ }
	1196	+ spin_unlock(&sighand->siglock);
	1197	+ rcu_read_unlock();
	1198	+ local_irq_restore(*flags);
1191	1199	}
1192		- rcu_read_unlock();
1193	1200
1194	1201	return sighand;
1195	1202	}
...	...	@@ -315,16 +315,24 @@
315	315	{
316	316	if (!force_irqthreads)
317	317	__do_softirq();
318		- else
	318	+ else {
	319	+ __local_bh_disable((unsigned long)__builtin_return_address(0),
	320	+ SOFTIRQ_OFFSET);
319	321	wakeup_softirqd();
	322	+ __local_bh_enable(SOFTIRQ_OFFSET);
	323	+ }
320	324	}
321	325	#else
322	326	static inline void invoke_softirq(void)
323	327	{
324	328	if (!force_irqthreads)
325	329	do_softirq();
326		- else
	330	+ else {
	331	+ __local_bh_disable((unsigned long)__builtin_return_address(0),
	332	+ SOFTIRQ_OFFSET);
327	333	wakeup_softirqd();
	334	+ __local_bh_enable(SOFTIRQ_OFFSET);
	335	+ }
328	336	}
329	337	#endif
330	338