rcu: Use smp_hotplug_thread facility for RCUs per-CPU kthread

Bring RCU into the new-age CPU-hotplug fold by modifying RCU's per-CPU kthread code to use the new smp_hotplug_thread facility. [ tglx: Adapted it to use callbacks and to the simplified rcu yield ] Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Namhyung Kim <namhyung@kernel.org> Link: http://lkml.kernel.org/r/20120716103948.673354828@linutronix.de Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

rcu: Use smp_hotplug_thread facility for RCUs per-CPU kthread
Bring RCU into the new-age CPU-hotplug fold by modifying RCU's per-CPU kthread code to use the new smp_hotplug_thread facility. [ tglx: Adapted it to use callbacks and to the simplified rcu yield ] Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Namhyung Kim <namhyung@kernel.org> Link: http://lkml.kernel.org/r/20120716103948.673354828@linutronix.de Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Paul E. McKenney · Thomas Gleixner
1 parent bcd951cf10
Showing 4 changed files with 41 additions and 177 deletions Side-by-side Diff
kernel/rcutree.c
kernel/rcutree.h
kernel/rcutree_plugin.h
kernel/rcutree_trace.c
@@ -133,7 +133,6 @@
  */
 static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
-DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu);
 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
 DEFINE_PER_CPU(char, rcu_cpu_has_work);
  
@@ -1468,7 +1467,6 @@
 	struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
  
 	/* Adjust any no-longer-needed kthreads. */
-	rcu_stop_cpu_kthread(cpu);
 	rcu_boost_kthread_setaffinity(rnp, -1);
  
 	/* Remove the dead CPU from the bitmasks in the rcu_node hierarchy. */
  
@@ -2595,11 +2593,9 @@
 	case CPU_ONLINE:
 	case CPU_DOWN_FAILED:
 		rcu_boost_kthread_setaffinity(rnp, -1);
-		rcu_cpu_kthread_setrt(cpu, 1);
 		break;
 	case CPU_DOWN_PREPARE:
 		rcu_boost_kthread_setaffinity(rnp, cpu);
-		rcu_cpu_kthread_setrt(cpu, 0);
 		break;
 	case CPU_DYING:
 	case CPU_DYING_FROZEN:
@@ -196,12 +196,6 @@
 				/* Refused to boost: not sure why, though. */
 				/*  This can happen due to race conditions. */
 #endif /* #ifdef CONFIG_RCU_BOOST */
-	struct task_struct *node_kthread_task;
-				/* kthread that takes care of this rcu_node */
-				/*  structure, for example, awakening the */
-				/*  per-CPU kthreads as needed. */
-	unsigned int node_kthread_status;
-				/* State of node_kthread_task for tracing. */
 } ____cacheline_internodealigned_in_smp;
  
 /*
@@ -468,7 +462,6 @@
 #ifdef CONFIG_HOTPLUG_CPU
 static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp,
 				      unsigned long flags);
-static void rcu_stop_cpu_kthread(int cpu);
 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
 static void rcu_print_detail_task_stall(struct rcu_state *rsp);
 static int rcu_print_task_stall(struct rcu_node *rnp);
@@ -494,7 +487,6 @@
 static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
 						 struct rcu_node *rnp);
 #endif /* #ifdef CONFIG_RCU_BOOST */
-static void rcu_cpu_kthread_setrt(int cpu, int to_rt);
 static void __cpuinit rcu_prepare_kthreads(int cpu);
 static void rcu_prepare_for_idle_init(int cpu);
 static void rcu_cleanup_after_idle(int cpu);
@@ -25,6 +25,7 @@
  */
  
 #include <linux/delay.h>
+#include <linux/smpboot.h>
  
 #define RCU_KTHREAD_PRIO 1
  
@@ -1292,25 +1293,6 @@
 	return 0;
 }
  
-#ifdef CONFIG_HOTPLUG_CPU
-
-/*
- * Stop the RCU's per-CPU kthread when its CPU goes offline,.
- */
-static void rcu_stop_cpu_kthread(int cpu)
-{
-	struct task_struct *t;
-
-	/* Stop the CPU's kthread. */
-	t = per_cpu(rcu_cpu_kthread_task, cpu);
-	if (t != NULL) {
-		per_cpu(rcu_cpu_kthread_task, cpu) = NULL;
-		kthread_stop(t);
-	}
-}
-
-#endif /* #ifdef CONFIG_HOTPLUG_CPU */
-
 static void rcu_kthread_do_work(void)
 {
 	rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data));
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -1318,159 +1300,61 @@
 	rcu_preempt_do_callbacks();
 }
  
-/*
- * Set the specified CPU's kthread to run RT or not, as specified by
- * the to_rt argument.  The CPU-hotplug locks are held, so the task
- * is not going away.
- */
-static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
+static void rcu_cpu_kthread_setup(unsigned int cpu)
 {
-	int policy;
 	struct sched_param sp;
-	struct task_struct *t;
  
-	t = per_cpu(rcu_cpu_kthread_task, cpu);
-	if (t == NULL)
-		return;
-	if (to_rt) {
-		policy = SCHED_FIFO;
-		sp.sched_priority = RCU_KTHREAD_PRIO;
-	} else {
-		policy = SCHED_NORMAL;
-		sp.sched_priority = 0;
-	}
-	sched_setscheduler_nocheck(t, policy, &sp);
+	sp.sched_priority = RCU_KTHREAD_PRIO;
+	sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
 }
  
-/*
- * Handle cases where the rcu_cpu_kthread() ends up on the wrong CPU.
- * This can happen while the corresponding CPU is either coming online
- * or going offline.  We cannot wait until the CPU is fully online
- * before starting the kthread, because the various notifier functions
- * can wait for RCU grace periods.  So we park rcu_cpu_kthread() until
- * the corresponding CPU is online.
- *
- * Return 1 if the kthread needs to stop, 0 otherwise.
- *
- * Caller must disable bh.  This function can momentarily enable it.
- */
-static int rcu_cpu_kthread_should_stop(int cpu)
+static void rcu_cpu_kthread_park(unsigned int cpu)
 {
-	while (cpu_is_offline(cpu) ||
-	       !cpumask_equal(&current->cpus_allowed, cpumask_of(cpu)) ||
-	       smp_processor_id() != cpu) {
-		if (kthread_should_stop())
-			return 1;
-		per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
-		per_cpu(rcu_cpu_kthread_cpu, cpu) = raw_smp_processor_id();
-		local_bh_enable();
-		schedule_timeout_uninterruptible(1);
-		if (!cpumask_equal(&current->cpus_allowed, cpumask_of(cpu)))
-			set_cpus_allowed_ptr(current, cpumask_of(cpu));
-		local_bh_disable();
-	}
-	per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
-	return 0;
+	per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
 }
  
+static int rcu_cpu_kthread_should_run(unsigned int cpu)
+{
+	return __get_cpu_var(rcu_cpu_has_work);
+}
+
 /*
  * Per-CPU kernel thread that invokes RCU callbacks.  This replaces the
  * RCU softirq used in flavors and configurations of RCU that do not
  * support RCU priority boosting.
  */
-static int rcu_cpu_kthread(void *arg)
+static void rcu_cpu_kthread(unsigned int cpu)
 {
-	int cpu = (int)(long)arg;
-	unsigned long flags;
-	int spincnt = 0;
-	unsigned int *statusp = &per_cpu(rcu_cpu_kthread_status, cpu);
-	char work;
-	char *workp = &per_cpu(rcu_cpu_has_work, cpu);
+	unsigned int *statusp = &__get_cpu_var(rcu_cpu_kthread_status);
+	char work, *workp = &__get_cpu_var(rcu_cpu_has_work);
+	int spincnt;
  
-	trace_rcu_utilization("Start CPU kthread@init");
-	for (;;) {
-		*statusp = RCU_KTHREAD_WAITING;
-		trace_rcu_utilization("End CPU kthread@rcu_wait");
-		rcu_wait(*workp != 0 || kthread_should_stop());
+	for (spincnt = 0; spincnt < 10; spincnt++) {
 		trace_rcu_utilization("Start CPU kthread@rcu_wait");
 		local_bh_disable();
-		if (rcu_cpu_kthread_should_stop(cpu)) {
-			local_bh_enable();
-			break;
-		}
 		*statusp = RCU_KTHREAD_RUNNING;
-		per_cpu(rcu_cpu_kthread_loops, cpu)++;
-		local_irq_save(flags);
+		this_cpu_inc(rcu_cpu_kthread_loops);
+		local_irq_disable();
 		work = *workp;
 		*workp = 0;
-		local_irq_restore(flags);
+		local_irq_enable();
 		if (work)
 			rcu_kthread_do_work();
 		local_bh_enable();
-		if (*workp != 0)
-			spincnt++;
-		else
-			spincnt = 0;
-		if (spincnt > 10) {
-			*statusp = RCU_KTHREAD_YIELDING;
-			trace_rcu_utilization("End CPU kthread@rcu_yield");
-			schedule_timeout_interruptible(2);
-			trace_rcu_utilization("Start CPU kthread@rcu_yield");
-			spincnt = 0;
+		if (*workp == 0) {
+			trace_rcu_utilization("End CPU kthread@rcu_wait");
+			*statusp = RCU_KTHREAD_WAITING;
+			return;
 		}
 	}
-	*statusp = RCU_KTHREAD_STOPPED;
-	trace_rcu_utilization("End CPU kthread@term");
-	return 0;
+	*statusp = RCU_KTHREAD_YIELDING;
+	trace_rcu_utilization("Start CPU kthread@rcu_yield");
+	schedule_timeout_interruptible(2);
+	trace_rcu_utilization("End CPU kthread@rcu_yield");
+	*statusp = RCU_KTHREAD_WAITING;
 }
  
 /*
- * Spawn a per-CPU kthread, setting up affinity and priority.
- * Because the CPU hotplug lock is held, no other CPU will be attempting
- * to manipulate rcu_cpu_kthread_task.  There might be another CPU
- * attempting to access it during boot, but the locking in kthread_bind()
- * will enforce sufficient ordering.
- *
- * Please note that we cannot simply refuse to wake up the per-CPU
- * kthread because kthreads are created in TASK_UNINTERRUPTIBLE state,
- * which can result in softlockup complaints if the task ends up being
- * idle for more than a couple of minutes.
- *
- * However, please note also that we cannot bind the per-CPU kthread to its
- * CPU until that CPU is fully online.  We also cannot wait until the
- * CPU is fully online before we create its per-CPU kthread, as this would
- * deadlock the system when CPU notifiers tried waiting for grace
- * periods.  So we bind the per-CPU kthread to its CPU only if the CPU
- * is online.  If its CPU is not yet fully online, then the code in
- * rcu_cpu_kthread() will wait until it is fully online, and then do
- * the binding.
- */
-static int __cpuinit rcu_spawn_one_cpu_kthread(int cpu)
-{
-	struct sched_param sp;
-	struct task_struct *t;
-
-	if (!rcu_scheduler_fully_active ||
-	    per_cpu(rcu_cpu_kthread_task, cpu) != NULL)
-		return 0;
-	t = kthread_create_on_node(rcu_cpu_kthread,
-				   (void *)(long)cpu,
-				   cpu_to_node(cpu),
-				   "rcuc/%d", cpu);
-	if (IS_ERR(t))
-		return PTR_ERR(t);
-	if (cpu_online(cpu))
-		kthread_bind(t, cpu);
-	per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
-	WARN_ON_ONCE(per_cpu(rcu_cpu_kthread_task, cpu) != NULL);
-	sp.sched_priority = RCU_KTHREAD_PRIO;
-	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
-	per_cpu(rcu_cpu_kthread_task, cpu) = t;
-	wake_up_process(t); /* Get to TASK_INTERRUPTIBLE quickly. */
-	return 0;
-}
-
-/*
  * Set the per-rcu_node kthread's affinity to cover all CPUs that are
  * served by the rcu_node in question.  The CPU hotplug lock is still
  * held, so the value of rnp->qsmaskinit will be stable.
@@ -1503,6 +1387,15 @@
 	free_cpumask_var(cm);
 }
  
+static struct smp_hotplug_thread rcu_cpu_thread_spec = {
+	.store			= &rcu_cpu_kthread_task,
+	.thread_should_run	= rcu_cpu_kthread_should_run,
+	.thread_fn		= rcu_cpu_kthread,
+	.thread_comm		= "rcuc/%u",
+	.setup			= rcu_cpu_kthread_setup,
+	.park			= rcu_cpu_kthread_park,
+};
+
 /*
  * Spawn all kthreads -- called as soon as the scheduler is running.
  */
  
@@ -1512,11 +1405,9 @@
 	int cpu;
  
 	rcu_scheduler_fully_active = 1;
-	for_each_possible_cpu(cpu) {
+	for_each_possible_cpu(cpu)
 		per_cpu(rcu_cpu_has_work, cpu) = 0;
-		if (cpu_online(cpu))
-			(void)rcu_spawn_one_cpu_kthread(cpu);
-	}
+	BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
 	rnp = rcu_get_root(rcu_state);
 	(void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
 	if (NUM_RCU_NODES > 1) {
  
@@ -1533,10 +1424,8 @@
 	struct rcu_node *rnp = rdp->mynode;
  
 	/* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
-	if (rcu_scheduler_fully_active) {
-		(void)rcu_spawn_one_cpu_kthread(cpu);
+	if (rcu_scheduler_fully_active)
 		(void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
-	}
 }
  
 #else /* #ifdef CONFIG_RCU_BOOST */
  
@@ -1560,19 +1449,7 @@
 {
 }
  
-#ifdef CONFIG_HOTPLUG_CPU
-
-static void rcu_stop_cpu_kthread(int cpu)
-{
-}
-
-#endif /* #ifdef CONFIG_HOTPLUG_CPU */
-
 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
-{
-}
-
-static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
 {
 }
  
@@ -108,11 +108,10 @@
 			rdp->nxttail[RCU_WAIT_TAIL]],
 		   ".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]]);
 #ifdef CONFIG_RCU_BOOST
-	seq_printf(m, " kt=%d/%c/%d ktl=%x",
+	seq_printf(m, " kt=%d/%c ktl=%x",
 		   per_cpu(rcu_cpu_has_work, rdp->cpu),
 		   convert_kthread_status(per_cpu(rcu_cpu_kthread_status,
 					  rdp->cpu)),
-		   per_cpu(rcu_cpu_kthread_cpu, rdp->cpu),
 		   per_cpu(rcu_cpu_kthread_loops, rdp->cpu) & 0xffff);
 #endif /* #ifdef CONFIG_RCU_BOOST */
 	seq_printf(m, " b=%ld", rdp->blimit);
...	...	@@ -133,7 +133,6 @@
133	133	*/
134	134	static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
135	135	DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
136		-DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu);
137	136	DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
138	137	DEFINE_PER_CPU(char, rcu_cpu_has_work);
139	138
...	...	@@ -1468,7 +1467,6 @@
1468	1467	struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
1469	1468
1470	1469	/* Adjust any no-longer-needed kthreads. */
1471		- rcu_stop_cpu_kthread(cpu);
1472	1470	rcu_boost_kthread_setaffinity(rnp, -1);
1473	1471
1474	1472	/* Remove the dead CPU from the bitmasks in the rcu_node hierarchy. */
1475	1473
...	...	@@ -2595,11 +2593,9 @@
2595	2593	case CPU_ONLINE:
2596	2594	case CPU_DOWN_FAILED:
2597	2595	rcu_boost_kthread_setaffinity(rnp, -1);
2598		- rcu_cpu_kthread_setrt(cpu, 1);
2599	2596	break;
2600	2597	case CPU_DOWN_PREPARE:
2601	2598	rcu_boost_kthread_setaffinity(rnp, cpu);
2602		- rcu_cpu_kthread_setrt(cpu, 0);
2603	2599	break;
2604	2600	case CPU_DYING:
2605	2601	case CPU_DYING_FROZEN:
...	...	@@ -196,12 +196,6 @@
196	196	/* Refused to boost: not sure why, though. */
197	197	/* This can happen due to race conditions. */
198	198	#endif /* #ifdef CONFIG_RCU_BOOST */
199		- struct task_struct *node_kthread_task;
200		- /* kthread that takes care of this rcu_node */
201		- /* structure, for example, awakening the */
202		- /* per-CPU kthreads as needed. */
203		- unsigned int node_kthread_status;
204		- /* State of node_kthread_task for tracing. */
205	199	} ____cacheline_internodealigned_in_smp;
206	200
207	201	/*
...	...	@@ -468,7 +462,6 @@
468	462	#ifdef CONFIG_HOTPLUG_CPU
469	463	static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp,
470	464	unsigned long flags);
471		-static void rcu_stop_cpu_kthread(int cpu);
472	465	#endif /* #ifdef CONFIG_HOTPLUG_CPU */
473	466	static void rcu_print_detail_task_stall(struct rcu_state *rsp);
474	467	static int rcu_print_task_stall(struct rcu_node *rnp);
...	...	@@ -494,7 +487,6 @@
494	487	static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
495	488	struct rcu_node *rnp);
496	489	#endif /* #ifdef CONFIG_RCU_BOOST */
497		-static void rcu_cpu_kthread_setrt(int cpu, int to_rt);
498	490	static void __cpuinit rcu_prepare_kthreads(int cpu);
499	491	static void rcu_prepare_for_idle_init(int cpu);
500	492	static void rcu_cleanup_after_idle(int cpu);
...	...	@@ -25,6 +25,7 @@
25	25	*/
26	26
27	27	#include <linux/delay.h>
	28	+#include <linux/smpboot.h>
28	29
29	30	#define RCU_KTHREAD_PRIO 1
30	31
...	...	@@ -1292,25 +1293,6 @@
1292	1293	return 0;
1293	1294	}
1294	1295
1295		-#ifdef CONFIG_HOTPLUG_CPU
1296		-
1297		-/*
1298		- * Stop the RCU's per-CPU kthread when its CPU goes offline,.
1299		- */
1300		-static void rcu_stop_cpu_kthread(int cpu)
1301		-{
1302		- struct task_struct *t;
1303		-
1304		- /* Stop the CPU's kthread. */
1305		- t = per_cpu(rcu_cpu_kthread_task, cpu);
1306		- if (t != NULL) {
1307		- per_cpu(rcu_cpu_kthread_task, cpu) = NULL;
1308		- kthread_stop(t);
1309		- }
1310		-}
1311		-
1312		-#endif /* #ifdef CONFIG_HOTPLUG_CPU */
1313		-
1314	1296	static void rcu_kthread_do_work(void)
1315	1297	{
1316	1298	rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data));
1317	1299
1318	1300
1319	1301
1320	1302
1321	1303
1322	1304
1323	1305
1324	1306
1325	1307
1326	1308
1327	1309
1328	1310
1329	1311
1330	1312
1331	1313
...	...	@@ -1318,159 +1300,61 @@
1318	1300	rcu_preempt_do_callbacks();
1319	1301	}
1320	1302
1321		-/*
1322		- * Set the specified CPU's kthread to run RT or not, as specified by
1323		- * the to_rt argument. The CPU-hotplug locks are held, so the task
1324		- * is not going away.
1325		- */
1326		-static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
	1303	+static void rcu_cpu_kthread_setup(unsigned int cpu)
1327	1304	{
1328		- int policy;
1329	1305	struct sched_param sp;
1330		- struct task_struct *t;
1331	1306
1332		- t = per_cpu(rcu_cpu_kthread_task, cpu);
1333		- if (t == NULL)
1334		- return;
1335		- if (to_rt) {
1336		- policy = SCHED_FIFO;
1337		- sp.sched_priority = RCU_KTHREAD_PRIO;
1338		- } else {
1339		- policy = SCHED_NORMAL;
1340		- sp.sched_priority = 0;
1341		- }
1342		- sched_setscheduler_nocheck(t, policy, &sp);
	1307	+ sp.sched_priority = RCU_KTHREAD_PRIO;
	1308	+ sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1343	1309	}
1344	1310
1345		-/*
1346		- * Handle cases where the rcu_cpu_kthread() ends up on the wrong CPU.
1347		- * This can happen while the corresponding CPU is either coming online
1348		- * or going offline. We cannot wait until the CPU is fully online
1349		- * before starting the kthread, because the various notifier functions
1350		- * can wait for RCU grace periods. So we park rcu_cpu_kthread() until
1351		- * the corresponding CPU is online.
1352		- *
1353		- * Return 1 if the kthread needs to stop, 0 otherwise.
1354		- *
1355		- * Caller must disable bh. This function can momentarily enable it.
1356		- */
1357		-static int rcu_cpu_kthread_should_stop(int cpu)
	1311	+static void rcu_cpu_kthread_park(unsigned int cpu)
1358	1312	{
1359		- while (cpu_is_offline(cpu) \|\|
1360		- !cpumask_equal(&current->cpus_allowed, cpumask_of(cpu)) \|\|
1361		- smp_processor_id() != cpu) {
1362		- if (kthread_should_stop())
1363		- return 1;
1364		- per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1365		- per_cpu(rcu_cpu_kthread_cpu, cpu) = raw_smp_processor_id();
1366		- local_bh_enable();
1367		- schedule_timeout_uninterruptible(1);
1368		- if (!cpumask_equal(&current->cpus_allowed, cpumask_of(cpu)))
1369		- set_cpus_allowed_ptr(current, cpumask_of(cpu));
1370		- local_bh_disable();
1371		- }
1372		- per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
1373		- return 0;
	1313	+ per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
1374	1314	}
1375	1315
	1316	+static int rcu_cpu_kthread_should_run(unsigned int cpu)
	1317	+{
	1318	+ return __get_cpu_var(rcu_cpu_has_work);
	1319	+}
	1320	+
1376	1321	/*
1377	1322	* Per-CPU kernel thread that invokes RCU callbacks. This replaces the
1378	1323	* RCU softirq used in flavors and configurations of RCU that do not
1379	1324	* support RCU priority boosting.
1380	1325	*/
1381		-static int rcu_cpu_kthread(void *arg)
	1326	+static void rcu_cpu_kthread(unsigned int cpu)
1382	1327	{
1383		- int cpu = (int)(long)arg;
1384		- unsigned long flags;
1385		- int spincnt = 0;
1386		- unsigned int *statusp = &per_cpu(rcu_cpu_kthread_status, cpu);
1387		- char work;
1388		- char *workp = &per_cpu(rcu_cpu_has_work, cpu);
	1328	+ unsigned int *statusp = &__get_cpu_var(rcu_cpu_kthread_status);
	1329	+ char work, *workp = &__get_cpu_var(rcu_cpu_has_work);
	1330	+ int spincnt;
1389	1331
1390		- trace_rcu_utilization("Start CPU kthread@init");
1391		- for (;;) {
1392		- *statusp = RCU_KTHREAD_WAITING;
1393		- trace_rcu_utilization("End CPU kthread@rcu_wait");
1394		- rcu_wait(*workp != 0 \|\| kthread_should_stop());
	1332	+ for (spincnt = 0; spincnt < 10; spincnt++) {
1395	1333	trace_rcu_utilization("Start CPU kthread@rcu_wait");
1396	1334	local_bh_disable();
1397		- if (rcu_cpu_kthread_should_stop(cpu)) {
1398		- local_bh_enable();
1399		- break;
1400		- }
1401	1335	*statusp = RCU_KTHREAD_RUNNING;
1402		- per_cpu(rcu_cpu_kthread_loops, cpu)++;
1403		- local_irq_save(flags);
	1336	+ this_cpu_inc(rcu_cpu_kthread_loops);
	1337	+ local_irq_disable();
1404	1338	work = *workp;
1405	1339	*workp = 0;
1406		- local_irq_restore(flags);
	1340	+ local_irq_enable();
1407	1341	if (work)
1408	1342	rcu_kthread_do_work();
1409	1343	local_bh_enable();
1410		- if (*workp != 0)
1411		- spincnt++;
1412		- else
1413		- spincnt = 0;
1414		- if (spincnt > 10) {
1415		- *statusp = RCU_KTHREAD_YIELDING;
1416		- trace_rcu_utilization("End CPU kthread@rcu_yield");
1417		- schedule_timeout_interruptible(2);
1418		- trace_rcu_utilization("Start CPU kthread@rcu_yield");
1419		- spincnt = 0;
	1344	+ if (*workp == 0) {
	1345	+ trace_rcu_utilization("End CPU kthread@rcu_wait");
	1346	+ *statusp = RCU_KTHREAD_WAITING;
	1347	+ return;
1420	1348	}
1421	1349	}
1422		- *statusp = RCU_KTHREAD_STOPPED;
1423		- trace_rcu_utilization("End CPU kthread@term");
1424		- return 0;
	1350	+ *statusp = RCU_KTHREAD_YIELDING;
	1351	+ trace_rcu_utilization("Start CPU kthread@rcu_yield");
	1352	+ schedule_timeout_interruptible(2);
	1353	+ trace_rcu_utilization("End CPU kthread@rcu_yield");
	1354	+ *statusp = RCU_KTHREAD_WAITING;
1425	1355	}
1426	1356
1427	1357	/*
1428		- * Spawn a per-CPU kthread, setting up affinity and priority.
1429		- * Because the CPU hotplug lock is held, no other CPU will be attempting
1430		- * to manipulate rcu_cpu_kthread_task. There might be another CPU
1431		- * attempting to access it during boot, but the locking in kthread_bind()
1432		- * will enforce sufficient ordering.
1433		- *
1434		- * Please note that we cannot simply refuse to wake up the per-CPU
1435		- * kthread because kthreads are created in TASK_UNINTERRUPTIBLE state,
1436		- * which can result in softlockup complaints if the task ends up being
1437		- * idle for more than a couple of minutes.
1438		- *
1439		- * However, please note also that we cannot bind the per-CPU kthread to its
1440		- * CPU until that CPU is fully online. We also cannot wait until the
1441		- * CPU is fully online before we create its per-CPU kthread, as this would
1442		- * deadlock the system when CPU notifiers tried waiting for grace
1443		- * periods. So we bind the per-CPU kthread to its CPU only if the CPU
1444		- * is online. If its CPU is not yet fully online, then the code in
1445		- * rcu_cpu_kthread() will wait until it is fully online, and then do
1446		- * the binding.
1447		- */
1448		-static int __cpuinit rcu_spawn_one_cpu_kthread(int cpu)
1449		-{
1450		- struct sched_param sp;
1451		- struct task_struct *t;
1452		-
1453		- if (!rcu_scheduler_fully_active \|\|
1454		- per_cpu(rcu_cpu_kthread_task, cpu) != NULL)
1455		- return 0;
1456		- t = kthread_create_on_node(rcu_cpu_kthread,
1457		- (void *)(long)cpu,
1458		- cpu_to_node(cpu),
1459		- "rcuc/%d", cpu);
1460		- if (IS_ERR(t))
1461		- return PTR_ERR(t);
1462		- if (cpu_online(cpu))
1463		- kthread_bind(t, cpu);
1464		- per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
1465		- WARN_ON_ONCE(per_cpu(rcu_cpu_kthread_task, cpu) != NULL);
1466		- sp.sched_priority = RCU_KTHREAD_PRIO;
1467		- sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1468		- per_cpu(rcu_cpu_kthread_task, cpu) = t;
1469		- wake_up_process(t); /* Get to TASK_INTERRUPTIBLE quickly. */
1470		- return 0;
1471		-}
1472		-
1473		-/*
1474	1358	* Set the per-rcu_node kthread's affinity to cover all CPUs that are
1475	1359	* served by the rcu_node in question. The CPU hotplug lock is still
1476	1360	* held, so the value of rnp->qsmaskinit will be stable.
...	...	@@ -1503,6 +1387,15 @@
1503	1387	free_cpumask_var(cm);
1504	1388	}
1505	1389
	1390	+static struct smp_hotplug_thread rcu_cpu_thread_spec = {
	1391	+ .store = &rcu_cpu_kthread_task,
	1392	+ .thread_should_run = rcu_cpu_kthread_should_run,
	1393	+ .thread_fn = rcu_cpu_kthread,
	1394	+ .thread_comm = "rcuc/%u",
	1395	+ .setup = rcu_cpu_kthread_setup,
	1396	+ .park = rcu_cpu_kthread_park,
	1397	+};
	1398	+
1506	1399	/*
1507	1400	* Spawn all kthreads -- called as soon as the scheduler is running.
1508	1401	*/
1509	1402
...	...	@@ -1512,11 +1405,9 @@
1512	1405	int cpu;
1513	1406
1514	1407	rcu_scheduler_fully_active = 1;
1515		- for_each_possible_cpu(cpu) {
	1408	+ for_each_possible_cpu(cpu)
1516	1409	per_cpu(rcu_cpu_has_work, cpu) = 0;
1517		- if (cpu_online(cpu))
1518		- (void)rcu_spawn_one_cpu_kthread(cpu);
1519		- }
	1410	+ BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1520	1411	rnp = rcu_get_root(rcu_state);
1521	1412	(void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1522	1413	if (NUM_RCU_NODES > 1) {
1523	1414
...	...	@@ -1533,10 +1424,8 @@
1533	1424	struct rcu_node *rnp = rdp->mynode;
1534	1425
1535	1426	/* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
1536		- if (rcu_scheduler_fully_active) {
1537		- (void)rcu_spawn_one_cpu_kthread(cpu);
	1427	+ if (rcu_scheduler_fully_active)
1538	1428	(void)rcu_spawn_one_boost_kthread(rcu_state, rnp);
1539		- }
1540	1429	}
1541	1430
1542	1431	#else /* #ifdef CONFIG_RCU_BOOST */
1543	1432
...	...	@@ -1560,19 +1449,7 @@
1560	1449	{
1561	1450	}
1562	1451
1563		-#ifdef CONFIG_HOTPLUG_CPU
1564		-
1565		-static void rcu_stop_cpu_kthread(int cpu)
1566		-{
1567		-}
1568		-
1569		-#endif /* #ifdef CONFIG_HOTPLUG_CPU */
1570		-
1571	1452	static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1572		-{
1573		-}
1574		-
1575		-static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
1576	1453	{
1577	1454	}
1578	1455
...	...	@@ -108,11 +108,10 @@
108	108	rdp->nxttail[RCU_WAIT_TAIL]],
109	109	".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]]);
110	110	#ifdef CONFIG_RCU_BOOST
111		- seq_printf(m, " kt=%d/%c/%d ktl=%x",
	111	+ seq_printf(m, " kt=%d/%c ktl=%x",
112	112	per_cpu(rcu_cpu_has_work, rdp->cpu),
113	113	convert_kthread_status(per_cpu(rcu_cpu_kthread_status,
114	114	rdp->cpu)),
115		- per_cpu(rcu_cpu_kthread_cpu, rdp->cpu),
116	115	per_cpu(rcu_cpu_kthread_loops, rdp->cpu) & 0xffff);
117	116	#endif /* #ifdef CONFIG_RCU_BOOST */
118	117	seq_printf(m, " b=%ld", rdp->blimit);