rcu: Split hierarchical RCU initialization into boot-time and CPU-online pieces

This patch divides the rcutree initialization into boot-time and hotplug-time components, so that the tree data structures are guaranteed to be fully linked at boot time regardless of what might happen in CPU hotplug operations. This makes RCU more resilient against CPU hotplug misbehavior (and vice versa), but more importantly, does a better job of compartmentalizing the code. Reported-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: laijs@cn.fujitsu.com Cc: dipankar@in.ibm.com Cc: josht@linux.vnet.ibm.com Cc: akpm@linux-foundation.org Cc: mathieu.desnoyers@polymtl.ca Cc: dvhltc@us.ibm.com Cc: niv@us.ibm.com Cc: peterz@infradead.org Cc: rostedt@goodmis.org Cc: hugh.dickins@tiscali.co.uk Cc: benh@kernel.crashing.org LKML-Reference: <1250355231152-git-send-email-> Signed-off-by: Ingo Molnar <mingo@elte.hu>

rcu: Split hierarchical RCU initialization into boot-time and CPU-online pieces
This patch divides the rcutree initialization into boot-time and hotplug-time components, so that the tree data structures are guaranteed to be fully linked at boot time regardless of what might happen in CPU hotplug operations. This makes RCU more resilient against CPU hotplug misbehavior (and vice versa), but more importantly, does a better job of compartmentalizing the code. Reported-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: laijs@cn.fujitsu.com Cc: dipankar@in.ibm.com Cc: josht@linux.vnet.ibm.com Cc: akpm@linux-foundation.org Cc: mathieu.desnoyers@polymtl.ca Cc: dvhltc@us.ibm.com Cc: niv@us.ibm.com Cc: peterz@infradead.org Cc: rostedt@goodmis.org Cc: hugh.dickins@tiscali.co.uk Cc: benh@kernel.crashing.org LKML-Reference: <1250355231152-git-send-email-> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Paul E. McKenney · Ingo Molnar
1 parent fa08661af8
Showing 1 changed file with 33 additions and 20 deletions Side-by-side Diff
kernel/rcutree.c
@@ -1325,22 +1325,40 @@
 }
  
 /*
- * Initialize a CPU's per-CPU RCU data.  We take this "scorched earth"
- * approach so that we don't have to worry about how long the CPU has
- * been gone, or whether it ever was online previously.  We do trust the
- * ->mynode field, as it is constant for a given struct rcu_data and
- * initialized during early boot.
- *
- * Note that only one online or offline event can be happening at a given
- * time.  Note also that we can accept some slop in the rsp->completed
- * access due to the fact that this CPU cannot possibly have any RCU
- * callbacks in flight yet.
+ * Do boot-time initialization of a CPU's per-CPU RCU data.
  */
+static void __init
+rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
+{
+	unsigned long flags;
+	int i;
+	struct rcu_data *rdp = rsp->rda[cpu];
+	struct rcu_node *rnp = rcu_get_root(rsp);
+
+	/* Set up local state, ensuring consistent view of global state. */
+	spin_lock_irqsave(&rnp->lock, flags);
+	rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
+	rdp->nxtlist = NULL;
+	for (i = 0; i < RCU_NEXT_SIZE; i++)
+		rdp->nxttail[i] = &rdp->nxtlist;
+	rdp->qlen = 0;
+#ifdef CONFIG_NO_HZ
+	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
+#endif /* #ifdef CONFIG_NO_HZ */
+	rdp->cpu = cpu;
+	spin_unlock_irqrestore(&rnp->lock, flags);
+}
+
+/*
+ * Initialize a CPU's per-CPU RCU data.  Note that only one online or
+ * offline event can be happening at a given time.  Note also that we
+ * can accept some slop in the rsp->completed access due to the fact
+ * that this CPU cannot possibly have any RCU callbacks in flight yet.
+ */
 static void __cpuinit
 rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 {
 	unsigned long flags;
-	int i;
 	long lastcomp;
 	unsigned long mask;
 	struct rcu_data *rdp = rsp->rda[cpu];
  
@@ -1355,16 +1373,7 @@
 	rdp->qs_pending = 1;	 /*  so set up to respond to current GP. */
 	rdp->beenonline = 1;	 /* We have now been online. */
 	rdp->passed_quiesc_completed = lastcomp - 1;
-	rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
-	rdp->nxtlist = NULL;
-	for (i = 0; i < RCU_NEXT_SIZE; i++)
-		rdp->nxttail[i] = &rdp->nxtlist;
-	rdp->qlen = 0;
 	rdp->blimit = blimit;
-#ifdef CONFIG_NO_HZ
-	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
-#endif /* #ifdef CONFIG_NO_HZ */
-	rdp->cpu = cpu;
 	spin_unlock(&rnp->lock);		/* irqs remain disabled. */
  
 	/*
  
@@ -1539,8 +1548,12 @@
 #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
 	rcu_init_one(&rcu_state);
 	RCU_DATA_PTR_INIT(&rcu_state, rcu_data);
+	for_each_possible_cpu(i)
+		rcu_boot_init_percpu_data(i, &rcu_state);
 	rcu_init_one(&rcu_bh_state);
 	RCU_DATA_PTR_INIT(&rcu_bh_state, rcu_bh_data);
+	for_each_possible_cpu(i)
+		rcu_boot_init_percpu_data(i, &rcu_bh_state);
  
 	for_each_online_cpu(i)
 		rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i);
...	...	@@ -1325,22 +1325,40 @@
1325	1325	}
1326	1326
1327	1327	/*
1328		- * Initialize a CPU's per-CPU RCU data. We take this "scorched earth"
1329		- * approach so that we don't have to worry about how long the CPU has
1330		- * been gone, or whether it ever was online previously. We do trust the
1331		- * ->mynode field, as it is constant for a given struct rcu_data and
1332		- * initialized during early boot.
1333		- *
1334		- * Note that only one online or offline event can be happening at a given
1335		- * time. Note also that we can accept some slop in the rsp->completed
1336		- * access due to the fact that this CPU cannot possibly have any RCU
1337		- * callbacks in flight yet.
	1328	+ * Do boot-time initialization of a CPU's per-CPU RCU data.
1338	1329	*/
	1330	+static void __init
	1331	+rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
	1332	+{
	1333	+ unsigned long flags;
	1334	+ int i;
	1335	+ struct rcu_data *rdp = rsp->rda[cpu];
	1336	+ struct rcu_node *rnp = rcu_get_root(rsp);
	1337	+
	1338	+ /* Set up local state, ensuring consistent view of global state. */
	1339	+ spin_lock_irqsave(&rnp->lock, flags);
	1340	+ rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
	1341	+ rdp->nxtlist = NULL;
	1342	+ for (i = 0; i < RCU_NEXT_SIZE; i++)
	1343	+ rdp->nxttail[i] = &rdp->nxtlist;
	1344	+ rdp->qlen = 0;
	1345	+#ifdef CONFIG_NO_HZ
	1346	+ rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
	1347	+#endif /* #ifdef CONFIG_NO_HZ */
	1348	+ rdp->cpu = cpu;
	1349	+ spin_unlock_irqrestore(&rnp->lock, flags);
	1350	+}
	1351	+
	1352	+/*
	1353	+ * Initialize a CPU's per-CPU RCU data. Note that only one online or
	1354	+ * offline event can be happening at a given time. Note also that we
	1355	+ * can accept some slop in the rsp->completed access due to the fact
	1356	+ * that this CPU cannot possibly have any RCU callbacks in flight yet.
	1357	+ */
1339	1358	static void __cpuinit
1340	1359	rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
1341	1360	{
1342	1361	unsigned long flags;
1343		- int i;
1344	1362	long lastcomp;
1345	1363	unsigned long mask;
1346	1364	struct rcu_data *rdp = rsp->rda[cpu];
1347	1365
...	...	@@ -1355,16 +1373,7 @@
1355	1373	rdp->qs_pending = 1; /* so set up to respond to current GP. */
1356	1374	rdp->beenonline = 1; /* We have now been online. */
1357	1375	rdp->passed_quiesc_completed = lastcomp - 1;
1358		- rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
1359		- rdp->nxtlist = NULL;
1360		- for (i = 0; i < RCU_NEXT_SIZE; i++)
1361		- rdp->nxttail[i] = &rdp->nxtlist;
1362		- rdp->qlen = 0;
1363	1376	rdp->blimit = blimit;
1364		-#ifdef CONFIG_NO_HZ
1365		- rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
1366		-#endif /* #ifdef CONFIG_NO_HZ */
1367		- rdp->cpu = cpu;
1368	1377	spin_unlock(&rnp->lock); /* irqs remain disabled. */
1369	1378
1370	1379	/*
1371	1380
...	...	@@ -1539,8 +1548,12 @@
1539	1548	#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
1540	1549	rcu_init_one(&rcu_state);
1541	1550	RCU_DATA_PTR_INIT(&rcu_state, rcu_data);
	1551	+ for_each_possible_cpu(i)
	1552	+ rcu_boot_init_percpu_data(i, &rcu_state);
1542	1553	rcu_init_one(&rcu_bh_state);
1543	1554	RCU_DATA_PTR_INIT(&rcu_bh_state, rcu_bh_data);
	1555	+ for_each_possible_cpu(i)
	1556	+ rcu_boot_init_percpu_data(i, &rcu_bh_state);
1544	1557
1545	1558	for_each_online_cpu(i)
1546	1559	rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i);