rcu: simplify the usage of percpu data

&percpu_data is compatible with allocated percpu data. And we use it and remove the "->rda[NR_CPUS]" array, saving significant storage on systems with large numbers of CPUs. This does add an additional level of indirection and thus an additional cache line referenced, but because ->rda is not used on the read side, this is OK. Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com> Reviewed-by: Tejun Heo <tj@kernel.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>

rcu: simplify the usage of percpu data
&percpu_data is compatible with allocated percpu data. And we use it and remove the "->rda[NR_CPUS]" array, saving significant storage on systems with large numbers of CPUs. This does add an additional level of indirection and thus an additional cache line referenced, but because ->rda is not used on the read side, this is OK. Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com> Reviewed-by: Tejun Heo <tj@kernel.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Lai Jiangshan · Paul E. McKenney
1 parent e546f485e1
Showing 4 changed files with 19 additions and 31 deletions Side-by-side Diff
kernel/rcutree.c
kernel/rcutree.h
kernel/rcutree_plugin.h
kernel/rcutree_trace.c
@@ -712,7 +712,7 @@
 rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
 	__releases(rcu_get_root(rsp)->lock)
 {
-	struct rcu_data *rdp = rsp->rda[smp_processor_id()];
+	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
 	struct rcu_node *rnp = rcu_get_root(rsp);
  
 	if (!cpu_needs_another_gp(rsp, rdp) || rsp->fqs_active) {
@@ -960,7 +960,7 @@
 static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)
 {
 	int i;
-	struct rcu_data *rdp = rsp->rda[smp_processor_id()];
+	struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
  
 	if (rdp->nxtlist == NULL)
 		return;  /* irqs disabled, so comparison is stable. */
@@ -984,7 +984,7 @@
 	struct rcu_data *rdp;
  
 	raw_spin_lock_irqsave(&rsp->onofflock, flags);
-	rdp = rsp->rda[smp_processor_id()];
+	rdp = this_cpu_ptr(rsp->rda);
 	if (rsp->orphan_cbs_list == NULL) {
 		raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
 		return;
@@ -1007,7 +1007,7 @@
 	unsigned long flags;
 	unsigned long mask;
 	int need_report = 0;
-	struct rcu_data *rdp = rsp->rda[cpu];
+	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
 	struct rcu_node *rnp;
  
 	/* Exclude any attempts to start a new grace period. */
@@ -1226,7 +1226,8 @@
 		cpu = rnp->grplo;
 		bit = 1;
 		for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
-			if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))
+			if ((rnp->qsmask & bit) != 0 &&
+			    f(per_cpu_ptr(rsp->rda, cpu)))
 				mask |= bit;
 		}
 		if (mask != 0) {
@@ -1402,7 +1403,7 @@
 	 * a quiescent state betweentimes.
 	 */
 	local_irq_save(flags);
-	rdp = rsp->rda[smp_processor_id()];
+	rdp = this_cpu_ptr(rsp->rda);
 	rcu_process_gp_end(rsp, rdp);
 	check_for_new_grace_period(rsp, rdp);
  
@@ -1701,7 +1702,7 @@
 {
 	unsigned long flags;
 	int i;
-	struct rcu_data *rdp = rsp->rda[cpu];
+	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
 	struct rcu_node *rnp = rcu_get_root(rsp);
  
 	/* Set up local state, ensuring consistent view of global state. */
@@ -1729,7 +1730,7 @@
 {
 	unsigned long flags;
 	unsigned long mask;
-	struct rcu_data *rdp = rsp->rda[cpu];
+	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
 	struct rcu_node *rnp = rcu_get_root(rsp);
  
 	/* Set up local state, ensuring consistent view of global state. */
@@ -1865,7 +1866,8 @@
 /*
  * Helper function for rcu_init() that initializes one rcu_state structure.
  */
-static void __init rcu_init_one(struct rcu_state *rsp)
+static void __init rcu_init_one(struct rcu_state *rsp,
+		struct rcu_data __percpu *rda)
 {
 	static char *buf[] = { "rcu_node_level_0",
 			       "rcu_node_level_1",
  
  
  
@@ -1918,37 +1920,23 @@
 		}
 	}
  
+	rsp->rda = rda;
 	rnp = rsp->level[NUM_RCU_LVLS - 1];
 	for_each_possible_cpu(i) {
 		while (i > rnp->grphi)
 			rnp++;
-		rsp->rda[i]->mynode = rnp;
+		per_cpu_ptr(rsp->rda, i)->mynode = rnp;
 		rcu_boot_init_percpu_data(i, rsp);
 	}
 }
  
-/*
- * Helper macro for __rcu_init() and __rcu_init_preempt().  To be used
- * nowhere else!  Assigns leaf node pointers into each CPU's rcu_data
- * structure.
- */
-#define RCU_INIT_FLAVOR(rsp, rcu_data) \
-do { \
-	int i; \
-	\
-	for_each_possible_cpu(i) { \
-		(rsp)->rda[i] = &per_cpu(rcu_data, i); \
-	} \
-	rcu_init_one(rsp); \
-} while (0)
-
 void __init rcu_init(void)
 {
 	int cpu;
  
 	rcu_bootup_announce();
-	RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data);
-	RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data);
+	rcu_init_one(&rcu_sched_state, &rcu_sched_data);
+	rcu_init_one(&rcu_bh_state, &rcu_bh_data);
 	__rcu_init_preempt();
 	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
  
@@ -283,7 +283,7 @@
 	struct rcu_node *level[NUM_RCU_LVLS];	/* Hierarchy levels. */
 	u32 levelcnt[MAX_RCU_LVLS + 1];		/* # nodes in each level. */
 	u8 levelspread[NUM_RCU_LVLS];		/* kids/node in each level. */
-	struct rcu_data *rda[NR_CPUS];		/* array of rdp pointers. */
+	struct rcu_data __percpu *rda;		/* pointer of percu rcu_data. */
  
 	/* The following fields are guarded by the root rcu_node's lock. */
  
@@ -154,7 +154,7 @@
 	    (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
  
 		/* Possibly blocking in an RCU read-side critical section. */
-		rdp = rcu_preempt_state.rda[cpu];
+		rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
 		rnp = rdp->mynode;
 		raw_spin_lock_irqsave(&rnp->lock, flags);
 		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
@@ -771,7 +771,7 @@
  */
 static void __init __rcu_init_preempt(void)
 {
-	RCU_INIT_FLAVOR(&rcu_preempt_state, rcu_preempt_data);
+	rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
 }
  
 /*
@@ -262,7 +262,7 @@
 	struct rcu_data *rdp;
  
 	for_each_possible_cpu(cpu) {
-		rdp = rsp->rda[cpu];
+		rdp = per_cpu_ptr(rsp->rda, cpu);
 		if (rdp->beenonline)
 			print_one_rcu_pending(m, rdp);
 	}
...	...	@@ -712,7 +712,7 @@
712	712	rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
713	713	__releases(rcu_get_root(rsp)->lock)
714	714	{
715		- struct rcu_data *rdp = rsp->rda[smp_processor_id()];
	715	+ struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
716	716	struct rcu_node *rnp = rcu_get_root(rsp);
717	717
718	718	if (!cpu_needs_another_gp(rsp, rdp) \|\| rsp->fqs_active) {
...	...	@@ -960,7 +960,7 @@
960	960	static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)
961	961	{
962	962	int i;
963		- struct rcu_data *rdp = rsp->rda[smp_processor_id()];
	963	+ struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
964	964
965	965	if (rdp->nxtlist == NULL)
966	966	return; /* irqs disabled, so comparison is stable. */
...	...	@@ -984,7 +984,7 @@
984	984	struct rcu_data *rdp;
985	985
986	986	raw_spin_lock_irqsave(&rsp->onofflock, flags);
987		- rdp = rsp->rda[smp_processor_id()];
	987	+ rdp = this_cpu_ptr(rsp->rda);
988	988	if (rsp->orphan_cbs_list == NULL) {
989	989	raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
990	990	return;
...	...	@@ -1007,7 +1007,7 @@
1007	1007	unsigned long flags;
1008	1008	unsigned long mask;
1009	1009	int need_report = 0;
1010		- struct rcu_data *rdp = rsp->rda[cpu];
	1010	+ struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1011	1011	struct rcu_node *rnp;
1012	1012
1013	1013	/* Exclude any attempts to start a new grace period. */
...	...	@@ -1226,7 +1226,8 @@
1226	1226	cpu = rnp->grplo;
1227	1227	bit = 1;
1228	1228	for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
1229		- if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu]))
	1229	+ if ((rnp->qsmask & bit) != 0 &&
	1230	+ f(per_cpu_ptr(rsp->rda, cpu)))
1230	1231	mask \|= bit;
1231	1232	}
1232	1233	if (mask != 0) {
...	...	@@ -1402,7 +1403,7 @@
1402	1403	* a quiescent state betweentimes.
1403	1404	*/
1404	1405	local_irq_save(flags);
1405		- rdp = rsp->rda[smp_processor_id()];
	1406	+ rdp = this_cpu_ptr(rsp->rda);
1406	1407	rcu_process_gp_end(rsp, rdp);
1407	1408	check_for_new_grace_period(rsp, rdp);
1408	1409
...	...	@@ -1701,7 +1702,7 @@
1701	1702	{
1702	1703	unsigned long flags;
1703	1704	int i;
1704		- struct rcu_data *rdp = rsp->rda[cpu];
	1705	+ struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1705	1706	struct rcu_node *rnp = rcu_get_root(rsp);
1706	1707
1707	1708	/* Set up local state, ensuring consistent view of global state. */
...	...	@@ -1729,7 +1730,7 @@
1729	1730	{
1730	1731	unsigned long flags;
1731	1732	unsigned long mask;
1732		- struct rcu_data *rdp = rsp->rda[cpu];
	1733	+ struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1733	1734	struct rcu_node *rnp = rcu_get_root(rsp);
1734	1735
1735	1736	/* Set up local state, ensuring consistent view of global state. */
...	...	@@ -1865,7 +1866,8 @@
1865	1866	/*
1866	1867	* Helper function for rcu_init() that initializes one rcu_state structure.
1867	1868	*/
1868		-static void __init rcu_init_one(struct rcu_state *rsp)
	1869	+static void __init rcu_init_one(struct rcu_state *rsp,
	1870	+ struct rcu_data __percpu *rda)
1869	1871	{
1870	1872	static char *buf[] = { "rcu_node_level_0",
1871	1873	"rcu_node_level_1",
1872	1874
1873	1875
1874	1876
...	...	@@ -1918,37 +1920,23 @@
1918	1920	}
1919	1921	}
1920	1922
	1923	+ rsp->rda = rda;
1921	1924	rnp = rsp->level[NUM_RCU_LVLS - 1];
1922	1925	for_each_possible_cpu(i) {
1923	1926	while (i > rnp->grphi)
1924	1927	rnp++;
1925		- rsp->rda[i]->mynode = rnp;
	1928	+ per_cpu_ptr(rsp->rda, i)->mynode = rnp;
1926	1929	rcu_boot_init_percpu_data(i, rsp);
1927	1930	}
1928	1931	}
1929	1932
1930		-/*
1931		- * Helper macro for __rcu_init() and __rcu_init_preempt(). To be used
1932		- * nowhere else! Assigns leaf node pointers into each CPU's rcu_data
1933		- * structure.
1934		- */
1935		-#define RCU_INIT_FLAVOR(rsp, rcu_data) \
1936		-do { \
1937		- int i; \
1938		- \
1939		- for_each_possible_cpu(i) { \
1940		- (rsp)->rda[i] = &per_cpu(rcu_data, i); \
1941		- } \
1942		- rcu_init_one(rsp); \
1943		-} while (0)
1944		-
1945	1933	void __init rcu_init(void)
1946	1934	{
1947	1935	int cpu;
1948	1936
1949	1937	rcu_bootup_announce();
1950		- RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data);
1951		- RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data);
	1938	+ rcu_init_one(&rcu_sched_state, &rcu_sched_data);
	1939	+ rcu_init_one(&rcu_bh_state, &rcu_bh_data);
1952	1940	__rcu_init_preempt();
1953	1941	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
1954	1942
...	...	@@ -283,7 +283,7 @@
283	283	struct rcu_node level[NUM_RCU_LVLS]; / Hierarchy levels. */
284	284	u32 levelcnt[MAX_RCU_LVLS + 1]; /* # nodes in each level. */
285	285	u8 levelspread[NUM_RCU_LVLS]; /* kids/node in each level. */
286		- struct rcu_data rda[NR_CPUS]; / array of rdp pointers. */
	286	+ struct rcu_data __percpu rda; / pointer of percu rcu_data. */
287	287
288	288	/* The following fields are guarded by the root rcu_node's lock. */
289	289
...	...	@@ -154,7 +154,7 @@
154	154	(t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
155	155
156	156	/* Possibly blocking in an RCU read-side critical section. */
157		- rdp = rcu_preempt_state.rda[cpu];
	157	+ rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
158	158	rnp = rdp->mynode;
159	159	raw_spin_lock_irqsave(&rnp->lock, flags);
160	160	t->rcu_read_unlock_special \|= RCU_READ_UNLOCK_BLOCKED;
...	...	@@ -771,7 +771,7 @@
771	771	*/
772	772	static void __init __rcu_init_preempt(void)
773	773	{
774		- RCU_INIT_FLAVOR(&rcu_preempt_state, rcu_preempt_data);
	774	+ rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
775	775	}
776	776
777	777	/*
...	...	@@ -262,7 +262,7 @@
262	262	struct rcu_data *rdp;
263	263
264	264	for_each_possible_cpu(cpu) {
265		- rdp = rsp->rda[cpu];
	265	+ rdp = per_cpu_ptr(rsp->rda, cpu);
266	266	if (rdp->beenonline)
267	267	print_one_rcu_pending(m, rdp);
268	268	}