rcu: Add boosting to TREE_PREEMPT_RCU tracing

Includes total number of tasks boosted, number boosted on behalf of each of normal and expedited grace periods, and statistics on attempts to initiate boosting that failed for various reasons. Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>

rcu: Add boosting to TREE_PREEMPT_RCU tracing
Includes total number of tasks boosted, number boosted on behalf of each of normal and expedited grace periods, and statistics on attempts to initiate boosting that failed for various reasons. Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Paul E. McKenney · Paul E. McKenney
1 parent 67b98dba47
Showing 3 changed files with 125 additions and 4 deletions Side-by-side Diff
kernel/rcutree.h
kernel/rcutree_plugin.h
kernel/rcutree_trace.c
@@ -152,6 +152,25 @@
 	wait_queue_head_t boost_wq;
 				/* Wait queue on which to park the boost */
 				/*  kthread. */
+	unsigned long n_tasks_boosted;
+				/* Total number of tasks boosted. */
+	unsigned long n_exp_boosts;
+				/* Number of tasks boosted for expedited GP. */
+	unsigned long n_normal_boosts;
+				/* Number of tasks boosted for normal GP. */
+	unsigned long n_balk_blkd_tasks;
+				/* Refused to boost: no blocked tasks. */
+	unsigned long n_balk_exp_gp_tasks;
+				/* Refused to boost: nothing blocking GP. */
+	unsigned long n_balk_boost_tasks;
+				/* Refused to boost: already boosting. */
+	unsigned long n_balk_notblocked;
+				/* Refused to boost: RCU RS CS still running. */
+	unsigned long n_balk_notyet;
+				/* Refused to boost: not yet time. */
+	unsigned long n_balk_nos;
+				/* Refused to boost: not sure why, though. */
+				/*  This can happen due to race conditions. */
 #endif /* #ifdef CONFIG_RCU_BOOST */
 	struct task_struct *node_kthread_task;
 				/* kthread that takes care of this rcu_node */
@@ -1073,6 +1073,33 @@
  
 #include "rtmutex_common.h"
  
+#ifdef CONFIG_RCU_TRACE
+
+static void rcu_initiate_boost_trace(struct rcu_node *rnp)
+{
+	if (list_empty(&rnp->blkd_tasks))
+		rnp->n_balk_blkd_tasks++;
+	else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
+		rnp->n_balk_exp_gp_tasks++;
+	else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
+		rnp->n_balk_boost_tasks++;
+	else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
+		rnp->n_balk_notblocked++;
+	else if (rnp->gp_tasks != NULL &&
+		 ULONG_CMP_GE(jiffies, rnp->boost_time))
+		rnp->n_balk_notyet++;
+	else
+		rnp->n_balk_nos++;
+}
+
+#else /* #ifdef CONFIG_RCU_TRACE */
+
+static void rcu_initiate_boost_trace(struct rcu_node *rnp)
+{
+}
+
+#endif /* #else #ifdef CONFIG_RCU_TRACE */
+
 /*
  * Carry out RCU priority boosting on the task indicated by ->exp_tasks
  * or ->boost_tasks, advancing the pointer to the next task in the
  
  
@@ -1108,10 +1135,14 @@
 	 * expedited grace period must boost all blocked tasks, including
 	 * those blocking the pre-existing normal grace period.
 	 */
-	if (rnp->exp_tasks != NULL)
+	if (rnp->exp_tasks != NULL) {
 		tb = rnp->exp_tasks;
-	else
+		rnp->n_exp_boosts++;
+	} else {
 		tb = rnp->boost_tasks;
+		rnp->n_normal_boosts++;
+	}
+	rnp->n_tasks_boosted++;
  
 	/*
 	 * We boost task t by manufacturing an rt_mutex that appears to
  
@@ -1197,8 +1228,10 @@
 {
 	struct task_struct *t;
  
-	if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL)
+	if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
+		rnp->n_balk_exp_gp_tasks++;
 		return;
+	}
 	if (rnp->exp_tasks != NULL ||
 	    (rnp->gp_tasks != NULL &&
 	     rnp->boost_tasks == NULL &&
@@ -1209,7 +1242,8 @@
 		t = rnp->boost_kthread_task;
 		if (t != NULL)
 			wake_up_process(t);
-	}
+	} else
+		rcu_initiate_boost_trace(rnp);
 }
  
 /*
@@ -157,6 +157,71 @@
 	.release = single_release,
 };
  
+#ifdef CONFIG_RCU_BOOST
+
+static void print_one_rcu_node_boost(struct seq_file *m, struct rcu_node *rnp)
+{
+	seq_printf(m,  "%d:%d tasks=%c%c%c%c ntb=%lu neb=%lu nnb=%lu "
+		   "j=%04x bt=%04x\n",
+		   rnp->grplo, rnp->grphi,
+		   "T."[list_empty(&rnp->blkd_tasks)],
+		   "N."[!rnp->gp_tasks],
+		   "E."[!rnp->exp_tasks],
+		   "B."[!rnp->boost_tasks],
+		   rnp->n_tasks_boosted, rnp->n_exp_boosts,
+		   rnp->n_normal_boosts,
+		   (int)(jiffies & 0xffff),
+		   (int)(rnp->boost_time & 0xffff));
+	seq_printf(m, "%s: nt=%lu egt=%lu bt=%lu nb=%lu ny=%lu nos=%lu\n",
+		   "     balk",
+		   rnp->n_balk_blkd_tasks,
+		   rnp->n_balk_exp_gp_tasks,
+		   rnp->n_balk_boost_tasks,
+		   rnp->n_balk_notblocked,
+		   rnp->n_balk_notyet,
+		   rnp->n_balk_nos);
+}
+
+static int show_rcu_node_boost(struct seq_file *m, void *unused)
+{
+	struct rcu_node *rnp;
+
+	rcu_for_each_leaf_node(&rcu_preempt_state, rnp)
+		print_one_rcu_node_boost(m, rnp);
+	return 0;
+}
+
+static int rcu_node_boost_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, show_rcu_node_boost, NULL);
+}
+
+static const struct file_operations rcu_node_boost_fops = {
+	.owner = THIS_MODULE,
+	.open = rcu_node_boost_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+/*
+ * Create the rcuboost debugfs entry.  Standard error return.
+ */
+static int rcu_boost_trace_create_file(struct dentry *rcudir)
+{
+	return !debugfs_create_file("rcuboost", 0444, rcudir, NULL,
+				    &rcu_node_boost_fops);
+}
+
+#else /* #ifdef CONFIG_RCU_BOOST */
+
+static int rcu_boost_trace_create_file(struct dentry *rcudir)
+{
+	return 0;  /* There cannot be an error if we didn't create it! */
+}
+
+#endif /* #else #ifdef CONFIG_RCU_BOOST */
+
 static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
 {
 	unsigned long gpnum;
@@ -313,6 +378,9 @@
 	retval = debugfs_create_file("rcudata.csv", 0444, rcudir,
 						NULL, &rcudata_csv_fops);
 	if (!retval)
+		goto free_out;
+
+	if (rcu_boost_trace_create_file(rcudir))
 		goto free_out;
  
 	retval = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
...	...	@@ -152,6 +152,25 @@
152	152	wait_queue_head_t boost_wq;
153	153	/* Wait queue on which to park the boost */
154	154	/* kthread. */
	155	+ unsigned long n_tasks_boosted;
	156	+ /* Total number of tasks boosted. */
	157	+ unsigned long n_exp_boosts;
	158	+ /* Number of tasks boosted for expedited GP. */
	159	+ unsigned long n_normal_boosts;
	160	+ /* Number of tasks boosted for normal GP. */
	161	+ unsigned long n_balk_blkd_tasks;
	162	+ /* Refused to boost: no blocked tasks. */
	163	+ unsigned long n_balk_exp_gp_tasks;
	164	+ /* Refused to boost: nothing blocking GP. */
	165	+ unsigned long n_balk_boost_tasks;
	166	+ /* Refused to boost: already boosting. */
	167	+ unsigned long n_balk_notblocked;
	168	+ /* Refused to boost: RCU RS CS still running. */
	169	+ unsigned long n_balk_notyet;
	170	+ /* Refused to boost: not yet time. */
	171	+ unsigned long n_balk_nos;
	172	+ /* Refused to boost: not sure why, though. */
	173	+ /* This can happen due to race conditions. */
155	174	#endif /* #ifdef CONFIG_RCU_BOOST */
156	175	struct task_struct *node_kthread_task;
157	176	/* kthread that takes care of this rcu_node */
...	...	@@ -1073,6 +1073,33 @@
1073	1073
1074	1074	#include "rtmutex_common.h"
1075	1075
	1076	+#ifdef CONFIG_RCU_TRACE
	1077	+
	1078	+static void rcu_initiate_boost_trace(struct rcu_node *rnp)
	1079	+{
	1080	+ if (list_empty(&rnp->blkd_tasks))
	1081	+ rnp->n_balk_blkd_tasks++;
	1082	+ else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
	1083	+ rnp->n_balk_exp_gp_tasks++;
	1084	+ else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
	1085	+ rnp->n_balk_boost_tasks++;
	1086	+ else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
	1087	+ rnp->n_balk_notblocked++;
	1088	+ else if (rnp->gp_tasks != NULL &&
	1089	+ ULONG_CMP_GE(jiffies, rnp->boost_time))
	1090	+ rnp->n_balk_notyet++;
	1091	+ else
	1092	+ rnp->n_balk_nos++;
	1093	+}
	1094	+
	1095	+#else /* #ifdef CONFIG_RCU_TRACE */
	1096	+
	1097	+static void rcu_initiate_boost_trace(struct rcu_node *rnp)
	1098	+{
	1099	+}
	1100	+
	1101	+#endif /* #else #ifdef CONFIG_RCU_TRACE */
	1102	+
1076	1103	/*
1077	1104	* Carry out RCU priority boosting on the task indicated by ->exp_tasks
1078	1105	* or ->boost_tasks, advancing the pointer to the next task in the
1079	1106
1080	1107
...	...	@@ -1108,10 +1135,14 @@
1108	1135	* expedited grace period must boost all blocked tasks, including
1109	1136	* those blocking the pre-existing normal grace period.
1110	1137	*/
1111		- if (rnp->exp_tasks != NULL)
	1138	+ if (rnp->exp_tasks != NULL) {
1112	1139	tb = rnp->exp_tasks;
1113		- else
	1140	+ rnp->n_exp_boosts++;
	1141	+ } else {
1114	1142	tb = rnp->boost_tasks;
	1143	+ rnp->n_normal_boosts++;
	1144	+ }
	1145	+ rnp->n_tasks_boosted++;
1115	1146
1116	1147	/*
1117	1148	* We boost task t by manufacturing an rt_mutex that appears to
1118	1149
...	...	@@ -1197,8 +1228,10 @@
1197	1228	{
1198	1229	struct task_struct *t;
1199	1230
1200		- if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL)
	1231	+ if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
	1232	+ rnp->n_balk_exp_gp_tasks++;
1201	1233	return;
	1234	+ }
1202	1235	if (rnp->exp_tasks != NULL \|\|
1203	1236	(rnp->gp_tasks != NULL &&
1204	1237	rnp->boost_tasks == NULL &&
...	...	@@ -1209,7 +1242,8 @@
1209	1242	t = rnp->boost_kthread_task;
1210	1243	if (t != NULL)
1211	1244	wake_up_process(t);
1212		- }
	1245	+ } else
	1246	+ rcu_initiate_boost_trace(rnp);
1213	1247	}
1214	1248
1215	1249	/*
...	...	@@ -157,6 +157,71 @@
157	157	.release = single_release,
158	158	};
159	159
	160	+#ifdef CONFIG_RCU_BOOST
	161	+
	162	+static void print_one_rcu_node_boost(struct seq_file m, struct rcu_node rnp)
	163	+{
	164	+ seq_printf(m, "%d:%d tasks=%c%c%c%c ntb=%lu neb=%lu nnb=%lu "
	165	+ "j=%04x bt=%04x\n",
	166	+ rnp->grplo, rnp->grphi,
	167	+ "T."[list_empty(&rnp->blkd_tasks)],
	168	+ "N."[!rnp->gp_tasks],
	169	+ "E."[!rnp->exp_tasks],
	170	+ "B."[!rnp->boost_tasks],
	171	+ rnp->n_tasks_boosted, rnp->n_exp_boosts,
	172	+ rnp->n_normal_boosts,
	173	+ (int)(jiffies & 0xffff),
	174	+ (int)(rnp->boost_time & 0xffff));
	175	+ seq_printf(m, "%s: nt=%lu egt=%lu bt=%lu nb=%lu ny=%lu nos=%lu\n",
	176	+ " balk",
	177	+ rnp->n_balk_blkd_tasks,
	178	+ rnp->n_balk_exp_gp_tasks,
	179	+ rnp->n_balk_boost_tasks,
	180	+ rnp->n_balk_notblocked,
	181	+ rnp->n_balk_notyet,
	182	+ rnp->n_balk_nos);
	183	+}
	184	+
	185	+static int show_rcu_node_boost(struct seq_file m, void unused)
	186	+{
	187	+ struct rcu_node *rnp;
	188	+
	189	+ rcu_for_each_leaf_node(&rcu_preempt_state, rnp)
	190	+ print_one_rcu_node_boost(m, rnp);
	191	+ return 0;
	192	+}
	193	+
	194	+static int rcu_node_boost_open(struct inode inode, struct file file)
	195	+{
	196	+ return single_open(file, show_rcu_node_boost, NULL);
	197	+}
	198	+
	199	+static const struct file_operations rcu_node_boost_fops = {
	200	+ .owner = THIS_MODULE,
	201	+ .open = rcu_node_boost_open,
	202	+ .read = seq_read,
	203	+ .llseek = seq_lseek,
	204	+ .release = single_release,
	205	+};
	206	+
	207	+/*
	208	+ * Create the rcuboost debugfs entry. Standard error return.
	209	+ */
	210	+static int rcu_boost_trace_create_file(struct dentry *rcudir)
	211	+{
	212	+ return !debugfs_create_file("rcuboost", 0444, rcudir, NULL,
	213	+ &rcu_node_boost_fops);
	214	+}
	215	+
	216	+#else /* #ifdef CONFIG_RCU_BOOST */
	217	+
	218	+static int rcu_boost_trace_create_file(struct dentry *rcudir)
	219	+{
	220	+ return 0; /* There cannot be an error if we didn't create it! */
	221	+}
	222	+
	223	+#endif /* #else #ifdef CONFIG_RCU_BOOST */
	224	+
160	225	static void print_one_rcu_state(struct seq_file m, struct rcu_state rsp)
161	226	{
162	227	unsigned long gpnum;
...	...	@@ -313,6 +378,9 @@
313	378	retval = debugfs_create_file("rcudata.csv", 0444, rcudir,
314	379	NULL, &rcudata_csv_fops);
315	380	if (!retval)
	381	+ goto free_out;
	382	+
	383	+ if (rcu_boost_trace_create_file(rcudir))
316	384	goto free_out;
317	385
318	386	retval = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);