rcu: Add grace-period, quiescent-state, and call_rcu trace events

Add trace events to record grace-period start and end, quiescent states, CPUs noticing grace-period start and end, grace-period initialization, call_rcu() invocation, tasks blocking in RCU read-side critical sections, tasks exiting those same critical sections, force_quiescent_state() detection of dyntick-idle and offline CPUs, CPUs entering and leaving dyntick-idle mode (except from NMIs), CPUs coming online and going offline, and CPUs being kicked for staying in dyntick-idle mode for too long (as in many weeks, even on 32-bit systems). Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> rcu: Add the rcu flavor to callback trace events The earlier trace events for registering RCU callbacks and for invoking them did not include the RCU flavor (rcu_bh, rcu_preempt, or rcu_sched). This commit adds the RCU flavor to those trace events. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

rcu: Add grace-period, quiescent-state, and call_rcu trace events
Add trace events to record grace-period start and end, quiescent states, CPUs noticing grace-period start and end, grace-period initialization, call_rcu() invocation, tasks blocking in RCU read-side critical sections, tasks exiting those same critical sections, force_quiescent_state() detection of dyntick-idle and offline CPUs, CPUs entering and leaving dyntick-idle mode (except from NMIs), CPUs coming online and going offline, and CPUs being kicked for staying in dyntick-idle mode for too long (as in many weeks, even on 32-bit systems). Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> rcu: Add the rcu flavor to callback trace events The earlier trace events for registering RCU callbacks and for invoking them did not include the RCU flavor (rcu_bh, rcu_preempt, or rcu_sched). This commit adds the RCU flavor to those trace events. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Paul E. McKenney · Paul E. McKenney
1 parent 965a002b4f
Showing 6 changed files with 399 additions and 24 deletions Side-by-side Diff
include/trace/events/rcu.h
kernel/rcu.h
kernel/rcutiny.c
kernel/rcutree.c
kernel/rcutree.h
kernel/rcutree_plugin.h
@@ -24,7 +24,7 @@
 	TP_ARGS(s),
  
 	TP_STRUCT__entry(
-		__field(char *,	s)
+		__field(char *, s)
 	),
  
 	TP_fast_assign(
  
@@ -34,7 +34,298 @@
 	TP_printk("%s", __entry->s)
 );
  
+#ifdef CONFIG_RCU_TRACE
+
+#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
+
 /*
+ * Tracepoint for grace-period events: starting and ending a grace
+ * period ("start" and "end", respectively), a CPU noting the start
+ * of a new grace period or the end of an old grace period ("cpustart"
+ * and "cpuend", respectively), a CPU passing through a quiescent
+ * state ("cpuqs"), a CPU coming online or going offline ("cpuonl"
+ * and "cpuofl", respectively), and a CPU being kicked for being too
+ * long in dyntick-idle mode ("kick").
+ */
+TRACE_EVENT(rcu_grace_period,
+
+	TP_PROTO(char *rcuname, unsigned long gpnum, char *gpevent),
+
+	TP_ARGS(rcuname, gpnum, gpevent),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(unsigned long, gpnum)
+		__field(char *, gpevent)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->gpnum = gpnum;
+		__entry->gpevent = gpevent;
+	),
+
+	TP_printk("%s %lu %s",
+		  __entry->rcuname, __entry->gpnum, __entry->gpevent)
+);
+
+/*
+ * Tracepoint for grace-period-initialization events.  These are
+ * distinguished by the type of RCU, the new grace-period number, the
+ * rcu_node structure level, the starting and ending CPU covered by the
+ * rcu_node structure, and the mask of CPUs that will be waited for.
+ * All but the type of RCU are extracted from the rcu_node structure.
+ */
+TRACE_EVENT(rcu_grace_period_init,
+
+	TP_PROTO(char *rcuname, unsigned long gpnum, u8 level,
+		 int grplo, int grphi, unsigned long qsmask),
+
+	TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(unsigned long, gpnum)
+		__field(u8, level)
+		__field(int, grplo)
+		__field(int, grphi)
+		__field(unsigned long, qsmask)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->gpnum = gpnum;
+		__entry->level = level;
+		__entry->grplo = grplo;
+		__entry->grphi = grphi;
+		__entry->qsmask = qsmask;
+	),
+
+	TP_printk("%s %lu %u %d %d %lx",
+		  __entry->rcuname, __entry->gpnum, __entry->level,
+		  __entry->grplo, __entry->grphi, __entry->qsmask)
+);
+
+/*
+ * Tracepoint for tasks blocking within preemptible-RCU read-side
+ * critical sections.  Track the type of RCU (which one day might
+ * include SRCU), the grace-period number that the task is blocking
+ * (the current or the next), and the task's PID.
+ */
+TRACE_EVENT(rcu_preempt_task,
+
+	TP_PROTO(char *rcuname, int pid, unsigned long gpnum),
+
+	TP_ARGS(rcuname, pid, gpnum),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(unsigned long, gpnum)
+		__field(int, pid)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->gpnum = gpnum;
+		__entry->pid = pid;
+	),
+
+	TP_printk("%s %lu %d",
+		  __entry->rcuname, __entry->gpnum, __entry->pid)
+);
+
+/*
+ * Tracepoint for tasks that blocked within a given preemptible-RCU
+ * read-side critical section exiting that critical section.  Track the
+ * type of RCU (which one day might include SRCU) and the task's PID.
+ */
+TRACE_EVENT(rcu_unlock_preempted_task,
+
+	TP_PROTO(char *rcuname, unsigned long gpnum, int pid),
+
+	TP_ARGS(rcuname, gpnum, pid),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(unsigned long, gpnum)
+		__field(int, pid)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->gpnum = gpnum;
+		__entry->pid = pid;
+	),
+
+	TP_printk("%s %lu %d", __entry->rcuname, __entry->gpnum, __entry->pid)
+);
+
+/*
+ * Tracepoint for quiescent-state-reporting events.  These are
+ * distinguished by the type of RCU, the grace-period number, the
+ * mask of quiescent lower-level entities, the rcu_node structure level,
+ * the starting and ending CPU covered by the rcu_node structure, and
+ * whether there are any blocked tasks blocking the current grace period.
+ * All but the type of RCU are extracted from the rcu_node structure.
+ */
+TRACE_EVENT(rcu_quiescent_state_report,
+
+	TP_PROTO(char *rcuname, unsigned long gpnum,
+		 unsigned long mask, unsigned long qsmask,
+		 u8 level, int grplo, int grphi, int gp_tasks),
+
+	TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(unsigned long, gpnum)
+		__field(unsigned long, mask)
+		__field(unsigned long, qsmask)
+		__field(u8, level)
+		__field(int, grplo)
+		__field(int, grphi)
+		__field(u8, gp_tasks)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->gpnum = gpnum;
+		__entry->mask = mask;
+		__entry->qsmask = qsmask;
+		__entry->level = level;
+		__entry->grplo = grplo;
+		__entry->grphi = grphi;
+		__entry->gp_tasks = gp_tasks;
+	),
+
+	TP_printk("%s %lu %lx>%lx %u %d %d %u",
+		  __entry->rcuname, __entry->gpnum,
+		  __entry->mask, __entry->qsmask, __entry->level,
+		  __entry->grplo, __entry->grphi, __entry->gp_tasks)
+);
+
+/*
+ * Tracepoint for quiescent states detected by force_quiescent_state().
+ * These trace events include the type of RCU, the grace-period number
+ * that was blocked by the CPU, the CPU itself, and the type of quiescent
+ * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
+ * or "kick" when kicking a CPU that has been in dyntick-idle mode for
+ * too long.
+ */
+TRACE_EVENT(rcu_fqs,
+
+	TP_PROTO(char *rcuname, unsigned long gpnum, int cpu, char *qsevent),
+
+	TP_ARGS(rcuname, gpnum, cpu, qsevent),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(unsigned long, gpnum)
+		__field(int, cpu)
+		__field(char *, qsevent)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->gpnum = gpnum;
+		__entry->cpu = cpu;
+		__entry->qsevent = qsevent;
+	),
+
+	TP_printk("%s %lu %d %s",
+		  __entry->rcuname, __entry->gpnum,
+		  __entry->cpu, __entry->qsevent)
+);
+
+#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) */
+
+/*
+ * Tracepoint for dyntick-idle entry/exit events.  These take a string
+ * as argument: "Start" for entering dyntick-idle mode and "End" for
+ * leaving it.
+ */
+TRACE_EVENT(rcu_dyntick,
+
+	TP_PROTO(char *polarity),
+
+	TP_ARGS(polarity),
+
+	TP_STRUCT__entry(
+		__field(char *, polarity)
+	),
+
+	TP_fast_assign(
+		__entry->polarity = polarity;
+	),
+
+	TP_printk("%s", __entry->polarity)
+);
+
+/*
+ * Tracepoint for the registration of a single RCU callback function.
+ * The first argument is the type of RCU, the second argument is
+ * a pointer to the RCU callback itself, and the third element is the
+ * new RCU callback queue length for the current CPU.
+ */
+TRACE_EVENT(rcu_callback,
+
+	TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen),
+
+	TP_ARGS(rcuname, rhp, qlen),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(void *, rhp)
+		__field(void *, func)
+		__field(long, qlen)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->rhp = rhp;
+		__entry->func = rhp->func;
+		__entry->qlen = qlen;
+	),
+
+	TP_printk("%s rhp=%p func=%pf %ld",
+		  __entry->rcuname, __entry->rhp, __entry->func, __entry->qlen)
+);
+
+/*
+ * Tracepoint for the registration of a single RCU callback of the special
+ * kfree() form.  The first argument is the RCU type, the second argument
+ * is a pointer to the RCU callback, the third argument is the offset
+ * of the callback within the enclosing RCU-protected data structure,
+ * and the fourth argument is the new RCU callback queue length for the
+ * current CPU.
+ */
+TRACE_EVENT(rcu_kfree_callback,
+
+	TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,
+		 long qlen),
+
+	TP_ARGS(rcuname, rhp, offset, qlen),
+
+	TP_STRUCT__entry(
+		__field(char *, rcuname)
+		__field(void *, rhp)
+		__field(unsigned long, offset)
+		__field(long, qlen)
+	),
+
+	TP_fast_assign(
+		__entry->rcuname = rcuname;
+		__entry->rhp = rhp;
+		__entry->offset = offset;
+		__entry->qlen = qlen;
+	),
+
+	TP_printk("%s rhp=%p func=%ld %ld",
+		  __entry->rcuname, __entry->rhp, __entry->offset,
+		  __entry->qlen)
+);
+
+/*
  * Tracepoint for marking the beginning rcu_do_batch, performed to start
  * RCU callback invocation.  The first argument is the RCU flavor,
  * the second is the total number of callbacks (including those that
  
  
  
  
  
  
  
  
  
  
  
@@ -65,50 +356,58 @@
  
 /*
  * Tracepoint for the invocation of a single RCU callback function.
- * The argument is a pointer to the RCU callback itself.
+ * The first argument is the type of RCU, and the second argument is
+ * a pointer to the RCU callback itself.
  */
 TRACE_EVENT(rcu_invoke_callback,
  
-	TP_PROTO(struct rcu_head *rhp),
+	TP_PROTO(char *rcuname, struct rcu_head *rhp),
  
-	TP_ARGS(rhp),
+	TP_ARGS(rcuname, rhp),
  
 	TP_STRUCT__entry(
-		__field(void *,	rhp)
-		__field(void *,	func)
+		__field(char *, rcuname)
+		__field(void *, rhp)
+		__field(void *, func)
 	),
  
 	TP_fast_assign(
+		__entry->rcuname = rcuname;
 		__entry->rhp = rhp;
 		__entry->func = rhp->func;
 	),
  
-	TP_printk("rhp=%p func=%pf", __entry->rhp, __entry->func)
+	TP_printk("%s rhp=%p func=%pf",
+		  __entry->rcuname, __entry->rhp, __entry->func)
 );
  
 /*
  * Tracepoint for the invocation of a single RCU callback of the special
- * kfree() form.  The first argument is a pointer to the RCU callback
- * and the second argument is the offset of the callback within the
- * enclosing RCU-protected data structure.
+ * kfree() form.  The first argument is the RCU flavor, the second
+ * argument is a pointer to the RCU callback, and the third argument
+ * is the offset of the callback within the enclosing RCU-protected
+ * data structure.
  */
 TRACE_EVENT(rcu_invoke_kfree_callback,
  
-	TP_PROTO(struct rcu_head *rhp, unsigned long offset),
+	TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset),
  
-	TP_ARGS(rhp, offset),
+	TP_ARGS(rcuname, rhp, offset),
  
 	TP_STRUCT__entry(
-		__field(void *,	rhp)
+		__field(char *, rcuname)
+		__field(void *, rhp)
 		__field(unsigned long, offset)
 	),
  
 	TP_fast_assign(
+		__entry->rcuname = rcuname;
 		__entry->rhp = rhp;
 		__entry->offset	= offset;
 	),
  
-	TP_printk("rhp=%p func=%ld", __entry->rhp, __entry->offset)
+	TP_printk("%s rhp=%p func=%ld",
+		  __entry->rcuname, __entry->rhp, __entry->offset)
 );
  
 /*
@@ -135,6 +434,24 @@
 	TP_printk("%s CBs-invoked=%d",
 		  __entry->rcuname, __entry->callbacks_invoked)
 );
+
+#else /* #ifdef CONFIG_RCU_TRACE */
+
+#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
+#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, qsmask) do { } while (0)
+#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
+#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
+#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks) do { } while (0)
+#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
+#define trace_rcu_dyntick(polarity) do { } while (0)
+#define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)
+#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)
+#define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
+#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
+#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
+#define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
+
+#endif /* #else #ifdef CONFIG_RCU_TRACE */
  
 #endif /* _TRACE_RCU_H */
  
@@ -69,15 +69,15 @@
  
 extern void kfree(const void *);
  
-static inline void __rcu_reclaim(struct rcu_head *head)
+static inline void __rcu_reclaim(char *rn, struct rcu_head *head)
 {
 	unsigned long offset = (unsigned long)head->func;
  
 	if (__is_kfree_rcu_offset(offset)) {
-		RCU_TRACE(trace_rcu_invoke_kfree_callback(head, offset));
+		RCU_TRACE(trace_rcu_invoke_kfree_callback(rn, head, offset));
 		kfree((void *)head - offset);
 	} else {
-		RCU_TRACE(trace_rcu_invoke_callback(head));
+		RCU_TRACE(trace_rcu_invoke_callback(rn, head));
 		head->func(head);
 	}
 }
@@ -147,6 +147,7 @@
  */
 static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
 {
+	char *rn = NULL;
 	struct rcu_head *next, *list;
 	unsigned long flags;
 	RCU_TRACE(int cb_count = 0);
  
@@ -171,12 +172,13 @@
 	local_irq_restore(flags);
  
 	/* Invoke the callbacks on the local list. */
+	RCU_TRACE(rn = rcp->name);
 	while (list) {
 		next = list->next;
 		prefetch(next);
 		debug_rcu_head_unqueue(list);
 		local_bh_disable();
-		__rcu_reclaim(list);
+		__rcu_reclaim(rn, list);
 		local_bh_enable();
 		list = next;
 		RCU_TRACE(cb_count++);
@@ -166,6 +166,8 @@
  
 	rdp->passed_quiesc_completed = rdp->gpnum - 1;
 	barrier();
+	if (rdp->passed_quiesc == 0)
+		trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs");
 	rdp->passed_quiesc = 1;
 }
  
@@ -175,6 +177,8 @@
  
 	rdp->passed_quiesc_completed = rdp->gpnum - 1;
 	barrier();
+	if (rdp->passed_quiesc == 0)
+		trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs");
 	rdp->passed_quiesc = 1;
 }
  
@@ -319,6 +323,7 @@
 	 * trust its state not to change because interrupts are disabled.
 	 */
 	if (cpu_is_offline(rdp->cpu)) {
+		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "ofl");
 		rdp->offline_fqs++;
 		return 1;
 	}
@@ -359,6 +364,7 @@
 		local_irq_restore(flags);
 		return;
 	}
+	trace_rcu_dyntick("Start");
 	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
 	smp_mb__before_atomic_inc();  /* See above. */
 	atomic_inc(&rdtp->dynticks);
@@ -396,6 +402,7 @@
 	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
 	smp_mb__after_atomic_inc();  /* See above. */
 	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+	trace_rcu_dyntick("End");
 	local_irq_restore(flags);
 }
  
@@ -501,6 +508,7 @@
 	 * of the current RCU grace period.
 	 */
 	if ((curr & 0x1) == 0 || ULONG_CMP_GE(curr, snap + 2)) {
+		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "dti");
 		rdp->dynticks_fqs++;
 		return 1;
 	}
@@ -683,6 +691,7 @@
 		 * go looking for one.
 		 */
 		rdp->gpnum = rnp->gpnum;
+		trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
 		if (rnp->qsmask & rdp->grpmask) {
 			rdp->qs_pending = 1;
 			rdp->passed_quiesc = 0;
@@ -746,6 +755,7 @@
  
 		/* Remember that we saw this grace-period completion. */
 		rdp->completed = rnp->completed;
+		trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuend");
  
 		/*
 		 * If we were in an extended quiescent state, we may have
@@ -856,6 +866,7 @@
  
 	/* Advance to a new grace period and initialize state. */
 	rsp->gpnum++;
+	trace_rcu_grace_period(rsp->name, rsp->gpnum, "start");
 	WARN_ON_ONCE(rsp->signaled == RCU_GP_INIT);
 	rsp->signaled = RCU_GP_INIT; /* Hold off force_quiescent_state. */
 	rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
@@ -870,6 +881,9 @@
 		rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */
 		rcu_start_gp_per_cpu(rsp, rnp, rdp);
 		rcu_preempt_boost_start_gp(rnp);
+		trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
+					    rnp->level, rnp->grplo,
+					    rnp->grphi, rnp->qsmask);
 		raw_spin_unlock_irqrestore(&rnp->lock, flags);
 		return;
 	}
@@ -906,6 +920,9 @@
 		if (rnp == rdp->mynode)
 			rcu_start_gp_per_cpu(rsp, rnp, rdp);
 		rcu_preempt_boost_start_gp(rnp);
+		trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
+					    rnp->level, rnp->grplo,
+					    rnp->grphi, rnp->qsmask);
 		raw_spin_unlock(&rnp->lock);	/* irqs remain disabled. */
 	}
  
@@ -939,6 +956,7 @@
 	if (gp_duration > rsp->gp_max)
 		rsp->gp_max = gp_duration;
 	rsp->completed = rsp->gpnum;
+	trace_rcu_grace_period(rsp->name, rsp->completed, "end");
 	rsp->signaled = RCU_GP_IDLE;
 	rcu_start_gp(rsp, flags);  /* releases root node's rnp->lock. */
 }
@@ -967,6 +985,10 @@
 			return;
 		}
 		rnp->qsmask &= ~mask;
+		trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,
+						 mask, rnp->qsmask, rnp->level,
+						 rnp->grplo, rnp->grphi,
+						 !!rnp->gp_tasks);
 		if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
  
 			/* Other bits still set at this level, so done. */
  
  
@@ -1135,11 +1157,20 @@
 		if (rnp->qsmaskinit != 0) {
 			if (rnp != rdp->mynode)
 				raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
+			else
+				trace_rcu_grace_period(rsp->name,
+						       rnp->gpnum + 1 -
+						       !!(rnp->qsmask & mask),
+						       "cpuofl");
 			break;
 		}
-		if (rnp == rdp->mynode)
+		if (rnp == rdp->mynode) {
+			trace_rcu_grace_period(rsp->name,
+					       rnp->gpnum + 1 -
+					       !!(rnp->qsmask & mask),
+					       "cpuofl");
 			need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);
-		else
+		} else
 			raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
 		mask = rnp->grpmask;
 		rnp = rnp->parent;
@@ -1226,7 +1257,7 @@
 		next = list->next;
 		prefetch(next);
 		debug_rcu_head_unqueue(list);
-		__rcu_reclaim(list);
+		__rcu_reclaim(rsp->name, list);
 		list = next;
 		if (++count >= bl)
 			break;
@@ -1552,6 +1583,12 @@
 	rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
 	rdp->qlen++;
  
+	if (__is_kfree_rcu_offset((unsigned long)func))
+		trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
+					 rdp->qlen);
+	else
+		trace_rcu_callback(rsp->name, head, rdp->qlen);
+
 	/* If interrupts were disabled, don't dive into RCU core. */
 	if (irqs_disabled_flags(flags)) {
 		local_irq_restore(flags);
@@ -1850,6 +1887,7 @@
 	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
 #endif /* #ifdef CONFIG_NO_HZ */
 	rdp->cpu = cpu;
+	rdp->rsp = rsp;
 	raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
  
@@ -1898,6 +1936,7 @@
 			rdp->gpnum = rnp->completed; /* if GP in progress... */
 			rdp->completed = rnp->completed;
 			rdp->passed_quiesc_completed = rnp->completed - 1;
+			trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuonl");
 		}
 		raw_spin_unlock(&rnp->lock); /* irqs already disabled. */
 		rnp = rnp->parent;
@@ -299,6 +299,7 @@
 	unsigned long n_rp_need_nothing;
  
 	int cpu;
+	struct rcu_state *rsp;
 };
  
 /* Values for signaled field in struct rcu_state. */
@@ -124,6 +124,8 @@
  
 	rdp->passed_quiesc_completed = rdp->gpnum - 1;
 	barrier();
+	if (rdp->passed_quiesc == 0)
+		trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
 	rdp->passed_quiesc = 1;
 	current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
 }
@@ -190,6 +192,11 @@
 			if (rnp->qsmask & rdp->grpmask)
 				rnp->gp_tasks = &t->rcu_node_entry;
 		}
+		trace_rcu_preempt_task(rdp->rsp->name,
+				       t->pid,
+				       (rnp->qsmask & rdp->grpmask)
+				       ? rnp->gpnum
+				       : rnp->gpnum + 1);
 		raw_spin_unlock_irqrestore(&rnp->lock, flags);
 	} else if (t->rcu_read_lock_nesting < 0 &&
 		   t->rcu_read_unlock_special) {
@@ -344,6 +351,8 @@
 		smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
 		np = rcu_next_node_entry(t, rnp);
 		list_del_init(&t->rcu_node_entry);
+		trace_rcu_unlock_preempted_task("rcu_preempt",
+						rnp->gpnum, t->pid);
 		if (&t->rcu_node_entry == rnp->gp_tasks)
 			rnp->gp_tasks = np;
 		if (&t->rcu_node_entry == rnp->exp_tasks)
  
@@ -364,10 +373,17 @@
 		 * we aren't waiting on any CPUs, report the quiescent state.
 		 * Note that rcu_report_unblock_qs_rnp() releases rnp->lock.
 		 */
-		if (empty)
-			raw_spin_unlock_irqrestore(&rnp->lock, flags);
-		else
+		if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
+			trace_rcu_quiescent_state_report("preempt_rcu",
+							 rnp->gpnum,
+							 0, rnp->qsmask,
+							 rnp->level,
+							 rnp->grplo,
+							 rnp->grphi,
+							 !!rnp->gp_tasks);
 			rcu_report_unblock_qs_rnp(rnp, flags);
+		} else
+			raw_spin_unlock_irqrestore(&rnp->lock, flags);
  
 #ifdef CONFIG_RCU_BOOST
 		/* Unboost if we were boosted. */
...	...	@@ -24,7 +24,7 @@
24	24	TP_ARGS(s),
25	25
26	26	TP_STRUCT__entry(
27		- __field(char *, s)
	27	+ __field(char *, s)
28	28	),
29	29
30	30	TP_fast_assign(
31	31
...	...	@@ -34,7 +34,298 @@
34	34	TP_printk("%s", __entry->s)
35	35	);
36	36
	37	+#ifdef CONFIG_RCU_TRACE
	38	+
	39	+#if defined(CONFIG_TREE_RCU) \|\| defined(CONFIG_TREE_PREEMPT_RCU)
	40	+
37	41	/*
	42	+ * Tracepoint for grace-period events: starting and ending a grace
	43	+ * period ("start" and "end", respectively), a CPU noting the start
	44	+ * of a new grace period or the end of an old grace period ("cpustart"
	45	+ * and "cpuend", respectively), a CPU passing through a quiescent
	46	+ * state ("cpuqs"), a CPU coming online or going offline ("cpuonl"
	47	+ * and "cpuofl", respectively), and a CPU being kicked for being too
	48	+ * long in dyntick-idle mode ("kick").
	49	+ */
	50	+TRACE_EVENT(rcu_grace_period,
	51	+
	52	+ TP_PROTO(char rcuname, unsigned long gpnum, char gpevent),
	53	+
	54	+ TP_ARGS(rcuname, gpnum, gpevent),
	55	+
	56	+ TP_STRUCT__entry(
	57	+ __field(char *, rcuname)
	58	+ __field(unsigned long, gpnum)
	59	+ __field(char *, gpevent)
	60	+ ),
	61	+
	62	+ TP_fast_assign(
	63	+ __entry->rcuname = rcuname;
	64	+ __entry->gpnum = gpnum;
	65	+ __entry->gpevent = gpevent;
	66	+ ),
	67	+
	68	+ TP_printk("%s %lu %s",
	69	+ __entry->rcuname, __entry->gpnum, __entry->gpevent)
	70	+);
	71	+
	72	+/*
	73	+ * Tracepoint for grace-period-initialization events. These are
	74	+ * distinguished by the type of RCU, the new grace-period number, the
	75	+ * rcu_node structure level, the starting and ending CPU covered by the
	76	+ * rcu_node structure, and the mask of CPUs that will be waited for.
	77	+ * All but the type of RCU are extracted from the rcu_node structure.
	78	+ */
	79	+TRACE_EVENT(rcu_grace_period_init,
	80	+
	81	+ TP_PROTO(char *rcuname, unsigned long gpnum, u8 level,
	82	+ int grplo, int grphi, unsigned long qsmask),
	83	+
	84	+ TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
	85	+
	86	+ TP_STRUCT__entry(
	87	+ __field(char *, rcuname)
	88	+ __field(unsigned long, gpnum)
	89	+ __field(u8, level)
	90	+ __field(int, grplo)
	91	+ __field(int, grphi)
	92	+ __field(unsigned long, qsmask)
	93	+ ),
	94	+
	95	+ TP_fast_assign(
	96	+ __entry->rcuname = rcuname;
	97	+ __entry->gpnum = gpnum;
	98	+ __entry->level = level;
	99	+ __entry->grplo = grplo;
	100	+ __entry->grphi = grphi;
	101	+ __entry->qsmask = qsmask;
	102	+ ),
	103	+
	104	+ TP_printk("%s %lu %u %d %d %lx",
	105	+ __entry->rcuname, __entry->gpnum, __entry->level,
	106	+ __entry->grplo, __entry->grphi, __entry->qsmask)
	107	+);
	108	+
	109	+/*
	110	+ * Tracepoint for tasks blocking within preemptible-RCU read-side
	111	+ * critical sections. Track the type of RCU (which one day might
	112	+ * include SRCU), the grace-period number that the task is blocking
	113	+ * (the current or the next), and the task's PID.
	114	+ */
	115	+TRACE_EVENT(rcu_preempt_task,
	116	+
	117	+ TP_PROTO(char *rcuname, int pid, unsigned long gpnum),
	118	+
	119	+ TP_ARGS(rcuname, pid, gpnum),
	120	+
	121	+ TP_STRUCT__entry(
	122	+ __field(char *, rcuname)
	123	+ __field(unsigned long, gpnum)
	124	+ __field(int, pid)
	125	+ ),
	126	+
	127	+ TP_fast_assign(
	128	+ __entry->rcuname = rcuname;
	129	+ __entry->gpnum = gpnum;
	130	+ __entry->pid = pid;
	131	+ ),
	132	+
	133	+ TP_printk("%s %lu %d",
	134	+ __entry->rcuname, __entry->gpnum, __entry->pid)
	135	+);
	136	+
	137	+/*
	138	+ * Tracepoint for tasks that blocked within a given preemptible-RCU
	139	+ * read-side critical section exiting that critical section. Track the
	140	+ * type of RCU (which one day might include SRCU) and the task's PID.
	141	+ */
	142	+TRACE_EVENT(rcu_unlock_preempted_task,
	143	+
	144	+ TP_PROTO(char *rcuname, unsigned long gpnum, int pid),
	145	+
	146	+ TP_ARGS(rcuname, gpnum, pid),
	147	+
	148	+ TP_STRUCT__entry(
	149	+ __field(char *, rcuname)
	150	+ __field(unsigned long, gpnum)
	151	+ __field(int, pid)
	152	+ ),
	153	+
	154	+ TP_fast_assign(
	155	+ __entry->rcuname = rcuname;
	156	+ __entry->gpnum = gpnum;
	157	+ __entry->pid = pid;
	158	+ ),
	159	+
	160	+ TP_printk("%s %lu %d", __entry->rcuname, __entry->gpnum, __entry->pid)
	161	+);
	162	+
	163	+/*
	164	+ * Tracepoint for quiescent-state-reporting events. These are
	165	+ * distinguished by the type of RCU, the grace-period number, the
	166	+ * mask of quiescent lower-level entities, the rcu_node structure level,
	167	+ * the starting and ending CPU covered by the rcu_node structure, and
	168	+ * whether there are any blocked tasks blocking the current grace period.
	169	+ * All but the type of RCU are extracted from the rcu_node structure.
	170	+ */
	171	+TRACE_EVENT(rcu_quiescent_state_report,
	172	+
	173	+ TP_PROTO(char *rcuname, unsigned long gpnum,
	174	+ unsigned long mask, unsigned long qsmask,
	175	+ u8 level, int grplo, int grphi, int gp_tasks),
	176	+
	177	+ TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
	178	+
	179	+ TP_STRUCT__entry(
	180	+ __field(char *, rcuname)
	181	+ __field(unsigned long, gpnum)
	182	+ __field(unsigned long, mask)
	183	+ __field(unsigned long, qsmask)
	184	+ __field(u8, level)
	185	+ __field(int, grplo)
	186	+ __field(int, grphi)
	187	+ __field(u8, gp_tasks)
	188	+ ),
	189	+
	190	+ TP_fast_assign(
	191	+ __entry->rcuname = rcuname;
	192	+ __entry->gpnum = gpnum;
	193	+ __entry->mask = mask;
	194	+ __entry->qsmask = qsmask;
	195	+ __entry->level = level;
	196	+ __entry->grplo = grplo;
	197	+ __entry->grphi = grphi;
	198	+ __entry->gp_tasks = gp_tasks;
	199	+ ),
	200	+
	201	+ TP_printk("%s %lu %lx>%lx %u %d %d %u",
	202	+ __entry->rcuname, __entry->gpnum,
	203	+ __entry->mask, __entry->qsmask, __entry->level,
	204	+ __entry->grplo, __entry->grphi, __entry->gp_tasks)
	205	+);
	206	+
	207	+/*
	208	+ * Tracepoint for quiescent states detected by force_quiescent_state().
	209	+ * These trace events include the type of RCU, the grace-period number
	210	+ * that was blocked by the CPU, the CPU itself, and the type of quiescent
	211	+ * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
	212	+ * or "kick" when kicking a CPU that has been in dyntick-idle mode for
	213	+ * too long.
	214	+ */
	215	+TRACE_EVENT(rcu_fqs,
	216	+
	217	+ TP_PROTO(char rcuname, unsigned long gpnum, int cpu, char qsevent),
	218	+
	219	+ TP_ARGS(rcuname, gpnum, cpu, qsevent),
	220	+
	221	+ TP_STRUCT__entry(
	222	+ __field(char *, rcuname)
	223	+ __field(unsigned long, gpnum)
	224	+ __field(int, cpu)
	225	+ __field(char *, qsevent)
	226	+ ),
	227	+
	228	+ TP_fast_assign(
	229	+ __entry->rcuname = rcuname;
	230	+ __entry->gpnum = gpnum;
	231	+ __entry->cpu = cpu;
	232	+ __entry->qsevent = qsevent;
	233	+ ),
	234	+
	235	+ TP_printk("%s %lu %d %s",
	236	+ __entry->rcuname, __entry->gpnum,
	237	+ __entry->cpu, __entry->qsevent)
	238	+);
	239	+
	240	+#endif /* #if defined(CONFIG_TREE_RCU) \|\| defined(CONFIG_TREE_PREEMPT_RCU) */
	241	+
	242	+/*
	243	+ * Tracepoint for dyntick-idle entry/exit events. These take a string
	244	+ * as argument: "Start" for entering dyntick-idle mode and "End" for
	245	+ * leaving it.
	246	+ */
	247	+TRACE_EVENT(rcu_dyntick,
	248	+
	249	+ TP_PROTO(char *polarity),
	250	+
	251	+ TP_ARGS(polarity),
	252	+
	253	+ TP_STRUCT__entry(
	254	+ __field(char *, polarity)
	255	+ ),
	256	+
	257	+ TP_fast_assign(
	258	+ __entry->polarity = polarity;
	259	+ ),
	260	+
	261	+ TP_printk("%s", __entry->polarity)
	262	+);
	263	+
	264	+/*
	265	+ * Tracepoint for the registration of a single RCU callback function.
	266	+ * The first argument is the type of RCU, the second argument is
	267	+ * a pointer to the RCU callback itself, and the third element is the
	268	+ * new RCU callback queue length for the current CPU.
	269	+ */
	270	+TRACE_EVENT(rcu_callback,
	271	+
	272	+ TP_PROTO(char rcuname, struct rcu_head rhp, long qlen),
	273	+
	274	+ TP_ARGS(rcuname, rhp, qlen),
	275	+
	276	+ TP_STRUCT__entry(
	277	+ __field(char *, rcuname)
	278	+ __field(void *, rhp)
	279	+ __field(void *, func)
	280	+ __field(long, qlen)
	281	+ ),
	282	+
	283	+ TP_fast_assign(
	284	+ __entry->rcuname = rcuname;
	285	+ __entry->rhp = rhp;
	286	+ __entry->func = rhp->func;
	287	+ __entry->qlen = qlen;
	288	+ ),
	289	+
	290	+ TP_printk("%s rhp=%p func=%pf %ld",
	291	+ __entry->rcuname, __entry->rhp, __entry->func, __entry->qlen)
	292	+);
	293	+
	294	+/*
	295	+ * Tracepoint for the registration of a single RCU callback of the special
	296	+ * kfree() form. The first argument is the RCU type, the second argument
	297	+ * is a pointer to the RCU callback, the third argument is the offset
	298	+ * of the callback within the enclosing RCU-protected data structure,
	299	+ * and the fourth argument is the new RCU callback queue length for the
	300	+ * current CPU.
	301	+ */
	302	+TRACE_EVENT(rcu_kfree_callback,
	303	+
	304	+ TP_PROTO(char rcuname, struct rcu_head rhp, unsigned long offset,
	305	+ long qlen),
	306	+
	307	+ TP_ARGS(rcuname, rhp, offset, qlen),
	308	+
	309	+ TP_STRUCT__entry(
	310	+ __field(char *, rcuname)
	311	+ __field(void *, rhp)
	312	+ __field(unsigned long, offset)
	313	+ __field(long, qlen)
	314	+ ),
	315	+
	316	+ TP_fast_assign(
	317	+ __entry->rcuname = rcuname;
	318	+ __entry->rhp = rhp;
	319	+ __entry->offset = offset;
	320	+ __entry->qlen = qlen;
	321	+ ),
	322	+
	323	+ TP_printk("%s rhp=%p func=%ld %ld",
	324	+ __entry->rcuname, __entry->rhp, __entry->offset,
	325	+ __entry->qlen)
	326	+);
	327	+
	328	+/*
38	329	* Tracepoint for marking the beginning rcu_do_batch, performed to start
39	330	* RCU callback invocation. The first argument is the RCU flavor,
40	331	* the second is the total number of callbacks (including those that
41	332
42	333
43	334
44	335
45	336
46	337
47	338
48	339
49	340
50	341
51	342
...	...	@@ -65,50 +356,58 @@
65	356
66	357	/*
67	358	* Tracepoint for the invocation of a single RCU callback function.
68		- * The argument is a pointer to the RCU callback itself.
	359	+ * The first argument is the type of RCU, and the second argument is
	360	+ * a pointer to the RCU callback itself.
69	361	*/
70	362	TRACE_EVENT(rcu_invoke_callback,
71	363
72		- TP_PROTO(struct rcu_head *rhp),
	364	+ TP_PROTO(char rcuname, struct rcu_head rhp),
73	365
74		- TP_ARGS(rhp),
	366	+ TP_ARGS(rcuname, rhp),
75	367
76	368	TP_STRUCT__entry(
77		- __field(void *, rhp)
78		- __field(void *, func)
	369	+ __field(char *, rcuname)
	370	+ __field(void *, rhp)
	371	+ __field(void *, func)
79	372	),
80	373
81	374	TP_fast_assign(
	375	+ __entry->rcuname = rcuname;
82	376	__entry->rhp = rhp;
83	377	__entry->func = rhp->func;
84	378	),
85	379
86		- TP_printk("rhp=%p func=%pf", __entry->rhp, __entry->func)
	380	+ TP_printk("%s rhp=%p func=%pf",
	381	+ __entry->rcuname, __entry->rhp, __entry->func)
87	382	);
88	383
89	384	/*
90	385	* Tracepoint for the invocation of a single RCU callback of the special
91		- * kfree() form. The first argument is a pointer to the RCU callback
92		- * and the second argument is the offset of the callback within the
93		- * enclosing RCU-protected data structure.
	386	+ * kfree() form. The first argument is the RCU flavor, the second
	387	+ * argument is a pointer to the RCU callback, and the third argument
	388	+ * is the offset of the callback within the enclosing RCU-protected
	389	+ * data structure.
94	390	*/
95	391	TRACE_EVENT(rcu_invoke_kfree_callback,
96	392
97		- TP_PROTO(struct rcu_head *rhp, unsigned long offset),
	393	+ TP_PROTO(char rcuname, struct rcu_head rhp, unsigned long offset),
98	394
99		- TP_ARGS(rhp, offset),
	395	+ TP_ARGS(rcuname, rhp, offset),
100	396
101	397	TP_STRUCT__entry(
102		- __field(void *, rhp)
	398	+ __field(char *, rcuname)
	399	+ __field(void *, rhp)
103	400	__field(unsigned long, offset)
104	401	),
105	402
106	403	TP_fast_assign(
	404	+ __entry->rcuname = rcuname;
107	405	__entry->rhp = rhp;
108	406	__entry->offset = offset;
109	407	),
110	408
111		- TP_printk("rhp=%p func=%ld", __entry->rhp, __entry->offset)
	409	+ TP_printk("%s rhp=%p func=%ld",
	410	+ __entry->rcuname, __entry->rhp, __entry->offset)
112	411	);
113	412
114	413	/*
...	...	@@ -135,6 +434,24 @@
135	434	TP_printk("%s CBs-invoked=%d",
136	435	__entry->rcuname, __entry->callbacks_invoked)
137	436	);
	437	+
	438	+#else /* #ifdef CONFIG_RCU_TRACE */
	439	+
	440	+#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
	441	+#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, qsmask) do { } while (0)
	442	+#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
	443	+#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
	444	+#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks) do { } while (0)
	445	+#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
	446	+#define trace_rcu_dyntick(polarity) do { } while (0)
	447	+#define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)
	448	+#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)
	449	+#define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
	450	+#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
	451	+#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
	452	+#define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
	453	+
	454	+#endif /* #else #ifdef CONFIG_RCU_TRACE */
138	455
139	456	#endif /* _TRACE_RCU_H */
140	457
...	...	@@ -69,15 +69,15 @@
69	69
70	70	extern void kfree(const void *);
71	71
72		-static inline void __rcu_reclaim(struct rcu_head *head)
	72	+static inline void __rcu_reclaim(char rn, struct rcu_head head)
73	73	{
74	74	unsigned long offset = (unsigned long)head->func;
75	75
76	76	if (__is_kfree_rcu_offset(offset)) {
77		- RCU_TRACE(trace_rcu_invoke_kfree_callback(head, offset));
	77	+ RCU_TRACE(trace_rcu_invoke_kfree_callback(rn, head, offset));
78	78	kfree((void *)head - offset);
79	79	} else {
80		- RCU_TRACE(trace_rcu_invoke_callback(head));
	80	+ RCU_TRACE(trace_rcu_invoke_callback(rn, head));
81	81	head->func(head);
82	82	}
83	83	}
...	...	@@ -147,6 +147,7 @@
147	147	*/
148	148	static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
149	149	{
	150	+ char *rn = NULL;
150	151	struct rcu_head next, list;
151	152	unsigned long flags;
152	153	RCU_TRACE(int cb_count = 0);
153	154
...	...	@@ -171,12 +172,13 @@
171	172	local_irq_restore(flags);
172	173
173	174	/* Invoke the callbacks on the local list. */
	175	+ RCU_TRACE(rn = rcp->name);
174	176	while (list) {
175	177	next = list->next;
176	178	prefetch(next);
177	179	debug_rcu_head_unqueue(list);
178	180	local_bh_disable();
179		- __rcu_reclaim(list);
	181	+ __rcu_reclaim(rn, list);
180	182	local_bh_enable();
181	183	list = next;
182	184	RCU_TRACE(cb_count++);
...	...	@@ -166,6 +166,8 @@
166	166
167	167	rdp->passed_quiesc_completed = rdp->gpnum - 1;
168	168	barrier();
	169	+ if (rdp->passed_quiesc == 0)
	170	+ trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs");
169	171	rdp->passed_quiesc = 1;
170	172	}
171	173
...	...	@@ -175,6 +177,8 @@
175	177
176	178	rdp->passed_quiesc_completed = rdp->gpnum - 1;
177	179	barrier();
	180	+ if (rdp->passed_quiesc == 0)
	181	+ trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs");
178	182	rdp->passed_quiesc = 1;
179	183	}
180	184
...	...	@@ -319,6 +323,7 @@
319	323	* trust its state not to change because interrupts are disabled.
320	324	*/
321	325	if (cpu_is_offline(rdp->cpu)) {
	326	+ trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "ofl");
322	327	rdp->offline_fqs++;
323	328	return 1;
324	329	}
...	...	@@ -359,6 +364,7 @@
359	364	local_irq_restore(flags);
360	365	return;
361	366	}
	367	+ trace_rcu_dyntick("Start");
362	368	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
363	369	smp_mb__before_atomic_inc(); /* See above. */
364	370	atomic_inc(&rdtp->dynticks);
...	...	@@ -396,6 +402,7 @@
396	402	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
397	403	smp_mb__after_atomic_inc(); /* See above. */
398	404	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
	405	+ trace_rcu_dyntick("End");
399	406	local_irq_restore(flags);
400	407	}
401	408
...	...	@@ -501,6 +508,7 @@
501	508	* of the current RCU grace period.
502	509	*/
503	510	if ((curr & 0x1) == 0 \|\| ULONG_CMP_GE(curr, snap + 2)) {
	511	+ trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "dti");
504	512	rdp->dynticks_fqs++;
505	513	return 1;
506	514	}
...	...	@@ -683,6 +691,7 @@
683	691	* go looking for one.
684	692	*/
685	693	rdp->gpnum = rnp->gpnum;
	694	+ trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
686	695	if (rnp->qsmask & rdp->grpmask) {
687	696	rdp->qs_pending = 1;
688	697	rdp->passed_quiesc = 0;
...	...	@@ -746,6 +755,7 @@
746	755
747	756	/* Remember that we saw this grace-period completion. */
748	757	rdp->completed = rnp->completed;
	758	+ trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuend");
749	759
750	760	/*
751	761	* If we were in an extended quiescent state, we may have
...	...	@@ -856,6 +866,7 @@
856	866
857	867	/* Advance to a new grace period and initialize state. */
858	868	rsp->gpnum++;
	869	+ trace_rcu_grace_period(rsp->name, rsp->gpnum, "start");
859	870	WARN_ON_ONCE(rsp->signaled == RCU_GP_INIT);
860	871	rsp->signaled = RCU_GP_INIT; /* Hold off force_quiescent_state. */
861	872	rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
...	...	@@ -870,6 +881,9 @@
870	881	rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */
871	882	rcu_start_gp_per_cpu(rsp, rnp, rdp);
872	883	rcu_preempt_boost_start_gp(rnp);
	884	+ trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
	885	+ rnp->level, rnp->grplo,
	886	+ rnp->grphi, rnp->qsmask);
873	887	raw_spin_unlock_irqrestore(&rnp->lock, flags);
874	888	return;
875	889	}
...	...	@@ -906,6 +920,9 @@
906	920	if (rnp == rdp->mynode)
907	921	rcu_start_gp_per_cpu(rsp, rnp, rdp);
908	922	rcu_preempt_boost_start_gp(rnp);
	923	+ trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
	924	+ rnp->level, rnp->grplo,
	925	+ rnp->grphi, rnp->qsmask);
909	926	raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
910	927	}
911	928
...	...	@@ -939,6 +956,7 @@
939	956	if (gp_duration > rsp->gp_max)
940	957	rsp->gp_max = gp_duration;
941	958	rsp->completed = rsp->gpnum;
	959	+ trace_rcu_grace_period(rsp->name, rsp->completed, "end");
942	960	rsp->signaled = RCU_GP_IDLE;
943	961	rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */
944	962	}
...	...	@@ -967,6 +985,10 @@
967	985	return;
968	986	}
969	987	rnp->qsmask &= ~mask;
	988	+ trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,
	989	+ mask, rnp->qsmask, rnp->level,
	990	+ rnp->grplo, rnp->grphi,
	991	+ !!rnp->gp_tasks);
970	992	if (rnp->qsmask != 0 \|\| rcu_preempt_blocked_readers_cgp(rnp)) {
971	993
972	994	/* Other bits still set at this level, so done. */
973	995
974	996
...	...	@@ -1135,11 +1157,20 @@
1135	1157	if (rnp->qsmaskinit != 0) {
1136	1158	if (rnp != rdp->mynode)
1137	1159	raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
	1160	+ else
	1161	+ trace_rcu_grace_period(rsp->name,
	1162	+ rnp->gpnum + 1 -
	1163	+ !!(rnp->qsmask & mask),
	1164	+ "cpuofl");
1138	1165	break;
1139	1166	}
1140		- if (rnp == rdp->mynode)
	1167	+ if (rnp == rdp->mynode) {
	1168	+ trace_rcu_grace_period(rsp->name,
	1169	+ rnp->gpnum + 1 -
	1170	+ !!(rnp->qsmask & mask),
	1171	+ "cpuofl");
1141	1172	need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);
1142		- else
	1173	+ } else
1143	1174	raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
1144	1175	mask = rnp->grpmask;
1145	1176	rnp = rnp->parent;
...	...	@@ -1226,7 +1257,7 @@
1226	1257	next = list->next;
1227	1258	prefetch(next);
1228	1259	debug_rcu_head_unqueue(list);
1229		- __rcu_reclaim(list);
	1260	+ __rcu_reclaim(rsp->name, list);
1230	1261	list = next;
1231	1262	if (++count >= bl)
1232	1263	break;
...	...	@@ -1552,6 +1583,12 @@
1552	1583	rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
1553	1584	rdp->qlen++;
1554	1585
	1586	+ if (__is_kfree_rcu_offset((unsigned long)func))
	1587	+ trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
	1588	+ rdp->qlen);
	1589	+ else
	1590	+ trace_rcu_callback(rsp->name, head, rdp->qlen);
	1591	+
1555	1592	/* If interrupts were disabled, don't dive into RCU core. */
1556	1593	if (irqs_disabled_flags(flags)) {
1557	1594	local_irq_restore(flags);
...	...	@@ -1850,6 +1887,7 @@
1850	1887	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
1851	1888	#endif /* #ifdef CONFIG_NO_HZ */
1852	1889	rdp->cpu = cpu;
	1890	+ rdp->rsp = rsp;
1853	1891	raw_spin_unlock_irqrestore(&rnp->lock, flags);
1854	1892	}
1855	1893
...	...	@@ -1898,6 +1936,7 @@
1898	1936	rdp->gpnum = rnp->completed; /* if GP in progress... */
1899	1937	rdp->completed = rnp->completed;
1900	1938	rdp->passed_quiesc_completed = rnp->completed - 1;
	1939	+ trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpuonl");
1901	1940	}
1902	1941	raw_spin_unlock(&rnp->lock); /* irqs already disabled. */
1903	1942	rnp = rnp->parent;
...	...	@@ -299,6 +299,7 @@
299	299	unsigned long n_rp_need_nothing;
300	300
301	301	int cpu;
	302	+ struct rcu_state *rsp;
302	303	};
303	304
304	305	/* Values for signaled field in struct rcu_state. */
...	...	@@ -124,6 +124,8 @@
124	124
125	125	rdp->passed_quiesc_completed = rdp->gpnum - 1;
126	126	barrier();
	127	+ if (rdp->passed_quiesc == 0)
	128	+ trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
127	129	rdp->passed_quiesc = 1;
128	130	current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
129	131	}
...	...	@@ -190,6 +192,11 @@
190	192	if (rnp->qsmask & rdp->grpmask)
191	193	rnp->gp_tasks = &t->rcu_node_entry;
192	194	}
	195	+ trace_rcu_preempt_task(rdp->rsp->name,
	196	+ t->pid,
	197	+ (rnp->qsmask & rdp->grpmask)
	198	+ ? rnp->gpnum
	199	+ : rnp->gpnum + 1);
193	200	raw_spin_unlock_irqrestore(&rnp->lock, flags);
194	201	} else if (t->rcu_read_lock_nesting < 0 &&
195	202	t->rcu_read_unlock_special) {
...	...	@@ -344,6 +351,8 @@
344	351	smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
345	352	np = rcu_next_node_entry(t, rnp);
346	353	list_del_init(&t->rcu_node_entry);
	354	+ trace_rcu_unlock_preempted_task("rcu_preempt",
	355	+ rnp->gpnum, t->pid);
347	356	if (&t->rcu_node_entry == rnp->gp_tasks)
348	357	rnp->gp_tasks = np;
349	358	if (&t->rcu_node_entry == rnp->exp_tasks)
350	359
...	...	@@ -364,10 +373,17 @@
364	373	* we aren't waiting on any CPUs, report the quiescent state.
365	374	* Note that rcu_report_unblock_qs_rnp() releases rnp->lock.
366	375	*/
367		- if (empty)
368		- raw_spin_unlock_irqrestore(&rnp->lock, flags);
369		- else
	376	+ if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
	377	+ trace_rcu_quiescent_state_report("preempt_rcu",
	378	+ rnp->gpnum,
	379	+ 0, rnp->qsmask,
	380	+ rnp->level,
	381	+ rnp->grplo,
	382	+ rnp->grphi,
	383	+ !!rnp->gp_tasks);
370	384	rcu_report_unblock_qs_rnp(rnp, flags);
	385	+ } else
	386	+ raw_spin_unlock_irqrestore(&rnp->lock, flags);
371	387
372	388	#ifdef CONFIG_RCU_BOOST
373	389	/* Unboost if we were boosted. */