tree/tiny rcu: Add debug RCU head objects

Helps finding racy users of call_rcu(), which results in hangs because list entries are overwritten and/or skipped. Changelog since v4: - Bissectability is now OK - Now generate a WARN_ON_ONCE() for non-initialized rcu_head passed to call_rcu(). Statically initialized objects are detected with object_is_static(). - Rename rcu_head_init_on_stack to init_rcu_head_on_stack. - Remove init_rcu_head() completely. Changelog since v3: - Include comments from Lai Jiangshan This new patch version is based on the debugobjects with the newly introduced "active state" tracker. Non-initialized entries are all considered as "statically initialized". An activation fixup (triggered by call_rcu()) takes care of performing the debug object initialization without issuing any warning. Since we cannot increase the size of struct rcu_head, I don't see much room to put an identifier for statically initialized rcu_head structures. So for now, we have to live without "activation without explicit init" detection. But the main purpose of this debug option is to detect double-activations (double call_rcu() use of a rcu_head before the callback is executed), which is correctly addressed here. This also detects potential internal RCU callback corruption, which would cause the callbacks to be executed twice. Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> CC: David S. Miller <davem@davemloft.net> CC: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> CC: akpm@linux-foundation.org CC: mingo@elte.hu CC: laijs@cn.fujitsu.com CC: dipankar@in.ibm.com CC: josh@joshtriplett.org CC: dvhltc@us.ibm.com CC: niv@us.ibm.com CC: tglx@linutronix.de CC: peterz@infradead.org CC: rostedt@goodmis.org CC: Valdis.Kletnieks@vt.edu CC: dhowells@redhat.com CC: eric.dumazet@gmail.com CC: Alexey Dobriyan <adobriyan@gmail.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>

tree/tiny rcu: Add debug RCU head objects
Helps finding racy users of call_rcu(), which results in hangs because list entries are overwritten and/or skipped. Changelog since v4: - Bissectability is now OK - Now generate a WARN_ON_ONCE() for non-initialized rcu_head passed to call_rcu(). Statically initialized objects are detected with object_is_static(). - Rename rcu_head_init_on_stack to init_rcu_head_on_stack. - Remove init_rcu_head() completely. Changelog since v3: - Include comments from Lai Jiangshan This new patch version is based on the debugobjects with the newly introduced "active state" tracker. Non-initialized entries are all considered as "statically initialized". An activation fixup (triggered by call_rcu()) takes care of performing the debug object initialization without issuing any warning. Since we cannot increase the size of struct rcu_head, I don't see much room to put an identifier for statically initialized rcu_head structures. So for now, we have to live without "activation without explicit init" detection. But the main purpose of this debug option is to detect double-activations (double call_rcu() use of a rcu_head before the callback is executed), which is correctly addressed here. This also detects potential internal RCU callback corruption, which would cause the callbacks to be executed twice. Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> CC: David S. Miller <davem@davemloft.net> CC: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> CC: akpm@linux-foundation.org CC: mingo@elte.hu CC: laijs@cn.fujitsu.com CC: dipankar@in.ibm.com CC: josh@joshtriplett.org CC: dvhltc@us.ibm.com CC: niv@us.ibm.com CC: tglx@linutronix.de CC: peterz@infradead.org CC: rostedt@goodmis.org CC: Valdis.Kletnieks@vt.edu CC: dhowells@redhat.com CC: eric.dumazet@gmail.com CC: Alexey Dobriyan <adobriyan@gmail.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Mathieu Desnoyers · Paul E. McKenney
1 parent 875352c942
Showing 5 changed files with 219 additions and 0 deletions Side-by-side Diff
include/linux/rcupdate.h
kernel/rcupdate.c
kernel/rcutiny.c
kernel/rcutree.c
lib/Kconfig.debug
@@ -40,6 +40,7 @@
 #include <linux/seqlock.h>
 #include <linux/lockdep.h>
 #include <linux/completion.h>
+#include <linux/debugobjects.h>
  
 #ifdef CONFIG_RCU_TORTURE_TEST
 extern int rcutorture_runnable; /* for sysctl */
@@ -79,6 +80,16 @@
        (ptr)->next = NULL; (ptr)->func = NULL; \
 } while (0)
  
+/*
+ * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
+ * initialization and destruction of rcu_head on the stack. rcu_head structures
+ * allocated dynamically in the heap or defined statically don't need any
+ * initialization.
+ */
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+extern void init_rcu_head_on_stack(struct rcu_head *head);
+extern void destroy_rcu_head_on_stack(struct rcu_head *head);
+#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 static inline void init_rcu_head_on_stack(struct rcu_head *head)
 {
 }
@@ -86,6 +97,7 @@
 static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
 {
 }
+#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
  
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  
@@ -516,6 +528,43 @@
  */
 extern void call_rcu_bh(struct rcu_head *head,
 			void (*func)(struct rcu_head *head));
+
+/*
+ * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
+ * by call_rcu() and rcu callback execution, and are therefore not part of the
+ * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.
+ */
+
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+# define STATE_RCU_HEAD_READY	0
+# define STATE_RCU_HEAD_QUEUED	1
+
+extern struct debug_obj_descr rcuhead_debug_descr;
+
+static inline void debug_rcu_head_queue(struct rcu_head *head)
+{
+	debug_object_activate(head, &rcuhead_debug_descr);
+	debug_object_active_state(head, &rcuhead_debug_descr,
+				  STATE_RCU_HEAD_READY,
+				  STATE_RCU_HEAD_QUEUED);
+}
+
+static inline void debug_rcu_head_unqueue(struct rcu_head *head)
+{
+	debug_object_active_state(head, &rcuhead_debug_descr,
+				  STATE_RCU_HEAD_QUEUED,
+				  STATE_RCU_HEAD_READY);
+	debug_object_deactivate(head, &rcuhead_debug_descr);
+}
+#else	/* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+static inline void debug_rcu_head_queue(struct rcu_head *head)
+{
+}
+
+static inline void debug_rcu_head_unqueue(struct rcu_head *head)
+{
+}
+#endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
  
 #endif /* __LINUX_RCUPDATE_H */
@@ -114,4 +114,164 @@
 }
 EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty);
 #endif /* #ifdef CONFIG_PROVE_RCU */
+
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+static inline void debug_init_rcu_head(struct rcu_head *head)
+{
+	debug_object_init(head, &rcuhead_debug_descr);
+}
+
+static inline void debug_rcu_head_free(struct rcu_head *head)
+{
+	debug_object_free(head, &rcuhead_debug_descr);
+}
+
+/*
+ * fixup_init is called when:
+ * - an active object is initialized
+ */
+static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)
+{
+	struct rcu_head *head = addr;
+
+	switch (state) {
+	case ODEBUG_STATE_ACTIVE:
+		/*
+		 * Ensure that queued callbacks are all executed.
+		 * If we detect that we are nested in a RCU read-side critical
+		 * section, we should simply fail, otherwise we would deadlock.
+		 */
+		if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
+		    irqs_disabled()) {
+			WARN_ON(1);
+			return 0;
+		}
+		rcu_barrier();
+		rcu_barrier_sched();
+		rcu_barrier_bh();
+		debug_object_init(head, &rcuhead_debug_descr);
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * fixup_activate is called when:
+ * - an active object is activated
+ * - an unknown object is activated (might be a statically initialized object)
+ * Activation is performed internally by call_rcu().
+ */
+static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
+{
+	struct rcu_head *head = addr;
+
+	switch (state) {
+
+	case ODEBUG_STATE_NOTAVAILABLE:
+		/*
+		 * This is not really a fixup. We just make sure that it is
+		 * tracked in the object tracker.
+		 */
+		debug_object_init(head, &rcuhead_debug_descr);
+		debug_object_activate(head, &rcuhead_debug_descr);
+		return 0;
+
+	case ODEBUG_STATE_ACTIVE:
+		/*
+		 * Ensure that queued callbacks are all executed.
+		 * If we detect that we are nested in a RCU read-side critical
+		 * section, we should simply fail, otherwise we would deadlock.
+		 */
+		if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
+		    irqs_disabled()) {
+			WARN_ON(1);
+			return 0;
+		}
+		rcu_barrier();
+		rcu_barrier_sched();
+		rcu_barrier_bh();
+		debug_object_activate(head, &rcuhead_debug_descr);
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * fixup_free is called when:
+ * - an active object is freed
+ */
+static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)
+{
+	struct rcu_head *head = addr;
+
+	switch (state) {
+	case ODEBUG_STATE_ACTIVE:
+		/*
+		 * Ensure that queued callbacks are all executed.
+		 * If we detect that we are nested in a RCU read-side critical
+		 * section, we should simply fail, otherwise we would deadlock.
+		 */
+#ifndef CONFIG_PREEMPT
+		WARN_ON(1);
+		return 0;
+#else
+		if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
+		    irqs_disabled()) {
+			WARN_ON(1);
+			return 0;
+		}
+		rcu_barrier();
+		rcu_barrier_sched();
+		rcu_barrier_bh();
+		debug_object_free(head, &rcuhead_debug_descr);
+		return 1;
+#endif
+	default:
+		return 0;
+	}
+}
+
+/**
+ * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects
+ * @head: pointer to rcu_head structure to be initialized
+ *
+ * This function informs debugobjects of a new rcu_head structure that
+ * has been allocated as an auto variable on the stack.  This function
+ * is not required for rcu_head structures that are statically defined or
+ * that are dynamically allocated on the heap.  This function has no
+ * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
+ */
+void init_rcu_head_on_stack(struct rcu_head *head)
+{
+	debug_object_init_on_stack(head, &rcuhead_debug_descr);
+}
+EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
+
+/**
+ * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects
+ * @head: pointer to rcu_head structure to be initialized
+ *
+ * This function informs debugobjects that an on-stack rcu_head structure
+ * is about to go out of scope.  As with init_rcu_head_on_stack(), this
+ * function is not required for rcu_head structures that are statically
+ * defined or that are dynamically allocated on the heap.  Also as with
+ * init_rcu_head_on_stack(), this function has no effect for
+ * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
+ */
+void destroy_rcu_head_on_stack(struct rcu_head *head)
+{
+	debug_object_free(head, &rcuhead_debug_descr);
+}
+EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
+
+struct debug_obj_descr rcuhead_debug_descr = {
+	.name = "rcu_head",
+	.fixup_init = rcuhead_fixup_init,
+	.fixup_activate = rcuhead_fixup_activate,
+	.fixup_free = rcuhead_fixup_free,
+};
+EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
+#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
@@ -169,6 +169,7 @@
 	while (list) {
 		next = list->next;
 		prefetch(next);
+		debug_rcu_head_unqueue(list);
 		list->func(list);
 		list = next;
 	}
@@ -211,6 +212,7 @@
 {
 	unsigned long flags;
  
+	debug_rcu_head_queue(head);
 	head->func = func;
 	head->next = NULL;
  
@@ -1112,6 +1112,7 @@
 	while (list) {
 		next = list->next;
 		prefetch(next);
+		debug_rcu_head_unqueue(list);
 		list->func(list);
 		list = next;
 		if (++count >= rdp->blimit)
@@ -1388,6 +1389,7 @@
 	unsigned long flags;
 	struct rcu_data *rdp;
  
+	debug_rcu_head_queue(head);
 	head->func = func;
 	head->next = NULL;
  
@@ -307,6 +307,12 @@
 	  work queue routines to track the life time of work objects and
 	  validate the work operations.
  
+config DEBUG_OBJECTS_RCU_HEAD
+	bool "Debug RCU callbacks objects"
+	depends on DEBUG_OBJECTS && PREEMPT
+	help
+	  Enable this to turn on debugging of RCU list heads (call_rcu() usage).
+
 config DEBUG_OBJECTS_ENABLE_DEFAULT
 	int "debug_objects bootup default value (0-1)"
         range 0 1
...	...	@@ -40,6 +40,7 @@
40	40	#include <linux/seqlock.h>
41	41	#include <linux/lockdep.h>
42	42	#include <linux/completion.h>
	43	+#include <linux/debugobjects.h>
43	44
44	45	#ifdef CONFIG_RCU_TORTURE_TEST
45	46	extern int rcutorture_runnable; /* for sysctl */
...	...	@@ -79,6 +80,16 @@
79	80	(ptr)->next = NULL; (ptr)->func = NULL; \
80	81	} while (0)
81	82
	83	+/*
	84	+ * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
	85	+ * initialization and destruction of rcu_head on the stack. rcu_head structures
	86	+ * allocated dynamically in the heap or defined statically don't need any
	87	+ * initialization.
	88	+ */
	89	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	90	+extern void init_rcu_head_on_stack(struct rcu_head *head);
	91	+extern void destroy_rcu_head_on_stack(struct rcu_head *head);
	92	+#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
82	93	static inline void init_rcu_head_on_stack(struct rcu_head *head)
83	94	{
84	95	}
...	...	@@ -86,6 +97,7 @@
86	97	static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
87	98	{
88	99	}
	100	+#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
89	101
90	102	#ifdef CONFIG_DEBUG_LOCK_ALLOC
91	103
...	...	@@ -516,6 +528,43 @@
516	528	*/
517	529	extern void call_rcu_bh(struct rcu_head *head,
518	530	void (func)(struct rcu_head head));
	531	+
	532	+/*
	533	+ * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
	534	+ * by call_rcu() and rcu callback execution, and are therefore not part of the
	535	+ * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.
	536	+ */
	537	+
	538	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	539	+# define STATE_RCU_HEAD_READY 0
	540	+# define STATE_RCU_HEAD_QUEUED 1
	541	+
	542	+extern struct debug_obj_descr rcuhead_debug_descr;
	543	+
	544	+static inline void debug_rcu_head_queue(struct rcu_head *head)
	545	+{
	546	+ debug_object_activate(head, &rcuhead_debug_descr);
	547	+ debug_object_active_state(head, &rcuhead_debug_descr,
	548	+ STATE_RCU_HEAD_READY,
	549	+ STATE_RCU_HEAD_QUEUED);
	550	+}
	551	+
	552	+static inline void debug_rcu_head_unqueue(struct rcu_head *head)
	553	+{
	554	+ debug_object_active_state(head, &rcuhead_debug_descr,
	555	+ STATE_RCU_HEAD_QUEUED,
	556	+ STATE_RCU_HEAD_READY);
	557	+ debug_object_deactivate(head, &rcuhead_debug_descr);
	558	+}
	559	+#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
	560	+static inline void debug_rcu_head_queue(struct rcu_head *head)
	561	+{
	562	+}
	563	+
	564	+static inline void debug_rcu_head_unqueue(struct rcu_head *head)
	565	+{
	566	+}
	567	+#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
519	568
520	569	#endif /* __LINUX_RCUPDATE_H */
...	...	@@ -114,4 +114,164 @@
114	114	}
115	115	EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty);
116	116	#endif /* #ifdef CONFIG_PROVE_RCU */
	117	+
	118	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	119	+static inline void debug_init_rcu_head(struct rcu_head *head)
	120	+{
	121	+ debug_object_init(head, &rcuhead_debug_descr);
	122	+}
	123	+
	124	+static inline void debug_rcu_head_free(struct rcu_head *head)
	125	+{
	126	+ debug_object_free(head, &rcuhead_debug_descr);
	127	+}
	128	+
	129	+/*
	130	+ * fixup_init is called when:
	131	+ * - an active object is initialized
	132	+ */
	133	+static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)
	134	+{
	135	+ struct rcu_head *head = addr;
	136	+
	137	+ switch (state) {
	138	+ case ODEBUG_STATE_ACTIVE:
	139	+ /*
	140	+ * Ensure that queued callbacks are all executed.
	141	+ * If we detect that we are nested in a RCU read-side critical
	142	+ * section, we should simply fail, otherwise we would deadlock.
	143	+ */
	144	+ if (rcu_preempt_depth() != 0 \|\| preempt_count() != 0 \|\|
	145	+ irqs_disabled()) {
	146	+ WARN_ON(1);
	147	+ return 0;
	148	+ }
	149	+ rcu_barrier();
	150	+ rcu_barrier_sched();
	151	+ rcu_barrier_bh();
	152	+ debug_object_init(head, &rcuhead_debug_descr);
	153	+ return 1;
	154	+ default:
	155	+ return 0;
	156	+ }
	157	+}
	158	+
	159	+/*
	160	+ * fixup_activate is called when:
	161	+ * - an active object is activated
	162	+ * - an unknown object is activated (might be a statically initialized object)
	163	+ * Activation is performed internally by call_rcu().
	164	+ */
	165	+static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
	166	+{
	167	+ struct rcu_head *head = addr;
	168	+
	169	+ switch (state) {
	170	+
	171	+ case ODEBUG_STATE_NOTAVAILABLE:
	172	+ /*
	173	+ * This is not really a fixup. We just make sure that it is
	174	+ * tracked in the object tracker.
	175	+ */
	176	+ debug_object_init(head, &rcuhead_debug_descr);
	177	+ debug_object_activate(head, &rcuhead_debug_descr);
	178	+ return 0;
	179	+
	180	+ case ODEBUG_STATE_ACTIVE:
	181	+ /*
	182	+ * Ensure that queued callbacks are all executed.
	183	+ * If we detect that we are nested in a RCU read-side critical
	184	+ * section, we should simply fail, otherwise we would deadlock.
	185	+ */
	186	+ if (rcu_preempt_depth() != 0 \|\| preempt_count() != 0 \|\|
	187	+ irqs_disabled()) {
	188	+ WARN_ON(1);
	189	+ return 0;
	190	+ }
	191	+ rcu_barrier();
	192	+ rcu_barrier_sched();
	193	+ rcu_barrier_bh();
	194	+ debug_object_activate(head, &rcuhead_debug_descr);
	195	+ return 1;
	196	+ default:
	197	+ return 0;
	198	+ }
	199	+}
	200	+
	201	+/*
	202	+ * fixup_free is called when:
	203	+ * - an active object is freed
	204	+ */
	205	+static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)
	206	+{
	207	+ struct rcu_head *head = addr;
	208	+
	209	+ switch (state) {
	210	+ case ODEBUG_STATE_ACTIVE:
	211	+ /*
	212	+ * Ensure that queued callbacks are all executed.
	213	+ * If we detect that we are nested in a RCU read-side critical
	214	+ * section, we should simply fail, otherwise we would deadlock.
	215	+ */
	216	+#ifndef CONFIG_PREEMPT
	217	+ WARN_ON(1);
	218	+ return 0;
	219	+#else
	220	+ if (rcu_preempt_depth() != 0 \|\| preempt_count() != 0 \|\|
	221	+ irqs_disabled()) {
	222	+ WARN_ON(1);
	223	+ return 0;
	224	+ }
	225	+ rcu_barrier();
	226	+ rcu_barrier_sched();
	227	+ rcu_barrier_bh();
	228	+ debug_object_free(head, &rcuhead_debug_descr);
	229	+ return 1;
	230	+#endif
	231	+ default:
	232	+ return 0;
	233	+ }
	234	+}
	235	+
	236	+/**
	237	+ * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects
	238	+ * @head: pointer to rcu_head structure to be initialized
	239	+ *
	240	+ * This function informs debugobjects of a new rcu_head structure that
	241	+ * has been allocated as an auto variable on the stack. This function
	242	+ * is not required for rcu_head structures that are statically defined or
	243	+ * that are dynamically allocated on the heap. This function has no
	244	+ * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
	245	+ */
	246	+void init_rcu_head_on_stack(struct rcu_head *head)
	247	+{
	248	+ debug_object_init_on_stack(head, &rcuhead_debug_descr);
	249	+}
	250	+EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
	251	+
	252	+/**
	253	+ * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects
	254	+ * @head: pointer to rcu_head structure to be initialized
	255	+ *
	256	+ * This function informs debugobjects that an on-stack rcu_head structure
	257	+ * is about to go out of scope. As with init_rcu_head_on_stack(), this
	258	+ * function is not required for rcu_head structures that are statically
	259	+ * defined or that are dynamically allocated on the heap. Also as with
	260	+ * init_rcu_head_on_stack(), this function has no effect for
	261	+ * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
	262	+ */
	263	+void destroy_rcu_head_on_stack(struct rcu_head *head)
	264	+{
	265	+ debug_object_free(head, &rcuhead_debug_descr);
	266	+}
	267	+EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
	268	+
	269	+struct debug_obj_descr rcuhead_debug_descr = {
	270	+ .name = "rcu_head",
	271	+ .fixup_init = rcuhead_fixup_init,
	272	+ .fixup_activate = rcuhead_fixup_activate,
	273	+ .fixup_free = rcuhead_fixup_free,
	274	+};
	275	+EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
	276	+#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
...	...	@@ -169,6 +169,7 @@
169	169	while (list) {
170	170	next = list->next;
171	171	prefetch(next);
	172	+ debug_rcu_head_unqueue(list);
172	173	list->func(list);
173	174	list = next;
174	175	}
...	...	@@ -211,6 +212,7 @@
211	212	{
212	213	unsigned long flags;
213	214
	215	+ debug_rcu_head_queue(head);
214	216	head->func = func;
215	217	head->next = NULL;
216	218
...	...	@@ -1112,6 +1112,7 @@
1112	1112	while (list) {
1113	1113	next = list->next;
1114	1114	prefetch(next);
	1115	+ debug_rcu_head_unqueue(list);
1115	1116	list->func(list);
1116	1117	list = next;
1117	1118	if (++count >= rdp->blimit)
...	...	@@ -1388,6 +1389,7 @@
1388	1389	unsigned long flags;
1389	1390	struct rcu_data *rdp;
1390	1391
	1392	+ debug_rcu_head_queue(head);
1391	1393	head->func = func;
1392	1394	head->next = NULL;
1393	1395
...	...	@@ -307,6 +307,12 @@
307	307	work queue routines to track the life time of work objects and
308	308	validate the work operations.
309	309
	310	+config DEBUG_OBJECTS_RCU_HEAD
	311	+ bool "Debug RCU callbacks objects"
	312	+ depends on DEBUG_OBJECTS && PREEMPT
	313	+ help
	314	+ Enable this to turn on debugging of RCU list heads (call_rcu() usage).
	315	+
310	316	config DEBUG_OBJECTS_ENABLE_DEFAULT
311	317	int "debug_objects bootup default value (0-1)"
312	318	range 0 1