slab, lockdep: Annotate slab -> rcu -> debug_object -> slab

Lockdep thinks there's lock recursion through: kmem_cache_free() cache_flusharray() spin_lock(&l3->list_lock) <----------------. free_block() | slab_destroy() | call_rcu() | debug_object_activate() | debug_object_init() | __debug_object_init() | kmem_cache_alloc() | cache_alloc_refill() | spin_lock(&l3->list_lock) --' Now debug objects doesn't use SLAB_DESTROY_BY_RCU and hence there is no actual possibility of recursing. Luckily debug objects marks it slab with SLAB_DEBUG_OBJECTS so we can identify the thing. Mark all SLAB_DEBUG_OBJECTS (all one!) slab caches with a special lockdep key so that lockdep sees its a different cachep. Also add a WARN on trying to create a SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS cache, to avoid possible future trouble. Reported-and-tested-by: Sebastian Siewior <sebastian@breakpoint.cc> [ fixes to the initial patch ] Reported-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Pekka Enberg <penberg@kernel.org> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/1311341165.27400.58.camel@twins Signed-off-by: Ingo Molnar <mingo@elte.hu>

slab, lockdep: Annotate slab -> rcu -> debug_object -> slab
Lockdep thinks there's lock recursion through: kmem_cache_free() cache_flusharray() spin_lock(&l3->list_lock) <----------------. free_block() | slab_destroy() | call_rcu() | debug_object_activate() | debug_object_init() | __debug_object_init() | kmem_cache_alloc() | cache_alloc_refill() | spin_lock(&l3->list_lock) --' Now debug objects doesn't use SLAB_DESTROY_BY_RCU and hence there is no actual possibility of recursing. Luckily debug objects marks it slab with SLAB_DEBUG_OBJECTS so we can identify the thing. Mark all SLAB_DEBUG_OBJECTS (all one!) slab caches with a special lockdep key so that lockdep sees its a different cachep. Also add a WARN on trying to create a SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS cache, to avoid possible future trouble. Reported-and-tested-by: Sebastian Siewior <sebastian@breakpoint.cc> [ fixes to the initial patch ] Reported-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Pekka Enberg <penberg@kernel.org> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/1311341165.27400.58.camel@twins Signed-off-by: Ingo Molnar <mingo@elte.hu>
Peter Zijlstra · Ingo Molnar
1 parent 70a0686a72
Showing 1 changed file with 68 additions and 18 deletions Side-by-side Diff
mm/slab.c
@@ -622,6 +622,51 @@
 static struct lock_class_key on_slab_l3_key;
 static struct lock_class_key on_slab_alc_key;
  
+static struct lock_class_key debugobj_l3_key;
+static struct lock_class_key debugobj_alc_key;
+
+static void slab_set_lock_classes(struct kmem_cache *cachep,
+		struct lock_class_key *l3_key, struct lock_class_key *alc_key,
+		int q)
+{
+	struct array_cache **alc;
+	struct kmem_list3 *l3;
+	int r;
+
+	l3 = cachep->nodelists[q];
+	if (!l3)
+		return;
+
+	lockdep_set_class(&l3->list_lock, l3_key);
+	alc = l3->alien;
+	/*
+	 * FIXME: This check for BAD_ALIEN_MAGIC
+	 * should go away when common slab code is taught to
+	 * work even without alien caches.
+	 * Currently, non NUMA code returns BAD_ALIEN_MAGIC
+	 * for alloc_alien_cache,
+	 */
+	if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
+		return;
+	for_each_node(r) {
+		if (alc[r])
+			lockdep_set_class(&alc[r]->lock, alc_key);
+	}
+}
+
+static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
+{
+	slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
+}
+
+static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
+{
+	int node;
+
+	for_each_online_node(node)
+		slab_set_debugobj_lock_classes_node(cachep, node);
+}
+
 static void init_node_lock_keys(int q)
 {
 	struct cache_sizes *s = malloc_sizes;
  
  
@@ -630,29 +675,14 @@
 		return;
  
 	for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
-		struct array_cache **alc;
 		struct kmem_list3 *l3;
-		int r;
  
 		l3 = s->cs_cachep->nodelists[q];
 		if (!l3 || OFF_SLAB(s->cs_cachep))
 			continue;
-		lockdep_set_class(&l3->list_lock, &on_slab_l3_key);
-		alc = l3->alien;
-		/*
-		 * FIXME: This check for BAD_ALIEN_MAGIC
-		 * should go away when common slab code is taught to
-		 * work even without alien caches.
-		 * Currently, non NUMA code returns BAD_ALIEN_MAGIC
-		 * for alloc_alien_cache,
-		 */
-		if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
-			continue;
-		for_each_node(r) {
-			if (alc[r])
-				lockdep_set_class(&alc[r]->lock,
-					&on_slab_alc_key);
-		}
+
+		slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
+				&on_slab_alc_key, q);
 	}
 }
  
@@ -671,6 +701,14 @@
 static inline void init_lock_keys(void)
 {
 }
+
+static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
+{
+}
+
+static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
+{
+}
 #endif
  
 /*
@@ -1264,6 +1302,8 @@
 		spin_unlock_irq(&l3->list_lock);
 		kfree(shared);
 		free_alien_cache(alien);
+		if (cachep->flags & SLAB_DEBUG_OBJECTS)
+			slab_set_debugobj_lock_classes_node(cachep, node);
 	}
 	init_node_lock_keys(node);
  
@@ -2424,6 +2464,16 @@
 		__kmem_cache_destroy(cachep);
 		cachep = NULL;
 		goto oops;
+	}
+
+	if (flags & SLAB_DEBUG_OBJECTS) {
+		/*
+		 * Would deadlock through slab_destroy()->call_rcu()->
+		 * debug_object_activate()->kmem_cache_alloc().
+		 */
+		WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU);
+
+		slab_set_debugobj_lock_classes(cachep);
 	}
  
 	/* cache setup completed, link it into the list */
...	...	@@ -622,6 +622,51 @@
622	622	static struct lock_class_key on_slab_l3_key;
623	623	static struct lock_class_key on_slab_alc_key;
624	624
	625	+static struct lock_class_key debugobj_l3_key;
	626	+static struct lock_class_key debugobj_alc_key;
	627	+
	628	+static void slab_set_lock_classes(struct kmem_cache *cachep,
	629	+ struct lock_class_key l3_key, struct lock_class_key alc_key,
	630	+ int q)
	631	+{
	632	+ struct array_cache **alc;
	633	+ struct kmem_list3 *l3;
	634	+ int r;
	635	+
	636	+ l3 = cachep->nodelists[q];
	637	+ if (!l3)
	638	+ return;
	639	+
	640	+ lockdep_set_class(&l3->list_lock, l3_key);
	641	+ alc = l3->alien;
	642	+ /*
	643	+ * FIXME: This check for BAD_ALIEN_MAGIC
	644	+ * should go away when common slab code is taught to
	645	+ * work even without alien caches.
	646	+ * Currently, non NUMA code returns BAD_ALIEN_MAGIC
	647	+ * for alloc_alien_cache,
	648	+ */
	649	+ if (!alc \|\| (unsigned long)alc == BAD_ALIEN_MAGIC)
	650	+ return;
	651	+ for_each_node(r) {
	652	+ if (alc[r])
	653	+ lockdep_set_class(&alc[r]->lock, alc_key);
	654	+ }
	655	+}
	656	+
	657	+static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
	658	+{
	659	+ slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
	660	+}
	661	+
	662	+static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
	663	+{
	664	+ int node;
	665	+
	666	+ for_each_online_node(node)
	667	+ slab_set_debugobj_lock_classes_node(cachep, node);
	668	+}
	669	+
625	670	static void init_node_lock_keys(int q)
626	671	{
627	672	struct cache_sizes *s = malloc_sizes;
628	673
629	674
...	...	@@ -630,29 +675,14 @@
630	675	return;
631	676
632	677	for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
633		- struct array_cache **alc;
634	678	struct kmem_list3 *l3;
635		- int r;
636	679
637	680	l3 = s->cs_cachep->nodelists[q];
638	681	if (!l3 \|\| OFF_SLAB(s->cs_cachep))
639	682	continue;
640		- lockdep_set_class(&l3->list_lock, &on_slab_l3_key);
641		- alc = l3->alien;
642		- /*
643		- * FIXME: This check for BAD_ALIEN_MAGIC
644		- * should go away when common slab code is taught to
645		- * work even without alien caches.
646		- * Currently, non NUMA code returns BAD_ALIEN_MAGIC
647		- * for alloc_alien_cache,
648		- */
649		- if (!alc \|\| (unsigned long)alc == BAD_ALIEN_MAGIC)
650		- continue;
651		- for_each_node(r) {
652		- if (alc[r])
653		- lockdep_set_class(&alc[r]->lock,
654		- &on_slab_alc_key);
655		- }
	683	+
	684	+ slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
	685	+ &on_slab_alc_key, q);
656	686	}
657	687	}
658	688
...	...	@@ -671,6 +701,14 @@
671	701	static inline void init_lock_keys(void)
672	702	{
673	703	}
	704	+
	705	+static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
	706	+{
	707	+}
	708	+
	709	+static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
	710	+{
	711	+}
674	712	#endif
675	713
676	714	/*
...	...	@@ -1264,6 +1302,8 @@
1264	1302	spin_unlock_irq(&l3->list_lock);
1265	1303	kfree(shared);
1266	1304	free_alien_cache(alien);
	1305	+ if (cachep->flags & SLAB_DEBUG_OBJECTS)
	1306	+ slab_set_debugobj_lock_classes_node(cachep, node);
1267	1307	}
1268	1308	init_node_lock_keys(node);
1269	1309
...	...	@@ -2424,6 +2464,16 @@
2424	2464	__kmem_cache_destroy(cachep);
2425	2465	cachep = NULL;
2426	2466	goto oops;
	2467	+ }
	2468	+
	2469	+ if (flags & SLAB_DEBUG_OBJECTS) {
	2470	+ /*
	2471	+ * Would deadlock through slab_destroy()->call_rcu()->
	2472	+ * debug_object_activate()->kmem_cache_alloc().
	2473	+ */
	2474	+ WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU);
	2475	+
	2476	+ slab_set_debugobj_lock_classes(cachep);
2427	2477	}
2428	2478
2429	2479	/* cache setup completed, link it into the list */