percpu, module: implement and use is_kernel/module_percpu_address()

lockdep has custom code to check whether a pointer belongs to static percpu area which is somewhat broken. Implement proper is_kernel/module_percpu_address() and replace the custom code. On UP, percpu variables are regular static variables and can't be distinguished from them. Always return %false on UP. Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Ingo Molnar <mingo@redhat.com>

percpu, module: implement and use is_kernel/module_percpu_address()
lockdep has custom code to check whether a pointer belongs to static percpu area which is somewhat broken. Implement proper is_kernel/module_percpu_address() and replace the custom code. On UP, percpu variables are regular static variables and can't be distinguished from them. Always return %false on UP. Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Ingo Molnar <mingo@redhat.com>
Tejun Heo
1 parent 259354deaa
Showing 5 changed files with 77 additions and 16 deletions Side-by-side Diff
include/linux/module.h
include/linux/percpu.h
kernel/lockdep.c
kernel/module.c
mm/percpu.c
@@ -395,6 +395,7 @@
 struct module *__module_text_address(unsigned long addr);
 struct module *__module_address(unsigned long addr);
 bool is_module_address(unsigned long addr);
+bool is_module_percpu_address(unsigned long addr);
 bool is_module_text_address(unsigned long addr);
  
 static inline int within_module_core(unsigned long addr, struct module *mod)
@@ -137,6 +137,7 @@
 extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
 extern void __percpu *__alloc_percpu(size_t size, size_t align);
 extern void free_percpu(void __percpu *__pdata);
+extern bool is_kernel_percpu_address(unsigned long addr);
 extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
  
 #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
@@ -161,6 +162,12 @@
 static inline void free_percpu(void __percpu *p)
 {
 	kfree(p);
+}
+
+/* can't distinguish from other static vars, always false */
+static inline bool is_kernel_percpu_address(unsigned long addr)
+{
+	return false;
 }
  
 static inline phys_addr_t per_cpu_ptr_to_phys(void *addr)
@@ -582,9 +582,6 @@
 	unsigned long start = (unsigned long) &_stext,
 		      end   = (unsigned long) &_end,
 		      addr  = (unsigned long) obj;
-#ifdef CONFIG_SMP
-	int i;
-#endif
  
 	/*
 	 * static variable?
  
  
  
  
  
@@ -595,24 +592,16 @@
 	if (arch_is_kernel_data(addr))
 		return 1;
  
-#ifdef CONFIG_SMP
 	/*
-	 * percpu var?
+	 * in-kernel percpu var?
 	 */
-	for_each_possible_cpu(i) {
-		start = (unsigned long) &__per_cpu_start + per_cpu_offset(i);
-		end   = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM
-					+ per_cpu_offset(i);
+	if (is_kernel_percpu_address(addr))
+		return 1;
  
-		if ((addr >= start) && (addr < end))
-			return 1;
-	}
-#endif
-
 	/*
-	 * module var?
+	 * module static or percpu var?
 	 */
-	return is_module_address(addr);
+	return is_module_address(addr) || is_module_percpu_address(addr);
 }
  
 /*
@@ -415,6 +415,40 @@
 		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
 }
  
+/**
+ * is_module_percpu_address - test whether address is from module static percpu
+ * @addr: address to test
+ *
+ * Test whether @addr belongs to module static percpu area.
+ *
+ * RETURNS:
+ * %true if @addr is from module static percpu area
+ */
+bool is_module_percpu_address(unsigned long addr)
+{
+	struct module *mod;
+	unsigned int cpu;
+
+	preempt_disable();
+
+	list_for_each_entry_rcu(mod, &modules, list) {
+		if (!mod->percpu_size)
+			continue;
+		for_each_possible_cpu(cpu) {
+			void *start = per_cpu_ptr(mod->percpu, cpu);
+
+			if ((void *)addr >= start &&
+			    (void *)addr < start + mod->percpu_size) {
+				preempt_enable();
+				return true;
+			}
+		}
+	}
+
+	preempt_enable();
+	return false;
+}
+
 #else /* ... !CONFIG_SMP */
  
 static inline void __percpu *mod_percpu(struct module *mod)
@@ -440,6 +474,10 @@
 {
 	/* pcpusec should be 0, and size of that section should be 0. */
 	BUG_ON(size != 0);
+}
+bool is_module_percpu_address(unsigned long addr)
+{
+	return false;
 }
  
 #endif /* CONFIG_SMP */
@@ -1304,6 +1304,32 @@
 EXPORT_SYMBOL_GPL(free_percpu);
  
 /**
+ * is_kernel_percpu_address - test whether address is from static percpu area
+ * @addr: address to test
+ *
+ * Test whether @addr belongs to in-kernel static percpu area.  Module
+ * static percpu areas are not considered.  For those, use
+ * is_module_percpu_address().
+ *
+ * RETURNS:
+ * %true if @addr is from in-kernel static percpu area, %false otherwise.
+ */
+bool is_kernel_percpu_address(unsigned long addr)
+{
+	const size_t static_size = __per_cpu_end - __per_cpu_start;
+	void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
+	unsigned int cpu;
+
+	for_each_possible_cpu(cpu) {
+		void *start = per_cpu_ptr(base, cpu);
+
+		if ((void *)addr >= start && (void *)addr < start + static_size)
+			return true;
+        }
+	return false;
+}
+
+/**
  * per_cpu_ptr_to_phys - convert translated percpu address to physical address
  * @addr: the address to be converted to physical address
  *
...	...	@@ -395,6 +395,7 @@
395	395	struct module *__module_text_address(unsigned long addr);
396	396	struct module *__module_address(unsigned long addr);
397	397	bool is_module_address(unsigned long addr);
	398	+bool is_module_percpu_address(unsigned long addr);
398	399	bool is_module_text_address(unsigned long addr);
399	400
400	401	static inline int within_module_core(unsigned long addr, struct module *mod)
...	...	@@ -137,6 +137,7 @@
137	137	extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
138	138	extern void __percpu *__alloc_percpu(size_t size, size_t align);
139	139	extern void free_percpu(void __percpu *__pdata);
	140	+extern bool is_kernel_percpu_address(unsigned long addr);
140	141	extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
141	142
142	143	#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
...	...	@@ -161,6 +162,12 @@
161	162	static inline void free_percpu(void __percpu *p)
162	163	{
163	164	kfree(p);
	165	+}
	166	+
	167	+/* can't distinguish from other static vars, always false */
	168	+static inline bool is_kernel_percpu_address(unsigned long addr)
	169	+{
	170	+ return false;
164	171	}
165	172
166	173	static inline phys_addr_t per_cpu_ptr_to_phys(void *addr)
...	...	@@ -582,9 +582,6 @@
582	582	unsigned long start = (unsigned long) &_stext,
583	583	end = (unsigned long) &_end,
584	584	addr = (unsigned long) obj;
585		-#ifdef CONFIG_SMP
586		- int i;
587		-#endif
588	585
589	586	/*
590	587	* static variable?
591	588
592	589
593	590
594	591
595	592
...	...	@@ -595,24 +592,16 @@
595	592	if (arch_is_kernel_data(addr))
596	593	return 1;
597	594
598		-#ifdef CONFIG_SMP
599	595	/*
600		- * percpu var?
	596	+ * in-kernel percpu var?
601	597	*/
602		- for_each_possible_cpu(i) {
603		- start = (unsigned long) &__per_cpu_start + per_cpu_offset(i);
604		- end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM
605		- + per_cpu_offset(i);
	598	+ if (is_kernel_percpu_address(addr))
	599	+ return 1;
606	600
607		- if ((addr >= start) && (addr < end))
608		- return 1;
609		- }
610		-#endif
611		-
612	601	/*
613		- * module var?
	602	+ * module static or percpu var?
614	603	*/
615		- return is_module_address(addr);
	604	+ return is_module_address(addr) \|\| is_module_percpu_address(addr);
616	605	}
617	606
618	607	/*
...	...	@@ -415,6 +415,40 @@
415	415	memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
416	416	}
417	417
	418	+/**
	419	+ * is_module_percpu_address - test whether address is from module static percpu
	420	+ * @addr: address to test
	421	+ *
	422	+ * Test whether @addr belongs to module static percpu area.
	423	+ *
	424	+ * RETURNS:
	425	+ * %true if @addr is from module static percpu area
	426	+ */
	427	+bool is_module_percpu_address(unsigned long addr)
	428	+{
	429	+ struct module *mod;
	430	+ unsigned int cpu;
	431	+
	432	+ preempt_disable();
	433	+
	434	+ list_for_each_entry_rcu(mod, &modules, list) {
	435	+ if (!mod->percpu_size)
	436	+ continue;
	437	+ for_each_possible_cpu(cpu) {
	438	+ void *start = per_cpu_ptr(mod->percpu, cpu);
	439	+
	440	+ if ((void *)addr >= start &&
	441	+ (void *)addr < start + mod->percpu_size) {
	442	+ preempt_enable();
	443	+ return true;
	444	+ }
	445	+ }
	446	+ }
	447	+
	448	+ preempt_enable();
	449	+ return false;
	450	+}
	451	+
418	452	#else /* ... !CONFIG_SMP */
419	453
420	454	static inline void __percpu mod_percpu(struct module mod)
...	...	@@ -440,6 +474,10 @@
440	474	{
441	475	/* pcpusec should be 0, and size of that section should be 0. */
442	476	BUG_ON(size != 0);
	477	+}
	478	+bool is_module_percpu_address(unsigned long addr)
	479	+{
	480	+ return false;
443	481	}
444	482
445	483	#endif /* CONFIG_SMP */
...	...	@@ -1304,6 +1304,32 @@
1304	1304	EXPORT_SYMBOL_GPL(free_percpu);
1305	1305
1306	1306	/**
	1307	+ * is_kernel_percpu_address - test whether address is from static percpu area
	1308	+ * @addr: address to test
	1309	+ *
	1310	+ * Test whether @addr belongs to in-kernel static percpu area. Module
	1311	+ * static percpu areas are not considered. For those, use
	1312	+ * is_module_percpu_address().
	1313	+ *
	1314	+ * RETURNS:
	1315	+ * %true if @addr is from in-kernel static percpu area, %false otherwise.
	1316	+ */
	1317	+bool is_kernel_percpu_address(unsigned long addr)
	1318	+{
	1319	+ const size_t static_size = __per_cpu_end - __per_cpu_start;
	1320	+ void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
	1321	+ unsigned int cpu;
	1322	+
	1323	+ for_each_possible_cpu(cpu) {
	1324	+ void *start = per_cpu_ptr(base, cpu);
	1325	+
	1326	+ if ((void )addr >= start && (void )addr < start + static_size)
	1327	+ return true;
	1328	+ }
	1329	+ return false;
	1330	+}
	1331	+
	1332	+/**
1307	1333	* per_cpu_ptr_to_phys - convert translated percpu address to physical address
1308	1334	* @addr: the address to be converted to physical address
1309	1335	*