module: do percpu allocation after uniqueness check. No, really!

v3.8-rc1-5-g1fb9341 was supposed to stop parallel kvm loads exhausting percpu memory on large machines: Now we have a new state MODULE_STATE_UNFORMED, we can insert the module into the list (and thus guarantee its uniqueness) before we allocate the per-cpu region. In my defence, it didn't actually say the patch did this. Just that we "can". This patch actually *does* it. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> Tested-by: Jim Hull <jim.hull@hp.com> Cc: stable@kernel.org # 3.8

module: do percpu allocation after uniqueness check. No, really!
v3.8-rc1-5-g1fb9341 was supposed to stop parallel kvm loads exhausting percpu memory on large machines: Now we have a new state MODULE_STATE_UNFORMED, we can insert the module into the list (and thus guarantee its uniqueness) before we allocate the per-cpu region. In my defence, it didn't actually say the patch did this. Just that we "can". This patch actually *does* it. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> Tested-by: Jim Hull <jim.hull@hp.com> Cc: stable@kernel.org # 3.8
Rusty Russell
1 parent 54041d8a73
Showing 1 changed file with 18 additions and 16 deletions Side-by-side Diff
kernel/module.c
@@ -2940,7 +2940,6 @@
 {
 	/* Module within temporary copy. */
 	struct module *mod;
-	Elf_Shdr *pcpusec;
 	int err;
  
 	mod = setup_load_info(info, flags);
  
@@ -2955,17 +2954,10 @@
 	err = module_frob_arch_sections(info->hdr, info->sechdrs,
 					info->secstrings, mod);
 	if (err < 0)
-		goto out;
+		return ERR_PTR(err);
  
-	pcpusec = &info->sechdrs[info->index.pcpu];
-	if (pcpusec->sh_size) {
-		/* We have a special allocation for this section. */
-		err = percpu_modalloc(mod,
-				      pcpusec->sh_size, pcpusec->sh_addralign);
-		if (err)
-			goto out;
-		pcpusec->sh_flags &= ~(unsigned long)SHF_ALLOC;
-	}
+	/* We will do a special allocation for per-cpu sections later. */
+	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
  
 	/* Determine total sizes, and put offsets in sh_entsize.  For now
 	   this is done generically; there doesn't appear to be any
  
  
@@ -2976,17 +2968,22 @@
 	/* Allocate and move to the final place */
 	err = move_module(mod, info);
 	if (err)
-		goto free_percpu;
+		return ERR_PTR(err);
  
 	/* Module has been copied to its final place now: return it. */
 	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
 	kmemleak_load_module(mod, info);
 	return mod;
+}
  
-free_percpu:
-	percpu_modfree(mod);
-out:
-	return ERR_PTR(err);
+static int alloc_module_percpu(struct module *mod, struct load_info *info)
+{
+	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
+	if (!pcpusec->sh_size)
+		return 0;
+
+	/* We have a special allocation for this section. */
+	return percpu_modalloc(mod, pcpusec->sh_size, pcpusec->sh_addralign);
 }
  
 /* mod is no longer valid after this! */
@@ -3261,6 +3258,11 @@
 		add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_STILL_OK);
 	}
 #endif
+
+	/* To avoid stressing percpu allocator, do this once we're unique. */
+	err = alloc_module_percpu(mod, info);
+	if (err)
+		goto unlink_mod;
  
 	/* Now module is in final location, initialize linked lists, etc. */
 	err = module_unload_init(mod);
...	...	@@ -2940,7 +2940,6 @@
2940	2940	{
2941	2941	/* Module within temporary copy. */
2942	2942	struct module *mod;
2943		- Elf_Shdr *pcpusec;
2944	2943	int err;
2945	2944
2946	2945	mod = setup_load_info(info, flags);
2947	2946
...	...	@@ -2955,17 +2954,10 @@
2955	2954	err = module_frob_arch_sections(info->hdr, info->sechdrs,
2956	2955	info->secstrings, mod);
2957	2956	if (err < 0)
2958		- goto out;
	2957	+ return ERR_PTR(err);
2959	2958
2960		- pcpusec = &info->sechdrs[info->index.pcpu];
2961		- if (pcpusec->sh_size) {
2962		- /* We have a special allocation for this section. */
2963		- err = percpu_modalloc(mod,
2964		- pcpusec->sh_size, pcpusec->sh_addralign);
2965		- if (err)
2966		- goto out;
2967		- pcpusec->sh_flags &= ~(unsigned long)SHF_ALLOC;
2968		- }
	2959	+ /* We will do a special allocation for per-cpu sections later. */
	2960	+ info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
2969	2961
2970	2962	/* Determine total sizes, and put offsets in sh_entsize. For now
2971	2963	this is done generically; there doesn't appear to be any
2972	2964
2973	2965
...	...	@@ -2976,17 +2968,22 @@
2976	2968	/* Allocate and move to the final place */
2977	2969	err = move_module(mod, info);
2978	2970	if (err)
2979		- goto free_percpu;
	2971	+ return ERR_PTR(err);
2980	2972
2981	2973	/* Module has been copied to its final place now: return it. */
2982	2974	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2983	2975	kmemleak_load_module(mod, info);
2984	2976	return mod;
	2977	+}
2985	2978
2986		-free_percpu:
2987		- percpu_modfree(mod);
2988		-out:
2989		- return ERR_PTR(err);
	2979	+static int alloc_module_percpu(struct module mod, struct load_info info)
	2980	+{
	2981	+ Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
	2982	+ if (!pcpusec->sh_size)
	2983	+ return 0;
	2984	+
	2985	+ /* We have a special allocation for this section. */
	2986	+ return percpu_modalloc(mod, pcpusec->sh_size, pcpusec->sh_addralign);
2990	2987	}
2991	2988
2992	2989	/* mod is no longer valid after this! */
...	...	@@ -3261,6 +3258,11 @@
3261	3258	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_STILL_OK);
3262	3259	}
3263	3260	#endif
	3261	+
	3262	+ /* To avoid stressing percpu allocator, do this once we're unique. */
	3263	+ err = alloc_module_percpu(mod, info);
	3264	+ if (err)
	3265	+ goto unlink_mod;
3264	3266
3265	3267	/* Now module is in final location, initialize linked lists, etc. */
3266	3268	err = module_unload_init(mod);