Merge branch 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux…

…/kernel/git/jeremy/xen * 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen: jump-label: initialize jump-label subsystem much earlier x86/jump_label: add arch_jump_label_transform_static() s390/jump-label: add arch_jump_label_transform_static() jump_label: add arch_jump_label_transform_static() to optimise non-live code updates sparc/jump_label: drop arch_jump_label_text_poke_early() x86/jump_label: drop arch_jump_label_text_poke_early() jump_label: if a key has already been initialized, don't nop it out stop_machine: make stop_machine safe and efficient to call early jump_label: use proper atomic_t initializer Conflicts: - arch/x86/kernel/jump_label.c Added __init_or_module to arch_jump_label_text_poke_early vs removal of that function entirely - kernel/stop_machine.c same patch ("stop_machine: make stop_machine safe and efficient to call early") merged twice, with whitespace fix in one version

Merge branch 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux…
…/kernel/git/jeremy/xen * 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen: jump-label: initialize jump-label subsystem much earlier x86/jump_label: add arch_jump_label_transform_static() s390/jump-label: add arch_jump_label_transform_static() jump_label: add arch_jump_label_transform_static() to optimise non-live code updates sparc/jump_label: drop arch_jump_label_text_poke_early() x86/jump_label: drop arch_jump_label_text_poke_early() jump_label: if a key has already been initialized, don't nop it out stop_machine: make stop_machine safe and efficient to call early jump_label: use proper atomic_t initializer Conflicts: - arch/x86/kernel/jump_label.c Added __init_or_module to arch_jump_label_text_poke_early vs removal of that function entirely - kernel/stop_machine.c same patch ("stop_machine: make stop_machine safe and efficient to call early") merged twice, with whitespace fix in one version
Linus Torvalds
2 parents 403299a851 97ce2c88f9
Showing 6 changed files Side-by-side Diff
arch/s390/kernel/jump_label.c
arch/sparc/kernel/jump_label.c
arch/x86/kernel/jump_label.c
include/linux/jump_label.h
init/main.c
kernel/jump_label.c
@@ -18,27 +18,16 @@
 } __packed;
  
 struct insn_args {
-	unsigned long *target;
-	struct insn *insn;
-	ssize_t size;
+	struct jump_entry *entry;
+	enum jump_label_type type;
 };
  
-static int __arch_jump_label_transform(void *data)
+static void __jump_label_transform(struct jump_entry *entry,
+				   enum jump_label_type type)
 {
-	struct insn_args *args = data;
+	struct insn insn;
 	int rc;
  
-	rc = probe_kernel_write(args->target, args->insn, args->size);
-	WARN_ON_ONCE(rc < 0);
-	return 0;
-}
-
-void arch_jump_label_transform(struct jump_entry *entry,
-			       enum jump_label_type type)
-{
-	struct insn_args args;
-	struct insn insn;
-
 	if (type == JUMP_LABEL_ENABLE) {
 		/* brcl 15,offset */
 		insn.opcode = 0xc0f4;
  
@@ -49,11 +38,33 @@
 		insn.offset = 0;
 	}
  
-	args.target = (void *) entry->code;
-	args.insn = &insn;
-	args.size = JUMP_LABEL_NOP_SIZE;
+	rc = probe_kernel_write((void *)entry->code, &insn, JUMP_LABEL_NOP_SIZE);
+	WARN_ON_ONCE(rc < 0);
+}
  
-	stop_machine(__arch_jump_label_transform, &args, NULL);
+static int __sm_arch_jump_label_transform(void *data)
+{
+	struct insn_args *args = data;
+
+	__jump_label_transform(args->entry, args->type);
+	return 0;
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+			       enum jump_label_type type)
+{
+	struct insn_args args;
+
+	args.entry = entry;
+	args.type = type;
+
+	stop_machine(__sm_arch_jump_label_transform, &args, NULL);
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+				      enum jump_label_type type)
+{
+	__jump_label_transform(entry, type);
 }
  
 #endif
@@ -36,13 +36,5 @@
 	put_online_cpus();
 }
  
-void arch_jump_label_text_poke_early(jump_label_t addr)
-{
-	u32 *insn_p = (u32 *) (unsigned long) addr;
-
-	*insn_p = 0x01000000;
-	flushi(insn_p);
-}
-
 #endif
@@ -24,8 +24,9 @@
 	} __attribute__((packed));
 };
  
-void arch_jump_label_transform(struct jump_entry *entry,
-			       enum jump_label_type type)
+static void __jump_label_transform(struct jump_entry *entry,
+				   enum jump_label_type type,
+				   void *(*poker)(void *, const void *, size_t))
 {
 	union jump_code_union code;
  
  
  
  
@@ -35,17 +36,24 @@
 				(entry->code + JUMP_LABEL_NOP_SIZE);
 	} else
 		memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+
+	(*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+			       enum jump_label_type type)
+{
 	get_online_cpus();
 	mutex_lock(&text_mutex);
-	text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+	__jump_label_transform(entry, type, text_poke_smp);
 	mutex_unlock(&text_mutex);
 	put_online_cpus();
 }
  
-void __init_or_module arch_jump_label_text_poke_early(jump_label_t addr)
+void arch_jump_label_transform_static(struct jump_entry *entry,
+				      enum jump_label_type type)
 {
-	text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],
-			JUMP_LABEL_NOP_SIZE);
+	__jump_label_transform(entry, type, text_poke_early);
 }
  
 #endif
@@ -16,7 +16,7 @@
  
 # include <asm/jump_label.h>
 # define HAVE_JUMP_LABEL
-#endif
+#endif	/* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
  
 enum jump_label_type {
 	JUMP_LABEL_DISABLE = 0,
  
@@ -28,9 +28,9 @@
 #ifdef HAVE_JUMP_LABEL
  
 #ifdef CONFIG_MODULES
-#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
 #else
-#define JUMP_LABEL_INIT {{ 0 }, NULL}
+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
 #endif
  
 static __always_inline bool static_branch(struct jump_label_key *key)
  
  
@@ -41,18 +41,20 @@
 extern struct jump_entry __start___jump_table[];
 extern struct jump_entry __stop___jump_table[];
  
+extern void jump_label_init(void);
 extern void jump_label_lock(void);
 extern void jump_label_unlock(void);
 extern void arch_jump_label_transform(struct jump_entry *entry,
-				 enum jump_label_type type);
-extern void arch_jump_label_text_poke_early(jump_label_t addr);
+				      enum jump_label_type type);
+extern void arch_jump_label_transform_static(struct jump_entry *entry,
+					     enum jump_label_type type);
 extern int jump_label_text_reserved(void *start, void *end);
 extern void jump_label_inc(struct jump_label_key *key);
 extern void jump_label_dec(struct jump_label_key *key);
 extern bool jump_label_enabled(struct jump_label_key *key);
 extern void jump_label_apply_nops(struct module *mod);
  
-#else
+#else  /* !HAVE_JUMP_LABEL */
  
 #include <linux/atomic.h>
  
@@ -62,6 +64,10 @@
 	atomic_t enabled;
 };
  
+static __always_inline void jump_label_init(void)
+{
+}
+
 static __always_inline bool static_branch(struct jump_label_key *key)
 {
 	if (unlikely(atomic_read(&key->enabled)))
  
@@ -96,8 +102,7 @@
 {
 	return 0;
 }
+#endif	/* HAVE_JUMP_LABEL */
  
-#endif
-
-#endif
+#endif	/* _LINUX_JUMP_LABEL_H */
@@ -512,6 +512,9 @@
 	parse_args("Booting kernel", static_command_line, __start___param,
 		   __stop___param - __start___param,
 		   &unknown_bootoption);
+
+	jump_label_init();
+
 	/*
 	 * These use large bootmem allocations and must precede
 	 * kmem_cache_init()
@@ -104,6 +104,18 @@
 	return 0;
 }
  
+/* 
+ * Update code which is definitely not currently executing.
+ * Architectures which need heavyweight synchronization to modify
+ * running code can override this to make the non-live update case
+ * cheaper.
+ */
+void __weak arch_jump_label_transform_static(struct jump_entry *entry,
+					    enum jump_label_type type)
+{
+	arch_jump_label_transform(entry, type);	
+}
+
 static void __jump_label_update(struct jump_label_key *key,
 				struct jump_entry *entry,
 				struct jump_entry *stop, int enable)
  
@@ -121,15 +133,8 @@
 	}
 }
  
-/*
- * Not all archs need this.
- */
-void __weak arch_jump_label_text_poke_early(jump_label_t addr)
+void __init jump_label_init(void)
 {
-}
-
-static __init int jump_label_init(void)
-{
 	struct jump_entry *iter_start = __start___jump_table;
 	struct jump_entry *iter_stop = __stop___jump_table;
 	struct jump_label_key *key = NULL;
  
  
  
@@ -139,22 +144,22 @@
 	jump_label_sort_entries(iter_start, iter_stop);
  
 	for (iter = iter_start; iter < iter_stop; iter++) {
-		arch_jump_label_text_poke_early(iter->code);
-		if (iter->key == (jump_label_t)(unsigned long)key)
+		struct jump_label_key *iterk;
+
+		iterk = (struct jump_label_key *)(unsigned long)iter->key;
+		arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
+						 JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+		if (iterk == key)
 			continue;
  
-		key = (struct jump_label_key *)(unsigned long)iter->key;
-		atomic_set(&key->enabled, 0);
+		key = iterk;
 		key->entries = iter;
 #ifdef CONFIG_MODULES
 		key->next = NULL;
 #endif
 	}
 	jump_label_unlock();
-
-	return 0;
 }
-early_initcall(jump_label_init);
  
 #ifdef CONFIG_MODULES
  
@@ -212,7 +217,7 @@
 		return;
  
 	for (iter = iter_start; iter < iter_stop; iter++)
-		arch_jump_label_text_poke_early(iter->code);
+		arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
 }
  
 static int jump_label_add_module(struct module *mod)
...	...	@@ -18,27 +18,16 @@
18	18	} __packed;
19	19
20	20	struct insn_args {
21		- unsigned long *target;
22		- struct insn *insn;
23		- ssize_t size;
	21	+ struct jump_entry *entry;
	22	+ enum jump_label_type type;
24	23	};
25	24
26		-static int __arch_jump_label_transform(void *data)
	25	+static void __jump_label_transform(struct jump_entry *entry,
	26	+ enum jump_label_type type)
27	27	{
28		- struct insn_args *args = data;
	28	+ struct insn insn;
29	29	int rc;
30	30
31		- rc = probe_kernel_write(args->target, args->insn, args->size);
32		- WARN_ON_ONCE(rc < 0);
33		- return 0;
34		-}
35		-
36		-void arch_jump_label_transform(struct jump_entry *entry,
37		- enum jump_label_type type)
38		-{
39		- struct insn_args args;
40		- struct insn insn;
41		-
42	31	if (type == JUMP_LABEL_ENABLE) {
43	32	/* brcl 15,offset */
44	33	insn.opcode = 0xc0f4;
45	34
...	...	@@ -49,11 +38,33 @@
49	38	insn.offset = 0;
50	39	}
51	40
52		- args.target = (void *) entry->code;
53		- args.insn = &insn;
54		- args.size = JUMP_LABEL_NOP_SIZE;
	41	+ rc = probe_kernel_write((void *)entry->code, &insn, JUMP_LABEL_NOP_SIZE);
	42	+ WARN_ON_ONCE(rc < 0);
	43	+}
55	44
56		- stop_machine(__arch_jump_label_transform, &args, NULL);
	45	+static int __sm_arch_jump_label_transform(void *data)
	46	+{
	47	+ struct insn_args *args = data;
	48	+
	49	+ __jump_label_transform(args->entry, args->type);
	50	+ return 0;
	51	+}
	52	+
	53	+void arch_jump_label_transform(struct jump_entry *entry,
	54	+ enum jump_label_type type)
	55	+{
	56	+ struct insn_args args;
	57	+
	58	+ args.entry = entry;
	59	+ args.type = type;
	60	+
	61	+ stop_machine(__sm_arch_jump_label_transform, &args, NULL);
	62	+}
	63	+
	64	+void arch_jump_label_transform_static(struct jump_entry *entry,
	65	+ enum jump_label_type type)
	66	+{
	67	+ __jump_label_transform(entry, type);
57	68	}
58	69
59	70	#endif
...	...	@@ -36,13 +36,5 @@
36	36	put_online_cpus();
37	37	}
38	38
39		-void arch_jump_label_text_poke_early(jump_label_t addr)
40		-{
41		- u32 insn_p = (u32 ) (unsigned long) addr;
42		-
43		- *insn_p = 0x01000000;
44		- flushi(insn_p);
45		-}
46		-
47	39	#endif
...	...	@@ -24,8 +24,9 @@
24	24	} __attribute__((packed));
25	25	};
26	26
27		-void arch_jump_label_transform(struct jump_entry *entry,
28		- enum jump_label_type type)
	27	+static void __jump_label_transform(struct jump_entry *entry,
	28	+ enum jump_label_type type,
	29	+ void (poker)(void , const void , size_t))
29	30	{
30	31	union jump_code_union code;
31	32
32	33
33	34
34	35
...	...	@@ -35,17 +36,24 @@
35	36	(entry->code + JUMP_LABEL_NOP_SIZE);
36	37	} else
37	38	memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
	39	+
	40	+ (poker)((void )entry->code, &code, JUMP_LABEL_NOP_SIZE);
	41	+}
	42	+
	43	+void arch_jump_label_transform(struct jump_entry *entry,
	44	+ enum jump_label_type type)
	45	+{
38	46	get_online_cpus();
39	47	mutex_lock(&text_mutex);
40		- text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
	48	+ __jump_label_transform(entry, type, text_poke_smp);
41	49	mutex_unlock(&text_mutex);
42	50	put_online_cpus();
43	51	}
44	52
45		-void __init_or_module arch_jump_label_text_poke_early(jump_label_t addr)
	53	+void arch_jump_label_transform_static(struct jump_entry *entry,
	54	+ enum jump_label_type type)
46	55	{
47		- text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],
48		- JUMP_LABEL_NOP_SIZE);
	56	+ __jump_label_transform(entry, type, text_poke_early);
49	57	}
50	58
51	59	#endif
...	...	@@ -16,7 +16,7 @@
16	16
17	17	# include <asm/jump_label.h>
18	18	# define HAVE_JUMP_LABEL
19		-#endif
	19	+#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
20	20
21	21	enum jump_label_type {
22	22	JUMP_LABEL_DISABLE = 0,
23	23
...	...	@@ -28,9 +28,9 @@
28	28	#ifdef HAVE_JUMP_LABEL
29	29
30	30	#ifdef CONFIG_MODULES
31		-#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
	31	+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
32	32	#else
33		-#define JUMP_LABEL_INIT {{ 0 }, NULL}
	33	+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
34	34	#endif
35	35
36	36	static __always_inline bool static_branch(struct jump_label_key *key)
37	37
38	38
...	...	@@ -41,18 +41,20 @@
41	41	extern struct jump_entry __start___jump_table[];
42	42	extern struct jump_entry __stop___jump_table[];
43	43
	44	+extern void jump_label_init(void);
44	45	extern void jump_label_lock(void);
45	46	extern void jump_label_unlock(void);
46	47	extern void arch_jump_label_transform(struct jump_entry *entry,
47		- enum jump_label_type type);
48		-extern void arch_jump_label_text_poke_early(jump_label_t addr);
	48	+ enum jump_label_type type);
	49	+extern void arch_jump_label_transform_static(struct jump_entry *entry,
	50	+ enum jump_label_type type);
49	51	extern int jump_label_text_reserved(void start, void end);
50	52	extern void jump_label_inc(struct jump_label_key *key);
51	53	extern void jump_label_dec(struct jump_label_key *key);
52	54	extern bool jump_label_enabled(struct jump_label_key *key);
53	55	extern void jump_label_apply_nops(struct module *mod);
54	56
55		-#else
	57	+#else /* !HAVE_JUMP_LABEL */
56	58
57	59	#include <linux/atomic.h>
58	60
...	...	@@ -62,6 +64,10 @@
62	64	atomic_t enabled;
63	65	};
64	66
	67	+static __always_inline void jump_label_init(void)
	68	+{
	69	+}
	70	+
65	71	static __always_inline bool static_branch(struct jump_label_key *key)
66	72	{
67	73	if (unlikely(atomic_read(&key->enabled)))
68	74
...	...	@@ -96,8 +102,7 @@
96	102	{
97	103	return 0;
98	104	}
	105	+#endif /* HAVE_JUMP_LABEL */
99	106
100		-#endif
101		-
102		-#endif
	107	+#endif /* _LINUX_JUMP_LABEL_H */
...	...	@@ -512,6 +512,9 @@
512	512	parse_args("Booting kernel", static_command_line, __start___param,
513	513	__stop___param - __start___param,
514	514	&unknown_bootoption);
	515	+
	516	+ jump_label_init();
	517	+
515	518	/*
516	519	* These use large bootmem allocations and must precede
517	520	* kmem_cache_init()
...	...	@@ -104,6 +104,18 @@
104	104	return 0;
105	105	}
106	106
	107	+/*
	108	+ * Update code which is definitely not currently executing.
	109	+ * Architectures which need heavyweight synchronization to modify
	110	+ * running code can override this to make the non-live update case
	111	+ * cheaper.
	112	+ */
	113	+void __weak arch_jump_label_transform_static(struct jump_entry *entry,
	114	+ enum jump_label_type type)
	115	+{
	116	+ arch_jump_label_transform(entry, type);
	117	+}
	118	+
107	119	static void __jump_label_update(struct jump_label_key *key,
108	120	struct jump_entry *entry,
109	121	struct jump_entry *stop, int enable)
110	122
...	...	@@ -121,15 +133,8 @@
121	133	}
122	134	}
123	135
124		-/*
125		- * Not all archs need this.
126		- */
127		-void __weak arch_jump_label_text_poke_early(jump_label_t addr)
	136	+void __init jump_label_init(void)
128	137	{
129		-}
130		-
131		-static __init int jump_label_init(void)
132		-{
133	138	struct jump_entry *iter_start = __start___jump_table;
134	139	struct jump_entry *iter_stop = __stop___jump_table;
135	140	struct jump_label_key *key = NULL;
136	141
137	142
138	143
...	...	@@ -139,22 +144,22 @@
139	144	jump_label_sort_entries(iter_start, iter_stop);
140	145
141	146	for (iter = iter_start; iter < iter_stop; iter++) {
142		- arch_jump_label_text_poke_early(iter->code);
143		- if (iter->key == (jump_label_t)(unsigned long)key)
	147	+ struct jump_label_key *iterk;
	148	+
	149	+ iterk = (struct jump_label_key *)(unsigned long)iter->key;
	150	+ arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
	151	+ JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
	152	+ if (iterk == key)
144	153	continue;
145	154
146		- key = (struct jump_label_key *)(unsigned long)iter->key;
147		- atomic_set(&key->enabled, 0);
	155	+ key = iterk;
148	156	key->entries = iter;
149	157	#ifdef CONFIG_MODULES
150	158	key->next = NULL;
151	159	#endif
152	160	}
153	161	jump_label_unlock();
154		-
155		- return 0;
156	162	}
157		-early_initcall(jump_label_init);
158	163
159	164	#ifdef CONFIG_MODULES
160	165
...	...	@@ -212,7 +217,7 @@
212	217	return;
213	218
214	219	for (iter = iter_start; iter < iter_stop; iter++)
215		- arch_jump_label_text_poke_early(iter->code);
	220	+ arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
216	221	}
217	222
218	223	static int jump_label_add_module(struct module *mod)