Eric Lee / linux-smarc-t335x-v3.2

Download as
Browse Code ยป

Commit 3caa37519ccbb200c7fbbf6ff4fb306a30f29425

Authored by Masami Hiramatsu
Committed by Ingo Molnar
1 parent 087a4eb559
Exists in master and in 4 other branches v3.2_AM335xPSP_04.06.00.11, v3.2_NEWT335xPSP_04.06.00.11, v3.2_SBC_SMARTMEN, v3.2_SMARCT335xPSP_04.06.00.11

x86: Use __stop_machine() in text_poke_smp() 

Use __stop_machine() in text_poke_smp() because the caller
must get online_cpus before calling text_poke_smp(), but
stop_machine() do it again. We don't need it.

Signed-off-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: 2nddept-manager@sdl.hitachi.co.jp
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
LKML-Reference: <20101014031036.4100.83989.stgit@ltc236.sdl.hitachi.co.jp>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

Showing 1 changed file with 2 additions and 1 deletions Inline Diff

arch/x86/kernel/alternative.c
Diff comments   View file @ 3caa375
1 #include <linux/module.h> 1 #include <linux/module.h>
2 #include <linux/sched.h> 2 #include <linux/sched.h>
3 #include <linux/mutex.h> 3 #include <linux/mutex.h>
4 #include <linux/list.h> 4 #include <linux/list.h>
5 #include <linux/stringify.h> 5 #include <linux/stringify.h>
6 #include <linux/kprobes.h> 6 #include <linux/kprobes.h>
7 #include <linux/mm.h> 7 #include <linux/mm.h>
8 #include <linux/vmalloc.h> 8 #include <linux/vmalloc.h>
9 #include <linux/memory.h> 9 #include <linux/memory.h>
10 #include <linux/stop_machine.h> 10 #include <linux/stop_machine.h>
11 #include <linux/slab.h> 11 #include <linux/slab.h>
12 #include <asm/alternative.h> 12 #include <asm/alternative.h>
13 #include <asm/sections.h> 13 #include <asm/sections.h>
14 #include <asm/pgtable.h> 14 #include <asm/pgtable.h>
15 #include <asm/mce.h> 15 #include <asm/mce.h>
16 #include <asm/nmi.h> 16 #include <asm/nmi.h>
17 #include <asm/vsyscall.h> 17 #include <asm/vsyscall.h>
18 #include <asm/cacheflush.h> 18 #include <asm/cacheflush.h>
19 #include <asm/tlbflush.h> 19 #include <asm/tlbflush.h>
20 #include <asm/io.h> 20 #include <asm/io.h>
21 #include <asm/fixmap.h> 21 #include <asm/fixmap.h>
22 22
23 #define MAX_PATCH_LEN (255-1) 23 #define MAX_PATCH_LEN (255-1)
24 24
25 #ifdef CONFIG_HOTPLUG_CPU 25 #ifdef CONFIG_HOTPLUG_CPU
26 static int smp_alt_once; 26 static int smp_alt_once;
27 27
28 static int __init bootonly(char *str) 28 static int __init bootonly(char *str)
29 { 29 {
30 smp_alt_once = 1; 30 smp_alt_once = 1;
31 return 1; 31 return 1;
32 } 32 }
33 __setup("smp-alt-boot", bootonly); 33 __setup("smp-alt-boot", bootonly);
34 #else 34 #else
35 #define smp_alt_once 1 35 #define smp_alt_once 1
36 #endif 36 #endif
37 37
38 static int __initdata_or_module debug_alternative; 38 static int __initdata_or_module debug_alternative;
39 39
40 static int __init debug_alt(char *str) 40 static int __init debug_alt(char *str)
41 { 41 {
42 debug_alternative = 1; 42 debug_alternative = 1;
43 return 1; 43 return 1;
44 } 44 }
45 __setup("debug-alternative", debug_alt); 45 __setup("debug-alternative", debug_alt);
46 46
47 static int noreplace_smp; 47 static int noreplace_smp;
48 48
49 static int __init setup_noreplace_smp(char *str) 49 static int __init setup_noreplace_smp(char *str)
50 { 50 {
51 noreplace_smp = 1; 51 noreplace_smp = 1;
52 return 1; 52 return 1;
53 } 53 }
54 __setup("noreplace-smp", setup_noreplace_smp); 54 __setup("noreplace-smp", setup_noreplace_smp);
55 55
56 #ifdef CONFIG_PARAVIRT 56 #ifdef CONFIG_PARAVIRT
57 static int __initdata_or_module noreplace_paravirt = 0; 57 static int __initdata_or_module noreplace_paravirt = 0;
58 58
59 static int __init setup_noreplace_paravirt(char *str) 59 static int __init setup_noreplace_paravirt(char *str)
60 { 60 {
61 noreplace_paravirt = 1; 61 noreplace_paravirt = 1;
62 return 1; 62 return 1;
63 } 63 }
64 __setup("noreplace-paravirt", setup_noreplace_paravirt); 64 __setup("noreplace-paravirt", setup_noreplace_paravirt);
65 #endif 65 #endif
66 66
67 #define DPRINTK(fmt, args...) if (debug_alternative) \ 67 #define DPRINTK(fmt, args...) if (debug_alternative) \
68 printk(KERN_DEBUG fmt, args) 68 printk(KERN_DEBUG fmt, args)
69 69
70 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) 70 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
71 /* Use inline assembly to define this because the nops are defined 71 /* Use inline assembly to define this because the nops are defined
72 as inline assembly strings in the include files and we cannot 72 as inline assembly strings in the include files and we cannot
73 get them easily into strings. */ 73 get them easily into strings. */
74 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: " 74 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: "
75 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6 75 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
76 GENERIC_NOP7 GENERIC_NOP8 76 GENERIC_NOP7 GENERIC_NOP8
77 "\t.previous"); 77 "\t.previous");
78 extern const unsigned char intelnops[]; 78 extern const unsigned char intelnops[];
79 static const unsigned char *const __initconst_or_module 79 static const unsigned char *const __initconst_or_module
80 intel_nops[ASM_NOP_MAX+1] = { 80 intel_nops[ASM_NOP_MAX+1] = {
81 NULL, 81 NULL,
82 intelnops, 82 intelnops,
83 intelnops + 1, 83 intelnops + 1,
84 intelnops + 1 + 2, 84 intelnops + 1 + 2,
85 intelnops + 1 + 2 + 3, 85 intelnops + 1 + 2 + 3,
86 intelnops + 1 + 2 + 3 + 4, 86 intelnops + 1 + 2 + 3 + 4,
87 intelnops + 1 + 2 + 3 + 4 + 5, 87 intelnops + 1 + 2 + 3 + 4 + 5,
88 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 88 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
89 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 89 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
90 }; 90 };
91 #endif 91 #endif
92 92
93 #ifdef K8_NOP1 93 #ifdef K8_NOP1
94 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: " 94 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: "
95 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6 95 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
96 K8_NOP7 K8_NOP8 96 K8_NOP7 K8_NOP8
97 "\t.previous"); 97 "\t.previous");
98 extern const unsigned char k8nops[]; 98 extern const unsigned char k8nops[];
99 static const unsigned char *const __initconst_or_module 99 static const unsigned char *const __initconst_or_module
100 k8_nops[ASM_NOP_MAX+1] = { 100 k8_nops[ASM_NOP_MAX+1] = {
101 NULL, 101 NULL,
102 k8nops, 102 k8nops,
103 k8nops + 1, 103 k8nops + 1,
104 k8nops + 1 + 2, 104 k8nops + 1 + 2,
105 k8nops + 1 + 2 + 3, 105 k8nops + 1 + 2 + 3,
106 k8nops + 1 + 2 + 3 + 4, 106 k8nops + 1 + 2 + 3 + 4,
107 k8nops + 1 + 2 + 3 + 4 + 5, 107 k8nops + 1 + 2 + 3 + 4 + 5,
108 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 108 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
109 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 109 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
110 }; 110 };
111 #endif 111 #endif
112 112
113 #if defined(K7_NOP1) && !defined(CONFIG_X86_64) 113 #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
114 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: " 114 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: "
115 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6 115 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
116 K7_NOP7 K7_NOP8 116 K7_NOP7 K7_NOP8
117 "\t.previous"); 117 "\t.previous");
118 extern const unsigned char k7nops[]; 118 extern const unsigned char k7nops[];
119 static const unsigned char *const __initconst_or_module 119 static const unsigned char *const __initconst_or_module
120 k7_nops[ASM_NOP_MAX+1] = { 120 k7_nops[ASM_NOP_MAX+1] = {
121 NULL, 121 NULL,
122 k7nops, 122 k7nops,
123 k7nops + 1, 123 k7nops + 1,
124 k7nops + 1 + 2, 124 k7nops + 1 + 2,
125 k7nops + 1 + 2 + 3, 125 k7nops + 1 + 2 + 3,
126 k7nops + 1 + 2 + 3 + 4, 126 k7nops + 1 + 2 + 3 + 4,
127 k7nops + 1 + 2 + 3 + 4 + 5, 127 k7nops + 1 + 2 + 3 + 4 + 5,
128 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 128 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
129 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 129 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
130 }; 130 };
131 #endif 131 #endif
132 132
133 #ifdef P6_NOP1 133 #ifdef P6_NOP1
134 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: " 134 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: "
135 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6 135 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6
136 P6_NOP7 P6_NOP8 136 P6_NOP7 P6_NOP8
137 "\t.previous"); 137 "\t.previous");
138 extern const unsigned char p6nops[]; 138 extern const unsigned char p6nops[];
139 static const unsigned char *const __initconst_or_module 139 static const unsigned char *const __initconst_or_module
140 p6_nops[ASM_NOP_MAX+1] = { 140 p6_nops[ASM_NOP_MAX+1] = {
141 NULL, 141 NULL,
142 p6nops, 142 p6nops,
143 p6nops + 1, 143 p6nops + 1,
144 p6nops + 1 + 2, 144 p6nops + 1 + 2,
145 p6nops + 1 + 2 + 3, 145 p6nops + 1 + 2 + 3,
146 p6nops + 1 + 2 + 3 + 4, 146 p6nops + 1 + 2 + 3 + 4,
147 p6nops + 1 + 2 + 3 + 4 + 5, 147 p6nops + 1 + 2 + 3 + 4 + 5,
148 p6nops + 1 + 2 + 3 + 4 + 5 + 6, 148 p6nops + 1 + 2 + 3 + 4 + 5 + 6,
149 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 149 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
150 }; 150 };
151 #endif 151 #endif
152 152
153 #ifdef CONFIG_X86_64 153 #ifdef CONFIG_X86_64
154 154
155 extern char __vsyscall_0; 155 extern char __vsyscall_0;
156 static const unsigned char *const *__init_or_module find_nop_table(void) 156 static const unsigned char *const *__init_or_module find_nop_table(void)
157 { 157 {
158 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 158 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
159 boot_cpu_has(X86_FEATURE_NOPL)) 159 boot_cpu_has(X86_FEATURE_NOPL))
160 return p6_nops; 160 return p6_nops;
161 else 161 else
162 return k8_nops; 162 return k8_nops;
163 } 163 }
164 164
165 #else /* CONFIG_X86_64 */ 165 #else /* CONFIG_X86_64 */
166 166
167 static const unsigned char *const *__init_or_module find_nop_table(void) 167 static const unsigned char *const *__init_or_module find_nop_table(void)
168 { 168 {
169 if (boot_cpu_has(X86_FEATURE_K8)) 169 if (boot_cpu_has(X86_FEATURE_K8))
170 return k8_nops; 170 return k8_nops;
171 else if (boot_cpu_has(X86_FEATURE_K7)) 171 else if (boot_cpu_has(X86_FEATURE_K7))
172 return k7_nops; 172 return k7_nops;
173 else if (boot_cpu_has(X86_FEATURE_NOPL)) 173 else if (boot_cpu_has(X86_FEATURE_NOPL))
174 return p6_nops; 174 return p6_nops;
175 else 175 else
176 return intel_nops; 176 return intel_nops;
177 } 177 }
178 178
179 #endif /* CONFIG_X86_64 */ 179 #endif /* CONFIG_X86_64 */
180 180
181 /* Use this to add nops to a buffer, then text_poke the whole buffer. */ 181 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
182 static void __init_or_module add_nops(void *insns, unsigned int len) 182 static void __init_or_module add_nops(void *insns, unsigned int len)
183 { 183 {
184 const unsigned char *const *noptable = find_nop_table(); 184 const unsigned char *const *noptable = find_nop_table();
185 185
186 while (len > 0) { 186 while (len > 0) {
187 unsigned int noplen = len; 187 unsigned int noplen = len;
188 if (noplen > ASM_NOP_MAX) 188 if (noplen > ASM_NOP_MAX)
189 noplen = ASM_NOP_MAX; 189 noplen = ASM_NOP_MAX;
190 memcpy(insns, noptable[noplen], noplen); 190 memcpy(insns, noptable[noplen], noplen);
191 insns += noplen; 191 insns += noplen;
192 len -= noplen; 192 len -= noplen;
193 } 193 }
194 } 194 }
195 195
196 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 196 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
197 extern s32 __smp_locks[], __smp_locks_end[]; 197 extern s32 __smp_locks[], __smp_locks_end[];
198 void *text_poke_early(void *addr, const void *opcode, size_t len); 198 void *text_poke_early(void *addr, const void *opcode, size_t len);
199 199
200 /* Replace instructions with better alternatives for this CPU type. 200 /* Replace instructions with better alternatives for this CPU type.
201 This runs before SMP is initialized to avoid SMP problems with 201 This runs before SMP is initialized to avoid SMP problems with
202 self modifying code. This implies that assymetric systems where 202 self modifying code. This implies that assymetric systems where
203 APs have less capabilities than the boot processor are not handled. 203 APs have less capabilities than the boot processor are not handled.
204 Tough. Make sure you disable such features by hand. */ 204 Tough. Make sure you disable such features by hand. */
205 205
206 void __init_or_module apply_alternatives(struct alt_instr *start, 206 void __init_or_module apply_alternatives(struct alt_instr *start,
207 struct alt_instr *end) 207 struct alt_instr *end)
208 { 208 {
209 struct alt_instr *a; 209 struct alt_instr *a;
210 u8 insnbuf[MAX_PATCH_LEN]; 210 u8 insnbuf[MAX_PATCH_LEN];
211 211
212 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); 212 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
213 for (a = start; a < end; a++) { 213 for (a = start; a < end; a++) {
214 u8 *instr = a->instr; 214 u8 *instr = a->instr;
215 BUG_ON(a->replacementlen > a->instrlen); 215 BUG_ON(a->replacementlen > a->instrlen);
216 BUG_ON(a->instrlen > sizeof(insnbuf)); 216 BUG_ON(a->instrlen > sizeof(insnbuf));
217 BUG_ON(a->cpuid >= NCAPINTS*32); 217 BUG_ON(a->cpuid >= NCAPINTS*32);
218 if (!boot_cpu_has(a->cpuid)) 218 if (!boot_cpu_has(a->cpuid))
219 continue; 219 continue;
220 #ifdef CONFIG_X86_64 220 #ifdef CONFIG_X86_64
221 /* vsyscall code is not mapped yet. resolve it manually. */ 221 /* vsyscall code is not mapped yet. resolve it manually. */
222 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) { 222 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
223 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0)); 223 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
224 DPRINTK("%s: vsyscall fixup: %p => %p\n", 224 DPRINTK("%s: vsyscall fixup: %p => %p\n",
225 __func__, a->instr, instr); 225 __func__, a->instr, instr);
226 } 226 }
227 #endif 227 #endif
228 memcpy(insnbuf, a->replacement, a->replacementlen); 228 memcpy(insnbuf, a->replacement, a->replacementlen);
229 if (*insnbuf == 0xe8 && a->replacementlen == 5) 229 if (*insnbuf == 0xe8 && a->replacementlen == 5)
230 *(s32 *)(insnbuf + 1) += a->replacement - a->instr; 230 *(s32 *)(insnbuf + 1) += a->replacement - a->instr;
231 add_nops(insnbuf + a->replacementlen, 231 add_nops(insnbuf + a->replacementlen,
232 a->instrlen - a->replacementlen); 232 a->instrlen - a->replacementlen);
233 text_poke_early(instr, insnbuf, a->instrlen); 233 text_poke_early(instr, insnbuf, a->instrlen);
234 } 234 }
235 } 235 }
236 236
237 #ifdef CONFIG_SMP 237 #ifdef CONFIG_SMP
238 238
239 static void alternatives_smp_lock(const s32 *start, const s32 *end, 239 static void alternatives_smp_lock(const s32 *start, const s32 *end,
240 u8 *text, u8 *text_end) 240 u8 *text, u8 *text_end)
241 { 241 {
242 const s32 *poff; 242 const s32 *poff;
243 243
244 mutex_lock(&text_mutex); 244 mutex_lock(&text_mutex);
245 for (poff = start; poff < end; poff++) { 245 for (poff = start; poff < end; poff++) {
246 u8 *ptr = (u8 *)poff + *poff; 246 u8 *ptr = (u8 *)poff + *poff;
247 247
248 if (!*poff || ptr < text || ptr >= text_end) 248 if (!*poff || ptr < text || ptr >= text_end)
249 continue; 249 continue;
250 /* turn DS segment override prefix into lock prefix */ 250 /* turn DS segment override prefix into lock prefix */
251 if (*ptr == 0x3e) 251 if (*ptr == 0x3e)
252 text_poke(ptr, ((unsigned char []){0xf0}), 1); 252 text_poke(ptr, ((unsigned char []){0xf0}), 1);
253 }; 253 };
254 mutex_unlock(&text_mutex); 254 mutex_unlock(&text_mutex);
255 } 255 }
256 256
257 static void alternatives_smp_unlock(const s32 *start, const s32 *end, 257 static void alternatives_smp_unlock(const s32 *start, const s32 *end,
258 u8 *text, u8 *text_end) 258 u8 *text, u8 *text_end)
259 { 259 {
260 const s32 *poff; 260 const s32 *poff;
261 261
262 if (noreplace_smp) 262 if (noreplace_smp)
263 return; 263 return;
264 264
265 mutex_lock(&text_mutex); 265 mutex_lock(&text_mutex);
266 for (poff = start; poff < end; poff++) { 266 for (poff = start; poff < end; poff++) {
267 u8 *ptr = (u8 *)poff + *poff; 267 u8 *ptr = (u8 *)poff + *poff;
268 268
269 if (!*poff || ptr < text || ptr >= text_end) 269 if (!*poff || ptr < text || ptr >= text_end)
270 continue; 270 continue;
271 /* turn lock prefix into DS segment override prefix */ 271 /* turn lock prefix into DS segment override prefix */
272 if (*ptr == 0xf0) 272 if (*ptr == 0xf0)
273 text_poke(ptr, ((unsigned char []){0x3E}), 1); 273 text_poke(ptr, ((unsigned char []){0x3E}), 1);
274 }; 274 };
275 mutex_unlock(&text_mutex); 275 mutex_unlock(&text_mutex);
276 } 276 }
277 277
278 struct smp_alt_module { 278 struct smp_alt_module {
279 /* what is this ??? */ 279 /* what is this ??? */
280 struct module *mod; 280 struct module *mod;
281 char *name; 281 char *name;
282 282
283 /* ptrs to lock prefixes */ 283 /* ptrs to lock prefixes */
284 const s32 *locks; 284 const s32 *locks;
285 const s32 *locks_end; 285 const s32 *locks_end;
286 286
287 /* .text segment, needed to avoid patching init code ;) */ 287 /* .text segment, needed to avoid patching init code ;) */
288 u8 *text; 288 u8 *text;
289 u8 *text_end; 289 u8 *text_end;
290 290
291 struct list_head next; 291 struct list_head next;
292 }; 292 };
293 static LIST_HEAD(smp_alt_modules); 293 static LIST_HEAD(smp_alt_modules);
294 static DEFINE_MUTEX(smp_alt); 294 static DEFINE_MUTEX(smp_alt);
295 static int smp_mode = 1; /* protected by smp_alt */ 295 static int smp_mode = 1; /* protected by smp_alt */
296 296
297 void __init_or_module alternatives_smp_module_add(struct module *mod, 297 void __init_or_module alternatives_smp_module_add(struct module *mod,
298 char *name, 298 char *name,
299 void *locks, void *locks_end, 299 void *locks, void *locks_end,
300 void *text, void *text_end) 300 void *text, void *text_end)
301 { 301 {
302 struct smp_alt_module *smp; 302 struct smp_alt_module *smp;
303 303
304 if (noreplace_smp) 304 if (noreplace_smp)
305 return; 305 return;
306 306
307 if (smp_alt_once) { 307 if (smp_alt_once) {
308 if (boot_cpu_has(X86_FEATURE_UP)) 308 if (boot_cpu_has(X86_FEATURE_UP))
309 alternatives_smp_unlock(locks, locks_end, 309 alternatives_smp_unlock(locks, locks_end,
310 text, text_end); 310 text, text_end);
311 return; 311 return;
312 } 312 }
313 313
314 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 314 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
315 if (NULL == smp) 315 if (NULL == smp)
316 return; /* we'll run the (safe but slow) SMP code then ... */ 316 return; /* we'll run the (safe but slow) SMP code then ... */
317 317
318 smp->mod = mod; 318 smp->mod = mod;
319 smp->name = name; 319 smp->name = name;
320 smp->locks = locks; 320 smp->locks = locks;
321 smp->locks_end = locks_end; 321 smp->locks_end = locks_end;
322 smp->text = text; 322 smp->text = text;
323 smp->text_end = text_end; 323 smp->text_end = text_end;
324 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", 324 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
325 __func__, smp->locks, smp->locks_end, 325 __func__, smp->locks, smp->locks_end,
326 smp->text, smp->text_end, smp->name); 326 smp->text, smp->text_end, smp->name);
327 327
328 mutex_lock(&smp_alt); 328 mutex_lock(&smp_alt);
329 list_add_tail(&smp->next, &smp_alt_modules); 329 list_add_tail(&smp->next, &smp_alt_modules);
330 if (boot_cpu_has(X86_FEATURE_UP)) 330 if (boot_cpu_has(X86_FEATURE_UP))
331 alternatives_smp_unlock(smp->locks, smp->locks_end, 331 alternatives_smp_unlock(smp->locks, smp->locks_end,
332 smp->text, smp->text_end); 332 smp->text, smp->text_end);
333 mutex_unlock(&smp_alt); 333 mutex_unlock(&smp_alt);
334 } 334 }
335 335
336 void __init_or_module alternatives_smp_module_del(struct module *mod) 336 void __init_or_module alternatives_smp_module_del(struct module *mod)
337 { 337 {
338 struct smp_alt_module *item; 338 struct smp_alt_module *item;
339 339
340 if (smp_alt_once || noreplace_smp) 340 if (smp_alt_once || noreplace_smp)
341 return; 341 return;
342 342
343 mutex_lock(&smp_alt); 343 mutex_lock(&smp_alt);
344 list_for_each_entry(item, &smp_alt_modules, next) { 344 list_for_each_entry(item, &smp_alt_modules, next) {
345 if (mod != item->mod) 345 if (mod != item->mod)
346 continue; 346 continue;
347 list_del(&item->next); 347 list_del(&item->next);
348 mutex_unlock(&smp_alt); 348 mutex_unlock(&smp_alt);
349 DPRINTK("%s: %s\n", __func__, item->name); 349 DPRINTK("%s: %s\n", __func__, item->name);
350 kfree(item); 350 kfree(item);
351 return; 351 return;
352 } 352 }
353 mutex_unlock(&smp_alt); 353 mutex_unlock(&smp_alt);
354 } 354 }
355 355
356 void alternatives_smp_switch(int smp) 356 void alternatives_smp_switch(int smp)
357 { 357 {
358 struct smp_alt_module *mod; 358 struct smp_alt_module *mod;
359 359
360 #ifdef CONFIG_LOCKDEP 360 #ifdef CONFIG_LOCKDEP
361 /* 361 /*
362 * Older binutils section handling bug prevented 362 * Older binutils section handling bug prevented
363 * alternatives-replacement from working reliably. 363 * alternatives-replacement from working reliably.
364 * 364 *
365 * If this still occurs then you should see a hang 365 * If this still occurs then you should see a hang
366 * or crash shortly after this line: 366 * or crash shortly after this line:
367 */ 367 */
368 printk("lockdep: fixing up alternatives.\n"); 368 printk("lockdep: fixing up alternatives.\n");
369 #endif 369 #endif
370 370
371 if (noreplace_smp || smp_alt_once) 371 if (noreplace_smp || smp_alt_once)
372 return; 372 return;
373 BUG_ON(!smp && (num_online_cpus() > 1)); 373 BUG_ON(!smp && (num_online_cpus() > 1));
374 374
375 mutex_lock(&smp_alt); 375 mutex_lock(&smp_alt);
376 376
377 /* 377 /*
378 * Avoid unnecessary switches because it forces JIT based VMs to 378 * Avoid unnecessary switches because it forces JIT based VMs to
379 * throw away all cached translations, which can be quite costly. 379 * throw away all cached translations, which can be quite costly.
380 */ 380 */
381 if (smp == smp_mode) { 381 if (smp == smp_mode) {
382 /* nothing */ 382 /* nothing */
383 } else if (smp) { 383 } else if (smp) {
384 printk(KERN_INFO "SMP alternatives: switching to SMP code\n"); 384 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
385 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 385 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
386 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 386 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
387 list_for_each_entry(mod, &smp_alt_modules, next) 387 list_for_each_entry(mod, &smp_alt_modules, next)
388 alternatives_smp_lock(mod->locks, mod->locks_end, 388 alternatives_smp_lock(mod->locks, mod->locks_end,
389 mod->text, mod->text_end); 389 mod->text, mod->text_end);
390 } else { 390 } else {
391 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 391 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
392 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 392 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
393 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 393 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
394 list_for_each_entry(mod, &smp_alt_modules, next) 394 list_for_each_entry(mod, &smp_alt_modules, next)
395 alternatives_smp_unlock(mod->locks, mod->locks_end, 395 alternatives_smp_unlock(mod->locks, mod->locks_end,
396 mod->text, mod->text_end); 396 mod->text, mod->text_end);
397 } 397 }
398 smp_mode = smp; 398 smp_mode = smp;
399 mutex_unlock(&smp_alt); 399 mutex_unlock(&smp_alt);
400 } 400 }
401 401
402 /* Return 1 if the address range is reserved for smp-alternatives */ 402 /* Return 1 if the address range is reserved for smp-alternatives */
403 int alternatives_text_reserved(void *start, void *end) 403 int alternatives_text_reserved(void *start, void *end)
404 { 404 {
405 struct smp_alt_module *mod; 405 struct smp_alt_module *mod;
406 const s32 *poff; 406 const s32 *poff;
407 u8 *text_start = start; 407 u8 *text_start = start;
408 u8 *text_end = end; 408 u8 *text_end = end;
409 409
410 list_for_each_entry(mod, &smp_alt_modules, next) { 410 list_for_each_entry(mod, &smp_alt_modules, next) {
411 if (mod->text > text_end || mod->text_end < text_start) 411 if (mod->text > text_end || mod->text_end < text_start)
412 continue; 412 continue;
413 for (poff = mod->locks; poff < mod->locks_end; poff++) { 413 for (poff = mod->locks; poff < mod->locks_end; poff++) {
414 const u8 *ptr = (const u8 *)poff + *poff; 414 const u8 *ptr = (const u8 *)poff + *poff;
415 415
416 if (text_start <= ptr && text_end > ptr) 416 if (text_start <= ptr && text_end > ptr)
417 return 1; 417 return 1;
418 } 418 }
419 } 419 }
420 420
421 return 0; 421 return 0;
422 } 422 }
423 #endif 423 #endif
424 424
425 #ifdef CONFIG_PARAVIRT 425 #ifdef CONFIG_PARAVIRT
426 void __init_or_module apply_paravirt(struct paravirt_patch_site *start, 426 void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
427 struct paravirt_patch_site *end) 427 struct paravirt_patch_site *end)
428 { 428 {
429 struct paravirt_patch_site *p; 429 struct paravirt_patch_site *p;
430 char insnbuf[MAX_PATCH_LEN]; 430 char insnbuf[MAX_PATCH_LEN];
431 431
432 if (noreplace_paravirt) 432 if (noreplace_paravirt)
433 return; 433 return;
434 434
435 for (p = start; p < end; p++) { 435 for (p = start; p < end; p++) {
436 unsigned int used; 436 unsigned int used;
437 437
438 BUG_ON(p->len > MAX_PATCH_LEN); 438 BUG_ON(p->len > MAX_PATCH_LEN);
439 /* prep the buffer with the original instructions */ 439 /* prep the buffer with the original instructions */
440 memcpy(insnbuf, p->instr, p->len); 440 memcpy(insnbuf, p->instr, p->len);
441 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf, 441 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
442 (unsigned long)p->instr, p->len); 442 (unsigned long)p->instr, p->len);
443 443
444 BUG_ON(used > p->len); 444 BUG_ON(used > p->len);
445 445
446 /* Pad the rest with nops */ 446 /* Pad the rest with nops */
447 add_nops(insnbuf + used, p->len - used); 447 add_nops(insnbuf + used, p->len - used);
448 text_poke_early(p->instr, insnbuf, p->len); 448 text_poke_early(p->instr, insnbuf, p->len);
449 } 449 }
450 } 450 }
451 extern struct paravirt_patch_site __start_parainstructions[], 451 extern struct paravirt_patch_site __start_parainstructions[],
452 __stop_parainstructions[]; 452 __stop_parainstructions[];
453 #endif /* CONFIG_PARAVIRT */ 453 #endif /* CONFIG_PARAVIRT */
454 454
455 void __init alternative_instructions(void) 455 void __init alternative_instructions(void)
456 { 456 {
457 /* The patching is not fully atomic, so try to avoid local interruptions 457 /* The patching is not fully atomic, so try to avoid local interruptions
458 that might execute the to be patched code. 458 that might execute the to be patched code.
459 Other CPUs are not running. */ 459 Other CPUs are not running. */
460 stop_nmi(); 460 stop_nmi();
461 461
462 /* 462 /*
463 * Don't stop machine check exceptions while patching. 463 * Don't stop machine check exceptions while patching.
464 * MCEs only happen when something got corrupted and in this 464 * MCEs only happen when something got corrupted and in this
465 * case we must do something about the corruption. 465 * case we must do something about the corruption.
466 * Ignoring it is worse than a unlikely patching race. 466 * Ignoring it is worse than a unlikely patching race.
467 * Also machine checks tend to be broadcast and if one CPU 467 * Also machine checks tend to be broadcast and if one CPU
468 * goes into machine check the others follow quickly, so we don't 468 * goes into machine check the others follow quickly, so we don't
469 * expect a machine check to cause undue problems during to code 469 * expect a machine check to cause undue problems during to code
470 * patching. 470 * patching.
471 */ 471 */
472 472
473 apply_alternatives(__alt_instructions, __alt_instructions_end); 473 apply_alternatives(__alt_instructions, __alt_instructions_end);
474 474
475 /* switch to patch-once-at-boottime-only mode and free the 475 /* switch to patch-once-at-boottime-only mode and free the
476 * tables in case we know the number of CPUs will never ever 476 * tables in case we know the number of CPUs will never ever
477 * change */ 477 * change */
478 #ifdef CONFIG_HOTPLUG_CPU 478 #ifdef CONFIG_HOTPLUG_CPU
479 if (num_possible_cpus() < 2) 479 if (num_possible_cpus() < 2)
480 smp_alt_once = 1; 480 smp_alt_once = 1;
481 #endif 481 #endif
482 482
483 #ifdef CONFIG_SMP 483 #ifdef CONFIG_SMP
484 if (smp_alt_once) { 484 if (smp_alt_once) {
485 if (1 == num_possible_cpus()) { 485 if (1 == num_possible_cpus()) {
486 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 486 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
487 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 487 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
488 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 488 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
489 489
490 alternatives_smp_unlock(__smp_locks, __smp_locks_end, 490 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
491 _text, _etext); 491 _text, _etext);
492 } 492 }
493 } else { 493 } else {
494 alternatives_smp_module_add(NULL, "core kernel", 494 alternatives_smp_module_add(NULL, "core kernel",
495 __smp_locks, __smp_locks_end, 495 __smp_locks, __smp_locks_end,
496 _text, _etext); 496 _text, _etext);
497 497
498 /* Only switch to UP mode if we don't immediately boot others */ 498 /* Only switch to UP mode if we don't immediately boot others */
499 if (num_present_cpus() == 1 || setup_max_cpus <= 1) 499 if (num_present_cpus() == 1 || setup_max_cpus <= 1)
500 alternatives_smp_switch(0); 500 alternatives_smp_switch(0);
501 } 501 }
502 #endif 502 #endif
503 apply_paravirt(__parainstructions, __parainstructions_end); 503 apply_paravirt(__parainstructions, __parainstructions_end);
504 504
505 if (smp_alt_once) 505 if (smp_alt_once)
506 free_init_pages("SMP alternatives", 506 free_init_pages("SMP alternatives",
507 (unsigned long)__smp_locks, 507 (unsigned long)__smp_locks,
508 (unsigned long)__smp_locks_end); 508 (unsigned long)__smp_locks_end);
509 509
510 restart_nmi(); 510 restart_nmi();
511 } 511 }
512 512
513 /** 513 /**
514 * text_poke_early - Update instructions on a live kernel at boot time 514 * text_poke_early - Update instructions on a live kernel at boot time
515 * @addr: address to modify 515 * @addr: address to modify
516 * @opcode: source of the copy 516 * @opcode: source of the copy
517 * @len: length to copy 517 * @len: length to copy
518 * 518 *
519 * When you use this code to patch more than one byte of an instruction 519 * When you use this code to patch more than one byte of an instruction
520 * you need to make sure that other CPUs cannot execute this code in parallel. 520 * you need to make sure that other CPUs cannot execute this code in parallel.
521 * Also no thread must be currently preempted in the middle of these 521 * Also no thread must be currently preempted in the middle of these
522 * instructions. And on the local CPU you need to be protected again NMI or MCE 522 * instructions. And on the local CPU you need to be protected again NMI or MCE
523 * handlers seeing an inconsistent instruction while you patch. 523 * handlers seeing an inconsistent instruction while you patch.
524 */ 524 */
525 void *__init_or_module text_poke_early(void *addr, const void *opcode, 525 void *__init_or_module text_poke_early(void *addr, const void *opcode,
526 size_t len) 526 size_t len)
527 { 527 {
528 unsigned long flags; 528 unsigned long flags;
529 local_irq_save(flags); 529 local_irq_save(flags);
530 memcpy(addr, opcode, len); 530 memcpy(addr, opcode, len);
531 sync_core(); 531 sync_core();
532 local_irq_restore(flags); 532 local_irq_restore(flags);
533 /* Could also do a CLFLUSH here to speed up CPU recovery; but 533 /* Could also do a CLFLUSH here to speed up CPU recovery; but
534 that causes hangs on some VIA CPUs. */ 534 that causes hangs on some VIA CPUs. */
535 return addr; 535 return addr;
536 } 536 }
537 537
538 /** 538 /**
539 * text_poke - Update instructions on a live kernel 539 * text_poke - Update instructions on a live kernel
540 * @addr: address to modify 540 * @addr: address to modify
541 * @opcode: source of the copy 541 * @opcode: source of the copy
542 * @len: length to copy 542 * @len: length to copy
543 * 543 *
544 * Only atomic text poke/set should be allowed when not doing early patching. 544 * Only atomic text poke/set should be allowed when not doing early patching.
545 * It means the size must be writable atomically and the address must be aligned 545 * It means the size must be writable atomically and the address must be aligned
546 * in a way that permits an atomic write. It also makes sure we fit on a single 546 * in a way that permits an atomic write. It also makes sure we fit on a single
547 * page. 547 * page.
548 * 548 *
549 * Note: Must be called under text_mutex. 549 * Note: Must be called under text_mutex.
550 */ 550 */
551 void *__kprobes text_poke(void *addr, const void *opcode, size_t len) 551 void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
552 { 552 {
553 unsigned long flags; 553 unsigned long flags;
554 char *vaddr; 554 char *vaddr;
555 struct page *pages[2]; 555 struct page *pages[2];
556 int i; 556 int i;
557 557
558 if (!core_kernel_text((unsigned long)addr)) { 558 if (!core_kernel_text((unsigned long)addr)) {
559 pages[0] = vmalloc_to_page(addr); 559 pages[0] = vmalloc_to_page(addr);
560 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 560 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
561 } else { 561 } else {
562 pages[0] = virt_to_page(addr); 562 pages[0] = virt_to_page(addr);
563 WARN_ON(!PageReserved(pages[0])); 563 WARN_ON(!PageReserved(pages[0]));
564 pages[1] = virt_to_page(addr + PAGE_SIZE); 564 pages[1] = virt_to_page(addr + PAGE_SIZE);
565 } 565 }
566 BUG_ON(!pages[0]); 566 BUG_ON(!pages[0]);
567 local_irq_save(flags); 567 local_irq_save(flags);
568 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0])); 568 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
569 if (pages[1]) 569 if (pages[1])
570 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1])); 570 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
571 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0); 571 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
572 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); 572 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
573 clear_fixmap(FIX_TEXT_POKE0); 573 clear_fixmap(FIX_TEXT_POKE0);
574 if (pages[1]) 574 if (pages[1])
575 clear_fixmap(FIX_TEXT_POKE1); 575 clear_fixmap(FIX_TEXT_POKE1);
576 local_flush_tlb(); 576 local_flush_tlb();
577 sync_core(); 577 sync_core();
578 /* Could also do a CLFLUSH here to speed up CPU recovery; but 578 /* Could also do a CLFLUSH here to speed up CPU recovery; but
579 that causes hangs on some VIA CPUs. */ 579 that causes hangs on some VIA CPUs. */
580 for (i = 0; i < len; i++) 580 for (i = 0; i < len; i++)
581 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); 581 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
582 local_irq_restore(flags); 582 local_irq_restore(flags);
583 return addr; 583 return addr;
584 } 584 }
585 585
586 /* 586 /*
587 * Cross-modifying kernel text with stop_machine(). 587 * Cross-modifying kernel text with stop_machine().
588 * This code originally comes from immediate value. 588 * This code originally comes from immediate value.
589 */ 589 */
590 static atomic_t stop_machine_first; 590 static atomic_t stop_machine_first;
591 static int wrote_text; 591 static int wrote_text;
592 592
593 struct text_poke_params { 593 struct text_poke_params {
594 void *addr; 594 void *addr;
595 const void *opcode; 595 const void *opcode;
596 size_t len; 596 size_t len;
597 }; 597 };
598 598
599 static int __kprobes stop_machine_text_poke(void *data) 599 static int __kprobes stop_machine_text_poke(void *data)
600 { 600 {
601 struct text_poke_params *tpp = data; 601 struct text_poke_params *tpp = data;
602 602
603 if (atomic_dec_and_test(&stop_machine_first)) { 603 if (atomic_dec_and_test(&stop_machine_first)) {
604 text_poke(tpp->addr, tpp->opcode, tpp->len); 604 text_poke(tpp->addr, tpp->opcode, tpp->len);
605 smp_wmb(); /* Make sure other cpus see that this has run */ 605 smp_wmb(); /* Make sure other cpus see that this has run */
606 wrote_text = 1; 606 wrote_text = 1;
607 } else { 607 } else {
608 while (!wrote_text) 608 while (!wrote_text)
609 cpu_relax(); 609 cpu_relax();
610 smp_mb(); /* Load wrote_text before following execution */ 610 smp_mb(); /* Load wrote_text before following execution */
611 } 611 }
612 612
613 flush_icache_range((unsigned long)tpp->addr, 613 flush_icache_range((unsigned long)tpp->addr,
614 (unsigned long)tpp->addr + tpp->len); 614 (unsigned long)tpp->addr + tpp->len);
615 return 0; 615 return 0;
616 } 616 }
617 617
618 /** 618 /**
619 * text_poke_smp - Update instructions on a live kernel on SMP 619 * text_poke_smp - Update instructions on a live kernel on SMP
620 * @addr: address to modify 620 * @addr: address to modify
621 * @opcode: source of the copy 621 * @opcode: source of the copy
622 * @len: length to copy 622 * @len: length to copy
623 * 623 *
624 * Modify multi-byte instruction by using stop_machine() on SMP. This allows 624 * Modify multi-byte instruction by using stop_machine() on SMP. This allows
625 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying 625 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying
626 * should be allowed, since stop_machine() does _not_ protect code against 626 * should be allowed, since stop_machine() does _not_ protect code against
627 * NMI and MCE. 627 * NMI and MCE.
628 * 628 *
629 * Note: Must be called under get_online_cpus() and text_mutex. 629 * Note: Must be called under get_online_cpus() and text_mutex.
630 */ 630 */
631 void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len) 631 void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
632 { 632 {
633 struct text_poke_params tpp; 633 struct text_poke_params tpp;
634 634
635 tpp.addr = addr; 635 tpp.addr = addr;
636 tpp.opcode = opcode; 636 tpp.opcode = opcode;
637 tpp.len = len; 637 tpp.len = len;
638 atomic_set(&stop_machine_first, 1); 638 atomic_set(&stop_machine_first, 1);
639 wrote_text = 0; 639 wrote_text = 0;
640 stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); 640 /* Use __stop_machine() because the caller already got online_cpus. */
641 __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);
641 return addr; 642 return addr;
642 } 643 }
643 644
644 #if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) 645 #if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
645 646
646 unsigned char ideal_nop5[IDEAL_NOP_SIZE_5]; 647 unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
647 648
648 void __init arch_init_ideal_nop5(void) 649 void __init arch_init_ideal_nop5(void)
649 { 650 {
650 extern const unsigned char ftrace_test_p6nop[]; 651 extern const unsigned char ftrace_test_p6nop[];
651 extern const unsigned char ftrace_test_nop5[]; 652 extern const unsigned char ftrace_test_nop5[];
652 extern const unsigned char ftrace_test_jmp[]; 653 extern const unsigned char ftrace_test_jmp[];
653 int faulted = 0; 654 int faulted = 0;
654 655
655 /* 656 /*
656 * There is no good nop for all x86 archs. 657 * There is no good nop for all x86 archs.
657 * We will default to using the P6_NOP5, but first we 658 * We will default to using the P6_NOP5, but first we
658 * will test to make sure that the nop will actually 659 * will test to make sure that the nop will actually
659 * work on this CPU. If it faults, we will then 660 * work on this CPU. If it faults, we will then
660 * go to a lesser efficient 5 byte nop. If that fails 661 * go to a lesser efficient 5 byte nop. If that fails
661 * we then just use a jmp as our nop. This isn't the most 662 * we then just use a jmp as our nop. This isn't the most
662 * efficient nop, but we can not use a multi part nop 663 * efficient nop, but we can not use a multi part nop
663 * since we would then risk being preempted in the middle 664 * since we would then risk being preempted in the middle
664 * of that nop, and if we enabled tracing then, it might 665 * of that nop, and if we enabled tracing then, it might
665 * cause a system crash. 666 * cause a system crash.
666 * 667 *
667 * TODO: check the cpuid to determine the best nop. 668 * TODO: check the cpuid to determine the best nop.
668 */ 669 */
669 asm volatile ( 670 asm volatile (
670 "ftrace_test_jmp:" 671 "ftrace_test_jmp:"
671 "jmp ftrace_test_p6nop\n" 672 "jmp ftrace_test_p6nop\n"
672 "nop\n" 673 "nop\n"
673 "nop\n" 674 "nop\n"
674 "nop\n" /* 2 byte jmp + 3 bytes */ 675 "nop\n" /* 2 byte jmp + 3 bytes */
675 "ftrace_test_p6nop:" 676 "ftrace_test_p6nop:"
676 P6_NOP5 677 P6_NOP5
677 "jmp 1f\n" 678 "jmp 1f\n"
678 "ftrace_test_nop5:" 679 "ftrace_test_nop5:"
679 ".byte 0x66,0x66,0x66,0x66,0x90\n" 680 ".byte 0x66,0x66,0x66,0x66,0x90\n"
680 "1:" 681 "1:"
681 ".section .fixup, \"ax\"\n" 682 ".section .fixup, \"ax\"\n"
682 "2: movl $1, %0\n" 683 "2: movl $1, %0\n"
683 " jmp ftrace_test_nop5\n" 684 " jmp ftrace_test_nop5\n"
684 "3: movl $2, %0\n" 685 "3: movl $2, %0\n"
685 " jmp 1b\n" 686 " jmp 1b\n"
686 ".previous\n" 687 ".previous\n"
687 _ASM_EXTABLE(ftrace_test_p6nop, 2b) 688 _ASM_EXTABLE(ftrace_test_p6nop, 2b)
688 _ASM_EXTABLE(ftrace_test_nop5, 3b) 689 _ASM_EXTABLE(ftrace_test_nop5, 3b)
689 : "=r"(faulted) : "0" (faulted)); 690 : "=r"(faulted) : "0" (faulted));
690 691
691 switch (faulted) { 692 switch (faulted) {
692 case 0: 693 case 0:
693 pr_info("converting mcount calls to 0f 1f 44 00 00\n"); 694 pr_info("converting mcount calls to 0f 1f 44 00 00\n");
694 memcpy(ideal_nop5, ftrace_test_p6nop, IDEAL_NOP_SIZE_5); 695 memcpy(ideal_nop5, ftrace_test_p6nop, IDEAL_NOP_SIZE_5);
695 break; 696 break;
696 case 1: 697 case 1:
697 pr_info("converting mcount calls to 66 66 66 66 90\n"); 698 pr_info("converting mcount calls to 66 66 66 66 90\n");
698 memcpy(ideal_nop5, ftrace_test_nop5, IDEAL_NOP_SIZE_5); 699 memcpy(ideal_nop5, ftrace_test_nop5, IDEAL_NOP_SIZE_5);
699 break; 700 break;
700 case 2: 701 case 2:
701 pr_info("converting mcount calls to jmp . + 5\n"); 702 pr_info("converting mcount calls to jmp . + 5\n");
702 memcpy(ideal_nop5, ftrace_test_jmp, IDEAL_NOP_SIZE_5); 703 memcpy(ideal_nop5, ftrace_test_jmp, IDEAL_NOP_SIZE_5);
703 break; 704 break;
704 } 705 }
705 706
706 } 707 }
707 #endif 708 #endif
708 709