Blame view

arch/x86/kernel/alternative.c 18.7 KB
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
1
  #include <linux/module.h>
f6a570333   Al Viro   [PATCH] severing ...
2
  #include <linux/sched.h>
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
3
  #include <linux/mutex.h>
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
4
  #include <linux/list.h>
8b5a10fc6   Jan Beulich   x86: properly ann...
5
  #include <linux/stringify.h>
19d36ccdc   Andi Kleen   x86: Fix alternat...
6
7
8
  #include <linux/kprobes.h>
  #include <linux/mm.h>
  #include <linux/vmalloc.h>
3945dab45   Masami Hiramatsu   tracing, Text Edi...
9
  #include <linux/memory.h>
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
10
  #include <linux/stop_machine.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
11
  #include <linux/slab.h>
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
12
13
  #include <asm/alternative.h>
  #include <asm/sections.h>
19d36ccdc   Andi Kleen   x86: Fix alternat...
14
  #include <asm/pgtable.h>
8f4e956b3   Andi Kleen   x86: Stop MCEs an...
15
16
  #include <asm/mce.h>
  #include <asm/nmi.h>
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
17
  #include <asm/cacheflush.h>
78ff7fae0   Masami Hiramatsu   x86: implement at...
18
  #include <asm/tlbflush.h>
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
19
  #include <asm/io.h>
78ff7fae0   Masami Hiramatsu   x86: implement at...
20
  #include <asm/fixmap.h>
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
21

ab144f5ec   Andi Kleen   i386: Make patchi...
22
  #define MAX_PATCH_LEN (255-1)
09488165d   Jan Beulich   i386: smp-alt-onc...
23
24
  #ifdef CONFIG_HOTPLUG_CPU
  static int smp_alt_once;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
25

d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
26
27
28
29
30
  static int __init bootonly(char *str)
  {
  	smp_alt_once = 1;
  	return 1;
  }
b7fb4af06   Jeremy Fitzhardinge   [PATCH] i386: All...
31
  __setup("smp-alt-boot", bootonly);
09488165d   Jan Beulich   i386: smp-alt-onc...
32
33
34
  #else
  #define smp_alt_once 1
  #endif
8b5a10fc6   Jan Beulich   x86: properly ann...
35
  static int __initdata_or_module debug_alternative;
b7fb4af06   Jeremy Fitzhardinge   [PATCH] i386: All...
36

d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
37
38
39
40
41
  static int __init debug_alt(char *str)
  {
  	debug_alternative = 1;
  	return 1;
  }
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
42
  __setup("debug-alternative", debug_alt);
09488165d   Jan Beulich   i386: smp-alt-onc...
43
  static int noreplace_smp;
b7fb4af06   Jeremy Fitzhardinge   [PATCH] i386: All...
44
45
46
47
48
49
  static int __init setup_noreplace_smp(char *str)
  {
  	noreplace_smp = 1;
  	return 1;
  }
  __setup("noreplace-smp", setup_noreplace_smp);
959b4fdfe   Jeremy Fitzhardinge   [PATCH] i386: PAR...
50
  #ifdef CONFIG_PARAVIRT
8b5a10fc6   Jan Beulich   x86: properly ann...
51
  static int __initdata_or_module noreplace_paravirt = 0;
959b4fdfe   Jeremy Fitzhardinge   [PATCH] i386: PAR...
52
53
54
55
56
57
58
59
  
  static int __init setup_noreplace_paravirt(char *str)
  {
  	noreplace_paravirt = 1;
  	return 1;
  }
  __setup("noreplace-paravirt", setup_noreplace_paravirt);
  #endif
b7fb4af06   Jeremy Fitzhardinge   [PATCH] i386: All...
60

d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
61
62
  #define DPRINTK(fmt, args...) if (debug_alternative) \
  	printk(KERN_DEBUG fmt, args)
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
63
64
65
66
67
68
69
70
71
  /*
   * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes
   * that correspond to that nop. Getting from one nop to the next, we
   * add to the array the offset that is equal to the sum of all sizes of
   * nops preceding the one we are after.
   *
   * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the
   * nice symmetry of sizes of the previous nops.
   */
8b5a10fc6   Jan Beulich   x86: properly ann...
72
  #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  static const unsigned char intelnops[] =
  {
  	GENERIC_NOP1,
  	GENERIC_NOP2,
  	GENERIC_NOP3,
  	GENERIC_NOP4,
  	GENERIC_NOP5,
  	GENERIC_NOP6,
  	GENERIC_NOP7,
  	GENERIC_NOP8,
  	GENERIC_NOP5_ATOMIC
  };
  static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
  {
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
87
88
89
90
91
92
93
94
95
  	NULL,
  	intelnops,
  	intelnops + 1,
  	intelnops + 1 + 2,
  	intelnops + 1 + 2 + 3,
  	intelnops + 1 + 2 + 3 + 4,
  	intelnops + 1 + 2 + 3 + 4 + 5,
  	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
  	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
96
  	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
97
  };
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
98
99
100
  #endif
  
  #ifdef K8_NOP1
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
101
102
103
104
105
106
107
108
109
110
111
112
113
114
  static const unsigned char k8nops[] =
  {
  	K8_NOP1,
  	K8_NOP2,
  	K8_NOP3,
  	K8_NOP4,
  	K8_NOP5,
  	K8_NOP6,
  	K8_NOP7,
  	K8_NOP8,
  	K8_NOP5_ATOMIC
  };
  static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
  {
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
115
116
117
118
119
120
121
122
123
  	NULL,
  	k8nops,
  	k8nops + 1,
  	k8nops + 1 + 2,
  	k8nops + 1 + 2 + 3,
  	k8nops + 1 + 2 + 3 + 4,
  	k8nops + 1 + 2 + 3 + 4 + 5,
  	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
  	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
124
  	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
125
  };
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
126
  #endif
8b5a10fc6   Jan Beulich   x86: properly ann...
127
  #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
128
129
130
131
132
133
134
135
136
137
138
139
140
141
  static const unsigned char k7nops[] =
  {
  	K7_NOP1,
  	K7_NOP2,
  	K7_NOP3,
  	K7_NOP4,
  	K7_NOP5,
  	K7_NOP6,
  	K7_NOP7,
  	K7_NOP8,
  	K7_NOP5_ATOMIC
  };
  static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
  {
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
142
143
144
145
146
147
148
149
150
  	NULL,
  	k7nops,
  	k7nops + 1,
  	k7nops + 1 + 2,
  	k7nops + 1 + 2 + 3,
  	k7nops + 1 + 2 + 3 + 4,
  	k7nops + 1 + 2 + 3 + 4 + 5,
  	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
  	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
151
  	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
152
  };
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
153
  #endif
32c464f5d   Jan Beulich   x86: multi-byte s...
154
  #ifdef P6_NOP1
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
155
156
157
158
159
160
161
162
163
164
165
166
167
168
  static const unsigned char  __initconst_or_module p6nops[] =
  {
  	P6_NOP1,
  	P6_NOP2,
  	P6_NOP3,
  	P6_NOP4,
  	P6_NOP5,
  	P6_NOP6,
  	P6_NOP7,
  	P6_NOP8,
  	P6_NOP5_ATOMIC
  };
  static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
  {
32c464f5d   Jan Beulich   x86: multi-byte s...
169
170
171
172
173
174
175
176
177
  	NULL,
  	p6nops,
  	p6nops + 1,
  	p6nops + 1 + 2,
  	p6nops + 1 + 2 + 3,
  	p6nops + 1 + 2 + 3 + 4,
  	p6nops + 1 + 2 + 3 + 4 + 5,
  	p6nops + 1 + 2 + 3 + 4 + 5 + 6,
  	p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
178
  	p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
32c464f5d   Jan Beulich   x86: multi-byte s...
179
180
  };
  #endif
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
181
  /* Initialize these to a safe default */
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
182
  #ifdef CONFIG_X86_64
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
183
184
185
186
  const unsigned char * const *ideal_nops = p6_nops;
  #else
  const unsigned char * const *ideal_nops = intel_nops;
  #endif
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
187

dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
188
  void __init arch_init_ideal_nops(void)
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
189
  {
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
190
191
  	switch (boot_cpu_data.x86_vendor) {
  	case X86_VENDOR_INTEL:
d8d9766c8   H. Peter Anvin   x86, cpu: Change ...
192
193
194
195
196
197
198
199
200
201
202
203
204
  		/*
  		 * Due to a decoder implementation quirk, some
  		 * specific Intel CPUs actually perform better with
  		 * the "k8_nops" than with the SDM-recommended NOPs.
  		 */
  		if (boot_cpu_data.x86 == 6 &&
  		    boot_cpu_data.x86_model >= 0x0f &&
  		    boot_cpu_data.x86_model != 0x1c &&
  		    boot_cpu_data.x86_model != 0x26 &&
  		    boot_cpu_data.x86_model != 0x27 &&
  		    boot_cpu_data.x86_model < 0x30) {
  			ideal_nops = k8_nops;
  		} else if (boot_cpu_has(X86_FEATURE_NOPL)) {
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
205
206
207
208
209
210
211
212
  			   ideal_nops = p6_nops;
  		} else {
  #ifdef CONFIG_X86_64
  			ideal_nops = k8_nops;
  #else
  			ideal_nops = intel_nops;
  #endif
  		}
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
213

dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
214
215
216
217
218
219
220
221
222
223
224
225
  	default:
  #ifdef CONFIG_X86_64
  		ideal_nops = k8_nops;
  #else
  		if (boot_cpu_has(X86_FEATURE_K8))
  			ideal_nops = k8_nops;
  		else if (boot_cpu_has(X86_FEATURE_K7))
  			ideal_nops = k7_nops;
  		else
  			ideal_nops = intel_nops;
  #endif
  	}
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
226
  }
ab144f5ec   Andi Kleen   i386: Make patchi...
227
  /* Use this to add nops to a buffer, then text_poke the whole buffer. */
8b5a10fc6   Jan Beulich   x86: properly ann...
228
  static void __init_or_module add_nops(void *insns, unsigned int len)
139ec7c41   Rusty Russell   [PATCH] paravirt:...
229
  {
139ec7c41   Rusty Russell   [PATCH] paravirt:...
230
231
232
233
  	while (len > 0) {
  		unsigned int noplen = len;
  		if (noplen > ASM_NOP_MAX)
  			noplen = ASM_NOP_MAX;
dc326fca2   H. Peter Anvin   x86, cpu: Clean u...
234
  		memcpy(insns, ideal_nops[noplen], noplen);
139ec7c41   Rusty Russell   [PATCH] paravirt:...
235
236
237
238
  		insns += noplen;
  		len -= noplen;
  	}
  }
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
239
  extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
5967ed87a   Jan Beulich   x86-64: Reduce SM...
240
  extern s32 __smp_locks[], __smp_locks_end[];
fa6f2cc77   Jason Baron   jump label: Make ...
241
  void *text_poke_early(void *addr, const void *opcode, size_t len);
d167a5187   Gerd Hoffmann   [PATCH] x86_64: x...
242

9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
243
244
  /* Replace instructions with better alternatives for this CPU type.
     This runs before SMP is initialized to avoid SMP problems with
0d2eb44f6   Lucas De Marchi   x86: Fix common m...
245
     self modifying code. This implies that asymmetric systems where
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
246
247
     APs have less capabilities than the boot processor are not handled.
     Tough. Make sure you disable such features by hand. */
8b5a10fc6   Jan Beulich   x86: properly ann...
248
249
  void __init_or_module apply_alternatives(struct alt_instr *start,
  					 struct alt_instr *end)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
250
  {
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
251
  	struct alt_instr *a;
59e97e4d6   Andy Lutomirski   x86: Make alterna...
252
  	u8 *instr, *replacement;
1b1d92581   Jan Beulich   x86-64: Modify co...
253
  	u8 insnbuf[MAX_PATCH_LEN];
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
254

77bf90ed6   Harvey Harrison   x86: replace rema...
255
256
  	DPRINTK("%s: alt table %p -> %p
  ", __func__, start, end);
509731336   Fenghua Yu   x86, alternative,...
257
258
259
260
261
262
263
264
265
  	/*
  	 * The scan order should be from start to end. A later scanned
  	 * alternative code can overwrite a previous scanned alternative code.
  	 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
  	 * patch code.
  	 *
  	 * So be careful if you want to change the scan order to any other
  	 * order.
  	 */
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
266
  	for (a = start; a < end; a++) {
59e97e4d6   Andy Lutomirski   x86: Make alterna...
267
268
  		instr = (u8 *)&a->instr_offset + a->instr_offset;
  		replacement = (u8 *)&a->repl_offset + a->repl_offset;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
269
  		BUG_ON(a->replacementlen > a->instrlen);
ab144f5ec   Andi Kleen   i386: Make patchi...
270
  		BUG_ON(a->instrlen > sizeof(insnbuf));
3b770a212   H. Peter Anvin   x86, alternatives...
271
  		BUG_ON(a->cpuid >= NCAPINTS*32);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
272
273
  		if (!boot_cpu_has(a->cpuid))
  			continue;
59e97e4d6   Andy Lutomirski   x86: Make alterna...
274
275
276
277
278
279
280
281
282
  
  		memcpy(insnbuf, replacement, a->replacementlen);
  
  		/* 0xe8 is a relative jump; fix the offset. */
  		if (*insnbuf == 0xe8 && a->replacementlen == 5)
  		    *(s32 *)(insnbuf + 1) += replacement - instr;
  
  		add_nops(insnbuf + a->replacementlen,
  			 a->instrlen - a->replacementlen);
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
283
  		text_poke_early(instr, insnbuf, a->instrlen);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
284
285
  	}
  }
8ec4d41f8   Gerd Hoffmann   [PATCH] SMP alter...
286
  #ifdef CONFIG_SMP
5967ed87a   Jan Beulich   x86-64: Reduce SM...
287
288
  static void alternatives_smp_lock(const s32 *start, const s32 *end,
  				  u8 *text, u8 *text_end)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
289
  {
5967ed87a   Jan Beulich   x86-64: Reduce SM...
290
  	const s32 *poff;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
291

3945dab45   Masami Hiramatsu   tracing, Text Edi...
292
  	mutex_lock(&text_mutex);
5967ed87a   Jan Beulich   x86-64: Reduce SM...
293
294
295
296
  	for (poff = start; poff < end; poff++) {
  		u8 *ptr = (u8 *)poff + *poff;
  
  		if (!*poff || ptr < text || ptr >= text_end)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
297
  			continue;
f88f07e0f   Mathieu Desnoyers   x86: alternatives...
298
  		/* turn DS segment override prefix into lock prefix */
d9c5841e2   H. Peter Anvin   Merge branch 'x86...
299
300
  		if (*ptr == 0x3e)
  			text_poke(ptr, ((unsigned char []){0xf0}), 1);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
301
  	};
3945dab45   Masami Hiramatsu   tracing, Text Edi...
302
  	mutex_unlock(&text_mutex);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
303
  }
5967ed87a   Jan Beulich   x86-64: Reduce SM...
304
305
  static void alternatives_smp_unlock(const s32 *start, const s32 *end,
  				    u8 *text, u8 *text_end)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
306
  {
5967ed87a   Jan Beulich   x86-64: Reduce SM...
307
  	const s32 *poff;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
308

b7fb4af06   Jeremy Fitzhardinge   [PATCH] i386: All...
309
310
  	if (noreplace_smp)
  		return;
3945dab45   Masami Hiramatsu   tracing, Text Edi...
311
  	mutex_lock(&text_mutex);
5967ed87a   Jan Beulich   x86-64: Reduce SM...
312
313
314
315
  	for (poff = start; poff < end; poff++) {
  		u8 *ptr = (u8 *)poff + *poff;
  
  		if (!*poff || ptr < text || ptr >= text_end)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
316
  			continue;
f88f07e0f   Mathieu Desnoyers   x86: alternatives...
317
  		/* turn lock prefix into DS segment override prefix */
d9c5841e2   H. Peter Anvin   Merge branch 'x86...
318
319
  		if (*ptr == 0xf0)
  			text_poke(ptr, ((unsigned char []){0x3E}), 1);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
320
  	};
3945dab45   Masami Hiramatsu   tracing, Text Edi...
321
  	mutex_unlock(&text_mutex);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
322
323
324
325
326
327
328
329
  }
  
  struct smp_alt_module {
  	/* what is this ??? */
  	struct module	*mod;
  	char		*name;
  
  	/* ptrs to lock prefixes */
5967ed87a   Jan Beulich   x86-64: Reduce SM...
330
331
  	const s32	*locks;
  	const s32	*locks_end;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
332
333
334
335
336
337
338
339
  
  	/* .text segment, needed to avoid patching init code ;) */
  	u8		*text;
  	u8		*text_end;
  
  	struct list_head next;
  };
  static LIST_HEAD(smp_alt_modules);
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
340
  static DEFINE_MUTEX(smp_alt);
ca74a6f84   Andi Kleen   x86: optimize loc...
341
  static int smp_mode = 1;	/* protected by smp_alt */
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
342

8b5a10fc6   Jan Beulich   x86: properly ann...
343
344
345
346
  void __init_or_module alternatives_smp_module_add(struct module *mod,
  						  char *name,
  						  void *locks, void *locks_end,
  						  void *text,  void *text_end)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
347
348
  {
  	struct smp_alt_module *smp;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
349

b7fb4af06   Jeremy Fitzhardinge   [PATCH] i386: All...
350
351
  	if (noreplace_smp)
  		return;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
  	if (smp_alt_once) {
  		if (boot_cpu_has(X86_FEATURE_UP))
  			alternatives_smp_unlock(locks, locks_end,
  						text, text_end);
  		return;
  	}
  
  	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
  	if (NULL == smp)
  		return; /* we'll run the (safe but slow) SMP code then ... */
  
  	smp->mod	= mod;
  	smp->name	= name;
  	smp->locks	= locks;
  	smp->locks_end	= locks_end;
  	smp->text	= text;
  	smp->text_end	= text_end;
  	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s
  ",
77bf90ed6   Harvey Harrison   x86: replace rema...
371
  		__func__, smp->locks, smp->locks_end,
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
372
  		smp->text, smp->text_end, smp->name);
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
373
  	mutex_lock(&smp_alt);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
374
375
376
377
  	list_add_tail(&smp->next, &smp_alt_modules);
  	if (boot_cpu_has(X86_FEATURE_UP))
  		alternatives_smp_unlock(smp->locks, smp->locks_end,
  					smp->text, smp->text_end);
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
378
  	mutex_unlock(&smp_alt);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
379
  }
8b5a10fc6   Jan Beulich   x86: properly ann...
380
  void __init_or_module alternatives_smp_module_del(struct module *mod)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
381
382
  {
  	struct smp_alt_module *item;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
383

b7fb4af06   Jeremy Fitzhardinge   [PATCH] i386: All...
384
  	if (smp_alt_once || noreplace_smp)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
385
  		return;
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
386
  	mutex_lock(&smp_alt);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
387
388
389
390
  	list_for_each_entry(item, &smp_alt_modules, next) {
  		if (mod != item->mod)
  			continue;
  		list_del(&item->next);
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
391
  		mutex_unlock(&smp_alt);
77bf90ed6   Harvey Harrison   x86: replace rema...
392
393
  		DPRINTK("%s: %s
  ", __func__, item->name);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
394
395
396
  		kfree(item);
  		return;
  	}
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
397
  	mutex_unlock(&smp_alt);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
398
  }
3fb82d56a   Suresh Siddha   x86, suspend: Avo...
399
  bool skip_smp_alternatives;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
400
401
402
  void alternatives_smp_switch(int smp)
  {
  	struct smp_alt_module *mod;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
403

3047e99ed   Ingo Molnar   [PATCH] lockdep: ...
404
405
  #ifdef CONFIG_LOCKDEP
  	/*
17abecfe6   Ingo Molnar   x86: fix up alter...
406
407
408
409
410
  	 * Older binutils section handling bug prevented
  	 * alternatives-replacement from working reliably.
  	 *
  	 * If this still occurs then you should see a hang
  	 * or crash shortly after this line:
3047e99ed   Ingo Molnar   [PATCH] lockdep: ...
411
  	 */
17abecfe6   Ingo Molnar   x86: fix up alter...
412
413
  	printk("lockdep: fixing up alternatives.
  ");
3047e99ed   Ingo Molnar   [PATCH] lockdep: ...
414
  #endif
3fb82d56a   Suresh Siddha   x86, suspend: Avo...
415
  	if (noreplace_smp || smp_alt_once || skip_smp_alternatives)
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
416
417
  		return;
  	BUG_ON(!smp && (num_online_cpus() > 1));
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
418
  	mutex_lock(&smp_alt);
ca74a6f84   Andi Kleen   x86: optimize loc...
419
420
421
422
423
424
425
426
  
  	/*
  	 * Avoid unnecessary switches because it forces JIT based VMs to
  	 * throw away all cached translations, which can be quite costly.
  	 */
  	if (smp == smp_mode) {
  		/* nothing */
  	} else if (smp) {
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
427
428
  		printk(KERN_INFO "SMP alternatives: switching to SMP code
  ");
53756d372   Jeremy Fitzhardinge   x86: add set/clea...
429
430
  		clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
  		clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
431
432
433
434
435
436
  		list_for_each_entry(mod, &smp_alt_modules, next)
  			alternatives_smp_lock(mod->locks, mod->locks_end,
  					      mod->text, mod->text_end);
  	} else {
  		printk(KERN_INFO "SMP alternatives: switching to UP code
  ");
53756d372   Jeremy Fitzhardinge   x86: add set/clea...
437
438
  		set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
  		set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
439
440
441
442
  		list_for_each_entry(mod, &smp_alt_modules, next)
  			alternatives_smp_unlock(mod->locks, mod->locks_end,
  						mod->text, mod->text_end);
  	}
ca74a6f84   Andi Kleen   x86: optimize loc...
443
  	smp_mode = smp;
2f1dafe50   Pekka Paalanen   x86: fix SMP alte...
444
  	mutex_unlock(&smp_alt);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
445
  }
2cfa19780   Masami Hiramatsu   ftrace/alternativ...
446
447
448
449
  /* Return 1 if the address range is reserved for smp-alternatives */
  int alternatives_text_reserved(void *start, void *end)
  {
  	struct smp_alt_module *mod;
5967ed87a   Jan Beulich   x86-64: Reduce SM...
450
  	const s32 *poff;
076dc4a65   Masami Hiramatsu   x86/alternatives:...
451
452
  	u8 *text_start = start;
  	u8 *text_end = end;
2cfa19780   Masami Hiramatsu   ftrace/alternativ...
453
454
  
  	list_for_each_entry(mod, &smp_alt_modules, next) {
076dc4a65   Masami Hiramatsu   x86/alternatives:...
455
  		if (mod->text > text_end || mod->text_end < text_start)
2cfa19780   Masami Hiramatsu   ftrace/alternativ...
456
  			continue;
5967ed87a   Jan Beulich   x86-64: Reduce SM...
457
458
459
460
  		for (poff = mod->locks; poff < mod->locks_end; poff++) {
  			const u8 *ptr = (const u8 *)poff + *poff;
  
  			if (text_start <= ptr && text_end > ptr)
2cfa19780   Masami Hiramatsu   ftrace/alternativ...
461
  				return 1;
5967ed87a   Jan Beulich   x86-64: Reduce SM...
462
  		}
2cfa19780   Masami Hiramatsu   ftrace/alternativ...
463
464
465
466
  	}
  
  	return 0;
  }
8ec4d41f8   Gerd Hoffmann   [PATCH] SMP alter...
467
  #endif
139ec7c41   Rusty Russell   [PATCH] paravirt:...
468
  #ifdef CONFIG_PARAVIRT
8b5a10fc6   Jan Beulich   x86: properly ann...
469
470
  void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
  				     struct paravirt_patch_site *end)
139ec7c41   Rusty Russell   [PATCH] paravirt:...
471
  {
98de032b6   Jeremy Fitzhardinge   [PATCH] i386: PAR...
472
  	struct paravirt_patch_site *p;
ab144f5ec   Andi Kleen   i386: Make patchi...
473
  	char insnbuf[MAX_PATCH_LEN];
139ec7c41   Rusty Russell   [PATCH] paravirt:...
474

959b4fdfe   Jeremy Fitzhardinge   [PATCH] i386: PAR...
475
476
  	if (noreplace_paravirt)
  		return;
139ec7c41   Rusty Russell   [PATCH] paravirt:...
477
478
  	for (p = start; p < end; p++) {
  		unsigned int used;
ab144f5ec   Andi Kleen   i386: Make patchi...
479
  		BUG_ON(p->len > MAX_PATCH_LEN);
d34fda4a8   Chris Wright   x86: properly ini...
480
481
  		/* prep the buffer with the original instructions */
  		memcpy(insnbuf, p->instr, p->len);
93b1eab3d   Jeremy Fitzhardinge   paravirt: refacto...
482
483
  		used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
  					 (unsigned long)p->instr, p->len);
7f63c41c6   Jeremy Fitzhardinge   [PATCH] i386: PAR...
484

63f70270c   Jeremy Fitzhardinge   [PATCH] i386: PAR...
485
  		BUG_ON(used > p->len);
139ec7c41   Rusty Russell   [PATCH] paravirt:...
486
  		/* Pad the rest with nops */
ab144f5ec   Andi Kleen   i386: Make patchi...
487
  		add_nops(insnbuf + used, p->len - used);
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
488
  		text_poke_early(p->instr, insnbuf, p->len);
139ec7c41   Rusty Russell   [PATCH] paravirt:...
489
  	}
139ec7c41   Rusty Russell   [PATCH] paravirt:...
490
  }
98de032b6   Jeremy Fitzhardinge   [PATCH] i386: PAR...
491
  extern struct paravirt_patch_site __start_parainstructions[],
139ec7c41   Rusty Russell   [PATCH] paravirt:...
492
493
  	__stop_parainstructions[];
  #endif	/* CONFIG_PARAVIRT */
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
494
495
  void __init alternative_instructions(void)
  {
8f4e956b3   Andi Kleen   x86: Stop MCEs an...
496
497
498
499
  	/* The patching is not fully atomic, so try to avoid local interruptions
  	   that might execute the to be patched code.
  	   Other CPUs are not running. */
  	stop_nmi();
123aa76ec   Andi Kleen   x86, mce: don't d...
500
501
502
503
504
505
506
507
508
509
510
  
  	/*
  	 * Don't stop machine check exceptions while patching.
  	 * MCEs only happen when something got corrupted and in this
  	 * case we must do something about the corruption.
  	 * Ignoring it is worse than a unlikely patching race.
  	 * Also machine checks tend to be broadcast and if one CPU
  	 * goes into machine check the others follow quickly, so we don't
  	 * expect a machine check to cause undue problems during to code
  	 * patching.
  	 */
8f4e956b3   Andi Kleen   x86: Stop MCEs an...
511

9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
512
513
514
515
516
517
518
519
  	apply_alternatives(__alt_instructions, __alt_instructions_end);
  
  	/* switch to patch-once-at-boottime-only mode and free the
  	 * tables in case we know the number of CPUs will never ever
  	 * change */
  #ifdef CONFIG_HOTPLUG_CPU
  	if (num_possible_cpus() < 2)
  		smp_alt_once = 1;
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
520
  #endif
8ec4d41f8   Gerd Hoffmann   [PATCH] SMP alter...
521
  #ifdef CONFIG_SMP
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
522
523
524
525
  	if (smp_alt_once) {
  		if (1 == num_possible_cpus()) {
  			printk(KERN_INFO "SMP alternatives: switching to UP code
  ");
53756d372   Jeremy Fitzhardinge   x86: add set/clea...
526
527
  			set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
  			set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
528
529
530
  			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
  						_text, _etext);
  		}
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
531
  	} else {
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
532
533
534
  		alternatives_smp_module_add(NULL, "core kernel",
  					    __smp_locks, __smp_locks_end,
  					    _text, _etext);
ca74a6f84   Andi Kleen   x86: optimize loc...
535
536
  
  		/* Only switch to UP mode if we don't immediately boot others */
649c6653f   Thomas Gleixner   x86: improve UP k...
537
  		if (num_present_cpus() == 1 || setup_max_cpus <= 1)
ca74a6f84   Andi Kleen   x86: optimize loc...
538
  			alternatives_smp_switch(0);
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
539
  	}
8ec4d41f8   Gerd Hoffmann   [PATCH] SMP alter...
540
  #endif
441d40dca   Jeremy Fitzhardinge   [PATCH] x86: PARA...
541
   	apply_paravirt(__parainstructions, __parainstructions_end);
8f4e956b3   Andi Kleen   x86: Stop MCEs an...
542

f68fd5f48   Fengguang Wu   x86: call free_in...
543
544
545
546
  	if (smp_alt_once)
  		free_init_pages("SMP alternatives",
  				(unsigned long)__smp_locks,
  				(unsigned long)__smp_locks_end);
8f4e956b3   Andi Kleen   x86: Stop MCEs an...
547
  	restart_nmi();
9a0b5817a   Gerd Hoffmann   [PATCH] x86: SMP ...
548
  }
19d36ccdc   Andi Kleen   x86: Fix alternat...
549

e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
550
551
552
553
554
555
  /**
   * text_poke_early - Update instructions on a live kernel at boot time
   * @addr: address to modify
   * @opcode: source of the copy
   * @len: length to copy
   *
19d36ccdc   Andi Kleen   x86: Fix alternat...
556
557
   * When you use this code to patch more than one byte of an instruction
   * you need to make sure that other CPUs cannot execute this code in parallel.
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
558
559
560
   * Also no thread must be currently preempted in the middle of these
   * instructions. And on the local CPU you need to be protected again NMI or MCE
   * handlers seeing an inconsistent instruction while you patch.
19d36ccdc   Andi Kleen   x86: Fix alternat...
561
   */
fa6f2cc77   Jason Baron   jump label: Make ...
562
  void *__init_or_module text_poke_early(void *addr, const void *opcode,
8b5a10fc6   Jan Beulich   x86: properly ann...
563
  					      size_t len)
19d36ccdc   Andi Kleen   x86: Fix alternat...
564
  {
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
565
566
  	unsigned long flags;
  	local_irq_save(flags);
19d36ccdc   Andi Kleen   x86: Fix alternat...
567
  	memcpy(addr, opcode, len);
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
568
  	sync_core();
5367b6887   Ben Hutchings   x86: Fix code pat...
569
  	local_irq_restore(flags);
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
  	/* Could also do a CLFLUSH here to speed up CPU recovery; but
  	   that causes hangs on some VIA CPUs. */
  	return addr;
  }
  
  /**
   * text_poke - Update instructions on a live kernel
   * @addr: address to modify
   * @opcode: source of the copy
   * @len: length to copy
   *
   * Only atomic text poke/set should be allowed when not doing early patching.
   * It means the size must be writable atomically and the address must be aligned
   * in a way that permits an atomic write. It also makes sure we fit on a single
   * page.
78ff7fae0   Masami Hiramatsu   x86: implement at...
585
586
   *
   * Note: Must be called under text_mutex.
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
587
588
589
   */
  void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
  {
78ff7fae0   Masami Hiramatsu   x86: implement at...
590
  	unsigned long flags;
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
591
  	char *vaddr;
b7b66baa8   Mathieu Desnoyers   x86: clean up tex...
592
593
  	struct page *pages[2];
  	int i;
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
594

b7b66baa8   Mathieu Desnoyers   x86: clean up tex...
595
596
597
  	if (!core_kernel_text((unsigned long)addr)) {
  		pages[0] = vmalloc_to_page(addr);
  		pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
15a601eb9   Mathieu Desnoyers   x86: fix test_pok...
598
  	} else {
b7b66baa8   Mathieu Desnoyers   x86: clean up tex...
599
  		pages[0] = virt_to_page(addr);
00c6b2d5d   Ingo Molnar   x86: harden kerne...
600
  		WARN_ON(!PageReserved(pages[0]));
b7b66baa8   Mathieu Desnoyers   x86: clean up tex...
601
  		pages[1] = virt_to_page(addr + PAGE_SIZE);
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
602
  	}
b7b66baa8   Mathieu Desnoyers   x86: clean up tex...
603
  	BUG_ON(!pages[0]);
7cf494270   Masami Hiramatsu   x86: expand irq-o...
604
  	local_irq_save(flags);
78ff7fae0   Masami Hiramatsu   x86: implement at...
605
606
607
608
  	set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
  	if (pages[1])
  		set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
  	vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
b7b66baa8   Mathieu Desnoyers   x86: clean up tex...
609
  	memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
78ff7fae0   Masami Hiramatsu   x86: implement at...
610
611
612
613
  	clear_fixmap(FIX_TEXT_POKE0);
  	if (pages[1])
  		clear_fixmap(FIX_TEXT_POKE1);
  	local_flush_tlb();
19d36ccdc   Andi Kleen   x86: Fix alternat...
614
  	sync_core();
a534b6791   Andi Kleen   x86_64: Remove CL...
615
616
  	/* Could also do a CLFLUSH here to speed up CPU recovery; but
  	   that causes hangs on some VIA CPUs. */
b7b66baa8   Mathieu Desnoyers   x86: clean up tex...
617
618
  	for (i = 0; i < len; i++)
  		BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
7cf494270   Masami Hiramatsu   x86: expand irq-o...
619
  	local_irq_restore(flags);
e587cadd8   Mathieu Desnoyers   x86: enhance DEBU...
620
  	return addr;
19d36ccdc   Andi Kleen   x86: Fix alternat...
621
  }
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
622
623
624
625
626
627
628
629
630
  
  /*
   * Cross-modifying kernel text with stop_machine().
   * This code originally comes from immediate value.
   */
  static atomic_t stop_machine_first;
  static int wrote_text;
  
  struct text_poke_params {
7deb18dcf   Masami Hiramatsu   x86: Introduce te...
631
632
  	struct text_poke_param *params;
  	int nparams;
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
633
634
635
636
637
  };
  
  static int __kprobes stop_machine_text_poke(void *data)
  {
  	struct text_poke_params *tpp = data;
7deb18dcf   Masami Hiramatsu   x86: Introduce te...
638
639
  	struct text_poke_param *p;
  	int i;
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
640
641
  
  	if (atomic_dec_and_test(&stop_machine_first)) {
7deb18dcf   Masami Hiramatsu   x86: Introduce te...
642
643
644
645
  		for (i = 0; i < tpp->nparams; i++) {
  			p = &tpp->params[i];
  			text_poke(p->addr, p->opcode, p->len);
  		}
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
646
647
648
649
  		smp_wmb();	/* Make sure other cpus see that this has run */
  		wrote_text = 1;
  	} else {
  		while (!wrote_text)
e5a110166   Masami Hiramatsu   x86: Issue at lea...
650
651
  			cpu_relax();
  		smp_mb();	/* Load wrote_text before following execution */
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
652
  	}
7deb18dcf   Masami Hiramatsu   x86: Introduce te...
653
654
655
656
657
  	for (i = 0; i < tpp->nparams; i++) {
  		p = &tpp->params[i];
  		flush_icache_range((unsigned long)p->addr,
  				   (unsigned long)p->addr + p->len);
  	}
0e00f7aed   Mathieu Desnoyers   x86: stop_machine...
658
659
660
661
662
663
  	/*
  	 * Intel Archiecture Software Developer's Manual section 7.1.3 specifies
  	 * that a core serializing instruction such as "cpuid" should be
  	 * executed on _each_ core before the new instruction is made visible.
  	 */
  	sync_core();
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
  	return 0;
  }
  
  /**
   * text_poke_smp - Update instructions on a live kernel on SMP
   * @addr: address to modify
   * @opcode: source of the copy
   * @len: length to copy
   *
   * Modify multi-byte instruction by using stop_machine() on SMP. This allows
   * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying
   * should be allowed, since stop_machine() does _not_ protect code against
   * NMI and MCE.
   *
   * Note: Must be called under get_online_cpus() and text_mutex.
   */
  void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
  {
  	struct text_poke_params tpp;
7deb18dcf   Masami Hiramatsu   x86: Introduce te...
683
  	struct text_poke_param p;
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
684

7deb18dcf   Masami Hiramatsu   x86: Introduce te...
685
686
687
688
689
  	p.addr = addr;
  	p.opcode = opcode;
  	p.len = len;
  	tpp.params = &p;
  	tpp.nparams = 1;
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
690
691
  	atomic_set(&stop_machine_first, 1);
  	wrote_text = 0;
3caa37519   Masami Hiramatsu   x86: Use __stop_m...
692
  	/* Use __stop_machine() because the caller already got online_cpus. */
404ba5d7b   Jason Baron   x86, alternative:...
693
  	__stop_machine(stop_machine_text_poke, (void *)&tpp, cpu_online_mask);
3d55cc8a0   Masami Hiramatsu   x86: Add text_pok...
694
695
  	return addr;
  }
7deb18dcf   Masami Hiramatsu   x86: Introduce te...
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
  /**
   * text_poke_smp_batch - Update instructions on a live kernel on SMP
   * @params: an array of text_poke parameters
   * @n: the number of elements in params.
   *
   * Modify multi-byte instruction by using stop_machine() on SMP. Since the
   * stop_machine() is heavy task, it is better to aggregate text_poke requests
   * and do it once if possible.
   *
   * Note: Must be called under get_online_cpus() and text_mutex.
   */
  void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n)
  {
  	struct text_poke_params tpp = {.params = params, .nparams = n};
  
  	atomic_set(&stop_machine_first, 1);
  	wrote_text = 0;
78345d2ed   Rabin Vincent   x86: Call stop_ma...
713
  	__stop_machine(stop_machine_text_poke, (void *)&tpp, cpu_online_mask);
7deb18dcf   Masami Hiramatsu   x86: Introduce te...
714
  }