Eric Lee / linux-smarc-t335x-v3.2

Commit 9f9d489a3e78b49d897734eaaf9dea568dbea66e

Authored by Jeremy Fitzhardinge 2008-06-25 12:19:32 +0800

Committed by Ingo Molnar 2008-07-08 19:15:58 +0800

Exists in master and in 4 other branches

x86/paravirt, 64-bit: make load_gs_index() a paravirt operation

Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Cc: xen-devel <xen-devel@lists.xensource.com>
Cc: Stephen Tweedie <sct@redhat.com>
Cc: Eduardo Habkost <ehabkost@redhat.com>
Cc: Mark McLoughlin <markmc@redhat.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

Showing 5 changed files with 18 additions and 4 deletions Inline Diff

arch/x86/kernel/entry_64.S
arch/x86/kernel/paravirt.c
include/asm-x86/elf.h
include/asm-x86/paravirt.h
include/asm-x86/system.h

arch/x86/kernel/entry_64.S

Diff comments View file @ 9f9d489

1	/*	1	/*
2	* linux/arch/x86_64/entry.S	2	* linux/arch/x86_64/entry.S
3	*	3	*
4	* Copyright (C) 1991, 1992 Linus Torvalds	4	* Copyright (C) 1991, 1992 Linus Torvalds
5	* Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs	5	* Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6	* Copyright (C) 2000 Pavel Machek <pavel@suse.cz>	6	* Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7	*/	7	*/
8		8
9	/*	9	/*
10	* entry.S contains the system-call and fault low-level handling routines.	10	* entry.S contains the system-call and fault low-level handling routines.
11	*	11	*
12	* NOTE: This code handles signal-recognition, which happens every time	12	* NOTE: This code handles signal-recognition, which happens every time
13	* after an interrupt and after each system call.	13	* after an interrupt and after each system call.
14	*	14	*
15	* Normal syscalls and interrupts don't save a full stack frame, this is	15	* Normal syscalls and interrupts don't save a full stack frame, this is
16	* only done for syscall tracing, signals or fork/exec et.al.	16	* only done for syscall tracing, signals or fork/exec et.al.
17	*	17	*
18	* A note on terminology:	18	* A note on terminology:
19	* - top of stack: Architecture defined interrupt frame from SS to RIP	19	* - top of stack: Architecture defined interrupt frame from SS to RIP
20	* at the top of the kernel process stack.	20	* at the top of the kernel process stack.
21	* - partial stack frame: partially saved registers upto R11.	21	* - partial stack frame: partially saved registers upto R11.
22	* - full stack frame: Like partial stack frame, but all register saved.	22	* - full stack frame: Like partial stack frame, but all register saved.
23	*	23	*
24	* Some macro usage:	24	* Some macro usage:
25	* - CFI macros are used to generate dwarf2 unwind information for better	25	* - CFI macros are used to generate dwarf2 unwind information for better
26	* backtraces. They don't change any code.	26	* backtraces. They don't change any code.
27	* - SAVE_ALL/RESTORE_ALL - Save/restore all registers	27	* - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28	* - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.	28	* - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29	* There are unfortunately lots of special cases where some registers	29	* There are unfortunately lots of special cases where some registers
30	* not touched. The macro is a big mess that should be cleaned up.	30	* not touched. The macro is a big mess that should be cleaned up.
31	* - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.	31	* - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32	* Gives a full stack frame.	32	* Gives a full stack frame.
33	* - ENTRY/END Define functions in the symbol table.	33	* - ENTRY/END Define functions in the symbol table.
34	* - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack	34	* - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35	* frame that is otherwise undefined after a SYSCALL	35	* frame that is otherwise undefined after a SYSCALL
36	* - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.	36	* - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37	* - errorentry/paranoidentry/zeroentry - Define exception entry points.	37	* - errorentry/paranoidentry/zeroentry - Define exception entry points.
38	*/	38	*/
39		39
40	#include <linux/linkage.h>	40	#include <linux/linkage.h>
41	#include <asm/segment.h>	41	#include <asm/segment.h>
42	#include <asm/cache.h>	42	#include <asm/cache.h>
43	#include <asm/errno.h>	43	#include <asm/errno.h>
44	#include <asm/dwarf2.h>	44	#include <asm/dwarf2.h>
45	#include <asm/calling.h>	45	#include <asm/calling.h>
46	#include <asm/asm-offsets.h>	46	#include <asm/asm-offsets.h>
47	#include <asm/msr.h>	47	#include <asm/msr.h>
48	#include <asm/unistd.h>	48	#include <asm/unistd.h>
49	#include <asm/thread_info.h>	49	#include <asm/thread_info.h>
50	#include <asm/hw_irq.h>	50	#include <asm/hw_irq.h>
51	#include <asm/page.h>	51	#include <asm/page.h>
52	#include <asm/irqflags.h>	52	#include <asm/irqflags.h>
53	#include <asm/paravirt.h>	53	#include <asm/paravirt.h>
54		54
55	.code64	55	.code64
56		56
57	#ifndef CONFIG_PREEMPT	57	#ifndef CONFIG_PREEMPT
58	#define retint_kernel retint_restore_args	58	#define retint_kernel retint_restore_args
59	#endif	59	#endif
60		60
61	#ifdef CONFIG_PARAVIRT	61	#ifdef CONFIG_PARAVIRT
62	ENTRY(native_usergs_sysret64)	62	ENTRY(native_usergs_sysret64)
63	swapgs	63	swapgs
64	sysretq	64	sysretq
65	#endif /* CONFIG_PARAVIRT */	65	#endif /* CONFIG_PARAVIRT */
66		66
67		67
68	.macro TRACE_IRQS_IRETQ offset=ARGOFFSET	68	.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
69	#ifdef CONFIG_TRACE_IRQFLAGS	69	#ifdef CONFIG_TRACE_IRQFLAGS
70	bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */	70	bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
71	jnc 1f	71	jnc 1f
72	TRACE_IRQS_ON	72	TRACE_IRQS_ON
73	1:	73	1:
74	#endif	74	#endif
75	.endm	75	.endm
76		76
77	/*	77	/*
78	* C code is not supposed to know about undefined top of stack. Every time	78	* C code is not supposed to know about undefined top of stack. Every time
79	* a C function with an pt_regs argument is called from the SYSCALL based	79	* a C function with an pt_regs argument is called from the SYSCALL based
80	* fast path FIXUP_TOP_OF_STACK is needed.	80	* fast path FIXUP_TOP_OF_STACK is needed.
81	* RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs	81	* RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
82	* manipulation.	82	* manipulation.
83	*/	83	*/
84		84
85	/* %rsp:at FRAMEEND */	85	/* %rsp:at FRAMEEND */
86	.macro FIXUP_TOP_OF_STACK tmp	86	.macro FIXUP_TOP_OF_STACK tmp
87	movq %gs:pda_oldrsp,\tmp	87	movq %gs:pda_oldrsp,\tmp
88	movq \tmp,RSP(%rsp)	88	movq \tmp,RSP(%rsp)
89	movq $__USER_DS,SS(%rsp)	89	movq $__USER_DS,SS(%rsp)
90	movq $__USER_CS,CS(%rsp)	90	movq $__USER_CS,CS(%rsp)
91	movq $-1,RCX(%rsp)	91	movq $-1,RCX(%rsp)
92	movq R11(%rsp),\tmp /* get eflags */	92	movq R11(%rsp),\tmp /* get eflags */
93	movq \tmp,EFLAGS(%rsp)	93	movq \tmp,EFLAGS(%rsp)
94	.endm	94	.endm
95		95
96	.macro RESTORE_TOP_OF_STACK tmp,offset=0	96	.macro RESTORE_TOP_OF_STACK tmp,offset=0
97	movq RSP-\offset(%rsp),\tmp	97	movq RSP-\offset(%rsp),\tmp
98	movq \tmp,%gs:pda_oldrsp	98	movq \tmp,%gs:pda_oldrsp
99	movq EFLAGS-\offset(%rsp),\tmp	99	movq EFLAGS-\offset(%rsp),\tmp
100	movq \tmp,R11-\offset(%rsp)	100	movq \tmp,R11-\offset(%rsp)
101	.endm	101	.endm
102		102
103	.macro FAKE_STACK_FRAME child_rip	103	.macro FAKE_STACK_FRAME child_rip
104	/* push in order ss, rsp, eflags, cs, rip */	104	/* push in order ss, rsp, eflags, cs, rip */
105	xorl %eax, %eax	105	xorl %eax, %eax
106	pushq $__KERNEL_DS /* ss */	106	pushq $__KERNEL_DS /* ss */
107	CFI_ADJUST_CFA_OFFSET 8	107	CFI_ADJUST_CFA_OFFSET 8
108	/CFI_REL_OFFSET ss,0/	108	/CFI_REL_OFFSET ss,0/
109	pushq %rax /* rsp */	109	pushq %rax /* rsp */
110	CFI_ADJUST_CFA_OFFSET 8	110	CFI_ADJUST_CFA_OFFSET 8
111	CFI_REL_OFFSET rsp,0	111	CFI_REL_OFFSET rsp,0
112	pushq $(1<<9) /* eflags - interrupts on */	112	pushq $(1<<9) /* eflags - interrupts on */
113	CFI_ADJUST_CFA_OFFSET 8	113	CFI_ADJUST_CFA_OFFSET 8
114	/CFI_REL_OFFSET rflags,0/	114	/CFI_REL_OFFSET rflags,0/
115	pushq $__KERNEL_CS /* cs */	115	pushq $__KERNEL_CS /* cs */
116	CFI_ADJUST_CFA_OFFSET 8	116	CFI_ADJUST_CFA_OFFSET 8
117	/CFI_REL_OFFSET cs,0/	117	/CFI_REL_OFFSET cs,0/
118	pushq \child_rip /* rip */	118	pushq \child_rip /* rip */
119	CFI_ADJUST_CFA_OFFSET 8	119	CFI_ADJUST_CFA_OFFSET 8
120	CFI_REL_OFFSET rip,0	120	CFI_REL_OFFSET rip,0
121	pushq %rax /* orig rax */	121	pushq %rax /* orig rax */
122	CFI_ADJUST_CFA_OFFSET 8	122	CFI_ADJUST_CFA_OFFSET 8
123	.endm	123	.endm
124		124
125	.macro UNFAKE_STACK_FRAME	125	.macro UNFAKE_STACK_FRAME
126	addq $8*6, %rsp	126	addq $8*6, %rsp
127	CFI_ADJUST_CFA_OFFSET -(6*8)	127	CFI_ADJUST_CFA_OFFSET -(6*8)
128	.endm	128	.endm
129		129
130	.macro CFI_DEFAULT_STACK start=1	130	.macro CFI_DEFAULT_STACK start=1
131	.if \start	131	.if \start
132	CFI_STARTPROC simple	132	CFI_STARTPROC simple
133	CFI_SIGNAL_FRAME	133	CFI_SIGNAL_FRAME
134	CFI_DEF_CFA rsp,SS+8	134	CFI_DEF_CFA rsp,SS+8
135	.else	135	.else
136	CFI_DEF_CFA_OFFSET SS+8	136	CFI_DEF_CFA_OFFSET SS+8
137	.endif	137	.endif
138	CFI_REL_OFFSET r15,R15	138	CFI_REL_OFFSET r15,R15
139	CFI_REL_OFFSET r14,R14	139	CFI_REL_OFFSET r14,R14
140	CFI_REL_OFFSET r13,R13	140	CFI_REL_OFFSET r13,R13
141	CFI_REL_OFFSET r12,R12	141	CFI_REL_OFFSET r12,R12
142	CFI_REL_OFFSET rbp,RBP	142	CFI_REL_OFFSET rbp,RBP
143	CFI_REL_OFFSET rbx,RBX	143	CFI_REL_OFFSET rbx,RBX
144	CFI_REL_OFFSET r11,R11	144	CFI_REL_OFFSET r11,R11
145	CFI_REL_OFFSET r10,R10	145	CFI_REL_OFFSET r10,R10
146	CFI_REL_OFFSET r9,R9	146	CFI_REL_OFFSET r9,R9
147	CFI_REL_OFFSET r8,R8	147	CFI_REL_OFFSET r8,R8
148	CFI_REL_OFFSET rax,RAX	148	CFI_REL_OFFSET rax,RAX
149	CFI_REL_OFFSET rcx,RCX	149	CFI_REL_OFFSET rcx,RCX
150	CFI_REL_OFFSET rdx,RDX	150	CFI_REL_OFFSET rdx,RDX
151	CFI_REL_OFFSET rsi,RSI	151	CFI_REL_OFFSET rsi,RSI
152	CFI_REL_OFFSET rdi,RDI	152	CFI_REL_OFFSET rdi,RDI
153	CFI_REL_OFFSET rip,RIP	153	CFI_REL_OFFSET rip,RIP
154	/CFI_REL_OFFSET cs,CS/	154	/CFI_REL_OFFSET cs,CS/
155	/CFI_REL_OFFSET rflags,EFLAGS/	155	/CFI_REL_OFFSET rflags,EFLAGS/
156	CFI_REL_OFFSET rsp,RSP	156	CFI_REL_OFFSET rsp,RSP
157	/CFI_REL_OFFSET ss,SS/	157	/CFI_REL_OFFSET ss,SS/
158	.endm	158	.endm
159	/*	159	/*
160	* A newly forked process directly context switches into this.	160	* A newly forked process directly context switches into this.
161	*/	161	*/
162	/* rdi: prev */	162	/* rdi: prev */
163	ENTRY(ret_from_fork)	163	ENTRY(ret_from_fork)
164	CFI_DEFAULT_STACK	164	CFI_DEFAULT_STACK
165	push kernel_eflags(%rip)	165	push kernel_eflags(%rip)
166	CFI_ADJUST_CFA_OFFSET 4	166	CFI_ADJUST_CFA_OFFSET 4
167	popf # reset kernel eflags	167	popf # reset kernel eflags
168	CFI_ADJUST_CFA_OFFSET -4	168	CFI_ADJUST_CFA_OFFSET -4
169	call schedule_tail	169	call schedule_tail
170	GET_THREAD_INFO(%rcx)	170	GET_THREAD_INFO(%rcx)
171	testl $(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)	171	testl $(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
172	jnz rff_trace	172	jnz rff_trace
173	rff_action:	173	rff_action:
174	RESTORE_REST	174	RESTORE_REST
175	testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?	175	testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
176	je int_ret_from_sys_call	176	je int_ret_from_sys_call
177	testl $_TIF_IA32,threadinfo_flags(%rcx)	177	testl $_TIF_IA32,threadinfo_flags(%rcx)
178	jnz int_ret_from_sys_call	178	jnz int_ret_from_sys_call
179	RESTORE_TOP_OF_STACK %rdi,ARGOFFSET	179	RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
180	jmp ret_from_sys_call	180	jmp ret_from_sys_call
181	rff_trace:	181	rff_trace:
182	movq %rsp,%rdi	182	movq %rsp,%rdi
183	call syscall_trace_leave	183	call syscall_trace_leave
184	GET_THREAD_INFO(%rcx)	184	GET_THREAD_INFO(%rcx)
185	jmp rff_action	185	jmp rff_action
186	CFI_ENDPROC	186	CFI_ENDPROC
187	END(ret_from_fork)	187	END(ret_from_fork)
188		188
189	/*	189	/*
190	* System call entry. Upto 6 arguments in registers are supported.	190	* System call entry. Upto 6 arguments in registers are supported.
191	*	191	*
192	* SYSCALL does not save anything on the stack and does not change the	192	* SYSCALL does not save anything on the stack and does not change the
193	* stack pointer.	193	* stack pointer.
194	*/	194	*/
195		195
196	/*	196	/*
197	* Register setup:	197	* Register setup:
198	* rax system call number	198	* rax system call number
199	* rdi arg0	199	* rdi arg0
200	* rcx return address for syscall/sysret, C arg3	200	* rcx return address for syscall/sysret, C arg3
201	* rsi arg1	201	* rsi arg1
202	* rdx arg2	202	* rdx arg2
203	* r10 arg3 (--> moved to rcx for C)	203	* r10 arg3 (--> moved to rcx for C)
204	* r8 arg4	204	* r8 arg4
205	* r9 arg5	205	* r9 arg5
206	* r11 eflags for syscall/sysret, temporary for C	206	* r11 eflags for syscall/sysret, temporary for C
207	* r12-r15,rbp,rbx saved by C code, not touched.	207	* r12-r15,rbp,rbx saved by C code, not touched.
208	*	208	*
209	* Interrupts are off on entry.	209	* Interrupts are off on entry.
210	* Only called from user space.	210	* Only called from user space.
211	*	211	*
212	* XXX if we had a free scratch register we could save the RSP into the stack frame	212	* XXX if we had a free scratch register we could save the RSP into the stack frame
213	* and report it properly in ps. Unfortunately we haven't.	213	* and report it properly in ps. Unfortunately we haven't.
214	*	214	*
215	* When user can change the frames always force IRET. That is because	215	* When user can change the frames always force IRET. That is because
216	* it deals with uncanonical addresses better. SYSRET has trouble	216	* it deals with uncanonical addresses better. SYSRET has trouble
217	* with them due to bugs in both AMD and Intel CPUs.	217	* with them due to bugs in both AMD and Intel CPUs.
218	*/	218	*/
219		219
220	ENTRY(system_call)	220	ENTRY(system_call)
221	CFI_STARTPROC simple	221	CFI_STARTPROC simple
222	CFI_SIGNAL_FRAME	222	CFI_SIGNAL_FRAME
223	CFI_DEF_CFA rsp,PDA_STACKOFFSET	223	CFI_DEF_CFA rsp,PDA_STACKOFFSET
224	CFI_REGISTER rip,rcx	224	CFI_REGISTER rip,rcx
225	/CFI_REGISTER rflags,r11/	225	/CFI_REGISTER rflags,r11/
226	SWAPGS_UNSAFE_STACK	226	SWAPGS_UNSAFE_STACK
227	/*	227	/*
228	* A hypervisor implementation might want to use a label	228	* A hypervisor implementation might want to use a label
229	* after the swapgs, so that it can do the swapgs	229	* after the swapgs, so that it can do the swapgs
230	* for the guest and jump here on syscall.	230	* for the guest and jump here on syscall.
231	*/	231	*/
232	ENTRY(system_call_after_swapgs)	232	ENTRY(system_call_after_swapgs)
233		233
234	movq %rsp,%gs:pda_oldrsp	234	movq %rsp,%gs:pda_oldrsp
235	movq %gs:pda_kernelstack,%rsp	235	movq %gs:pda_kernelstack,%rsp
236	/*	236	/*
237	* No need to follow this irqs off/on section - it's straight	237	* No need to follow this irqs off/on section - it's straight
238	* and short:	238	* and short:
239	*/	239	*/
240	ENABLE_INTERRUPTS(CLBR_NONE)	240	ENABLE_INTERRUPTS(CLBR_NONE)
241	SAVE_ARGS 8,1	241	SAVE_ARGS 8,1
242	movq %rax,ORIG_RAX-ARGOFFSET(%rsp)	242	movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
243	movq %rcx,RIP-ARGOFFSET(%rsp)	243	movq %rcx,RIP-ARGOFFSET(%rsp)
244	CFI_REL_OFFSET rip,RIP-ARGOFFSET	244	CFI_REL_OFFSET rip,RIP-ARGOFFSET
245	GET_THREAD_INFO(%rcx)	245	GET_THREAD_INFO(%rcx)
246	testl $(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT\|_TIF_SECCOMP),threadinfo_flags(%rcx)	246	testl $(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT\|_TIF_SECCOMP),threadinfo_flags(%rcx)
247	jnz tracesys	247	jnz tracesys
248	cmpq $__NR_syscall_max,%rax	248	cmpq $__NR_syscall_max,%rax
249	ja badsys	249	ja badsys
250	movq %r10,%rcx	250	movq %r10,%rcx
251	call *sys_call_table(,%rax,8) # XXX: rip relative	251	call *sys_call_table(,%rax,8) # XXX: rip relative
252	movq %rax,RAX-ARGOFFSET(%rsp)	252	movq %rax,RAX-ARGOFFSET(%rsp)
253	/*	253	/*
254	* Syscall return path ending with SYSRET (fast path)	254	* Syscall return path ending with SYSRET (fast path)
255	* Has incomplete stack frame and undefined top of stack.	255	* Has incomplete stack frame and undefined top of stack.
256	*/	256	*/
257	ret_from_sys_call:	257	ret_from_sys_call:
258	movl $_TIF_ALLWORK_MASK,%edi	258	movl $_TIF_ALLWORK_MASK,%edi
259	/* edi: flagmask */	259	/* edi: flagmask */
260	sysret_check:	260	sysret_check:
261	LOCKDEP_SYS_EXIT	261	LOCKDEP_SYS_EXIT
262	GET_THREAD_INFO(%rcx)	262	GET_THREAD_INFO(%rcx)
263	DISABLE_INTERRUPTS(CLBR_NONE)	263	DISABLE_INTERRUPTS(CLBR_NONE)
264	TRACE_IRQS_OFF	264	TRACE_IRQS_OFF
265	movl threadinfo_flags(%rcx),%edx	265	movl threadinfo_flags(%rcx),%edx
266	andl %edi,%edx	266	andl %edi,%edx
267	jnz sysret_careful	267	jnz sysret_careful
268	CFI_REMEMBER_STATE	268	CFI_REMEMBER_STATE
269	/*	269	/*
270	* sysretq will re-enable interrupts:	270	* sysretq will re-enable interrupts:
271	*/	271	*/
272	TRACE_IRQS_ON	272	TRACE_IRQS_ON
273	movq RIP-ARGOFFSET(%rsp),%rcx	273	movq RIP-ARGOFFSET(%rsp),%rcx
274	CFI_REGISTER rip,rcx	274	CFI_REGISTER rip,rcx
275	RESTORE_ARGS 0,-ARG_SKIP,1	275	RESTORE_ARGS 0,-ARG_SKIP,1
276	/CFI_REGISTER rflags,r11/	276	/CFI_REGISTER rflags,r11/
277	movq %gs:pda_oldrsp, %rsp	277	movq %gs:pda_oldrsp, %rsp
278	USERGS_SYSRET64	278	USERGS_SYSRET64
279		279
280	CFI_RESTORE_STATE	280	CFI_RESTORE_STATE
281	/* Handle reschedules */	281	/* Handle reschedules */
282	/* edx: work, edi: workmask */	282	/* edx: work, edi: workmask */
283	sysret_careful:	283	sysret_careful:
284	bt $TIF_NEED_RESCHED,%edx	284	bt $TIF_NEED_RESCHED,%edx
285	jnc sysret_signal	285	jnc sysret_signal
286	TRACE_IRQS_ON	286	TRACE_IRQS_ON
287	ENABLE_INTERRUPTS(CLBR_NONE)	287	ENABLE_INTERRUPTS(CLBR_NONE)
288	pushq %rdi	288	pushq %rdi
289	CFI_ADJUST_CFA_OFFSET 8	289	CFI_ADJUST_CFA_OFFSET 8
290	call schedule	290	call schedule
291	popq %rdi	291	popq %rdi
292	CFI_ADJUST_CFA_OFFSET -8	292	CFI_ADJUST_CFA_OFFSET -8
293	jmp sysret_check	293	jmp sysret_check
294		294
295	/* Handle a signal */	295	/* Handle a signal */
296	sysret_signal:	296	sysret_signal:
297	TRACE_IRQS_ON	297	TRACE_IRQS_ON
298	ENABLE_INTERRUPTS(CLBR_NONE)	298	ENABLE_INTERRUPTS(CLBR_NONE)
299	testl $_TIF_DO_NOTIFY_MASK,%edx	299	testl $_TIF_DO_NOTIFY_MASK,%edx
300	jz 1f	300	jz 1f
301		301
302	/* Really a signal */	302	/* Really a signal */
303	/* edx: work flags (arg3) */	303	/* edx: work flags (arg3) */
304	leaq do_notify_resume(%rip),%rax	304	leaq do_notify_resume(%rip),%rax
305	leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1	305	leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
306	xorl %esi,%esi # oldset -> arg2	306	xorl %esi,%esi # oldset -> arg2
307	call ptregscall_common	307	call ptregscall_common
308	1: movl $_TIF_NEED_RESCHED,%edi	308	1: movl $_TIF_NEED_RESCHED,%edi
309	/* Use IRET because user could have changed frame. This	309	/* Use IRET because user could have changed frame. This
310	works because ptregscall_common has called FIXUP_TOP_OF_STACK. */	310	works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
311	DISABLE_INTERRUPTS(CLBR_NONE)	311	DISABLE_INTERRUPTS(CLBR_NONE)
312	TRACE_IRQS_OFF	312	TRACE_IRQS_OFF
313	jmp int_with_check	313	jmp int_with_check
314		314
315	badsys:	315	badsys:
316	movq $-ENOSYS,RAX-ARGOFFSET(%rsp)	316	movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
317	jmp ret_from_sys_call	317	jmp ret_from_sys_call
318		318
319	/* Do syscall tracing */	319	/* Do syscall tracing */
320	tracesys:	320	tracesys:
321	SAVE_REST	321	SAVE_REST
322	movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */	322	movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
323	FIXUP_TOP_OF_STACK %rdi	323	FIXUP_TOP_OF_STACK %rdi
324	movq %rsp,%rdi	324	movq %rsp,%rdi
325	call syscall_trace_enter	325	call syscall_trace_enter
326	LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */	326	LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
327	RESTORE_REST	327	RESTORE_REST
328	cmpq $__NR_syscall_max,%rax	328	cmpq $__NR_syscall_max,%rax
329	ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */	329	ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
330	movq %r10,%rcx /* fixup for C */	330	movq %r10,%rcx /* fixup for C */
331	call *sys_call_table(,%rax,8)	331	call *sys_call_table(,%rax,8)
332	movq %rax,RAX-ARGOFFSET(%rsp)	332	movq %rax,RAX-ARGOFFSET(%rsp)
333	/* Use IRET because user could have changed frame */	333	/* Use IRET because user could have changed frame */
334		334
335	/*	335	/*
336	* Syscall return path ending with IRET.	336	* Syscall return path ending with IRET.
337	* Has correct top of stack, but partial stack frame.	337	* Has correct top of stack, but partial stack frame.
338	*/	338	*/
339	.globl int_ret_from_sys_call	339	.globl int_ret_from_sys_call
340	int_ret_from_sys_call:	340	int_ret_from_sys_call:
341	DISABLE_INTERRUPTS(CLBR_NONE)	341	DISABLE_INTERRUPTS(CLBR_NONE)
342	TRACE_IRQS_OFF	342	TRACE_IRQS_OFF
343	testl $3,CS-ARGOFFSET(%rsp)	343	testl $3,CS-ARGOFFSET(%rsp)
344	je retint_restore_args	344	je retint_restore_args
345	movl $_TIF_ALLWORK_MASK,%edi	345	movl $_TIF_ALLWORK_MASK,%edi
346	/* edi: mask to check */	346	/* edi: mask to check */
347	int_with_check:	347	int_with_check:
348	LOCKDEP_SYS_EXIT_IRQ	348	LOCKDEP_SYS_EXIT_IRQ
349	GET_THREAD_INFO(%rcx)	349	GET_THREAD_INFO(%rcx)
350	movl threadinfo_flags(%rcx),%edx	350	movl threadinfo_flags(%rcx),%edx
351	andl %edi,%edx	351	andl %edi,%edx
352	jnz int_careful	352	jnz int_careful
353	andl $~TS_COMPAT,threadinfo_status(%rcx)	353	andl $~TS_COMPAT,threadinfo_status(%rcx)
354	jmp retint_swapgs	354	jmp retint_swapgs
355		355
356	/* Either reschedule or signal or syscall exit tracking needed. */	356	/* Either reschedule or signal or syscall exit tracking needed. */
357	/* First do a reschedule test. */	357	/* First do a reschedule test. */
358	/* edx: work, edi: workmask */	358	/* edx: work, edi: workmask */
359	int_careful:	359	int_careful:
360	bt $TIF_NEED_RESCHED,%edx	360	bt $TIF_NEED_RESCHED,%edx
361	jnc int_very_careful	361	jnc int_very_careful
362	TRACE_IRQS_ON	362	TRACE_IRQS_ON
363	ENABLE_INTERRUPTS(CLBR_NONE)	363	ENABLE_INTERRUPTS(CLBR_NONE)
364	pushq %rdi	364	pushq %rdi
365	CFI_ADJUST_CFA_OFFSET 8	365	CFI_ADJUST_CFA_OFFSET 8
366	call schedule	366	call schedule
367	popq %rdi	367	popq %rdi
368	CFI_ADJUST_CFA_OFFSET -8	368	CFI_ADJUST_CFA_OFFSET -8
369	DISABLE_INTERRUPTS(CLBR_NONE)	369	DISABLE_INTERRUPTS(CLBR_NONE)
370	TRACE_IRQS_OFF	370	TRACE_IRQS_OFF
371	jmp int_with_check	371	jmp int_with_check
372		372
373	/* handle signals and tracing -- both require a full stack frame */	373	/* handle signals and tracing -- both require a full stack frame */
374	int_very_careful:	374	int_very_careful:
375	TRACE_IRQS_ON	375	TRACE_IRQS_ON
376	ENABLE_INTERRUPTS(CLBR_NONE)	376	ENABLE_INTERRUPTS(CLBR_NONE)
377	SAVE_REST	377	SAVE_REST
378	/* Check for syscall exit trace */	378	/* Check for syscall exit trace */
379	testl $(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT\|_TIF_SINGLESTEP),%edx	379	testl $(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT\|_TIF_SINGLESTEP),%edx
380	jz int_signal	380	jz int_signal
381	pushq %rdi	381	pushq %rdi
382	CFI_ADJUST_CFA_OFFSET 8	382	CFI_ADJUST_CFA_OFFSET 8
383	leaq 8(%rsp),%rdi # &ptregs -> arg1	383	leaq 8(%rsp),%rdi # &ptregs -> arg1
384	call syscall_trace_leave	384	call syscall_trace_leave
385	popq %rdi	385	popq %rdi
386	CFI_ADJUST_CFA_OFFSET -8	386	CFI_ADJUST_CFA_OFFSET -8
387	andl $~(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT\|_TIF_SINGLESTEP),%edi	387	andl $~(_TIF_SYSCALL_TRACE\|_TIF_SYSCALL_AUDIT\|_TIF_SINGLESTEP),%edi
388	jmp int_restore_rest	388	jmp int_restore_rest
389		389
390	int_signal:	390	int_signal:
391	testl $_TIF_DO_NOTIFY_MASK,%edx	391	testl $_TIF_DO_NOTIFY_MASK,%edx
392	jz 1f	392	jz 1f
393	movq %rsp,%rdi # &ptregs -> arg1	393	movq %rsp,%rdi # &ptregs -> arg1
394	xorl %esi,%esi # oldset -> arg2	394	xorl %esi,%esi # oldset -> arg2
395	call do_notify_resume	395	call do_notify_resume
396	1: movl $_TIF_NEED_RESCHED,%edi	396	1: movl $_TIF_NEED_RESCHED,%edi
397	int_restore_rest:	397	int_restore_rest:
398	RESTORE_REST	398	RESTORE_REST
399	DISABLE_INTERRUPTS(CLBR_NONE)	399	DISABLE_INTERRUPTS(CLBR_NONE)
400	TRACE_IRQS_OFF	400	TRACE_IRQS_OFF
401	jmp int_with_check	401	jmp int_with_check
402	CFI_ENDPROC	402	CFI_ENDPROC
403	END(system_call)	403	END(system_call)
404		404
405	/*	405	/*
406	* Certain special system calls that need to save a complete full stack frame.	406	* Certain special system calls that need to save a complete full stack frame.
407	*/	407	*/
408		408
409	.macro PTREGSCALL label,func,arg	409	.macro PTREGSCALL label,func,arg
410	.globl \label	410	.globl \label
411	\label:	411	\label:
412	leaq \func(%rip),%rax	412	leaq \func(%rip),%rax
413	leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */	413	leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
414	jmp ptregscall_common	414	jmp ptregscall_common
415	END(\label)	415	END(\label)
416	.endm	416	.endm
417		417
418	CFI_STARTPROC	418	CFI_STARTPROC
419		419
420	PTREGSCALL stub_clone, sys_clone, %r8	420	PTREGSCALL stub_clone, sys_clone, %r8
421	PTREGSCALL stub_fork, sys_fork, %rdi	421	PTREGSCALL stub_fork, sys_fork, %rdi
422	PTREGSCALL stub_vfork, sys_vfork, %rdi	422	PTREGSCALL stub_vfork, sys_vfork, %rdi
423	PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx	423	PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
424	PTREGSCALL stub_iopl, sys_iopl, %rsi	424	PTREGSCALL stub_iopl, sys_iopl, %rsi
425		425
426	ENTRY(ptregscall_common)	426	ENTRY(ptregscall_common)
427	popq %r11	427	popq %r11
428	CFI_ADJUST_CFA_OFFSET -8	428	CFI_ADJUST_CFA_OFFSET -8
429	CFI_REGISTER rip, r11	429	CFI_REGISTER rip, r11
430	SAVE_REST	430	SAVE_REST
431	movq %r11, %r15	431	movq %r11, %r15
432	CFI_REGISTER rip, r15	432	CFI_REGISTER rip, r15
433	FIXUP_TOP_OF_STACK %r11	433	FIXUP_TOP_OF_STACK %r11
434	call *%rax	434	call *%rax
435	RESTORE_TOP_OF_STACK %r11	435	RESTORE_TOP_OF_STACK %r11
436	movq %r15, %r11	436	movq %r15, %r11
437	CFI_REGISTER rip, r11	437	CFI_REGISTER rip, r11
438	RESTORE_REST	438	RESTORE_REST
439	pushq %r11	439	pushq %r11
440	CFI_ADJUST_CFA_OFFSET 8	440	CFI_ADJUST_CFA_OFFSET 8
441	CFI_REL_OFFSET rip, 0	441	CFI_REL_OFFSET rip, 0
442	ret	442	ret
443	CFI_ENDPROC	443	CFI_ENDPROC
444	END(ptregscall_common)	444	END(ptregscall_common)
445		445
446	ENTRY(stub_execve)	446	ENTRY(stub_execve)
447	CFI_STARTPROC	447	CFI_STARTPROC
448	popq %r11	448	popq %r11
449	CFI_ADJUST_CFA_OFFSET -8	449	CFI_ADJUST_CFA_OFFSET -8
450	CFI_REGISTER rip, r11	450	CFI_REGISTER rip, r11
451	SAVE_REST	451	SAVE_REST
452	FIXUP_TOP_OF_STACK %r11	452	FIXUP_TOP_OF_STACK %r11
453	movq %rsp, %rcx	453	movq %rsp, %rcx
454	call sys_execve	454	call sys_execve
455	RESTORE_TOP_OF_STACK %r11	455	RESTORE_TOP_OF_STACK %r11
456	movq %rax,RAX(%rsp)	456	movq %rax,RAX(%rsp)
457	RESTORE_REST	457	RESTORE_REST
458	jmp int_ret_from_sys_call	458	jmp int_ret_from_sys_call
459	CFI_ENDPROC	459	CFI_ENDPROC
460	END(stub_execve)	460	END(stub_execve)
461		461
462	/*	462	/*
463	* sigreturn is special because it needs to restore all registers on return.	463	* sigreturn is special because it needs to restore all registers on return.
464	* This cannot be done with SYSRET, so use the IRET return path instead.	464	* This cannot be done with SYSRET, so use the IRET return path instead.
465	*/	465	*/
466	ENTRY(stub_rt_sigreturn)	466	ENTRY(stub_rt_sigreturn)
467	CFI_STARTPROC	467	CFI_STARTPROC
468	addq $8, %rsp	468	addq $8, %rsp
469	CFI_ADJUST_CFA_OFFSET -8	469	CFI_ADJUST_CFA_OFFSET -8
470	SAVE_REST	470	SAVE_REST
471	movq %rsp,%rdi	471	movq %rsp,%rdi
472	FIXUP_TOP_OF_STACK %r11	472	FIXUP_TOP_OF_STACK %r11
473	call sys_rt_sigreturn	473	call sys_rt_sigreturn
474	movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer	474	movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
475	RESTORE_REST	475	RESTORE_REST
476	jmp int_ret_from_sys_call	476	jmp int_ret_from_sys_call
477	CFI_ENDPROC	477	CFI_ENDPROC
478	END(stub_rt_sigreturn)	478	END(stub_rt_sigreturn)
479		479
480	/*	480	/*
481	* initial frame state for interrupts and exceptions	481	* initial frame state for interrupts and exceptions
482	*/	482	*/
483	.macro _frame ref	483	.macro _frame ref
484	CFI_STARTPROC simple	484	CFI_STARTPROC simple
485	CFI_SIGNAL_FRAME	485	CFI_SIGNAL_FRAME
486	CFI_DEF_CFA rsp,SS+8-\ref	486	CFI_DEF_CFA rsp,SS+8-\ref
487	/CFI_REL_OFFSET ss,SS-\ref/	487	/CFI_REL_OFFSET ss,SS-\ref/
488	CFI_REL_OFFSET rsp,RSP-\ref	488	CFI_REL_OFFSET rsp,RSP-\ref
489	/CFI_REL_OFFSET rflags,EFLAGS-\ref/	489	/CFI_REL_OFFSET rflags,EFLAGS-\ref/
490	/CFI_REL_OFFSET cs,CS-\ref/	490	/CFI_REL_OFFSET cs,CS-\ref/
491	CFI_REL_OFFSET rip,RIP-\ref	491	CFI_REL_OFFSET rip,RIP-\ref
492	.endm	492	.endm
493		493
494	/* initial frame state for interrupts (and exceptions without error code) */	494	/* initial frame state for interrupts (and exceptions without error code) */
495	#define INTR_FRAME _frame RIP	495	#define INTR_FRAME _frame RIP
496	/* initial frame state for exceptions with error code (and interrupts with	496	/* initial frame state for exceptions with error code (and interrupts with
497	vector already pushed) */	497	vector already pushed) */
498	#define XCPT_FRAME _frame ORIG_RAX	498	#define XCPT_FRAME _frame ORIG_RAX
499		499
500	/*	500	/*
501	* Interrupt entry/exit.	501	* Interrupt entry/exit.
502	*	502	*
503	* Interrupt entry points save only callee clobbered registers in fast path.	503	* Interrupt entry points save only callee clobbered registers in fast path.
504	*	504	*
505	* Entry runs with interrupts off.	505	* Entry runs with interrupts off.
506	*/	506	*/
507		507
508	/* 0(%rsp): interrupt number */	508	/* 0(%rsp): interrupt number */
509	.macro interrupt func	509	.macro interrupt func
510	cld	510	cld
511	SAVE_ARGS	511	SAVE_ARGS
512	leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler	512	leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
513	pushq %rbp	513	pushq %rbp
514	CFI_ADJUST_CFA_OFFSET 8	514	CFI_ADJUST_CFA_OFFSET 8
515	CFI_REL_OFFSET rbp, 0	515	CFI_REL_OFFSET rbp, 0
516	movq %rsp,%rbp	516	movq %rsp,%rbp
517	CFI_DEF_CFA_REGISTER rbp	517	CFI_DEF_CFA_REGISTER rbp
518	testl $3,CS(%rdi)	518	testl $3,CS(%rdi)
519	je 1f	519	je 1f
520	SWAPGS	520	SWAPGS
521	/* irqcount is used to check if a CPU is already on an interrupt	521	/* irqcount is used to check if a CPU is already on an interrupt
522	stack or not. While this is essentially redundant with preempt_count	522	stack or not. While this is essentially redundant with preempt_count
523	it is a little cheaper to use a separate counter in the PDA	523	it is a little cheaper to use a separate counter in the PDA
524	(short of moving irq_enter into assembly, which would be too	524	(short of moving irq_enter into assembly, which would be too
525	much work) */	525	much work) */
526	1: incl %gs:pda_irqcount	526	1: incl %gs:pda_irqcount
527	cmoveq %gs:pda_irqstackptr,%rsp	527	cmoveq %gs:pda_irqstackptr,%rsp
528	push %rbp # backlink for old unwinder	528	push %rbp # backlink for old unwinder
529	/*	529	/*
530	* We entered an interrupt context - irqs are off:	530	* We entered an interrupt context - irqs are off:
531	*/	531	*/
532	TRACE_IRQS_OFF	532	TRACE_IRQS_OFF
533	call \func	533	call \func
534	.endm	534	.endm
535		535
536	ENTRY(common_interrupt)	536	ENTRY(common_interrupt)
537	XCPT_FRAME	537	XCPT_FRAME
538	interrupt do_IRQ	538	interrupt do_IRQ
539	/* 0(%rsp): oldrsp-ARGOFFSET */	539	/* 0(%rsp): oldrsp-ARGOFFSET */
540	ret_from_intr:	540	ret_from_intr:
541	DISABLE_INTERRUPTS(CLBR_NONE)	541	DISABLE_INTERRUPTS(CLBR_NONE)
542	TRACE_IRQS_OFF	542	TRACE_IRQS_OFF
543	decl %gs:pda_irqcount	543	decl %gs:pda_irqcount
544	leaveq	544	leaveq
545	CFI_DEF_CFA_REGISTER rsp	545	CFI_DEF_CFA_REGISTER rsp
546	CFI_ADJUST_CFA_OFFSET -8	546	CFI_ADJUST_CFA_OFFSET -8
547	exit_intr:	547	exit_intr:
548	GET_THREAD_INFO(%rcx)	548	GET_THREAD_INFO(%rcx)
549	testl $3,CS-ARGOFFSET(%rsp)	549	testl $3,CS-ARGOFFSET(%rsp)
550	je retint_kernel	550	je retint_kernel
551		551
552	/* Interrupt came from user space */	552	/* Interrupt came from user space */
553	/*	553	/*
554	* Has a correct top of stack, but a partial stack frame	554	* Has a correct top of stack, but a partial stack frame
555	* %rcx: thread info. Interrupts off.	555	* %rcx: thread info. Interrupts off.
556	*/	556	*/
557	retint_with_reschedule:	557	retint_with_reschedule:
558	movl $_TIF_WORK_MASK,%edi	558	movl $_TIF_WORK_MASK,%edi
559	retint_check:	559	retint_check:
560	LOCKDEP_SYS_EXIT_IRQ	560	LOCKDEP_SYS_EXIT_IRQ
561	movl threadinfo_flags(%rcx),%edx	561	movl threadinfo_flags(%rcx),%edx
562	andl %edi,%edx	562	andl %edi,%edx
563	CFI_REMEMBER_STATE	563	CFI_REMEMBER_STATE
564	jnz retint_careful	564	jnz retint_careful
565		565
566	retint_swapgs: /* return to user-space */	566	retint_swapgs: /* return to user-space */
567	/*	567	/*
568	* The iretq could re-enable interrupts:	568	* The iretq could re-enable interrupts:
569	*/	569	*/
570	DISABLE_INTERRUPTS(CLBR_ANY)	570	DISABLE_INTERRUPTS(CLBR_ANY)
571	TRACE_IRQS_IRETQ	571	TRACE_IRQS_IRETQ
572	SWAPGS	572	SWAPGS
573	jmp restore_args	573	jmp restore_args
574		574
575	retint_restore_args: /* return to kernel space */	575	retint_restore_args: /* return to kernel space */
576	DISABLE_INTERRUPTS(CLBR_ANY)	576	DISABLE_INTERRUPTS(CLBR_ANY)
577	/*	577	/*
578	* The iretq could re-enable interrupts:	578	* The iretq could re-enable interrupts:
579	*/	579	*/
580	TRACE_IRQS_IRETQ	580	TRACE_IRQS_IRETQ
581	restore_args:	581	restore_args:
582	RESTORE_ARGS 0,8,0	582	RESTORE_ARGS 0,8,0
583		583
584	irq_return:	584	irq_return:
585	INTERRUPT_RETURN	585	INTERRUPT_RETURN
586		586
587	.section __ex_table, "a"	587	.section __ex_table, "a"
588	.quad irq_return, bad_iret	588	.quad irq_return, bad_iret
589	.previous	589	.previous
590		590
591	#ifdef CONFIG_PARAVIRT	591	#ifdef CONFIG_PARAVIRT
592	ENTRY(native_iret)	592	ENTRY(native_iret)
593	iretq	593	iretq
594		594
595	.section __ex_table,"a"	595	.section __ex_table,"a"
596	.quad native_iret, bad_iret	596	.quad native_iret, bad_iret
597	.previous	597	.previous
598	#endif	598	#endif
599		599
600	.section .fixup,"ax"	600	.section .fixup,"ax"
601	bad_iret:	601	bad_iret:
602	/*	602	/*
603	* The iret traps when the %cs or %ss being restored is bogus.	603	* The iret traps when the %cs or %ss being restored is bogus.
604	* We've lost the original trap vector and error code.	604	* We've lost the original trap vector and error code.
605	* #GPF is the most likely one to get for an invalid selector.	605	* #GPF is the most likely one to get for an invalid selector.
606	* So pretend we completed the iret and took the #GPF in user mode.	606	* So pretend we completed the iret and took the #GPF in user mode.
607	*	607	*
608	* We are now running with the kernel GS after exception recovery.	608	* We are now running with the kernel GS after exception recovery.
609	* But error_entry expects us to have user GS to match the user %cs,	609	* But error_entry expects us to have user GS to match the user %cs,
610	* so swap back.	610	* so swap back.
611	*/	611	*/
612	pushq $0	612	pushq $0
613		613
614	SWAPGS	614	SWAPGS
615	jmp general_protection	615	jmp general_protection
616		616
617	.previous	617	.previous
618		618
619	/* edi: workmask, edx: work */	619	/* edi: workmask, edx: work */
620	retint_careful:	620	retint_careful:
621	CFI_RESTORE_STATE	621	CFI_RESTORE_STATE
622	bt $TIF_NEED_RESCHED,%edx	622	bt $TIF_NEED_RESCHED,%edx
623	jnc retint_signal	623	jnc retint_signal
624	TRACE_IRQS_ON	624	TRACE_IRQS_ON
625	ENABLE_INTERRUPTS(CLBR_NONE)	625	ENABLE_INTERRUPTS(CLBR_NONE)
626	pushq %rdi	626	pushq %rdi
627	CFI_ADJUST_CFA_OFFSET 8	627	CFI_ADJUST_CFA_OFFSET 8
628	call schedule	628	call schedule
629	popq %rdi	629	popq %rdi
630	CFI_ADJUST_CFA_OFFSET -8	630	CFI_ADJUST_CFA_OFFSET -8
631	GET_THREAD_INFO(%rcx)	631	GET_THREAD_INFO(%rcx)
632	DISABLE_INTERRUPTS(CLBR_NONE)	632	DISABLE_INTERRUPTS(CLBR_NONE)
633	TRACE_IRQS_OFF	633	TRACE_IRQS_OFF
634	jmp retint_check	634	jmp retint_check
635		635
636	retint_signal:	636	retint_signal:
637	testl $_TIF_DO_NOTIFY_MASK,%edx	637	testl $_TIF_DO_NOTIFY_MASK,%edx
638	jz retint_swapgs	638	jz retint_swapgs
639	TRACE_IRQS_ON	639	TRACE_IRQS_ON
640	ENABLE_INTERRUPTS(CLBR_NONE)	640	ENABLE_INTERRUPTS(CLBR_NONE)
641	SAVE_REST	641	SAVE_REST
642	movq $-1,ORIG_RAX(%rsp)	642	movq $-1,ORIG_RAX(%rsp)
643	xorl %esi,%esi # oldset	643	xorl %esi,%esi # oldset
644	movq %rsp,%rdi # &pt_regs	644	movq %rsp,%rdi # &pt_regs
645	call do_notify_resume	645	call do_notify_resume
646	RESTORE_REST	646	RESTORE_REST
647	DISABLE_INTERRUPTS(CLBR_NONE)	647	DISABLE_INTERRUPTS(CLBR_NONE)
648	TRACE_IRQS_OFF	648	TRACE_IRQS_OFF
649	movl $_TIF_NEED_RESCHED,%edi	649	movl $_TIF_NEED_RESCHED,%edi
650	GET_THREAD_INFO(%rcx)	650	GET_THREAD_INFO(%rcx)
651	jmp retint_check	651	jmp retint_check
652		652
653	#ifdef CONFIG_PREEMPT	653	#ifdef CONFIG_PREEMPT
654	/* Returning to kernel space. Check if we need preemption */	654	/* Returning to kernel space. Check if we need preemption */
655	/* rcx: threadinfo. interrupts off. */	655	/* rcx: threadinfo. interrupts off. */
656	ENTRY(retint_kernel)	656	ENTRY(retint_kernel)
657	cmpl $0,threadinfo_preempt_count(%rcx)	657	cmpl $0,threadinfo_preempt_count(%rcx)
658	jnz retint_restore_args	658	jnz retint_restore_args
659	bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)	659	bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
660	jnc retint_restore_args	660	jnc retint_restore_args
661	bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */	661	bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
662	jnc retint_restore_args	662	jnc retint_restore_args
663	call preempt_schedule_irq	663	call preempt_schedule_irq
664	jmp exit_intr	664	jmp exit_intr
665	#endif	665	#endif
666		666
667	CFI_ENDPROC	667	CFI_ENDPROC
668	END(common_interrupt)	668	END(common_interrupt)
669		669
670	/*	670	/*
671	* APIC interrupts.	671	* APIC interrupts.
672	*/	672	*/
673	.macro apicinterrupt num,func	673	.macro apicinterrupt num,func
674	INTR_FRAME	674	INTR_FRAME
675	pushq $~(\num)	675	pushq $~(\num)
676	CFI_ADJUST_CFA_OFFSET 8	676	CFI_ADJUST_CFA_OFFSET 8
677	interrupt \func	677	interrupt \func
678	jmp ret_from_intr	678	jmp ret_from_intr
679	CFI_ENDPROC	679	CFI_ENDPROC
680	.endm	680	.endm
681		681
682	ENTRY(thermal_interrupt)	682	ENTRY(thermal_interrupt)
683	apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt	683	apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
684	END(thermal_interrupt)	684	END(thermal_interrupt)
685		685
686	ENTRY(threshold_interrupt)	686	ENTRY(threshold_interrupt)
687	apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt	687	apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
688	END(threshold_interrupt)	688	END(threshold_interrupt)
689		689
690	#ifdef CONFIG_SMP	690	#ifdef CONFIG_SMP
691	ENTRY(reschedule_interrupt)	691	ENTRY(reschedule_interrupt)
692	apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt	692	apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
693	END(reschedule_interrupt)	693	END(reschedule_interrupt)
694		694
695	.macro INVALIDATE_ENTRY num	695	.macro INVALIDATE_ENTRY num
696	ENTRY(invalidate_interrupt\num)	696	ENTRY(invalidate_interrupt\num)
697	apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt	697	apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
698	END(invalidate_interrupt\num)	698	END(invalidate_interrupt\num)
699	.endm	699	.endm
700		700
701	INVALIDATE_ENTRY 0	701	INVALIDATE_ENTRY 0
702	INVALIDATE_ENTRY 1	702	INVALIDATE_ENTRY 1
703	INVALIDATE_ENTRY 2	703	INVALIDATE_ENTRY 2
704	INVALIDATE_ENTRY 3	704	INVALIDATE_ENTRY 3
705	INVALIDATE_ENTRY 4	705	INVALIDATE_ENTRY 4
706	INVALIDATE_ENTRY 5	706	INVALIDATE_ENTRY 5
707	INVALIDATE_ENTRY 6	707	INVALIDATE_ENTRY 6
708	INVALIDATE_ENTRY 7	708	INVALIDATE_ENTRY 7
709		709
710	ENTRY(call_function_interrupt)	710	ENTRY(call_function_interrupt)
711	apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt	711	apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
712	END(call_function_interrupt)	712	END(call_function_interrupt)
713	ENTRY(irq_move_cleanup_interrupt)	713	ENTRY(irq_move_cleanup_interrupt)
714	apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt	714	apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
715	END(irq_move_cleanup_interrupt)	715	END(irq_move_cleanup_interrupt)
716	#endif	716	#endif
717		717
718	ENTRY(apic_timer_interrupt)	718	ENTRY(apic_timer_interrupt)
719	apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt	719	apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
720	END(apic_timer_interrupt)	720	END(apic_timer_interrupt)
721		721
722	ENTRY(uv_bau_message_intr1)	722	ENTRY(uv_bau_message_intr1)
723	apicinterrupt 220,uv_bau_message_interrupt	723	apicinterrupt 220,uv_bau_message_interrupt
724	END(uv_bau_message_intr1)	724	END(uv_bau_message_intr1)
725		725
726	ENTRY(error_interrupt)	726	ENTRY(error_interrupt)
727	apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt	727	apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
728	END(error_interrupt)	728	END(error_interrupt)
729		729
730	ENTRY(spurious_interrupt)	730	ENTRY(spurious_interrupt)
731	apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt	731	apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
732	END(spurious_interrupt)	732	END(spurious_interrupt)
733		733
734	/*	734	/*
735	* Exception entry points.	735	* Exception entry points.
736	*/	736	*/
737	.macro zeroentry sym	737	.macro zeroentry sym
738	INTR_FRAME	738	INTR_FRAME
739	PARAVIRT_ADJUST_EXCEPTION_FRAME	739	PARAVIRT_ADJUST_EXCEPTION_FRAME
740	pushq $0 /* push error code/oldrax */	740	pushq $0 /* push error code/oldrax */
741	CFI_ADJUST_CFA_OFFSET 8	741	CFI_ADJUST_CFA_OFFSET 8
742	pushq %rax /* push real oldrax to the rdi slot */	742	pushq %rax /* push real oldrax to the rdi slot */
743	CFI_ADJUST_CFA_OFFSET 8	743	CFI_ADJUST_CFA_OFFSET 8
744	CFI_REL_OFFSET rax,0	744	CFI_REL_OFFSET rax,0
745	leaq \sym(%rip),%rax	745	leaq \sym(%rip),%rax
746	jmp error_entry	746	jmp error_entry
747	CFI_ENDPROC	747	CFI_ENDPROC
748	.endm	748	.endm
749		749
750	.macro errorentry sym	750	.macro errorentry sym
751	XCPT_FRAME	751	XCPT_FRAME
752	PARAVIRT_ADJUST_EXCEPTION_FRAME	752	PARAVIRT_ADJUST_EXCEPTION_FRAME
753	pushq %rax	753	pushq %rax
754	CFI_ADJUST_CFA_OFFSET 8	754	CFI_ADJUST_CFA_OFFSET 8
755	CFI_REL_OFFSET rax,0	755	CFI_REL_OFFSET rax,0
756	leaq \sym(%rip),%rax	756	leaq \sym(%rip),%rax
757	jmp error_entry	757	jmp error_entry
758	CFI_ENDPROC	758	CFI_ENDPROC
759	.endm	759	.endm
760		760
761	/* error code is on the stack already */	761	/* error code is on the stack already */
762	/* handle NMI like exceptions that can happen everywhere */	762	/* handle NMI like exceptions that can happen everywhere */
763	.macro paranoidentry sym, ist=0, irqtrace=1	763	.macro paranoidentry sym, ist=0, irqtrace=1
764	SAVE_ALL	764	SAVE_ALL
765	cld	765	cld
766	movl $1,%ebx	766	movl $1,%ebx
767	movl $MSR_GS_BASE,%ecx	767	movl $MSR_GS_BASE,%ecx
768	rdmsr	768	rdmsr
769	testl %edx,%edx	769	testl %edx,%edx
770	js 1f	770	js 1f
771	SWAPGS	771	SWAPGS
772	xorl %ebx,%ebx	772	xorl %ebx,%ebx
773	1:	773	1:
774	.if \ist	774	.if \ist
775	movq %gs:pda_data_offset, %rbp	775	movq %gs:pda_data_offset, %rbp
776	.endif	776	.endif
777	movq %rsp,%rdi	777	movq %rsp,%rdi
778	movq ORIG_RAX(%rsp),%rsi	778	movq ORIG_RAX(%rsp),%rsi
779	movq $-1,ORIG_RAX(%rsp)	779	movq $-1,ORIG_RAX(%rsp)
780	.if \ist	780	.if \ist
781	subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)	781	subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
782	.endif	782	.endif
783	call \sym	783	call \sym
784	.if \ist	784	.if \ist
785	addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)	785	addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
786	.endif	786	.endif
787	DISABLE_INTERRUPTS(CLBR_NONE)	787	DISABLE_INTERRUPTS(CLBR_NONE)
788	.if \irqtrace	788	.if \irqtrace
789	TRACE_IRQS_OFF	789	TRACE_IRQS_OFF
790	.endif	790	.endif
791	.endm	791	.endm
792		792
793	/*	793	/*
794	* "Paranoid" exit path from exception stack.	794	* "Paranoid" exit path from exception stack.
795	* Paranoid because this is used by NMIs and cannot take	795	* Paranoid because this is used by NMIs and cannot take
796	* any kernel state for granted.	796	* any kernel state for granted.
797	* We don't do kernel preemption checks here, because only	797	* We don't do kernel preemption checks here, because only
798	* NMI should be common and it does not enable IRQs and	798	* NMI should be common and it does not enable IRQs and
799	* cannot get reschedule ticks.	799	* cannot get reschedule ticks.
800	*	800	*
801	* "trace" is 0 for the NMI handler only, because irq-tracing	801	* "trace" is 0 for the NMI handler only, because irq-tracing
802	* is fundamentally NMI-unsafe. (we cannot change the soft and	802	* is fundamentally NMI-unsafe. (we cannot change the soft and
803	* hard flags at once, atomically)	803	* hard flags at once, atomically)
804	*/	804	*/
805	.macro paranoidexit trace=1	805	.macro paranoidexit trace=1
806	/* ebx: no swapgs flag */	806	/* ebx: no swapgs flag */
807	paranoid_exit\trace:	807	paranoid_exit\trace:
808	testl %ebx,%ebx /* swapgs needed? */	808	testl %ebx,%ebx /* swapgs needed? */
809	jnz paranoid_restore\trace	809	jnz paranoid_restore\trace
810	testl $3,CS(%rsp)	810	testl $3,CS(%rsp)
811	jnz paranoid_userspace\trace	811	jnz paranoid_userspace\trace
812	paranoid_swapgs\trace:	812	paranoid_swapgs\trace:
813	.if \trace	813	.if \trace
814	TRACE_IRQS_IRETQ 0	814	TRACE_IRQS_IRETQ 0
815	.endif	815	.endif
816	SWAPGS_UNSAFE_STACK	816	SWAPGS_UNSAFE_STACK
817	paranoid_restore\trace:	817	paranoid_restore\trace:
818	RESTORE_ALL 8	818	RESTORE_ALL 8
819	jmp irq_return	819	jmp irq_return
820	paranoid_userspace\trace:	820	paranoid_userspace\trace:
821	GET_THREAD_INFO(%rcx)	821	GET_THREAD_INFO(%rcx)
822	movl threadinfo_flags(%rcx),%ebx	822	movl threadinfo_flags(%rcx),%ebx
823	andl $_TIF_WORK_MASK,%ebx	823	andl $_TIF_WORK_MASK,%ebx
824	jz paranoid_swapgs\trace	824	jz paranoid_swapgs\trace
825	movq %rsp,%rdi /* &pt_regs */	825	movq %rsp,%rdi /* &pt_regs */
826	call sync_regs	826	call sync_regs
827	movq %rax,%rsp /* switch stack for scheduling */	827	movq %rax,%rsp /* switch stack for scheduling */
828	testl $_TIF_NEED_RESCHED,%ebx	828	testl $_TIF_NEED_RESCHED,%ebx
829	jnz paranoid_schedule\trace	829	jnz paranoid_schedule\trace
830	movl %ebx,%edx /* arg3: thread flags */	830	movl %ebx,%edx /* arg3: thread flags */
831	.if \trace	831	.if \trace
832	TRACE_IRQS_ON	832	TRACE_IRQS_ON
833	.endif	833	.endif
834	ENABLE_INTERRUPTS(CLBR_NONE)	834	ENABLE_INTERRUPTS(CLBR_NONE)
835	xorl %esi,%esi /* arg2: oldset */	835	xorl %esi,%esi /* arg2: oldset */
836	movq %rsp,%rdi /* arg1: &pt_regs */	836	movq %rsp,%rdi /* arg1: &pt_regs */
837	call do_notify_resume	837	call do_notify_resume
838	DISABLE_INTERRUPTS(CLBR_NONE)	838	DISABLE_INTERRUPTS(CLBR_NONE)
839	.if \trace	839	.if \trace
840	TRACE_IRQS_OFF	840	TRACE_IRQS_OFF
841	.endif	841	.endif
842	jmp paranoid_userspace\trace	842	jmp paranoid_userspace\trace
843	paranoid_schedule\trace:	843	paranoid_schedule\trace:
844	.if \trace	844	.if \trace
845	TRACE_IRQS_ON	845	TRACE_IRQS_ON
846	.endif	846	.endif
847	ENABLE_INTERRUPTS(CLBR_ANY)	847	ENABLE_INTERRUPTS(CLBR_ANY)
848	call schedule	848	call schedule
849	DISABLE_INTERRUPTS(CLBR_ANY)	849	DISABLE_INTERRUPTS(CLBR_ANY)
850	.if \trace	850	.if \trace
851	TRACE_IRQS_OFF	851	TRACE_IRQS_OFF
852	.endif	852	.endif
853	jmp paranoid_userspace\trace	853	jmp paranoid_userspace\trace
854	CFI_ENDPROC	854	CFI_ENDPROC
855	.endm	855	.endm
856		856
857	/*	857	/*
858	* Exception entry point. This expects an error code/orig_rax on the stack	858	* Exception entry point. This expects an error code/orig_rax on the stack
859	* and the exception handler in %rax.	859	* and the exception handler in %rax.
860	*/	860	*/
861	KPROBE_ENTRY(error_entry)	861	KPROBE_ENTRY(error_entry)
862	_frame RDI	862	_frame RDI
863	CFI_REL_OFFSET rax,0	863	CFI_REL_OFFSET rax,0
864	/* rdi slot contains rax, oldrax contains error code */	864	/* rdi slot contains rax, oldrax contains error code */
865	cld	865	cld
866	subq $14*8,%rsp	866	subq $14*8,%rsp
867	CFI_ADJUST_CFA_OFFSET (14*8)	867	CFI_ADJUST_CFA_OFFSET (14*8)
868	movq %rsi,13*8(%rsp)	868	movq %rsi,13*8(%rsp)
869	CFI_REL_OFFSET rsi,RSI	869	CFI_REL_OFFSET rsi,RSI
870	movq 148(%rsp),%rsi / load rax from rdi slot */	870	movq 148(%rsp),%rsi / load rax from rdi slot */
871	CFI_REGISTER rax,rsi	871	CFI_REGISTER rax,rsi
872	movq %rdx,12*8(%rsp)	872	movq %rdx,12*8(%rsp)
873	CFI_REL_OFFSET rdx,RDX	873	CFI_REL_OFFSET rdx,RDX
874	movq %rcx,11*8(%rsp)	874	movq %rcx,11*8(%rsp)
875	CFI_REL_OFFSET rcx,RCX	875	CFI_REL_OFFSET rcx,RCX
876	movq %rsi,108(%rsp) / store rax */	876	movq %rsi,108(%rsp) / store rax */
877	CFI_REL_OFFSET rax,RAX	877	CFI_REL_OFFSET rax,RAX
878	movq %r8, 9*8(%rsp)	878	movq %r8, 9*8(%rsp)
879	CFI_REL_OFFSET r8,R8	879	CFI_REL_OFFSET r8,R8
880	movq %r9, 8*8(%rsp)	880	movq %r9, 8*8(%rsp)
881	CFI_REL_OFFSET r9,R9	881	CFI_REL_OFFSET r9,R9
882	movq %r10,7*8(%rsp)	882	movq %r10,7*8(%rsp)
883	CFI_REL_OFFSET r10,R10	883	CFI_REL_OFFSET r10,R10
884	movq %r11,6*8(%rsp)	884	movq %r11,6*8(%rsp)
885	CFI_REL_OFFSET r11,R11	885	CFI_REL_OFFSET r11,R11
886	movq %rbx,5*8(%rsp)	886	movq %rbx,5*8(%rsp)
887	CFI_REL_OFFSET rbx,RBX	887	CFI_REL_OFFSET rbx,RBX
888	movq %rbp,4*8(%rsp)	888	movq %rbp,4*8(%rsp)
889	CFI_REL_OFFSET rbp,RBP	889	CFI_REL_OFFSET rbp,RBP
890	movq %r12,3*8(%rsp)	890	movq %r12,3*8(%rsp)
891	CFI_REL_OFFSET r12,R12	891	CFI_REL_OFFSET r12,R12
892	movq %r13,2*8(%rsp)	892	movq %r13,2*8(%rsp)
893	CFI_REL_OFFSET r13,R13	893	CFI_REL_OFFSET r13,R13
894	movq %r14,1*8(%rsp)	894	movq %r14,1*8(%rsp)
895	CFI_REL_OFFSET r14,R14	895	CFI_REL_OFFSET r14,R14
896	movq %r15,(%rsp)	896	movq %r15,(%rsp)
897	CFI_REL_OFFSET r15,R15	897	CFI_REL_OFFSET r15,R15
898	xorl %ebx,%ebx	898	xorl %ebx,%ebx
899	testl $3,CS(%rsp)	899	testl $3,CS(%rsp)
900	je error_kernelspace	900	je error_kernelspace
901	error_swapgs:	901	error_swapgs:
902	SWAPGS	902	SWAPGS
903	error_sti:	903	error_sti:
904	movq %rdi,RDI(%rsp)	904	movq %rdi,RDI(%rsp)
905	CFI_REL_OFFSET rdi,RDI	905	CFI_REL_OFFSET rdi,RDI
906	movq %rsp,%rdi	906	movq %rsp,%rdi
907	movq ORIG_RAX(%rsp),%rsi /* get error code */	907	movq ORIG_RAX(%rsp),%rsi /* get error code */
908	movq $-1,ORIG_RAX(%rsp)	908	movq $-1,ORIG_RAX(%rsp)
909	call *%rax	909	call *%rax
910	/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */	910	/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
911	error_exit:	911	error_exit:
912	movl %ebx,%eax	912	movl %ebx,%eax
913	RESTORE_REST	913	RESTORE_REST
914	DISABLE_INTERRUPTS(CLBR_NONE)	914	DISABLE_INTERRUPTS(CLBR_NONE)
915	TRACE_IRQS_OFF	915	TRACE_IRQS_OFF
916	GET_THREAD_INFO(%rcx)	916	GET_THREAD_INFO(%rcx)
917	testl %eax,%eax	917	testl %eax,%eax
918	jne retint_kernel	918	jne retint_kernel
919	LOCKDEP_SYS_EXIT_IRQ	919	LOCKDEP_SYS_EXIT_IRQ
920	movl threadinfo_flags(%rcx),%edx	920	movl threadinfo_flags(%rcx),%edx
921	movl $_TIF_WORK_MASK,%edi	921	movl $_TIF_WORK_MASK,%edi
922	andl %edi,%edx	922	andl %edi,%edx
923	jnz retint_careful	923	jnz retint_careful
924	jmp retint_swapgs	924	jmp retint_swapgs
925	CFI_ENDPROC	925	CFI_ENDPROC
926		926
927	error_kernelspace:	927	error_kernelspace:
928	incl %ebx	928	incl %ebx
929	/* There are two places in the kernel that can potentially fault with	929	/* There are two places in the kernel that can potentially fault with
930	usergs. Handle them here. The exception handlers after	930	usergs. Handle them here. The exception handlers after
931	iret run with kernel gs again, so don't set the user space flag.	931	iret run with kernel gs again, so don't set the user space flag.
932	B stepping K8s sometimes report an truncated RIP for IRET	932	B stepping K8s sometimes report an truncated RIP for IRET
933	exceptions returning to compat mode. Check for these here too. */	933	exceptions returning to compat mode. Check for these here too. */
934	leaq irq_return(%rip),%rcx	934	leaq irq_return(%rip),%rcx
935	cmpq %rcx,RIP(%rsp)	935	cmpq %rcx,RIP(%rsp)
936	je error_swapgs	936	je error_swapgs
937	movl %ecx,%ecx /* zero extend */	937	movl %ecx,%ecx /* zero extend */
938	cmpq %rcx,RIP(%rsp)	938	cmpq %rcx,RIP(%rsp)
939	je error_swapgs	939	je error_swapgs
940	cmpq $gs_change,RIP(%rsp)	940	cmpq $gs_change,RIP(%rsp)
941	je error_swapgs	941	je error_swapgs
942	jmp error_sti	942	jmp error_sti
943	KPROBE_END(error_entry)	943	KPROBE_END(error_entry)
944		944
945	/* Reload gs selector with exception handling */	945	/* Reload gs selector with exception handling */
946	/* edi: new selector */	946	/* edi: new selector */
947	ENTRY(load_gs_index)	947	ENTRY(native_load_gs_index)
948	CFI_STARTPROC	948	CFI_STARTPROC
949	pushf	949	pushf
950	CFI_ADJUST_CFA_OFFSET 8	950	CFI_ADJUST_CFA_OFFSET 8
951	DISABLE_INTERRUPTS(CLBR_ANY \| ~(CLBR_RDI))	951	DISABLE_INTERRUPTS(CLBR_ANY \| ~(CLBR_RDI))
952	SWAPGS	952	SWAPGS
953	gs_change:	953	gs_change:
954	movl %edi,%gs	954	movl %edi,%gs
955	2: mfence /* workaround */	955	2: mfence /* workaround */
956	SWAPGS	956	SWAPGS
957	popf	957	popf
958	CFI_ADJUST_CFA_OFFSET -8	958	CFI_ADJUST_CFA_OFFSET -8
959	ret	959	ret
960	CFI_ENDPROC	960	CFI_ENDPROC
961	ENDPROC(load_gs_index)	961	ENDPROC(native_load_gs_index)
962		962
963	.section __ex_table,"a"	963	.section __ex_table,"a"
964	.align 8	964	.align 8
965	.quad gs_change,bad_gs	965	.quad gs_change,bad_gs
966	.previous	966	.previous
967	.section .fixup,"ax"	967	.section .fixup,"ax"
968	/* running with kernelgs */	968	/* running with kernelgs */
969	bad_gs:	969	bad_gs:
970	SWAPGS /* switch back to user gs */	970	SWAPGS /* switch back to user gs */
971	xorl %eax,%eax	971	xorl %eax,%eax
972	movl %eax,%gs	972	movl %eax,%gs
973	jmp 2b	973	jmp 2b
974	.previous	974	.previous
975		975
976	/*	976	/*
977	* Create a kernel thread.	977	* Create a kernel thread.
978	*	978	*
979	* C extern interface:	979	* C extern interface:
980	* extern long kernel_thread(int (fn)(void ), void * arg, unsigned long flags)	980	* extern long kernel_thread(int (fn)(void ), void * arg, unsigned long flags)
981	*	981	*
982	* asm input arguments:	982	* asm input arguments:
983	* rdi: fn, rsi: arg, rdx: flags	983	* rdi: fn, rsi: arg, rdx: flags
984	*/	984	*/
985	ENTRY(kernel_thread)	985	ENTRY(kernel_thread)
986	CFI_STARTPROC	986	CFI_STARTPROC
987	FAKE_STACK_FRAME $child_rip	987	FAKE_STACK_FRAME $child_rip
988	SAVE_ALL	988	SAVE_ALL
989		989
990	# rdi: flags, rsi: usp, rdx: will be &pt_regs	990	# rdi: flags, rsi: usp, rdx: will be &pt_regs
991	movq %rdx,%rdi	991	movq %rdx,%rdi
992	orq kernel_thread_flags(%rip),%rdi	992	orq kernel_thread_flags(%rip),%rdi
993	movq $-1, %rsi	993	movq $-1, %rsi
994	movq %rsp, %rdx	994	movq %rsp, %rdx
995		995
996	xorl %r8d,%r8d	996	xorl %r8d,%r8d
997	xorl %r9d,%r9d	997	xorl %r9d,%r9d
998		998
999	# clone now	999	# clone now
1000	call do_fork	1000	call do_fork
1001	movq %rax,RAX(%rsp)	1001	movq %rax,RAX(%rsp)
1002	xorl %edi,%edi	1002	xorl %edi,%edi
1003		1003
1004	/*	1004	/*
1005	* It isn't worth to check for reschedule here,	1005	* It isn't worth to check for reschedule here,
1006	* so internally to the x86_64 port you can rely on kernel_thread()	1006	* so internally to the x86_64 port you can rely on kernel_thread()
1007	* not to reschedule the child before returning, this avoids the need	1007	* not to reschedule the child before returning, this avoids the need
1008	* of hacks for example to fork off the per-CPU idle tasks.	1008	* of hacks for example to fork off the per-CPU idle tasks.
1009	* [Hopefully no generic code relies on the reschedule -AK]	1009	* [Hopefully no generic code relies on the reschedule -AK]
1010	*/	1010	*/
1011	RESTORE_ALL	1011	RESTORE_ALL
1012	UNFAKE_STACK_FRAME	1012	UNFAKE_STACK_FRAME
1013	ret	1013	ret
1014	CFI_ENDPROC	1014	CFI_ENDPROC
1015	ENDPROC(kernel_thread)	1015	ENDPROC(kernel_thread)
1016		1016
1017	child_rip:	1017	child_rip:
1018	pushq $0 # fake return address	1018	pushq $0 # fake return address
1019	CFI_STARTPROC	1019	CFI_STARTPROC
1020	/*	1020	/*
1021	* Here we are in the child and the registers are set as they were	1021	* Here we are in the child and the registers are set as they were
1022	* at kernel_thread() invocation in the parent.	1022	* at kernel_thread() invocation in the parent.
1023	*/	1023	*/
1024	movq %rdi, %rax	1024	movq %rdi, %rax
1025	movq %rsi, %rdi	1025	movq %rsi, %rdi
1026	call *%rax	1026	call *%rax
1027	# exit	1027	# exit
1028	mov %eax, %edi	1028	mov %eax, %edi
1029	call do_exit	1029	call do_exit
1030	CFI_ENDPROC	1030	CFI_ENDPROC
1031	ENDPROC(child_rip)	1031	ENDPROC(child_rip)
1032		1032
1033	/*	1033	/*
1034	* execve(). This function needs to use IRET, not SYSRET, to set up all state properly.	1034	* execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1035	*	1035	*
1036	* C extern interface:	1036	* C extern interface:
1037	* extern long execve(char name, char argv, char *envp)	1037	* extern long execve(char name, char argv, char *envp)
1038	*	1038	*
1039	* asm input arguments:	1039	* asm input arguments:
1040	* rdi: name, rsi: argv, rdx: envp	1040	* rdi: name, rsi: argv, rdx: envp
1041	*	1041	*
1042	* We want to fallback into:	1042	* We want to fallback into:
1043	* extern long sys_execve(char name, char argv,char envp, struct pt_regs regs)	1043	* extern long sys_execve(char name, char argv,char envp, struct pt_regs regs)
1044	*	1044	*
1045	* do_sys_execve asm fallback arguments:	1045	* do_sys_execve asm fallback arguments:
1046	* rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack	1046	* rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1047	*/	1047	*/
1048	ENTRY(kernel_execve)	1048	ENTRY(kernel_execve)
1049	CFI_STARTPROC	1049	CFI_STARTPROC
1050	FAKE_STACK_FRAME $0	1050	FAKE_STACK_FRAME $0
1051	SAVE_ALL	1051	SAVE_ALL
1052	movq %rsp,%rcx	1052	movq %rsp,%rcx
1053	call sys_execve	1053	call sys_execve
1054	movq %rax, RAX(%rsp)	1054	movq %rax, RAX(%rsp)
1055	RESTORE_REST	1055	RESTORE_REST
1056	testq %rax,%rax	1056	testq %rax,%rax
1057	je int_ret_from_sys_call	1057	je int_ret_from_sys_call
1058	RESTORE_ARGS	1058	RESTORE_ARGS
1059	UNFAKE_STACK_FRAME	1059	UNFAKE_STACK_FRAME
1060	ret	1060	ret
1061	CFI_ENDPROC	1061	CFI_ENDPROC
1062	ENDPROC(kernel_execve)	1062	ENDPROC(kernel_execve)
1063		1063
1064	KPROBE_ENTRY(page_fault)	1064	KPROBE_ENTRY(page_fault)
1065	errorentry do_page_fault	1065	errorentry do_page_fault
1066	KPROBE_END(page_fault)	1066	KPROBE_END(page_fault)
1067		1067
1068	ENTRY(coprocessor_error)	1068	ENTRY(coprocessor_error)
1069	zeroentry do_coprocessor_error	1069	zeroentry do_coprocessor_error
1070	END(coprocessor_error)	1070	END(coprocessor_error)
1071		1071
1072	ENTRY(simd_coprocessor_error)	1072	ENTRY(simd_coprocessor_error)
1073	zeroentry do_simd_coprocessor_error	1073	zeroentry do_simd_coprocessor_error
1074	END(simd_coprocessor_error)	1074	END(simd_coprocessor_error)
1075		1075
1076	ENTRY(device_not_available)	1076	ENTRY(device_not_available)
1077	zeroentry math_state_restore	1077	zeroentry math_state_restore
1078	END(device_not_available)	1078	END(device_not_available)
1079		1079
1080	/* runs on exception stack */	1080	/* runs on exception stack */
1081	KPROBE_ENTRY(debug)	1081	KPROBE_ENTRY(debug)
1082	INTR_FRAME	1082	INTR_FRAME
1083	pushq $0	1083	pushq $0
1084	CFI_ADJUST_CFA_OFFSET 8	1084	CFI_ADJUST_CFA_OFFSET 8
1085	paranoidentry do_debug, DEBUG_STACK	1085	paranoidentry do_debug, DEBUG_STACK
1086	paranoidexit	1086	paranoidexit
1087	KPROBE_END(debug)	1087	KPROBE_END(debug)
1088		1088
1089	/* runs on exception stack */	1089	/* runs on exception stack */
1090	KPROBE_ENTRY(nmi)	1090	KPROBE_ENTRY(nmi)
1091	INTR_FRAME	1091	INTR_FRAME
1092	pushq $-1	1092	pushq $-1
1093	CFI_ADJUST_CFA_OFFSET 8	1093	CFI_ADJUST_CFA_OFFSET 8
1094	paranoidentry do_nmi, 0, 0	1094	paranoidentry do_nmi, 0, 0
1095	#ifdef CONFIG_TRACE_IRQFLAGS	1095	#ifdef CONFIG_TRACE_IRQFLAGS
1096	paranoidexit 0	1096	paranoidexit 0
1097	#else	1097	#else
1098	jmp paranoid_exit1	1098	jmp paranoid_exit1
1099	CFI_ENDPROC	1099	CFI_ENDPROC
1100	#endif	1100	#endif
1101	KPROBE_END(nmi)	1101	KPROBE_END(nmi)
1102		1102
1103	KPROBE_ENTRY(int3)	1103	KPROBE_ENTRY(int3)
1104	INTR_FRAME	1104	INTR_FRAME
1105	pushq $0	1105	pushq $0
1106	CFI_ADJUST_CFA_OFFSET 8	1106	CFI_ADJUST_CFA_OFFSET 8
1107	paranoidentry do_int3, DEBUG_STACK	1107	paranoidentry do_int3, DEBUG_STACK
1108	jmp paranoid_exit1	1108	jmp paranoid_exit1
1109	CFI_ENDPROC	1109	CFI_ENDPROC
1110	KPROBE_END(int3)	1110	KPROBE_END(int3)
1111		1111
1112	ENTRY(overflow)	1112	ENTRY(overflow)
1113	zeroentry do_overflow	1113	zeroentry do_overflow
1114	END(overflow)	1114	END(overflow)
1115		1115
1116	ENTRY(bounds)	1116	ENTRY(bounds)
1117	zeroentry do_bounds	1117	zeroentry do_bounds
1118	END(bounds)	1118	END(bounds)
1119		1119
1120	ENTRY(invalid_op)	1120	ENTRY(invalid_op)
1121	zeroentry do_invalid_op	1121	zeroentry do_invalid_op
1122	END(invalid_op)	1122	END(invalid_op)
1123		1123
1124	ENTRY(coprocessor_segment_overrun)	1124	ENTRY(coprocessor_segment_overrun)
1125	zeroentry do_coprocessor_segment_overrun	1125	zeroentry do_coprocessor_segment_overrun
1126	END(coprocessor_segment_overrun)	1126	END(coprocessor_segment_overrun)
1127		1127
1128	/* runs on exception stack */	1128	/* runs on exception stack */
1129	ENTRY(double_fault)	1129	ENTRY(double_fault)
1130	XCPT_FRAME	1130	XCPT_FRAME
1131	paranoidentry do_double_fault	1131	paranoidentry do_double_fault
1132	jmp paranoid_exit1	1132	jmp paranoid_exit1
1133	CFI_ENDPROC	1133	CFI_ENDPROC
1134	END(double_fault)	1134	END(double_fault)
1135		1135
1136	ENTRY(invalid_TSS)	1136	ENTRY(invalid_TSS)
1137	errorentry do_invalid_TSS	1137	errorentry do_invalid_TSS
1138	END(invalid_TSS)	1138	END(invalid_TSS)
1139		1139
1140	ENTRY(segment_not_present)	1140	ENTRY(segment_not_present)
1141	errorentry do_segment_not_present	1141	errorentry do_segment_not_present
1142	END(segment_not_present)	1142	END(segment_not_present)
1143		1143
1144	/* runs on exception stack */	1144	/* runs on exception stack */
1145	ENTRY(stack_segment)	1145	ENTRY(stack_segment)
1146	XCPT_FRAME	1146	XCPT_FRAME
1147	paranoidentry do_stack_segment	1147	paranoidentry do_stack_segment
1148	jmp paranoid_exit1	1148	jmp paranoid_exit1
1149	CFI_ENDPROC	1149	CFI_ENDPROC
1150	END(stack_segment)	1150	END(stack_segment)
1151		1151
1152	KPROBE_ENTRY(general_protection)	1152	KPROBE_ENTRY(general_protection)
1153	errorentry do_general_protection	1153	errorentry do_general_protection
1154	KPROBE_END(general_protection)	1154	KPROBE_END(general_protection)
1155		1155
1156	ENTRY(alignment_check)	1156	ENTRY(alignment_check)
1157	errorentry do_alignment_check	1157	errorentry do_alignment_check
1158	END(alignment_check)	1158	END(alignment_check)
1159		1159
1160	ENTRY(divide_error)	1160	ENTRY(divide_error)
1161	zeroentry do_divide_error	1161	zeroentry do_divide_error
1162	END(divide_error)	1162	END(divide_error)
1163		1163
1164	ENTRY(spurious_interrupt_bug)	1164	ENTRY(spurious_interrupt_bug)
1165	zeroentry do_spurious_interrupt_bug	1165	zeroentry do_spurious_interrupt_bug
1166	END(spurious_interrupt_bug)	1166	END(spurious_interrupt_bug)
1167		1167
1168	#ifdef CONFIG_X86_MCE	1168	#ifdef CONFIG_X86_MCE
1169	/* runs on exception stack */	1169	/* runs on exception stack */
1170	ENTRY(machine_check)	1170	ENTRY(machine_check)
1171	INTR_FRAME	1171	INTR_FRAME
1172	pushq $0	1172	pushq $0
1173	CFI_ADJUST_CFA_OFFSET 8	1173	CFI_ADJUST_CFA_OFFSET 8
1174	paranoidentry do_machine_check	1174	paranoidentry do_machine_check
1175	jmp paranoid_exit1	1175	jmp paranoid_exit1
1176	CFI_ENDPROC	1176	CFI_ENDPROC
1177	END(machine_check)	1177	END(machine_check)
1178	#endif	1178	#endif
1179		1179
1180	/* Call softirq on interrupt stack. Interrupts are off. */	1180	/* Call softirq on interrupt stack. Interrupts are off. */
1181	ENTRY(call_softirq)	1181	ENTRY(call_softirq)
1182	CFI_STARTPROC	1182	CFI_STARTPROC
1183	push %rbp	1183	push %rbp
1184	CFI_ADJUST_CFA_OFFSET 8	1184	CFI_ADJUST_CFA_OFFSET 8
1185	CFI_REL_OFFSET rbp,0	1185	CFI_REL_OFFSET rbp,0
1186	mov %rsp,%rbp	1186	mov %rsp,%rbp
1187	CFI_DEF_CFA_REGISTER rbp	1187	CFI_DEF_CFA_REGISTER rbp
1188	incl %gs:pda_irqcount	1188	incl %gs:pda_irqcount
1189	cmove %gs:pda_irqstackptr,%rsp	1189	cmove %gs:pda_irqstackptr,%rsp
1190	push %rbp # backlink for old unwinder	1190	push %rbp # backlink for old unwinder
1191	call __do_softirq	1191	call __do_softirq
1192	leaveq	1192	leaveq
1193	CFI_DEF_CFA_REGISTER rsp	1193	CFI_DEF_CFA_REGISTER rsp
1194	CFI_ADJUST_CFA_OFFSET -8	1194	CFI_ADJUST_CFA_OFFSET -8
1195	decl %gs:pda_irqcount	1195	decl %gs:pda_irqcount
1196	ret	1196	ret
1197	CFI_ENDPROC	1197	CFI_ENDPROC
1198	ENDPROC(call_softirq)	1198	ENDPROC(call_softirq)
1199		1199
1200	KPROBE_ENTRY(ignore_sysret)	1200	KPROBE_ENTRY(ignore_sysret)
1201	CFI_STARTPROC	1201	CFI_STARTPROC
1202	mov $-ENOSYS,%eax	1202	mov $-ENOSYS,%eax
1203	sysret	1203	sysret
1204	CFI_ENDPROC	1204	CFI_ENDPROC
1205	ENDPROC(ignore_sysret)	1205	ENDPROC(ignore_sysret)
1206		1206

arch/x86/kernel/paravirt.c

Diff comments View file @ 9f9d489

 /*  Paravirtualization interfaces
     Copyright (C) 2006 Rusty Russell IBM Corporation
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
     You should have received a copy of the GNU General Public License
     along with this program; if not, write to the Free Software
     Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
     2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
 */
 #include <linux/errno.h>
 #include <linux/module.h>
 #include <linux/efi.h>
 #include <linux/bcd.h>
 #include <linux/highmem.h>
 #include <asm/bug.h>
 #include <asm/paravirt.h>
 #include <asm/desc.h>
 #include <asm/setup.h>
 #include <asm/arch_hooks.h>
 #include <asm/time.h>
 #include <asm/pgalloc.h>
 #include <asm/irq.h>
 #include <asm/delay.h>
 #include <asm/fixmap.h>
 #include <asm/apic.h>
 #include <asm/tlbflush.h>
 #include <asm/timer.h>
 /* nop stub */
 void _paravirt_nop(void)
 {
 }
 static void __init default_banner(void)
 {
 	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
 	       pv_info.name);
 }
 char *memory_setup(void)
 {
 	return pv_init_ops.memory_setup();
 }
 /* Simple instruction patching code. */
 #define DEF_NATIVE(ops, name, code)					\
 	extern const char start_##ops##_##name[], end_##ops##_##name[];	\
 	asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
 /* Undefined instruction for dealing with missing ops pointers. */
 static const unsigned char ud2a[] = { 0x0f, 0x0b };
 unsigned paravirt_patch_nop(void)
 {
 	return 0;
 }
 unsigned paravirt_patch_ignore(unsigned len)
 {
 	return len;
 }
 struct branch {
 	unsigned char opcode;
 	u32 delta;
 } __attribute__((packed));
 unsigned paravirt_patch_call(void *insnbuf,
 			     const void *target, u16 tgt_clobbers,
 			     unsigned long addr, u16 site_clobbers,
 			     unsigned len)
 {
 	struct branch *b = insnbuf;
 	unsigned long delta = (unsigned long)target - (addr+5);
 	if (tgt_clobbers & ~site_clobbers)
 		return len;	/* target would clobber too much for this site */
 	if (len < 5)
 		return len;	/* call too long for patch site */
 	b->opcode = 0xe8; /* call */
 	b->delta = delta;
 	BUILD_BUG_ON(sizeof(*b) != 5);
 	return 5;
 }
 unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
 			    unsigned long addr, unsigned len)
 {
 	struct branch *b = insnbuf;
 	unsigned long delta = (unsigned long)target - (addr+5);
 	if (len < 5)
 		return len;	/* call too long for patch site */
 	b->opcode = 0xe9;	/* jmp */
 	b->delta = delta;
 	return 5;
 }
 /* Neat trick to map patch type back to the call within the
  * corresponding structure. */
 static void *get_call_destination(u8 type)
 {
 	struct paravirt_patch_template tmpl = {
 		.pv_init_ops = pv_init_ops,
 		.pv_time_ops = pv_time_ops,
 		.pv_cpu_ops = pv_cpu_ops,
 		.pv_irq_ops = pv_irq_ops,
 		.pv_apic_ops = pv_apic_ops,
 		.pv_mmu_ops = pv_mmu_ops,
 	};
 	return *((void **)&tmpl + type);
 }
 unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
 				unsigned long addr, unsigned len)
 {
 	void *opfunc = get_call_destination(type);
 	unsigned ret;
 	if (opfunc == NULL)
 		/* If there's no function, patch it with a ud2a (BUG) */
 		ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
 	else if (opfunc == paravirt_nop)
 		/* If the operation is a nop, then nop the callsite */
 		ret = paravirt_patch_nop();
 	else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
 		 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||
 		 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) ||
 		 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64))
 		/* If operation requires a jmp, then jmp */
 		ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
 	else
 		/* Otherwise call the function; assume target could
 		   clobber any caller-save reg */
 		ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY,
 					  addr, clobbers, len);
 	return ret;
 }
 unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
 			      const char *start, const char *end)
 {
 	unsigned insn_len = end - start;
 	if (insn_len > len || start == NULL)
 		insn_len = len;
 	else
 		memcpy(insnbuf, start, insn_len);
 	return insn_len;
 }
 void init_IRQ(void)
 {
 	pv_irq_ops.init_IRQ();
 }
 static void native_flush_tlb(void)
 {
 	__native_flush_tlb();
 }
 /*
  * Global pages have to be flushed a bit differently. Not a real
  * performance problem because this does not happen often.
  */
 static void native_flush_tlb_global(void)
 {
 	__native_flush_tlb_global();
 }
 static void native_flush_tlb_single(unsigned long addr)
 {
 	__native_flush_tlb_single(addr);
 }
 /* These are in entry.S */
 extern void native_iret(void);
 extern void native_irq_enable_sysexit(void);
 extern void native_usergs_sysret32(void);
 extern void native_usergs_sysret64(void);
 static int __init print_banner(void)
 {
 	pv_init_ops.banner();
 	return 0;
 }
 core_initcall(print_banner);
 static struct resource reserve_ioports = {
 	.start = 0,
 	.end = IO_SPACE_LIMIT,
 	.name = "paravirt-ioport",
 	.flags = IORESOURCE_IO | IORESOURCE_BUSY,
 };
 /*
  * Reserve the whole legacy IO space to prevent any legacy drivers
  * from wasting time probing for their hardware.  This is a fairly
  * brute-force approach to disabling all non-virtual drivers.
  *
  * Note that this must be called very early to have any effect.
  */
 int paravirt_disable_iospace(void)
 {
 	return request_resource(&ioport_resource, &reserve_ioports);
 }
 static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
 static inline void enter_lazy(enum paravirt_lazy_mode mode)
 {
 	BUG_ON(__get_cpu_var(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
 	BUG_ON(preemptible());
 	__get_cpu_var(paravirt_lazy_mode) = mode;
 }
 void paravirt_leave_lazy(enum paravirt_lazy_mode mode)
 {
 	BUG_ON(__get_cpu_var(paravirt_lazy_mode) != mode);
 	BUG_ON(preemptible());
 	__get_cpu_var(paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
 }
 void paravirt_enter_lazy_mmu(void)
 {
 	enter_lazy(PARAVIRT_LAZY_MMU);
 }
 void paravirt_leave_lazy_mmu(void)
 {
 	paravirt_leave_lazy(PARAVIRT_LAZY_MMU);
 }
 void paravirt_enter_lazy_cpu(void)
 {
 	enter_lazy(PARAVIRT_LAZY_CPU);
 }
 void paravirt_leave_lazy_cpu(void)
 {
 	paravirt_leave_lazy(PARAVIRT_LAZY_CPU);
 }
 enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
 {
 	return __get_cpu_var(paravirt_lazy_mode);
 }
 struct pv_info pv_info = {
 	.name = "bare hardware",
 	.paravirt_enabled = 0,
 	.kernel_rpl = 0,
 	.shared_kernel_pmd = 1,	/* Only used when CONFIG_X86_PAE is set */
 };
 struct pv_init_ops pv_init_ops = {
 	.patch = native_patch,
 	.banner = default_banner,
 	.arch_setup = paravirt_nop,
 	.memory_setup = machine_specific_memory_setup,
 };
 struct pv_time_ops pv_time_ops = {
 	.time_init = hpet_time_init,
 	.get_wallclock = native_get_wallclock,
 	.set_wallclock = native_set_wallclock,
 	.sched_clock = native_sched_clock,
 	.get_cpu_khz = native_calculate_cpu_khz,
 };
 struct pv_irq_ops pv_irq_ops = {
 	.init_IRQ = native_init_IRQ,
 	.save_fl = native_save_fl,
 	.restore_fl = native_restore_fl,
 	.irq_disable = native_irq_disable,
 	.irq_enable = native_irq_enable,
 	.safe_halt = native_safe_halt,
 	.halt = native_halt,
 #ifdef CONFIG_X86_64
 	.adjust_exception_frame = paravirt_nop,
 #endif
 };
 struct pv_cpu_ops pv_cpu_ops = {
 	.cpuid = native_cpuid,
 	.get_debugreg = native_get_debugreg,
 	.set_debugreg = native_set_debugreg,
 	.clts = native_clts,
 	.read_cr0 = native_read_cr0,
 	.write_cr0 = native_write_cr0,
 	.read_cr4 = native_read_cr4,
 	.read_cr4_safe = native_read_cr4_safe,
 	.write_cr4 = native_write_cr4,
 #ifdef CONFIG_X86_64
 	.read_cr8 = native_read_cr8,
 	.write_cr8 = native_write_cr8,
 #endif
 	.wbinvd = native_wbinvd,
 	.read_msr = native_read_msr_safe,
 	.write_msr = native_write_msr_safe,
 	.read_tsc = native_read_tsc,
 	.read_pmc = native_read_pmc,
 	.read_tscp = native_read_tscp,
 	.load_tr_desc = native_load_tr_desc,
 	.set_ldt = native_set_ldt,
 	.load_gdt = native_load_gdt,
 	.load_idt = native_load_idt,
 	.store_gdt = native_store_gdt,
 	.store_idt = native_store_idt,
 	.store_tr = native_store_tr,
 	.load_tls = native_load_tls,
+#ifdef CONFIG_X86_64
+	.load_gs_index = native_load_gs_index,
+#endif
 	.write_ldt_entry = native_write_ldt_entry,
 	.write_gdt_entry = native_write_gdt_entry,
 	.write_idt_entry = native_write_idt_entry,
 	.load_sp0 = native_load_sp0,
 	.irq_enable_sysexit = native_irq_enable_sysexit,
 #ifdef CONFIG_X86_64
 	.usergs_sysret32 = native_usergs_sysret32,
 	.usergs_sysret64 = native_usergs_sysret64,
 #endif
 	.iret = native_iret,
 	.swapgs = native_swapgs,
 	.set_iopl_mask = native_set_iopl_mask,
 	.io_delay = native_io_delay,
 	.lazy_mode = {
 		.enter = paravirt_nop,
 		.leave = paravirt_nop,
 	},
 };
 struct pv_apic_ops pv_apic_ops = {
 #ifdef CONFIG_X86_LOCAL_APIC
 	.apic_write = native_apic_write,
 	.apic_write_atomic = native_apic_write_atomic,
 	.apic_read = native_apic_read,
 	.setup_boot_clock = setup_boot_APIC_clock,
 	.setup_secondary_clock = setup_secondary_APIC_clock,
 	.startup_ipi_hook = paravirt_nop,
 #endif
 };
 struct pv_mmu_ops pv_mmu_ops = {
 #ifndef CONFIG_X86_64
 	.pagetable_setup_start = native_pagetable_setup_start,
 	.pagetable_setup_done = native_pagetable_setup_done,
 #endif
 	.read_cr2 = native_read_cr2,
 	.write_cr2 = native_write_cr2,
 	.read_cr3 = native_read_cr3,
 	.write_cr3 = native_write_cr3,
 	.flush_tlb_user = native_flush_tlb,
 	.flush_tlb_kernel = native_flush_tlb_global,
 	.flush_tlb_single = native_flush_tlb_single,
 	.flush_tlb_others = native_flush_tlb_others,
 	.pgd_alloc = __paravirt_pgd_alloc,
 	.pgd_free = paravirt_nop,
 	.alloc_pte = paravirt_nop,
 	.alloc_pmd = paravirt_nop,
 	.alloc_pmd_clone = paravirt_nop,
 	.alloc_pud = paravirt_nop,
 	.release_pte = paravirt_nop,
 	.release_pmd = paravirt_nop,
 	.release_pud = paravirt_nop,
 	.set_pte = native_set_pte,
 	.set_pte_at = native_set_pte_at,
 	.set_pmd = native_set_pmd,
 	.pte_update = paravirt_nop,
 	.pte_update_defer = paravirt_nop,
 	.ptep_modify_prot_start = __ptep_modify_prot_start,
 	.ptep_modify_prot_commit = __ptep_modify_prot_commit,
 #ifdef CONFIG_HIGHPTE
 	.kmap_atomic_pte = kmap_atomic,
 #endif
 #if PAGETABLE_LEVELS >= 3
 #ifdef CONFIG_X86_PAE
 	.set_pte_atomic = native_set_pte_atomic,
 	.set_pte_present = native_set_pte_present,
 	.pte_clear = native_pte_clear,
 	.pmd_clear = native_pmd_clear,
 #endif
 	.set_pud = native_set_pud,
 	.pmd_val = native_pmd_val,
 	.make_pmd = native_make_pmd,
 #if PAGETABLE_LEVELS == 4
 	.pud_val = native_pud_val,
 	.make_pud = native_make_pud,
 	.set_pgd = native_set_pgd,
 #endif
 #endif /* PAGETABLE_LEVELS >= 3 */
 	.pte_val = native_pte_val,
 	.pte_flags = native_pte_val,
 	.pgd_val = native_pgd_val,
 	.make_pte = native_make_pte,
 	.make_pgd = native_make_pgd,
 	.dup_mmap = paravirt_nop,
 	.exit_mmap = paravirt_nop,
 	.activate_mm = paravirt_nop,
 	.lazy_mode = {
 		.enter = paravirt_nop,
 		.leave = paravirt_nop,
 	},
 	.set_fixmap = native_set_fixmap,
 };
 EXPORT_SYMBOL_GPL(pv_time_ops);
 EXPORT_SYMBOL    (pv_cpu_ops);
 EXPORT_SYMBOL    (pv_mmu_ops);
 EXPORT_SYMBOL_GPL(pv_apic_ops);
 EXPORT_SYMBOL_GPL(pv_info);
 EXPORT_SYMBOL    (pv_irq_ops);

include/asm-x86/elf.h

Diff comments View file @ 9f9d489

 #ifndef _ASM_X86_ELF_H
 #define _ASM_X86_ELF_H
 /*
  * ELF register definitions..
  */
 #include <asm/ptrace.h>
 #include <asm/user.h>
 #include <asm/auxvec.h>
 typedef unsigned long elf_greg_t;
 #define ELF_NGREG (sizeof(struct user_regs_struct) / sizeof(elf_greg_t))
 typedef elf_greg_t elf_gregset_t[ELF_NGREG];
 typedef struct user_i387_struct elf_fpregset_t;
 #ifdef __i386__
 typedef struct user_fxsr_struct elf_fpxregset_t;
 #define R_386_NONE	0
 #define R_386_32	1
 #define R_386_PC32	2
 #define R_386_GOT32	3
 #define R_386_PLT32	4
 #define R_386_COPY	5
 #define R_386_GLOB_DAT	6
 #define R_386_JMP_SLOT	7
 #define R_386_RELATIVE	8
 #define R_386_GOTOFF	9
 #define R_386_GOTPC	10
 #define R_386_NUM	11
 /*
  * These are used to set parameters in the core dumps.
  */
 #define ELF_CLASS	ELFCLASS32
 #define ELF_DATA	ELFDATA2LSB
 #define ELF_ARCH	EM_386
 #else
 /* x86-64 relocation types */
 #define R_X86_64_NONE		0	/* No reloc */
 #define R_X86_64_64		1	/* Direct 64 bit  */
 #define R_X86_64_PC32		2	/* PC relative 32 bit signed */
 #define R_X86_64_GOT32		3	/* 32 bit GOT entry */
 #define R_X86_64_PLT32		4	/* 32 bit PLT address */
 #define R_X86_64_COPY		5	/* Copy symbol at runtime */
 #define R_X86_64_GLOB_DAT	6	/* Create GOT entry */
 #define R_X86_64_JUMP_SLOT	7	/* Create PLT entry */
 #define R_X86_64_RELATIVE	8	/* Adjust by program base */
 #define R_X86_64_GOTPCREL	9	/* 32 bit signed pc relative
 					   offset to GOT */
 #define R_X86_64_32		10	/* Direct 32 bit zero extended */
 #define R_X86_64_32S		11	/* Direct 32 bit sign extended */
 #define R_X86_64_16		12	/* Direct 16 bit zero extended */
 #define R_X86_64_PC16		13	/* 16 bit sign extended pc relative */
 #define R_X86_64_8		14	/* Direct 8 bit sign extended  */
 #define R_X86_64_PC8		15	/* 8 bit sign extended pc relative */
 #define R_X86_64_NUM		16
 /*
  * These are used to set parameters in the core dumps.
  */
 #define ELF_CLASS	ELFCLASS64
 #define ELF_DATA	ELFDATA2LSB
 #define ELF_ARCH	EM_X86_64
 #endif
 #include <asm/vdso.h>
 extern unsigned int vdso_enabled;
 /*
  * This is used to ensure we don't load something for the wrong architecture.
  */
 #define elf_check_arch_ia32(x) \
 	(((x)->e_machine == EM_386) || ((x)->e_machine == EM_486))
 #include <asm/processor.h>
+#include <asm/system.h>
 #ifdef CONFIG_X86_32
-#include <asm/system.h>		/* for savesegment */
 #include <asm/desc.h>
 #define elf_check_arch(x)	elf_check_arch_ia32(x)
 /* SVR4/i386 ABI (pages 3-31, 3-32) says that when the program starts %edx
    contains a pointer to a function which might be registered using `atexit'.
    This provides a mean for the dynamic linker to call DT_FINI functions for
    shared libraries that have been loaded before the code runs.
    A value of 0 tells we have no such handler.
    We might as well make sure everything else is cleared too (except for %esp),
    just to make things more deterministic.
  */
 #define ELF_PLAT_INIT(_r, load_addr)		\
 	do {					\
 	_r->bx = 0; _r->cx = 0; _r->dx = 0;	\
 	_r->si = 0; _r->di = 0; _r->bp = 0;	\
 	_r->ax = 0;				\
 } while (0)
 /*
  * regs is struct pt_regs, pr_reg is elf_gregset_t (which is
  * now struct_user_regs, they are different)
  */
 #define ELF_CORE_COPY_REGS(pr_reg, regs)	\
 do {						\
 	pr_reg[0] = regs->bx;			\
 	pr_reg[1] = regs->cx;			\
 	pr_reg[2] = regs->dx;			\
 	pr_reg[3] = regs->si;			\
 	pr_reg[4] = regs->di;			\
 	pr_reg[5] = regs->bp;			\
 	pr_reg[6] = regs->ax;			\
 	pr_reg[7] = regs->ds & 0xffff;		\
 	pr_reg[8] = regs->es & 0xffff;		\
 	pr_reg[9] = regs->fs & 0xffff;		\
 	savesegment(gs, pr_reg[10]);		\
 	pr_reg[11] = regs->orig_ax;		\
 	pr_reg[12] = regs->ip;			\
 	pr_reg[13] = regs->cs & 0xffff;		\
 	pr_reg[14] = regs->flags;		\
 	pr_reg[15] = regs->sp;			\
 	pr_reg[16] = regs->ss & 0xffff;		\
 } while (0);
 #define ELF_PLATFORM	(utsname()->machine)
 #define set_personality_64bit()	do { } while (0)
 #else /* CONFIG_X86_32 */
 /*
  * This is used to ensure we don't load something for the wrong architecture.
  */
 #define elf_check_arch(x)			\
 	((x)->e_machine == EM_X86_64)
 #define compat_elf_check_arch(x)	elf_check_arch_ia32(x)
 static inline void start_ia32_thread(struct pt_regs *regs, u32 ip, u32 sp)
 {
 	asm volatile("movl %0,%%fs" :: "r" (0));
 	asm volatile("movl %0,%%es; movl %0,%%ds" : : "r" (__USER32_DS));
 	load_gs_index(0);
 	regs->ip = ip;
 	regs->sp = sp;
 	regs->flags = X86_EFLAGS_IF;
 	regs->cs = __USER32_CS;
 	regs->ss = __USER32_DS;
 }
 static inline void elf_common_init(struct thread_struct *t,
 				   struct pt_regs *regs, const u16 ds)
 {
 	regs->ax = regs->bx = regs->cx = regs->dx = 0;
 	regs->si = regs->di = regs->bp = 0;
 	regs->r8 = regs->r9 = regs->r10 = regs->r11 = 0;
 	regs->r12 = regs->r13 = regs->r14 = regs->r15 = 0;
 	t->fs = t->gs = 0;
 	t->fsindex = t->gsindex = 0;
 	t->ds = t->es = ds;
 }
 #define ELF_PLAT_INIT(_r, load_addr)			\
 do {							\
 	elf_common_init(&current->thread, _r, 0);	\
 	clear_thread_flag(TIF_IA32);			\
 } while (0)
 #define	COMPAT_ELF_PLAT_INIT(regs, load_addr)		\
 	elf_common_init(&current->thread, regs, __USER_DS)
 #define	compat_start_thread(regs, ip, sp)		\
 do {							\
 	start_ia32_thread(regs, ip, sp);		\
 	set_fs(USER_DS);				\
 } while (0)
 #define COMPAT_SET_PERSONALITY(ex, ibcs2)		\
 do {							\
 	if (test_thread_flag(TIF_IA32))			\
 		clear_thread_flag(TIF_ABI_PENDING);	\
 	else						\
 		set_thread_flag(TIF_ABI_PENDING);	\
 	current->personality |= force_personality32;	\
 } while (0)
 #define COMPAT_ELF_PLATFORM			("i686")
 /*
  * regs is struct pt_regs, pr_reg is elf_gregset_t (which is
  * now struct_user_regs, they are different). Assumes current is the process
  * getting dumped.
  */
 #define ELF_CORE_COPY_REGS(pr_reg, regs)			\
 do {								\
 	unsigned v;						\
 	(pr_reg)[0] = (regs)->r15;				\
 	(pr_reg)[1] = (regs)->r14;				\
 	(pr_reg)[2] = (regs)->r13;				\
 	(pr_reg)[3] = (regs)->r12;				\
 	(pr_reg)[4] = (regs)->bp;				\
 	(pr_reg)[5] = (regs)->bx;				\
 	(pr_reg)[6] = (regs)->r11;				\
 	(pr_reg)[7] = (regs)->r10;				\
 	(pr_reg)[8] = (regs)->r9;				\
 	(pr_reg)[9] = (regs)->r8;				\
 	(pr_reg)[10] = (regs)->ax;				\
 	(pr_reg)[11] = (regs)->cx;				\
 	(pr_reg)[12] = (regs)->dx;				\
 	(pr_reg)[13] = (regs)->si;				\
 	(pr_reg)[14] = (regs)->di;				\
 	(pr_reg)[15] = (regs)->orig_ax;				\
 	(pr_reg)[16] = (regs)->ip;				\
 	(pr_reg)[17] = (regs)->cs;				\
 	(pr_reg)[18] = (regs)->flags;				\
 	(pr_reg)[19] = (regs)->sp;				\
 	(pr_reg)[20] = (regs)->ss;				\
 	(pr_reg)[21] = current->thread.fs;			\
 	(pr_reg)[22] = current->thread.gs;			\
 	asm("movl %%ds,%0" : "=r" (v)); (pr_reg)[23] = v;	\
 	asm("movl %%es,%0" : "=r" (v)); (pr_reg)[24] = v;	\
 	asm("movl %%fs,%0" : "=r" (v)); (pr_reg)[25] = v;	\
 	asm("movl %%gs,%0" : "=r" (v)); (pr_reg)[26] = v;	\
 } while (0);
 /* I'm not sure if we can use '-' here */
 #define ELF_PLATFORM       ("x86_64")
 extern void set_personality_64bit(void);
 extern unsigned int sysctl_vsyscall32;
 extern int force_personality32;
 #endif /* !CONFIG_X86_32 */
 #define CORE_DUMP_USE_REGSET
 #define USE_ELF_CORE_DUMP
 #define ELF_EXEC_PAGESIZE	4096
 /* This is the location that an ET_DYN program is loaded if exec'ed.  Typical
    use of this is to invoke "./ld.so someprog" to test out a new version of
    the loader.  We need to make sure that it is out of the way of the program
    that it will "exec", and that there is sufficient room for the brk.  */
 #define ELF_ET_DYN_BASE		(TASK_SIZE / 3 * 2)
 /* This yields a mask that user programs can use to figure out what
    instruction set this CPU supports.  This could be done in user space,
    but it's not easy, and we've already done it here.  */
 #define ELF_HWCAP		(boot_cpu_data.x86_capability[0])
 /* This yields a string that ld.so will use to load implementation
    specific libraries for optimization.  This is more specific in
    intent than poking at uname or /proc/cpuinfo.
    For the moment, we have only optimizations for the Intel generations,
    but that could change... */
 #define SET_PERSONALITY(ex, ibcs2) set_personality_64bit()
 /*
  * An executable for which elf_read_implies_exec() returns TRUE will
  * have the READ_IMPLIES_EXEC personality flag set automatically.
  */
 #define elf_read_implies_exec(ex, executable_stack)	\
 	(executable_stack != EXSTACK_DISABLE_X)
 struct task_struct;
 #define	ARCH_DLINFO_IA32(vdso_enabled)					\
 do {									\
 	if (vdso_enabled) {						\
 		NEW_AUX_ENT(AT_SYSINFO,	VDSO_ENTRY);			\
 		NEW_AUX_ENT(AT_SYSINFO_EHDR, VDSO_CURRENT_BASE);	\
 	}								\
 } while (0)
 #ifdef CONFIG_X86_32
 #define VDSO_HIGH_BASE		(__fix_to_virt(FIX_VDSO))
 #define ARCH_DLINFO		ARCH_DLINFO_IA32(vdso_enabled)
 /* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */
 #else /* CONFIG_X86_32 */
 #define VDSO_HIGH_BASE		0xffffe000U /* CONFIG_COMPAT_VDSO address */
 /* 1GB for 64bit, 8MB for 32bit */
 #define STACK_RND_MASK (test_thread_flag(TIF_IA32) ? 0x7ff : 0x3fffff)
 #define ARCH_DLINFO							\
 do {									\
 	if (vdso_enabled)						\
 		NEW_AUX_ENT(AT_SYSINFO_EHDR,				\
 			    (unsigned long)current->mm->context.vdso);	\
 } while (0)
 #define AT_SYSINFO		32
 #define COMPAT_ARCH_DLINFO	ARCH_DLINFO_IA32(sysctl_vsyscall32)
 #define COMPAT_ELF_ET_DYN_BASE	(TASK_UNMAPPED_BASE + 0x1000000)
 #endif /* !CONFIG_X86_32 */
 #define VDSO_CURRENT_BASE	((unsigned long)current->mm->context.vdso)
 #define VDSO_ENTRY							\
 	((unsigned long)VDSO32_SYMBOL(VDSO_CURRENT_BASE, vsyscall))
 struct linux_binprm;
 #define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
 extern int arch_setup_additional_pages(struct linux_binprm *bprm,
 				       int executable_stack);
 extern int syscall32_setup_pages(struct linux_binprm *, int exstack);
 #define compat_arch_setup_additional_pages	syscall32_setup_pages
 extern unsigned long arch_randomize_brk(struct mm_struct *mm);
 #define arch_randomize_brk arch_randomize_brk
 #endif

include/asm-x86/paravirt.h

Diff comments View file @ 9f9d489

 #ifndef __ASM_PARAVIRT_H
 #define __ASM_PARAVIRT_H
 /* Various instructions on x86 need to be replaced for
  * para-virtualization: those hooks are defined here. */
 #ifdef CONFIG_PARAVIRT
 #include <asm/page.h>
 #include <asm/asm.h>
 /* Bitmask of what can be clobbered: usually at least eax. */
 #define CLBR_NONE 0
 #define CLBR_EAX  (1 << 0)
 #define CLBR_ECX  (1 << 1)
 #define CLBR_EDX  (1 << 2)
 #ifdef CONFIG_X86_64
 #define CLBR_RSI  (1 << 3)
 #define CLBR_RDI  (1 << 4)
 #define CLBR_R8   (1 << 5)
 #define CLBR_R9   (1 << 6)
 #define CLBR_R10  (1 << 7)
 #define CLBR_R11  (1 << 8)
 #define CLBR_ANY  ((1 << 9) - 1)
 #include <asm/desc_defs.h>
 #else
 /* CLBR_ANY should match all regs platform has. For i386, that's just it */
 #define CLBR_ANY  ((1 << 3) - 1)
 #endif /* X86_64 */
 #ifndef __ASSEMBLY__
 #include <linux/types.h>
 #include <linux/cpumask.h>
 #include <asm/kmap_types.h>
 #include <asm/desc_defs.h>
 struct page;
 struct thread_struct;
 struct desc_ptr;
 struct tss_struct;
 struct mm_struct;
 struct desc_struct;
 /* general info */
 struct pv_info {
 	unsigned int kernel_rpl;
 	int shared_kernel_pmd;
 	int paravirt_enabled;
 	const char *name;
 };
 struct pv_init_ops {
 	/*
 	 * Patch may replace one of the defined code sequences with
 	 * arbitrary code, subject to the same register constraints.
 	 * This generally means the code is not free to clobber any
 	 * registers other than EAX.  The patch function should return
 	 * the number of bytes of code generated, as we nop pad the
 	 * rest in generic code.
 	 */
 	unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
 			  unsigned long addr, unsigned len);
 	/* Basic arch-specific setup */
 	void (*arch_setup)(void);
 	char *(*memory_setup)(void);
 	void (*post_allocator_init)(void);
 	/* Print a banner to identify the environment */
 	void (*banner)(void);
 };
 struct pv_lazy_ops {
 	/* Set deferred update mode, used for batching operations. */
 	void (*enter)(void);
 	void (*leave)(void);
 };
 struct pv_time_ops {
 	void (*time_init)(void);
 	/* Set and set time of day */
 	unsigned long (*get_wallclock)(void);
 	int (*set_wallclock)(unsigned long);
 	unsigned long long (*sched_clock)(void);
 	unsigned long (*get_cpu_khz)(void);
 };
 struct pv_cpu_ops {
 	/* hooks for various privileged instructions */
 	unsigned long (*get_debugreg)(int regno);
 	void (*set_debugreg)(int regno, unsigned long value);
 	void (*clts)(void);
 	unsigned long (*read_cr0)(void);
 	void (*write_cr0)(unsigned long);
 	unsigned long (*read_cr4_safe)(void);
 	unsigned long (*read_cr4)(void);
 	void (*write_cr4)(unsigned long);
 #ifdef CONFIG_X86_64
 	unsigned long (*read_cr8)(void);
 	void (*write_cr8)(unsigned long);
 #endif
 	/* Segment descriptor handling */
 	void (*load_tr_desc)(void);
 	void (*load_gdt)(const struct desc_ptr *);
 	void (*load_idt)(const struct desc_ptr *);
 	void (*store_gdt)(struct desc_ptr *);
 	void (*store_idt)(struct desc_ptr *);
 	void (*set_ldt)(const void *desc, unsigned entries);
 	unsigned long (*store_tr)(void);
 	void (*load_tls)(struct thread_struct *t, unsigned int cpu);
+#ifdef CONFIG_X86_64
+	void (*load_gs_index)(unsigned int idx);
+#endif
 	void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
 				const void *desc);
 	void (*write_gdt_entry)(struct desc_struct *,
 				int entrynum, const void *desc, int size);
 	void (*write_idt_entry)(gate_desc *,
 				int entrynum, const gate_desc *gate);
 	void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
 	void (*set_iopl_mask)(unsigned mask);
 	void (*wbinvd)(void);
 	void (*io_delay)(void);
 	/* cpuid emulation, mostly so that caps bits can be disabled */
 	void (*cpuid)(unsigned int *eax, unsigned int *ebx,
 		      unsigned int *ecx, unsigned int *edx);
 	/* MSR, PMC and TSR operations.
 	   err = 0/-EFAULT.  wrmsr returns 0/-EFAULT. */
 	u64 (*read_msr)(unsigned int msr, int *err);
 	int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
 	u64 (*read_tsc)(void);
 	u64 (*read_pmc)(int counter);
 	unsigned long long (*read_tscp)(unsigned int *aux);
 	/*
 	 * Atomically enable interrupts and return to userspace.  This
 	 * is only ever used to return to 32-bit processes; in a
 	 * 64-bit kernel, it's used for 32-on-64 compat processes, but
 	 * never native 64-bit processes.  (Jump, not call.)
 	 */
 	void (*irq_enable_sysexit)(void);
 	/*
 	 * Switch to usermode gs and return to 64-bit usermode using
 	 * sysret.  Only used in 64-bit kernels to return to 64-bit
 	 * processes.  Usermode register state, including %rsp, must
 	 * already be restored.
 	 */
 	void (*usergs_sysret64)(void);
 	/*
 	 * Switch to usermode gs and return to 32-bit usermode using
 	 * sysret.  Used to return to 32-on-64 compat processes.
 	 * Other usermode register state, including %esp, must already
 	 * be restored.
 	 */
 	void (*usergs_sysret32)(void);
 	/* Normal iret.  Jump to this with the standard iret stack
 	   frame set up. */
 	void (*iret)(void);
 	void (*swapgs)(void);
 	struct pv_lazy_ops lazy_mode;
 };
 struct pv_irq_ops {
 	void (*init_IRQ)(void);
 	/*
 	 * Get/set interrupt state.  save_fl and restore_fl are only
 	 * expected to use X86_EFLAGS_IF; all other bits
 	 * returned from save_fl are undefined, and may be ignored by
 	 * restore_fl.
 	 */
 	unsigned long (*save_fl)(void);
 	void (*restore_fl)(unsigned long);
 	void (*irq_disable)(void);
 	void (*irq_enable)(void);
 	void (*safe_halt)(void);
 	void (*halt)(void);
 #ifdef CONFIG_X86_64
 	void (*adjust_exception_frame)(void);
 #endif
 };
 struct pv_apic_ops {
 #ifdef CONFIG_X86_LOCAL_APIC
 	/*
 	 * Direct APIC operations, principally for VMI.  Ideally
 	 * these shouldn't be in this interface.
 	 */
 	void (*apic_write)(unsigned long reg, u32 v);
 	void (*apic_write_atomic)(unsigned long reg, u32 v);
 	u32 (*apic_read)(unsigned long reg);
 	void (*setup_boot_clock)(void);
 	void (*setup_secondary_clock)(void);
 	void (*startup_ipi_hook)(int phys_apicid,
 				 unsigned long start_eip,
 				 unsigned long start_esp);
 #endif
 };
 struct pv_mmu_ops {
 	/*
 	 * Called before/after init_mm pagetable setup. setup_start
 	 * may reset %cr3, and may pre-install parts of the pagetable;
 	 * pagetable setup is expected to preserve any existing
 	 * mapping.
 	 */
 	void (*pagetable_setup_start)(pgd_t *pgd_base);
 	void (*pagetable_setup_done)(pgd_t *pgd_base);
 	unsigned long (*read_cr2)(void);
 	void (*write_cr2)(unsigned long);
 	unsigned long (*read_cr3)(void);
 	void (*write_cr3)(unsigned long);
 	/*
 	 * Hooks for intercepting the creation/use/destruction of an
 	 * mm_struct.
 	 */
 	void (*activate_mm)(struct mm_struct *prev,
 			    struct mm_struct *next);
 	void (*dup_mmap)(struct mm_struct *oldmm,
 			 struct mm_struct *mm);
 	void (*exit_mmap)(struct mm_struct *mm);
 	/* TLB operations */
 	void (*flush_tlb_user)(void);
 	void (*flush_tlb_kernel)(void);
 	void (*flush_tlb_single)(unsigned long addr);
 	void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm,
 				 unsigned long va);
 	/* Hooks for allocating and freeing a pagetable top-level */
 	int  (*pgd_alloc)(struct mm_struct *mm);
 	void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
 	/*
 	 * Hooks for allocating/releasing pagetable pages when they're
 	 * attached to a pagetable
 	 */
 	void (*alloc_pte)(struct mm_struct *mm, u32 pfn);
 	void (*alloc_pmd)(struct mm_struct *mm, u32 pfn);
 	void (*alloc_pmd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count);
 	void (*alloc_pud)(struct mm_struct *mm, u32 pfn);
 	void (*release_pte)(u32 pfn);
 	void (*release_pmd)(u32 pfn);
 	void (*release_pud)(u32 pfn);
 	/* Pagetable manipulation functions */
 	void (*set_pte)(pte_t *ptep, pte_t pteval);
 	void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
 			   pte_t *ptep, pte_t pteval);
 	void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
 	void (*pte_update)(struct mm_struct *mm, unsigned long addr,
 			   pte_t *ptep);
 	void (*pte_update_defer)(struct mm_struct *mm,
 				 unsigned long addr, pte_t *ptep);
 	pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
 					pte_t *ptep);
 	void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
 					pte_t *ptep, pte_t pte);
 	pteval_t (*pte_val)(pte_t);
 	pteval_t (*pte_flags)(pte_t);
 	pte_t (*make_pte)(pteval_t pte);
 	pgdval_t (*pgd_val)(pgd_t);
 	pgd_t (*make_pgd)(pgdval_t pgd);
 #if PAGETABLE_LEVELS >= 3
 #ifdef CONFIG_X86_PAE
 	void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
 	void (*set_pte_present)(struct mm_struct *mm, unsigned long addr,
 				pte_t *ptep, pte_t pte);
 	void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
 			  pte_t *ptep);
 	void (*pmd_clear)(pmd_t *pmdp);
 #endif	/* CONFIG_X86_PAE */
 	void (*set_pud)(pud_t *pudp, pud_t pudval);
 	pmdval_t (*pmd_val)(pmd_t);
 	pmd_t (*make_pmd)(pmdval_t pmd);
 #if PAGETABLE_LEVELS == 4
 	pudval_t (*pud_val)(pud_t);
 	pud_t (*make_pud)(pudval_t pud);
 	void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
 #endif	/* PAGETABLE_LEVELS == 4 */
 #endif	/* PAGETABLE_LEVELS >= 3 */
 #ifdef CONFIG_HIGHPTE
 	void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
 #endif
 	struct pv_lazy_ops lazy_mode;
 	/* dom0 ops */
 	/* Sometimes the physical address is a pfn, and sometimes its
 	   an mfn.  We can tell which is which from the index. */
 	void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx,
 			   unsigned long phys, pgprot_t flags);
 };
 /* This contains all the paravirt structures: we get a convenient
  * number for each function using the offset which we use to indicate
  * what to patch. */
 struct paravirt_patch_template {
 	struct pv_init_ops pv_init_ops;
 	struct pv_time_ops pv_time_ops;
 	struct pv_cpu_ops pv_cpu_ops;
 	struct pv_irq_ops pv_irq_ops;
 	struct pv_apic_ops pv_apic_ops;
 	struct pv_mmu_ops pv_mmu_ops;
 };
 extern struct pv_info pv_info;
 extern struct pv_init_ops pv_init_ops;
 extern struct pv_time_ops pv_time_ops;
 extern struct pv_cpu_ops pv_cpu_ops;
 extern struct pv_irq_ops pv_irq_ops;
 extern struct pv_apic_ops pv_apic_ops;
 extern struct pv_mmu_ops pv_mmu_ops;
 #define PARAVIRT_PATCH(x)					\
 	(offsetof(struct paravirt_patch_template, x) / sizeof(void *))
 #define paravirt_type(op)				\
 	[paravirt_typenum] "i" (PARAVIRT_PATCH(op)),	\
 	[paravirt_opptr] "m" (op)
 #define paravirt_clobber(clobber)		\
 	[paravirt_clobber] "i" (clobber)
 /*
  * Generate some code, and mark it as patchable by the
  * apply_paravirt() alternate instruction patcher.
  */
 #define _paravirt_alt(insn_string, type, clobber)	\
 	"771:\n\t" insn_string "\n" "772:\n"		\
 	".pushsection .parainstructions,\"a\"\n"	\
 	_ASM_ALIGN "\n"					\
 	_ASM_PTR " 771b\n"				\
 	"  .byte " type "\n"				\
 	"  .byte 772b-771b\n"				\
 	"  .short " clobber "\n"			\
 	".popsection\n"
 /* Generate patchable code, with the default asm parameters. */
 #define paravirt_alt(insn_string)					\
 	_paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
 /* Simple instruction patching code. */
 #define DEF_NATIVE(ops, name, code) 					\
 	extern const char start_##ops##_##name[], end_##ops##_##name[];	\
 	asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
 unsigned paravirt_patch_nop(void);
 unsigned paravirt_patch_ignore(unsigned len);
 unsigned paravirt_patch_call(void *insnbuf,
 			     const void *target, u16 tgt_clobbers,
 			     unsigned long addr, u16 site_clobbers,
 			     unsigned len);
 unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
 			    unsigned long addr, unsigned len);
 unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
 				unsigned long addr, unsigned len);
 unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
 			      const char *start, const char *end);
 unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
 		      unsigned long addr, unsigned len);
 int paravirt_disable_iospace(void);
 /*
  * This generates an indirect call based on the operation type number.
  * The type number, computed in PARAVIRT_PATCH, is derived from the
  * offset into the paravirt_patch_template structure, and can therefore be
  * freely converted back into a structure offset.
  */
 #define PARAVIRT_CALL	"call *%[paravirt_opptr];"
 /*
  * These macros are intended to wrap calls through one of the paravirt
  * ops structs, so that they can be later identified and patched at
  * runtime.
  *
  * Normally, a call to a pv_op function is a simple indirect call:
  * (pv_op_struct.operations)(args...).
  *
  * Unfortunately, this is a relatively slow operation for modern CPUs,
  * because it cannot necessarily determine what the destination
  * address is.  In this case, the address is a runtime constant, so at
  * the very least we can patch the call to e a simple direct call, or
  * ideally, patch an inline implementation into the callsite.  (Direct
  * calls are essentially free, because the call and return addresses
  * are completely predictable.)
  *
  * For i386, these macros rely on the standard gcc "regparm(3)" calling
  * convention, in which the first three arguments are placed in %eax,
  * %edx, %ecx (in that order), and the remaining arguments are placed
  * on the stack.  All caller-save registers (eax,edx,ecx) are expected
  * to be modified (either clobbered or used for return values).
  * X86_64, on the other hand, already specifies a register-based calling
  * conventions, returning at %rax, with parameteres going on %rdi, %rsi,
  * %rdx, and %rcx. Note that for this reason, x86_64 does not need any
  * special handling for dealing with 4 arguments, unlike i386.
  * However, x86_64 also have to clobber all caller saved registers, which
  * unfortunately, are quite a bit (r8 - r11)
  *
  * The call instruction itself is marked by placing its start address
  * and size into the .parainstructions section, so that
  * apply_paravirt() in arch/i386/kernel/alternative.c can do the
  * appropriate patching under the control of the backend pv_init_ops
  * implementation.
  *
  * Unfortunately there's no way to get gcc to generate the args setup
  * for the call, and then allow the call itself to be generated by an
  * inline asm.  Because of this, we must do the complete arg setup and
  * return value handling from within these macros.  This is fairly
  * cumbersome.
  *
  * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments.
  * It could be extended to more arguments, but there would be little
  * to be gained from that.  For each number of arguments, there are
  * the two VCALL and CALL variants for void and non-void functions.
  *
  * When there is a return value, the invoker of the macro must specify
  * the return type.  The macro then uses sizeof() on that type to
  * determine whether its a 32 or 64 bit value, and places the return
  * in the right register(s) (just %eax for 32-bit, and %edx:%eax for
  * 64-bit). For x86_64 machines, it just returns at %rax regardless of
  * the return value size.
  *
  * 64-bit arguments are passed as a pair of adjacent 32-bit arguments
  * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments
  * in low,high order
  *
  * Small structures are passed and returned in registers.  The macro
  * calling convention can't directly deal with this, so the wrapper
  * functions must do this.
  *
  * These PVOP_* macros are only defined within this header.  This
  * means that all uses must be wrapped in inline functions.  This also
  * makes sure the incoming and outgoing types are always correct.
  */
 #ifdef CONFIG_X86_32
 #define PVOP_VCALL_ARGS			unsigned long __eax, __edx, __ecx
 #define PVOP_CALL_ARGS			PVOP_VCALL_ARGS
 #define PVOP_VCALL_CLOBBERS		"=a" (__eax), "=d" (__edx),	\
 					"=c" (__ecx)
 #define PVOP_CALL_CLOBBERS		PVOP_VCALL_CLOBBERS
 #define EXTRA_CLOBBERS
 #define VEXTRA_CLOBBERS
 #else
 #define PVOP_VCALL_ARGS		unsigned long __edi, __esi, __edx, __ecx
 #define PVOP_CALL_ARGS		PVOP_VCALL_ARGS, __eax
 #define PVOP_VCALL_CLOBBERS	"=D" (__edi),				\
 				"=S" (__esi), "=d" (__edx),		\
 				"=c" (__ecx)
 #define PVOP_CALL_CLOBBERS	PVOP_VCALL_CLOBBERS, "=a" (__eax)
 #define EXTRA_CLOBBERS	 , "r8", "r9", "r10", "r11"
 #define VEXTRA_CLOBBERS	 , "rax", "r8", "r9", "r10", "r11"
 #endif
 #ifdef CONFIG_PARAVIRT_DEBUG
 #define PVOP_TEST_NULL(op)	BUG_ON(op == NULL)
 #else
 #define PVOP_TEST_NULL(op)	((void)op)
 #endif
 #define __PVOP_CALL(rettype, op, pre, post, ...)			\
 	({								\
 		rettype __ret;						\
 		PVOP_CALL_ARGS;					\
 		PVOP_TEST_NULL(op);					\
 		/* This is 32-bit specific, but is okay in 64-bit */	\
 		/* since this condition will never hold */		\
 		if (sizeof(rettype) > sizeof(unsigned long)) {		\
 			asm volatile(pre				\
 				     paravirt_alt(PARAVIRT_CALL)	\
 				     post				\
 				     : PVOP_CALL_CLOBBERS		\
 				     : paravirt_type(op),		\
 				       paravirt_clobber(CLBR_ANY),	\
 				       ##__VA_ARGS__			\
 				     : "memory", "cc" EXTRA_CLOBBERS);	\
 			__ret = (rettype)((((u64)__edx) << 32) | __eax); \
 		} else {						\
 			asm volatile(pre				\
 				     paravirt_alt(PARAVIRT_CALL)	\
 				     post				\
 				     : PVOP_CALL_CLOBBERS		\
 				     : paravirt_type(op),		\
 				       paravirt_clobber(CLBR_ANY),	\
 				       ##__VA_ARGS__			\
 				     : "memory", "cc" EXTRA_CLOBBERS);	\
 			__ret = (rettype)__eax;				\
 		}							\
 		__ret;							\
 	})
 #define __PVOP_VCALL(op, pre, post, ...)				\
 	({								\
 		PVOP_VCALL_ARGS;					\
 		PVOP_TEST_NULL(op);					\
 		asm volatile(pre					\
 			     paravirt_alt(PARAVIRT_CALL)		\
 			     post					\
 			     : PVOP_VCALL_CLOBBERS			\
 			     : paravirt_type(op),			\
 			       paravirt_clobber(CLBR_ANY),		\
 			       ##__VA_ARGS__				\
 			     : "memory", "cc" VEXTRA_CLOBBERS);		\
 	})
 #define PVOP_CALL0(rettype, op)						\
 	__PVOP_CALL(rettype, op, "", "")
 #define PVOP_VCALL0(op)							\
 	__PVOP_VCALL(op, "", "")
 #define PVOP_CALL1(rettype, op, arg1)					\
 	__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)))
 #define PVOP_VCALL1(op, arg1)						\
 	__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)))
 #define PVOP_CALL2(rettype, op, arg1, arg2)				\
 	__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), 	\
 	"1" ((unsigned long)(arg2)))
 #define PVOP_VCALL2(op, arg1, arg2)					\
 	__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), 		\
 	"1" ((unsigned long)(arg2)))
 #define PVOP_CALL3(rettype, op, arg1, arg2, arg3)			\
 	__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)),	\
 	"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
 #define PVOP_VCALL3(op, arg1, arg2, arg3)				\
 	__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)),		\
 	"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
 /* This is the only difference in x86_64. We can make it much simpler */
 #ifdef CONFIG_X86_32
 #define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4)			\
 	__PVOP_CALL(rettype, op,					\
 		    "push %[_arg4];", "lea 4(%%esp),%%esp;",		\
 		    "0" ((u32)(arg1)), "1" ((u32)(arg2)),		\
 		    "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
 #define PVOP_VCALL4(op, arg1, arg2, arg3, arg4)				\
 	__PVOP_VCALL(op,						\
 		    "push %[_arg4];", "lea 4(%%esp),%%esp;",		\
 		    "0" ((u32)(arg1)), "1" ((u32)(arg2)),		\
 		    "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
 #else
 #define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4)			\
 	__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)),	\
 	"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)),		\
 	"3"((unsigned long)(arg4)))
 #define PVOP_VCALL4(op, arg1, arg2, arg3, arg4)				\
 	__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)),		\
 	"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)),		\
 	"3"((unsigned long)(arg4)))
 #endif
 static inline int paravirt_enabled(void)
 {
 	return pv_info.paravirt_enabled;
 }
 static inline void load_sp0(struct tss_struct *tss,
 			     struct thread_struct *thread)
 {
 	PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
 }
 #define ARCH_SETUP			pv_init_ops.arch_setup();
 static inline unsigned long get_wallclock(void)
 {
 	return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
 }
 static inline int set_wallclock(unsigned long nowtime)
 {
 	return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
 }
 static inline void (*choose_time_init(void))(void)
 {
 	return pv_time_ops.time_init;
 }
 /* The paravirtualized CPUID instruction. */
 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
 			   unsigned int *ecx, unsigned int *edx)
 {
 	PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
 }
 /*
  * These special macros can be used to get or set a debugging register
  */
 static inline unsigned long paravirt_get_debugreg(int reg)
 {
 	return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
 }
 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
 static inline void set_debugreg(unsigned long val, int reg)
 {
 	PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
 }
 static inline void clts(void)
 {
 	PVOP_VCALL0(pv_cpu_ops.clts);
 }
 static inline unsigned long read_cr0(void)
 {
 	return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
 }
 static inline void write_cr0(unsigned long x)
 {
 	PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
 }
 static inline unsigned long read_cr2(void)
 {
 	return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
 }
 static inline void write_cr2(unsigned long x)
 {
 	PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
 }
 static inline unsigned long read_cr3(void)
 {
 	return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
 }
 static inline void write_cr3(unsigned long x)
 {
 	PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
 }
 static inline unsigned long read_cr4(void)
 {
 	return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
 }
 static inline unsigned long read_cr4_safe(void)
 {
 	return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
 }
 static inline void write_cr4(unsigned long x)
 {
 	PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
 }
 #ifdef CONFIG_X86_64
 static inline unsigned long read_cr8(void)
 {
 	return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
 }
 static inline void write_cr8(unsigned long x)
 {
 	PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
 }
 #endif
 static inline void raw_safe_halt(void)
 {
 	PVOP_VCALL0(pv_irq_ops.safe_halt);
 }
 static inline void halt(void)
 {
 	PVOP_VCALL0(pv_irq_ops.safe_halt);
 }
 static inline void wbinvd(void)
 {
 	PVOP_VCALL0(pv_cpu_ops.wbinvd);
 }
 #define get_kernel_rpl()  (pv_info.kernel_rpl)
 static inline u64 paravirt_read_msr(unsigned msr, int *err)
 {
 	return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
 }
 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
 {
 	return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
 }
 /* These should all do BUG_ON(_err), but our headers are too tangled. */
 #define rdmsr(msr, val1, val2)			\
 do {						\
 	int _err;				\
 	u64 _l = paravirt_read_msr(msr, &_err);	\
 	val1 = (u32)_l;				\
 	val2 = _l >> 32;			\
 } while (0)
 #define wrmsr(msr, val1, val2)			\
 do {						\
 	paravirt_write_msr(msr, val1, val2);	\
 } while (0)
 #define rdmsrl(msr, val)			\
 do {						\
 	int _err;				\
 	val = paravirt_read_msr(msr, &_err);	\
 } while (0)
 #define wrmsrl(msr, val)	wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
 #define wrmsr_safe(msr, a, b)	paravirt_write_msr(msr, a, b)
 /* rdmsr with exception handling */
 #define rdmsr_safe(msr, a, b)			\
 ({						\
 	int _err;				\
 	u64 _l = paravirt_read_msr(msr, &_err);	\
 	(*a) = (u32)_l;				\
 	(*b) = _l >> 32;			\
 	_err;					\
 })
 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
 {
 	int err;
 	*p = paravirt_read_msr(msr, &err);
 	return err;
 }
 static inline u64 paravirt_read_tsc(void)
 {
 	return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
 }
 #define rdtscl(low)				\
 do {						\
 	u64 _l = paravirt_read_tsc();		\
 	low = (int)_l;				\
 } while (0)
 #define rdtscll(val) (val = paravirt_read_tsc())
 static inline unsigned long long paravirt_sched_clock(void)
 {
 	return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
 }
 #define calculate_cpu_khz() (pv_time_ops.get_cpu_khz())
 static inline unsigned long long paravirt_read_pmc(int counter)
 {
 	return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
 }
 #define rdpmc(counter, low, high)		\
 do {						\
 	u64 _l = paravirt_read_pmc(counter);	\
 	low = (u32)_l;				\
 	high = _l >> 32;			\
 } while (0)
 static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
 {
 	return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
 }
 #define rdtscp(low, high, aux)				\
 do {							\
 	int __aux;					\
 	unsigned long __val = paravirt_rdtscp(&__aux);	\
 	(low) = (u32)__val;				\
 	(high) = (u32)(__val >> 32);			\
 	(aux) = __aux;					\
 } while (0)
 #define rdtscpll(val, aux)				\
 do {							\
 	unsigned long __aux; 				\
 	val = paravirt_rdtscp(&__aux);			\
 	(aux) = __aux;					\
 } while (0)
 static inline void load_TR_desc(void)
 {
 	PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
 }
 static inline void load_gdt(const struct desc_ptr *dtr)
 {
 	PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
 }
 static inline void load_idt(const struct desc_ptr *dtr)
 {
 	PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
 }
 static inline void set_ldt(const void *addr, unsigned entries)
 {
 	PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
 }
 static inline void store_gdt(struct desc_ptr *dtr)
 {
 	PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
 }
 static inline void store_idt(struct desc_ptr *dtr)
 {
 	PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
 }
 static inline unsigned long paravirt_store_tr(void)
 {
 	return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
 }
 #define store_tr(tr)	((tr) = paravirt_store_tr())
 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
 {
 	PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
 }
+#ifdef CONFIG_X86_64
+static inline void load_gs_index(unsigned int gs)
+{
+	PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
+}
+#endif
 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
 				   const void *desc)
 {
 	PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
 }
 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
 				   void *desc, int type)
 {
 	PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
 }
 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
 {
 	PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
 }
 static inline void set_iopl_mask(unsigned mask)
 {
 	PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
 }
 /* The paravirtualized I/O functions */
 static inline void slow_down_io(void)
 {
 	pv_cpu_ops.io_delay();
 #ifdef REALLY_SLOW_IO
 	pv_cpu_ops.io_delay();
 	pv_cpu_ops.io_delay();
 	pv_cpu_ops.io_delay();
 #endif
 }
 #ifdef CONFIG_X86_LOCAL_APIC
 /*
  * Basic functions accessing APICs.
  */
 static inline void apic_write(unsigned long reg, u32 v)
 {
 	PVOP_VCALL2(pv_apic_ops.apic_write, reg, v);
 }
 static inline void apic_write_atomic(unsigned long reg, u32 v)
 {
 	PVOP_VCALL2(pv_apic_ops.apic_write_atomic, reg, v);
 }
 static inline u32 apic_read(unsigned long reg)
 {
 	return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg);
 }
 static inline void setup_boot_clock(void)
 {
 	PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
 }
 static inline void setup_secondary_clock(void)
 {
 	PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
 }
 #endif
 static inline void paravirt_post_allocator_init(void)
 {
 	if (pv_init_ops.post_allocator_init)
 		(*pv_init_ops.post_allocator_init)();
 }
 static inline void paravirt_pagetable_setup_start(pgd_t *base)
 {
 	(*pv_mmu_ops.pagetable_setup_start)(base);
 }
 static inline void paravirt_pagetable_setup_done(pgd_t *base)
 {
 	(*pv_mmu_ops.pagetable_setup_done)(base);
 }
 #ifdef CONFIG_SMP
 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
 				    unsigned long start_esp)
 {
 	PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
 		    phys_apicid, start_eip, start_esp);
 }
 #endif
 static inline void paravirt_activate_mm(struct mm_struct *prev,
 					struct mm_struct *next)
 {
 	PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
 }
 static inline void arch_dup_mmap(struct mm_struct *oldmm,
 				 struct mm_struct *mm)
 {
 	PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
 }
 static inline void arch_exit_mmap(struct mm_struct *mm)
 {
 	PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
 }
 static inline void __flush_tlb(void)
 {
 	PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
 }
 static inline void __flush_tlb_global(void)
 {
 	PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
 }
 static inline void __flush_tlb_single(unsigned long addr)
 {
 	PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
 }
 static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
 				    unsigned long va)
 {
 	PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va);
 }
 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
 {
 	return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
 }
 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
 	PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
 }
 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned pfn)
 {
 	PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
 }
 static inline void paravirt_release_pte(unsigned pfn)
 {
 	PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
 }
 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned pfn)
 {
 	PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
 }
 static inline void paravirt_alloc_pmd_clone(unsigned pfn, unsigned clonepfn,
 					    unsigned start, unsigned count)
 {
 	PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
 }
 static inline void paravirt_release_pmd(unsigned pfn)
 {
 	PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
 }
 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned pfn)
 {
 	PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
 }
 static inline void paravirt_release_pud(unsigned pfn)
 {
 	PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
 }
 #ifdef CONFIG_HIGHPTE
 static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
 {
 	unsigned long ret;
 	ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
 	return (void *)ret;
 }
 #endif
 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
 			      pte_t *ptep)
 {
 	PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
 }
 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
 				    pte_t *ptep)
 {
 	PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
 }
 static inline pte_t __pte(pteval_t val)
 {
 	pteval_t ret;
 	if (sizeof(pteval_t) > sizeof(long))
 		ret = PVOP_CALL2(pteval_t,
 				 pv_mmu_ops.make_pte,
 				 val, (u64)val >> 32);
 	else
 		ret = PVOP_CALL1(pteval_t,
 				 pv_mmu_ops.make_pte,
 				 val);
 	return (pte_t) { .pte = ret };
 }
 static inline pteval_t pte_val(pte_t pte)
 {
 	pteval_t ret;
 	if (sizeof(pteval_t) > sizeof(long))
 		ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_val,
 				 pte.pte, (u64)pte.pte >> 32);
 	else
 		ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_val,
 				 pte.pte);
 	return ret;
 }
 static inline pteval_t pte_flags(pte_t pte)
 {
 	pteval_t ret;
 	if (sizeof(pteval_t) > sizeof(long))
 		ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_flags,
 				 pte.pte, (u64)pte.pte >> 32);
 	else
 		ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_flags,
 				 pte.pte);
 	return ret;
 }
 static inline pgd_t __pgd(pgdval_t val)
 {
 	pgdval_t ret;
 	if (sizeof(pgdval_t) > sizeof(long))
 		ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.make_pgd,
 				 val, (u64)val >> 32);
 	else
 		ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.make_pgd,
 				 val);
 	return (pgd_t) { ret };
 }
 static inline pgdval_t pgd_val(pgd_t pgd)
 {
 	pgdval_t ret;
 	if (sizeof(pgdval_t) > sizeof(long))
 		ret =  PVOP_CALL2(pgdval_t, pv_mmu_ops.pgd_val,
 				  pgd.pgd, (u64)pgd.pgd >> 32);
 	else
 		ret =  PVOP_CALL1(pgdval_t, pv_mmu_ops.pgd_val,
 				  pgd.pgd);
 	return ret;
 }
 #define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
 					   pte_t *ptep)
 {
 	pteval_t ret;
 	ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
 			 mm, addr, ptep);
 	return (pte_t) { .pte = ret };
 }
 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 					   pte_t *ptep, pte_t pte)
 {
 	if (sizeof(pteval_t) > sizeof(long))
 		/* 5 arg words */
 		pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
 	else
 		PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
 			    mm, addr, ptep, pte.pte);
 }
 static inline void set_pte(pte_t *ptep, pte_t pte)
 {
 	if (sizeof(pteval_t) > sizeof(long))
 		PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
 			    pte.pte, (u64)pte.pte >> 32);
 	else
 		PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
 			    pte.pte);
 }
 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 			      pte_t *ptep, pte_t pte)
 {
 	if (sizeof(pteval_t) > sizeof(long))
 		/* 5 arg words */
 		pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
 	else
 		PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
 }
 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 {
 	pmdval_t val = native_pmd_val(pmd);
 	if (sizeof(pmdval_t) > sizeof(long))
 		PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
 	else
 		PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
 }
 #if PAGETABLE_LEVELS >= 3
 static inline pmd_t __pmd(pmdval_t val)
 {
 	pmdval_t ret;
 	if (sizeof(pmdval_t) > sizeof(long))
 		ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.make_pmd,
 				 val, (u64)val >> 32);
 	else
 		ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.make_pmd,
 				 val);
 	return (pmd_t) { ret };
 }
 static inline pmdval_t pmd_val(pmd_t pmd)
 {
 	pmdval_t ret;
 	if (sizeof(pmdval_t) > sizeof(long))
 		ret =  PVOP_CALL2(pmdval_t, pv_mmu_ops.pmd_val,
 				  pmd.pmd, (u64)pmd.pmd >> 32);
 	else
 		ret =  PVOP_CALL1(pmdval_t, pv_mmu_ops.pmd_val,
 				  pmd.pmd);
 	return ret;
 }
 static inline void set_pud(pud_t *pudp, pud_t pud)
 {
 	pudval_t val = native_pud_val(pud);
 	if (sizeof(pudval_t) > sizeof(long))
 		PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
 			    val, (u64)val >> 32);
 	else
 		PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
 			    val);
 }
 #if PAGETABLE_LEVELS == 4
 static inline pud_t __pud(pudval_t val)
 {
 	pudval_t ret;
 	if (sizeof(pudval_t) > sizeof(long))
 		ret = PVOP_CALL2(pudval_t, pv_mmu_ops.make_pud,
 				 val, (u64)val >> 32);
 	else
 		ret = PVOP_CALL1(pudval_t, pv_mmu_ops.make_pud,
 				 val);
 	return (pud_t) { ret };
 }
 static inline pudval_t pud_val(pud_t pud)
 {
 	pudval_t ret;
 	if (sizeof(pudval_t) > sizeof(long))
 		ret =  PVOP_CALL2(pudval_t, pv_mmu_ops.pud_val,
 				  pud.pud, (u64)pud.pud >> 32);
 	else
 		ret =  PVOP_CALL1(pudval_t, pv_mmu_ops.pud_val,
 				  pud.pud);
 	return ret;
 }
 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
 {
 	pgdval_t val = native_pgd_val(pgd);
 	if (sizeof(pgdval_t) > sizeof(long))
 		PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
 			    val, (u64)val >> 32);
 	else
 		PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
 			    val);
 }
 static inline void pgd_clear(pgd_t *pgdp)
 {
 	set_pgd(pgdp, __pgd(0));
 }
 static inline void pud_clear(pud_t *pudp)
 {
 	set_pud(pudp, __pud(0));
 }
 #endif	/* PAGETABLE_LEVELS == 4 */
 #endif	/* PAGETABLE_LEVELS >= 3 */
 #ifdef CONFIG_X86_PAE
 /* Special-case pte-setting operations for PAE, which can't update a
    64-bit pte atomically */
 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 {
 	PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
 		    pte.pte, pte.pte >> 32);
 }
 static inline void set_pte_present(struct mm_struct *mm, unsigned long addr,
 				   pte_t *ptep, pte_t pte)
 {
 	/* 5 arg words */
 	pv_mmu_ops.set_pte_present(mm, addr, ptep, pte);
 }
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 			     pte_t *ptep)
 {
 	PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
 }
 static inline void pmd_clear(pmd_t *pmdp)
 {
 	PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
 }
 #else  /* !CONFIG_X86_PAE */
 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 {
 	set_pte(ptep, pte);
 }
 static inline void set_pte_present(struct mm_struct *mm, unsigned long addr,
 				   pte_t *ptep, pte_t pte)
 {
 	set_pte(ptep, pte);
 }
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 			     pte_t *ptep)
 {
 	set_pte_at(mm, addr, ptep, __pte(0));
 }
 static inline void pmd_clear(pmd_t *pmdp)
 {
 	set_pmd(pmdp, __pmd(0));
 }
 #endif	/* CONFIG_X86_PAE */
 /* Lazy mode for batching updates / context switch */
 enum paravirt_lazy_mode {
 	PARAVIRT_LAZY_NONE,
 	PARAVIRT_LAZY_MMU,
 	PARAVIRT_LAZY_CPU,
 };
 enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
 void paravirt_enter_lazy_cpu(void);
 void paravirt_leave_lazy_cpu(void);
 void paravirt_enter_lazy_mmu(void);
 void paravirt_leave_lazy_mmu(void);
 void paravirt_leave_lazy(enum paravirt_lazy_mode mode);
 #define  __HAVE_ARCH_ENTER_LAZY_CPU_MODE
 static inline void arch_enter_lazy_cpu_mode(void)
 {
 	PVOP_VCALL0(pv_cpu_ops.lazy_mode.enter);
 }
 static inline void arch_leave_lazy_cpu_mode(void)
 {
 	PVOP_VCALL0(pv_cpu_ops.lazy_mode.leave);
 }
 static inline void arch_flush_lazy_cpu_mode(void)
 {
 	if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU)) {
 		arch_leave_lazy_cpu_mode();
 		arch_enter_lazy_cpu_mode();
 	}
 }
 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
 static inline void arch_enter_lazy_mmu_mode(void)
 {
 	PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
 }
 static inline void arch_leave_lazy_mmu_mode(void)
 {
 	PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
 }
 static inline void arch_flush_lazy_mmu_mode(void)
 {
 	if (unlikely(paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU)) {
 		arch_leave_lazy_mmu_mode();
 		arch_enter_lazy_mmu_mode();
 	}
 }
 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
 				unsigned long phys, pgprot_t flags)
 {
 	pv_mmu_ops.set_fixmap(idx, phys, flags);
 }
 void _paravirt_nop(void);
 #define paravirt_nop	((void *)_paravirt_nop)
 /* These all sit in the .parainstructions section to tell us what to patch. */
 struct paravirt_patch_site {
 	u8 *instr; 		/* original instructions */
 	u8 instrtype;		/* type of this instruction */
 	u8 len;			/* length of original instruction */
 	u16 clobbers;		/* what registers you may clobber */
 };
 extern struct paravirt_patch_site __parainstructions[],
 	__parainstructions_end[];
 #ifdef CONFIG_X86_32
 #define PV_SAVE_REGS "pushl %%ecx; pushl %%edx;"
 #define PV_RESTORE_REGS "popl %%edx; popl %%ecx"
 #define PV_FLAGS_ARG "0"
 #define PV_EXTRA_CLOBBERS
 #define PV_VEXTRA_CLOBBERS
 #else
 /* We save some registers, but all of them, that's too much. We clobber all
  * caller saved registers but the argument parameter */
 #define PV_SAVE_REGS "pushq %%rdi;"
 #define PV_RESTORE_REGS "popq %%rdi;"
 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx"
 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx"
 #define PV_FLAGS_ARG "D"
 #endif
 static inline unsigned long __raw_local_save_flags(void)
 {
 	unsigned long f;
 	asm volatile(paravirt_alt(PV_SAVE_REGS
 				  PARAVIRT_CALL
 				  PV_RESTORE_REGS)
 		     : "=a"(f)
 		     : paravirt_type(pv_irq_ops.save_fl),
 		       paravirt_clobber(CLBR_EAX)
 		     : "memory", "cc" PV_VEXTRA_CLOBBERS);
 	return f;
 }
 static inline void raw_local_irq_restore(unsigned long f)
 {
 	asm volatile(paravirt_alt(PV_SAVE_REGS
 				  PARAVIRT_CALL
 				  PV_RESTORE_REGS)
 		     : "=a"(f)
 		     : PV_FLAGS_ARG(f),
 		       paravirt_type(pv_irq_ops.restore_fl),
 		       paravirt_clobber(CLBR_EAX)
 		     : "memory", "cc" PV_EXTRA_CLOBBERS);
 }
 static inline void raw_local_irq_disable(void)
 {
 	asm volatile(paravirt_alt(PV_SAVE_REGS
 				  PARAVIRT_CALL
 				  PV_RESTORE_REGS)
 		     :
 		     : paravirt_type(pv_irq_ops.irq_disable),
 		       paravirt_clobber(CLBR_EAX)
 		     : "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
 }
 static inline void raw_local_irq_enable(void)
 {
 	asm volatile(paravirt_alt(PV_SAVE_REGS
 				  PARAVIRT_CALL
 				  PV_RESTORE_REGS)
 		     :
 		     : paravirt_type(pv_irq_ops.irq_enable),
 		       paravirt_clobber(CLBR_EAX)
 		     : "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
 }
 static inline unsigned long __raw_local_irq_save(void)
 {
 	unsigned long f;
 	f = __raw_local_save_flags();
 	raw_local_irq_disable();
 	return f;
 }
 /* Make sure as little as possible of this mess escapes. */
 #undef PARAVIRT_CALL
 #undef __PVOP_CALL
 #undef __PVOP_VCALL
 #undef PVOP_VCALL0
 #undef PVOP_CALL0
 #undef PVOP_VCALL1
 #undef PVOP_CALL1
 #undef PVOP_VCALL2
 #undef PVOP_CALL2
 #undef PVOP_VCALL3
 #undef PVOP_CALL3
 #undef PVOP_VCALL4
 #undef PVOP_CALL4
 #else  /* __ASSEMBLY__ */
 #define _PVSITE(ptype, clobbers, ops, word, algn)	\
 771:;						\
 	ops;					\
 772:;						\
 	.pushsection .parainstructions,"a";	\
 	 .align	algn;				\
 	 word 771b;				\
 	 .byte ptype;				\
 	 .byte 772b-771b;			\
 	 .short clobbers;			\
 	.popsection
 #ifdef CONFIG_X86_64
 #define PV_SAVE_REGS   pushq %rax; pushq %rdi; pushq %rcx; pushq %rdx
 #define PV_RESTORE_REGS popq %rdx; popq %rcx; popq %rdi; popq %rax
 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
 #define PARA_INDIRECT(addr)	*addr(%rip)
 #else
 #define PV_SAVE_REGS   pushl %eax; pushl %edi; pushl %ecx; pushl %edx
 #define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax
 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
 #define PARA_INDIRECT(addr)	*%cs:addr
 #endif
 #define INTERRUPT_RETURN						\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,	\
 		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
 #define DISABLE_INTERRUPTS(clobbers)					\
 	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
 		  PV_SAVE_REGS;						\
 		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);	\
 		  PV_RESTORE_REGS;)			\
 #define ENABLE_INTERRUPTS(clobbers)					\
 	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,	\
 		  PV_SAVE_REGS;						\
 		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);	\
 		  PV_RESTORE_REGS;)
 #define USERGS_SYSRET32							\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32),	\
 		  CLBR_NONE,						\
 		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
 #ifdef CONFIG_X86_32
 #define GET_CR0_INTO_EAX				\
 	push %ecx; push %edx;				\
 	call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0);	\
 	pop %edx; pop %ecx
 #define ENABLE_INTERRUPTS_SYSEXIT					\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),	\
 		  CLBR_NONE,						\
 		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
 #else	/* !CONFIG_X86_32 */
 /*
  * If swapgs is used while the userspace stack is still current,
  * there's no way to call a pvop.  The PV replacement *must* be
  * inlined, or the swapgs instruction must be trapped and emulated.
  */
 #define SWAPGS_UNSAFE_STACK						\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,	\
 		  swapgs)
 #define SWAPGS								\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,	\
 		  PV_SAVE_REGS;						\
 		  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs);		\
 		  PV_RESTORE_REGS					\
 		 )
 #define GET_CR2_INTO_RCX				\
 	call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2);	\
 	movq %rax, %rcx;				\
 	xorq %rax, %rax;
 #define PARAVIRT_ADJUST_EXCEPTION_FRAME					\
 	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
 		  CLBR_NONE,						\
 		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
 #define USERGS_SYSRET64							\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),	\
 		  CLBR_NONE,						\
 		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
 #define ENABLE_INTERRUPTS_SYSEXIT32					\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),	\
 		  CLBR_NONE,						\
 		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
 #endif	/* CONFIG_X86_32 */
 #endif /* __ASSEMBLY__ */
 #endif /* CONFIG_PARAVIRT */
 #endif	/* __ASM_PARAVIRT_H */

include/asm-x86/system.h

Diff comments View file @ 9f9d489

 #ifndef _ASM_X86_SYSTEM_H_
 #define _ASM_X86_SYSTEM_H_
 #include <asm/asm.h>
 #include <asm/segment.h>
 #include <asm/cpufeature.h>
 #include <asm/cmpxchg.h>
 #include <asm/nops.h>
 #include <linux/kernel.h>
 #include <linux/irqflags.h>
 /* entries in ARCH_DLINFO: */
 #ifdef CONFIG_IA32_EMULATION
 # define AT_VECTOR_SIZE_ARCH 2
 #else
 # define AT_VECTOR_SIZE_ARCH 1
 #endif
 #ifdef CONFIG_X86_32
 struct task_struct; /* one of the stranger aspects of C forward declarations */
 struct task_struct *__switch_to(struct task_struct *prev,
 				struct task_struct *next);
 /*
  * Saving eflags is important. It switches not only IOPL between tasks,
  * it also protects other tasks from NT leaking through sysenter etc.
  */
 #define switch_to(prev, next, last)					\
 do {									\
 	/*								\
 	 * Context-switching clobbers all registers, so we clobber	\
 	 * them explicitly, via unused output variables.		\
 	 * (EAX and EBP is not listed because EBP is saved/restored	\
 	 * explicitly for wchan access and EAX is the return value of	\
 	 * __switch_to())						\
 	 */								\
 	unsigned long ebx, ecx, edx, esi, edi;				\
 									\
 	asm volatile("pushfl\n\t"		/* save    flags */	\
 		     "pushl %%ebp\n\t"		/* save    EBP   */	\
 		     "movl %%esp,%[prev_sp]\n\t"	/* save    ESP   */ \
 		     "movl %[next_sp],%%esp\n\t"	/* restore ESP   */ \
 		     "movl $1f,%[prev_ip]\n\t"	/* save    EIP   */	\
 		     "pushl %[next_ip]\n\t"	/* restore EIP   */	\
 		     "jmp __switch_to\n"	/* regparm call  */	\
 		     "1:\t"						\
 		     "popl %%ebp\n\t"		/* restore EBP   */	\
 		     "popfl\n"			/* restore flags */	\
 									\
 		     /* output parameters */				\
 		     : [prev_sp] "=m" (prev->thread.sp),		\
 		       [prev_ip] "=m" (prev->thread.ip),		\
 		       "=a" (last),					\
 									\
 		       /* clobbered output registers: */		\
 		       "=b" (ebx), "=c" (ecx), "=d" (edx),		\
 		       "=S" (esi), "=D" (edi)				\
 		       							\
 		       /* input parameters: */				\
 		     : [next_sp]  "m" (next->thread.sp),		\
 		       [next_ip]  "m" (next->thread.ip),		\
 		       							\
 		       /* regparm parameters for __switch_to(): */	\
 		       [prev]     "a" (prev),				\
 		       [next]     "d" (next));				\
 } while (0)
 /*
  * disable hlt during certain critical i/o operations
  */
 #define HAVE_DISABLE_HLT
 #else
 #define __SAVE(reg, offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t"
 #define __RESTORE(reg, offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t"
 /* frame pointer must be last for get_wchan */
 #define SAVE_CONTEXT    "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
 #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t"
 #define __EXTRA_CLOBBER  \
 	, "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
 	  "r12", "r13", "r14", "r15"
 /* Save restore flags to clear handle leaking NT */
 #define switch_to(prev, next, last) \
 	asm volatile(SAVE_CONTEXT						    \
 	     "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */	  \
 	     "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */	  \
 	     "call __switch_to\n\t"					  \
 	     ".globl thread_return\n"					  \
 	     "thread_return:\n\t"					  \
 	     "movq %%gs:%P[pda_pcurrent],%%rsi\n\t"			  \
 	     "movq %P[thread_info](%%rsi),%%r8\n\t"			  \
 	     LOCK_PREFIX "btr  %[tif_fork],%P[ti_flags](%%r8)\n\t"	  \
 	     "movq %%rax,%%rdi\n\t" 					  \
 	     "jc   ret_from_fork\n\t"					  \
 	     RESTORE_CONTEXT						  \
 	     : "=a" (last)					  	  \
 	     : [next] "S" (next), [prev] "D" (prev),			  \
 	       [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
 	       [ti_flags] "i" (offsetof(struct thread_info, flags)),	  \
 	       [tif_fork] "i" (TIF_FORK),			  	  \
 	       [thread_info] "i" (offsetof(struct task_struct, stack)),   \
 	       [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent))  \
 	     : "memory", "cc" __EXTRA_CLOBBER)
 #endif
 #ifdef __KERNEL__
 #define _set_base(addr, base) do { unsigned long __pr; \
 __asm__ __volatile__ ("movw %%dx,%1\n\t" \
 	"rorl $16,%%edx\n\t" \
 	"movb %%dl,%2\n\t" \
 	"movb %%dh,%3" \
 	:"=&d" (__pr) \
 	:"m" (*((addr)+2)), \
 	 "m" (*((addr)+4)), \
 	 "m" (*((addr)+7)), \
 	 "0" (base) \
 	); } while (0)
 #define _set_limit(addr, limit) do { unsigned long __lr; \
 __asm__ __volatile__ ("movw %%dx,%1\n\t" \
 	"rorl $16,%%edx\n\t" \
 	"movb %2,%%dh\n\t" \
 	"andb $0xf0,%%dh\n\t" \
 	"orb %%dh,%%dl\n\t" \
 	"movb %%dl,%2" \
 	:"=&d" (__lr) \
 	:"m" (*(addr)), \
 	 "m" (*((addr)+6)), \
 	 "0" (limit) \
 	); } while (0)
 #define set_base(ldt, base) _set_base(((char *)&(ldt)) , (base))
 #define set_limit(ldt, limit) _set_limit(((char *)&(ldt)) , ((limit)-1))
-extern void load_gs_index(unsigned);
+extern void native_load_gs_index(unsigned);
 /*
  * Load a segment. Fall back on loading the zero
  * segment if something goes wrong..
  */
 #define loadsegment(seg, value)			\
 	asm volatile("\n"			\
 		     "1:\t"			\
 		     "movl %k0,%%" #seg "\n"	\
 		     "2:\n"			\
 		     ".section .fixup,\"ax\"\n"	\
 		     "3:\t"			\
 		     "movl %k1, %%" #seg "\n\t"	\
 		     "jmp 2b\n"			\
 		     ".previous\n"		\
 		     _ASM_EXTABLE(1b,3b)	\
 		     : :"r" (value), "r" (0) : "memory")
 /*
  * Save a segment register away
  */
 #define savesegment(seg, value)				\
 	asm("mov %%" #seg ",%0":"=rm" (value) : : "memory")
 static inline unsigned long get_limit(unsigned long segment)
 {
 	unsigned long __limit;
 	asm("lsll %1,%0" : "=r" (__limit) : "r" (segment));
 	return __limit + 1;
 }
 static inline void native_clts(void)
 {
 	asm volatile("clts");
 }
 /*
  * Volatile isn't enough to prevent the compiler from reordering the
  * read/write functions for the control registers and messing everything up.
  * A memory clobber would solve the problem, but would prevent reordering of
  * all loads stores around it, which can hurt performance. Solution is to
  * use a variable and mimic reads and writes to it to enforce serialization
  */
 static unsigned long __force_order;
 static inline unsigned long native_read_cr0(void)
 {
 	unsigned long val;
 	asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
 	return val;
 }
 static inline void native_write_cr0(unsigned long val)
 {
 	asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
 }
 static inline unsigned long native_read_cr2(void)
 {
 	unsigned long val;
 	asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
 	return val;
 }
 static inline void native_write_cr2(unsigned long val)
 {
 	asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
 }
 static inline unsigned long native_read_cr3(void)
 {
 	unsigned long val;
 	asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
 	return val;
 }
 static inline void native_write_cr3(unsigned long val)
 {
 	asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
 }
 static inline unsigned long native_read_cr4(void)
 {
 	unsigned long val;
 	asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
 	return val;
 }
 static inline unsigned long native_read_cr4_safe(void)
 {
 	unsigned long val;
 	/* This could fault if %cr4 does not exist. In x86_64, a cr4 always
 	 * exists, so it will never fail. */
 #ifdef CONFIG_X86_32
 	asm volatile("1: mov %%cr4, %0\n"
 		     "2:\n"
 		     _ASM_EXTABLE(1b, 2b)
 		     : "=r" (val), "=m" (__force_order) : "0" (0));
 #else
 	val = native_read_cr4();
 #endif
 	return val;
 }
 static inline void native_write_cr4(unsigned long val)
 {
 	asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
 }
 #ifdef CONFIG_X86_64
 static inline unsigned long native_read_cr8(void)
 {
 	unsigned long cr8;
 	asm volatile("movq %%cr8,%0" : "=r" (cr8));
 	return cr8;
 }
 static inline void native_write_cr8(unsigned long val)
 {
 	asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
 }
 #endif
 static inline void native_wbinvd(void)
 {
 	asm volatile("wbinvd": : :"memory");
 }
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
 #else
 #define read_cr0()	(native_read_cr0())
 #define write_cr0(x)	(native_write_cr0(x))
 #define read_cr2()	(native_read_cr2())
 #define write_cr2(x)	(native_write_cr2(x))
 #define read_cr3()	(native_read_cr3())
 #define write_cr3(x)	(native_write_cr3(x))
 #define read_cr4()	(native_read_cr4())
 #define read_cr4_safe()	(native_read_cr4_safe())
 #define write_cr4(x)	(native_write_cr4(x))
 #define wbinvd()	(native_wbinvd())
 #ifdef CONFIG_X86_64
 #define read_cr8()	(native_read_cr8())
 #define write_cr8(x)	(native_write_cr8(x))
+#define load_gs_index   native_load_gs_index
 #endif
 /* Clear the 'TS' bit */
 #define clts()		(native_clts())
 #endif/* CONFIG_PARAVIRT */
 #define stts() write_cr0(read_cr0() | X86_CR0_TS)
 #endif /* __KERNEL__ */
 static inline void clflush(volatile void *__p)
 {
 	asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
 }
 #define nop() asm volatile ("nop")
 void disable_hlt(void);
 void enable_hlt(void);
 void cpu_idle_wait(void);
 extern unsigned long arch_align_stack(unsigned long sp);
 extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
 void default_idle(void);
 /*
  * Force strict CPU ordering.
  * And yes, this is required on UP too when we're talking
  * to devices.
  */
 #ifdef CONFIG_X86_32
 /*
  * Some non-Intel clones support out of order store. wmb() ceases to be a
  * nop for these.
  */
 #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
 #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
 #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
 #else
 #define mb() 	asm volatile("mfence":::"memory")
 #define rmb()	asm volatile("lfence":::"memory")
 #define wmb()	asm volatile("sfence" ::: "memory")
 #endif
 /**
  * read_barrier_depends - Flush all pending reads that subsequents reads
  * depend on.
  *
  * No data-dependent reads from memory-like regions are ever reordered
  * over this barrier.  All reads preceding this primitive are guaranteed
  * to access memory (but not necessarily other CPUs' caches) before any
  * reads following this primitive that depend on the data return by
  * any of the preceding reads.  This primitive is much lighter weight than
  * rmb() on most CPUs, and is never heavier weight than is
  * rmb().
  *
  * These ordering constraints are respected by both the local CPU
  * and the compiler.
  *
  * Ordering is not guaranteed by anything other than these primitives,
  * not even by data dependencies.  See the documentation for
  * memory_barrier() for examples and URLs to more information.
  *
  * For example, the following code would force ordering (the initial
  * value of "a" is zero, "b" is one, and "p" is "&a"):
  *
  * <programlisting>
  *	CPU 0				CPU 1
  *
  *	b = 2;
  *	memory_barrier();
  *	p = &b;				q = p;
  *					read_barrier_depends();
  *					d = *q;
  * </programlisting>
  *
  * because the read of "*q" depends on the read of "p" and these
  * two reads are separated by a read_barrier_depends().  However,
  * the following code, with the same initial values for "a" and "b":
  *
  * <programlisting>
  *	CPU 0				CPU 1
  *
  *	a = 2;
  *	memory_barrier();
  *	b = 3;				y = b;
  *					read_barrier_depends();
  *					x = a;
  * </programlisting>
  *
  * does not enforce ordering, since there is no data dependency between
  * the read of "a" and the read of "b".  Therefore, on some CPUs, such
  * as Alpha, "y" could be set to 3 and "x" to 0.  Use rmb()
  * in cases like this where there are no data dependencies.
  **/
 #define read_barrier_depends()	do { } while (0)
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()
 #ifdef CONFIG_X86_PPRO_FENCE
 # define smp_rmb()	rmb()
 #else
 # define smp_rmb()	barrier()
 #endif
 #ifdef CONFIG_X86_OOSTORE
 # define smp_wmb() 	wmb()
 #else
 # define smp_wmb()	barrier()
 #endif
 #define smp_read_barrier_depends()	read_barrier_depends()
 #define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
 #define smp_read_barrier_depends()	do { } while (0)
 #define set_mb(var, value) do { var = value; barrier(); } while (0)
 #endif
 /*
  * Stop RDTSC speculation. This is needed when you need to use RDTSC
  * (or get_cycles or vread that possibly accesses the TSC) in a defined
  * code region.
  *
  * (Could use an alternative three way for this if there was one.)
  */
 static inline void rdtsc_barrier(void)
 {
 	alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
 	alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
 }
 #endif