Blame view
arch/x86/kernel/process_64.c
16.1 KB
1da177e4c Linux-2.6.12-rc2 |
1 |
/* |
1da177e4c Linux-2.6.12-rc2 |
2 3 4 5 |
* Copyright (C) 1995 Linus Torvalds * * Pentium III FXSR, SSE support * Gareth Hughes <gareth@valinux.com>, May 2000 |
6612538ca x86: clean up pro... |
6 |
* |
1da177e4c Linux-2.6.12-rc2 |
7 8 |
* X86-64 port * Andi Kleen. |
76e4f660d [PATCH] x86_64: C... |
9 10 |
* * CPU hotplug support - ashok.raj@intel.com |
1da177e4c Linux-2.6.12-rc2 |
11 12 13 14 15 |
*/ /* * This file handles the architecture-dependent parts of process handling.. */ |
420594296 x86: fix the stac... |
16 |
#include <linux/stackprotector.h> |
76e4f660d [PATCH] x86_64: C... |
17 |
#include <linux/cpu.h> |
1da177e4c Linux-2.6.12-rc2 |
18 19 |
#include <linux/errno.h> #include <linux/sched.h> |
6612538ca x86: clean up pro... |
20 |
#include <linux/fs.h> |
1da177e4c Linux-2.6.12-rc2 |
21 22 23 24 25 26 |
#include <linux/kernel.h> #include <linux/mm.h> #include <linux/elfcore.h> #include <linux/smp.h> #include <linux/slab.h> #include <linux/user.h> |
1da177e4c Linux-2.6.12-rc2 |
27 28 |
#include <linux/interrupt.h> #include <linux/delay.h> |
6612538ca x86: clean up pro... |
29 |
#include <linux/module.h> |
1da177e4c Linux-2.6.12-rc2 |
30 |
#include <linux/ptrace.h> |
95833c83f [PATCH] x86_64: A... |
31 |
#include <linux/notifier.h> |
c6fd91f0b [PATCH] kretprobe... |
32 |
#include <linux/kprobes.h> |
1eeb66a1b move die notifier... |
33 |
#include <linux/kdebug.h> |
022906833 x86_64: prepare i... |
34 |
#include <linux/tick.h> |
529e25f64 x86: implement pr... |
35 |
#include <linux/prctl.h> |
7de08b4e1 x86: coding style... |
36 37 |
#include <linux/uaccess.h> #include <linux/io.h> |
8b96f0119 tracing/function-... |
38 |
#include <linux/ftrace.h> |
a0bfa1373 cpuidle: stop dep... |
39 |
#include <linux/cpuidle.h> |
1da177e4c Linux-2.6.12-rc2 |
40 |
|
1da177e4c Linux-2.6.12-rc2 |
41 42 |
#include <asm/pgtable.h> #include <asm/system.h> |
1da177e4c Linux-2.6.12-rc2 |
43 44 45 |
#include <asm/processor.h> #include <asm/i387.h> #include <asm/mmu_context.h> |
1da177e4c Linux-2.6.12-rc2 |
46 |
#include <asm/prctl.h> |
1da177e4c Linux-2.6.12-rc2 |
47 48 49 |
#include <asm/desc.h> #include <asm/proto.h> #include <asm/ia32.h> |
95833c83f [PATCH] x86_64: A... |
50 |
#include <asm/idle.h> |
bbc1f698a x86: Introducing ... |
51 |
#include <asm/syscalls.h> |
66cb59172 hw-breakpoints: u... |
52 |
#include <asm/debugreg.h> |
b227e2339 x86, nmi: Add in ... |
53 |
#include <asm/nmi.h> |
1da177e4c Linux-2.6.12-rc2 |
54 55 |
asmlinkage extern void ret_from_fork(void); |
3d1e42a7c x86-64: Move oldr... |
56 |
DEFINE_PER_CPU(unsigned long, old_rsp); |
c2558e0eb x86-64: Move isid... |
57 |
static DEFINE_PER_CPU(unsigned char, is_idle); |
3d1e42a7c x86-64: Move oldr... |
58 |
|
e041c6834 [PATCH] Notifier ... |
59 |
static ATOMIC_NOTIFIER_HEAD(idle_notifier); |
95833c83f [PATCH] x86_64: A... |
60 61 62 |
void idle_notifier_register(struct notifier_block *n) { |
e041c6834 [PATCH] Notifier ... |
63 |
atomic_notifier_chain_register(&idle_notifier, n); |
95833c83f [PATCH] x86_64: A... |
64 |
} |
c7d87d79d x86 allow modules... |
65 66 67 68 69 70 71 |
EXPORT_SYMBOL_GPL(idle_notifier_register); void idle_notifier_unregister(struct notifier_block *n) { atomic_notifier_chain_unregister(&idle_notifier, n); } EXPORT_SYMBOL_GPL(idle_notifier_unregister); |
95833c83f [PATCH] x86_64: A... |
72 |
|
95833c83f [PATCH] x86_64: A... |
73 74 |
void enter_idle(void) { |
c2558e0eb x86-64: Move isid... |
75 |
percpu_write(is_idle, 1); |
e041c6834 [PATCH] Notifier ... |
76 |
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); |
95833c83f [PATCH] x86_64: A... |
77 78 79 80 |
} static void __exit_idle(void) { |
c2558e0eb x86-64: Move isid... |
81 |
if (x86_test_and_clear_bit_percpu(0, is_idle) == 0) |
a15da49de [PATCH] Fix idle ... |
82 |
return; |
e041c6834 [PATCH] Notifier ... |
83 |
atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); |
95833c83f [PATCH] x86_64: A... |
84 85 86 87 88 |
} /* Called from interrupts to signify idle end */ void exit_idle(void) { |
a15da49de [PATCH] Fix idle ... |
89 90 |
/* idle loop has pid 0 */ if (current->pid) |
95833c83f [PATCH] x86_64: A... |
91 92 93 |
return; __exit_idle(); } |
913da64b5 x86: build fix fo... |
94 |
#ifndef CONFIG_SMP |
76e4f660d [PATCH] x86_64: C... |
95 96 97 98 |
static inline void play_dead(void) { BUG(); } |
913da64b5 x86: build fix fo... |
99 |
#endif |
76e4f660d [PATCH] x86_64: C... |
100 |
|
1da177e4c Linux-2.6.12-rc2 |
101 102 103 104 105 106 |
/* * The idle thread. There's no useful work to be * done, so just try to conserve power and have a * low exit latency (ie sit in a loop waiting for * somebody to say that they'd like to reschedule) */ |
b10db7f0d time: more timer ... |
107 |
void cpu_idle(void) |
1da177e4c Linux-2.6.12-rc2 |
108 |
{ |
495ab9c04 [PATCH] i386/x86-... |
109 |
current_thread_info()->status |= TS_POLLING; |
ce22bd92c x86: setup stack ... |
110 |
|
ce22bd92c x86: setup stack ... |
111 |
/* |
5c79d2a51 x86: fix x86_32 s... |
112 113 114 115 116 |
* If we're the non-boot CPU, nothing set the stack canary up * for us. CPU0 already has it initialized but no harm in * doing it again. This is a good place for updating it, as * we wont ever return from this function (so the invalid * canaries already on the stack wont ever trigger). |
ce22bd92c x86: setup stack ... |
117 |
*/ |
18aa8bb12 stackprotector: a... |
118 |
boot_init_stack_canary(); |
1da177e4c Linux-2.6.12-rc2 |
119 120 |
/* endless idle loop with no priority at all */ while (1) { |
e37e112de x86: Enter rcu ex... |
121 |
tick_nohz_idle_enter(); |
1da177e4c Linux-2.6.12-rc2 |
122 |
while (!need_resched()) { |
1da177e4c Linux-2.6.12-rc2 |
123 |
|
1da177e4c Linux-2.6.12-rc2 |
124 |
rmb(); |
6ddd2a279 x86: simplify idl... |
125 |
|
76e4f660d [PATCH] x86_64: C... |
126 127 |
if (cpu_is_offline(smp_processor_id())) play_dead(); |
d331e739f [PATCH] x86-64: F... |
128 129 130 131 132 |
/* * Idle routines should keep interrupts disabled * from here on, until they go to idle. * Otherwise, idle callbacks can misfire. */ |
b227e2339 x86, nmi: Add in ... |
133 |
local_touch_nmi(); |
d331e739f [PATCH] x86-64: F... |
134 |
local_irq_disable(); |
95833c83f [PATCH] x86_64: A... |
135 |
enter_idle(); |
81d68a96a ftrace: trace irq... |
136 137 |
/* Don't trace irqs off for idle */ stop_critical_timings(); |
e37e112de x86: Enter rcu ex... |
138 139 140 |
/* enter_idle() needs rcu for notifiers */ rcu_idle_enter(); |
a0bfa1373 cpuidle: stop dep... |
141 142 |
if (cpuidle_idle_call()) pm_idle(); |
e37e112de x86: Enter rcu ex... |
143 144 |
rcu_idle_exit(); |
81d68a96a ftrace: trace irq... |
145 |
start_critical_timings(); |
c882e0feb x86, perf: Add po... |
146 |
|
a15da49de [PATCH] Fix idle ... |
147 148 149 |
/* In many cases the interrupt that ended idle has already called exit_idle. But some idle loops can be woken up without interrupt. */ |
95833c83f [PATCH] x86_64: A... |
150 |
__exit_idle(); |
1da177e4c Linux-2.6.12-rc2 |
151 |
} |
e37e112de x86: Enter rcu ex... |
152 |
tick_nohz_idle_exit(); |
5bfb5d690 [PATCH] sched: di... |
153 |
preempt_enable_no_resched(); |
1da177e4c Linux-2.6.12-rc2 |
154 |
schedule(); |
5bfb5d690 [PATCH] sched: di... |
155 |
preempt_disable(); |
1da177e4c Linux-2.6.12-rc2 |
156 157 |
} } |
6612538ca x86: clean up pro... |
158 |
/* Prints also some state that isn't saved in the pt_regs */ |
e2ce07c80 x86: __show_regis... |
159 |
void __show_regs(struct pt_regs *regs, int all) |
1da177e4c Linux-2.6.12-rc2 |
160 161 |
{ unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs; |
bb1995d52 x86: Make Alt-Sys... |
162 |
unsigned long d0, d1, d2, d3, d6, d7; |
6612538ca x86: clean up pro... |
163 164 |
unsigned int fsindex, gsindex; unsigned int ds, cs, es; |
814e2c84a x86: Factor dupli... |
165 166 |
show_regs_common(); |
d015a0929 x86: Use KERN_DEF... |
167 |
printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip); |
aafbd7eb2 x86: make printk_... |
168 |
printk_address(regs->ip, 1); |
d015a0929 x86: Use KERN_DEF... |
169 170 |
printk(KERN_DEFAULT "RSP: %04lx:%016lx EFLAGS: %08lx ", regs->ss, |
8092c654d x86: add KERN_INF... |
171 |
regs->sp, regs->flags); |
d015a0929 x86: Use KERN_DEF... |
172 173 |
printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx ", |
65ea5b034 x86: rename the s... |
174 |
regs->ax, regs->bx, regs->cx); |
d015a0929 x86: Use KERN_DEF... |
175 176 |
printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx ", |
65ea5b034 x86: rename the s... |
177 |
regs->dx, regs->si, regs->di); |
d015a0929 x86: Use KERN_DEF... |
178 179 |
printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx ", |
65ea5b034 x86: rename the s... |
180 |
regs->bp, regs->r8, regs->r9); |
d015a0929 x86: Use KERN_DEF... |
181 182 |
printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx ", |
7de08b4e1 x86: coding style... |
183 |
regs->r10, regs->r11, regs->r12); |
d015a0929 x86: Use KERN_DEF... |
184 185 |
printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx ", |
7de08b4e1 x86: coding style... |
186 |
regs->r13, regs->r14, regs->r15); |
1da177e4c Linux-2.6.12-rc2 |
187 |
|
7de08b4e1 x86: coding style... |
188 189 190 |
asm("movl %%ds,%0" : "=r" (ds)); asm("movl %%cs,%0" : "=r" (cs)); asm("movl %%es,%0" : "=r" (es)); |
1da177e4c Linux-2.6.12-rc2 |
191 192 193 194 |
asm("movl %%fs,%0" : "=r" (fsindex)); asm("movl %%gs,%0" : "=r" (gsindex)); rdmsrl(MSR_FS_BASE, fs); |
7de08b4e1 x86: coding style... |
195 196 |
rdmsrl(MSR_GS_BASE, gs); rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); |
1da177e4c Linux-2.6.12-rc2 |
197 |
|
e2ce07c80 x86: __show_regis... |
198 199 |
if (!all) return; |
1da177e4c Linux-2.6.12-rc2 |
200 |
|
f51c94528 x86_64: Use read ... |
201 202 203 204 |
cr0 = read_cr0(); cr2 = read_cr2(); cr3 = read_cr3(); cr4 = read_cr4(); |
1da177e4c Linux-2.6.12-rc2 |
205 |
|
d015a0929 x86: Use KERN_DEF... |
206 207 |
printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx ", |
7de08b4e1 x86: coding style... |
208 |
fs, fsindex, gs, gsindex, shadowgs); |
d015a0929 x86: Use KERN_DEF... |
209 210 |
printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx ", cs, ds, |
8092c654d x86: add KERN_INF... |
211 |
es, cr0); |
d015a0929 x86: Use KERN_DEF... |
212 213 |
printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx ", cr2, cr3, |
8092c654d x86: add KERN_INF... |
214 |
cr4); |
bb1995d52 x86: Make Alt-Sys... |
215 216 217 218 |
get_debugreg(d0, 0); get_debugreg(d1, 1); get_debugreg(d2, 2); |
d015a0929 x86: Use KERN_DEF... |
219 220 |
printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx ", d0, d1, d2); |
bb1995d52 x86: Make Alt-Sys... |
221 222 223 |
get_debugreg(d3, 3); get_debugreg(d6, 6); get_debugreg(d7, 7); |
d015a0929 x86: Use KERN_DEF... |
224 225 |
printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx ", d3, d6, d7); |
1da177e4c Linux-2.6.12-rc2 |
226 |
} |
1da177e4c Linux-2.6.12-rc2 |
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 |
void release_thread(struct task_struct *dead_task) { if (dead_task->mm) { if (dead_task->mm->context.size) { printk("WARNING: dead process %8s still has LDT? <%p/%d> ", dead_task->comm, dead_task->mm->context.ldt, dead_task->mm->context.size); BUG(); } } } static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr) { |
6612538ca x86: clean up pro... |
243 |
struct user_desc ud = { |
1da177e4c Linux-2.6.12-rc2 |
244 245 246 247 248 249 |
.base_addr = addr, .limit = 0xfffff, .seg_32bit = 1, .limit_in_pages = 1, .useable = 1, }; |
ade1af771 x86: remove unned... |
250 |
struct desc_struct *desc = t->thread.tls_array; |
1da177e4c Linux-2.6.12-rc2 |
251 |
desc += tls; |
80fbb69a8 x86: introduce fi... |
252 |
fill_ldt(desc, &ud); |
1da177e4c Linux-2.6.12-rc2 |
253 254 255 256 |
} static inline u32 read_32bit_tls(struct task_struct *t, int tls) { |
91394eb09 x86: use get_desc... |
257 |
return get_desc_base(&t->thread.tls_array[tls]); |
1da177e4c Linux-2.6.12-rc2 |
258 259 260 261 262 263 264 265 266 267 |
} /* * This gets called before we allocate a new thread and copy * the current task into it. */ void prepare_to_copy(struct task_struct *tsk) { unlazy_fpu(tsk); } |
6f2c55b84 Simplify copy_thr... |
268 |
int copy_thread(unsigned long clone_flags, unsigned long sp, |
1da177e4c Linux-2.6.12-rc2 |
269 |
unsigned long unused, |
7de08b4e1 x86: coding style... |
270 |
struct task_struct *p, struct pt_regs *regs) |
1da177e4c Linux-2.6.12-rc2 |
271 272 |
{ int err; |
7de08b4e1 x86: coding style... |
273 |
struct pt_regs *childregs; |
1da177e4c Linux-2.6.12-rc2 |
274 |
struct task_struct *me = current; |
a88cde13b [PATCH] x86_64: F... |
275 |
childregs = ((struct pt_regs *) |
57eafdc22 [PATCH] amd64: ta... |
276 |
(THREAD_SIZE + task_stack_page(p))) - 1; |
1da177e4c Linux-2.6.12-rc2 |
277 |
*childregs = *regs; |
65ea5b034 x86: rename the s... |
278 |
childregs->ax = 0; |
fa4b8f843 x86, 64-bit: Use ... |
279 280 281 |
if (user_mode(regs)) childregs->sp = sp; else |
65ea5b034 x86: rename the s... |
282 |
childregs->sp = (unsigned long)childregs; |
1da177e4c Linux-2.6.12-rc2 |
283 |
|
faca62273 x86: use generic ... |
284 285 286 |
p->thread.sp = (unsigned long) childregs; p->thread.sp0 = (unsigned long) (childregs+1); p->thread.usersp = me->thread.usersp; |
1da177e4c Linux-2.6.12-rc2 |
287 |
|
e4f17c436 [PATCH] amd64: ta... |
288 |
set_tsk_thread_flag(p, TIF_FORK); |
1da177e4c Linux-2.6.12-rc2 |
289 |
|
66cb59172 hw-breakpoints: u... |
290 |
p->thread.io_bitmap_ptr = NULL; |
1da177e4c Linux-2.6.12-rc2 |
291 |
|
ada857082 x86: remove open-... |
292 |
savesegment(gs, p->thread.gsindex); |
7ce5a2b9b x86-64: Clear a 6... |
293 |
p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs; |
ada857082 x86: remove open-... |
294 |
savesegment(fs, p->thread.fsindex); |
7ce5a2b9b x86-64: Clear a 6... |
295 |
p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs; |
ada857082 x86: remove open-... |
296 297 |
savesegment(es, p->thread.es); savesegment(ds, p->thread.ds); |
1da177e4c Linux-2.6.12-rc2 |
298 |
|
66cb59172 hw-breakpoints: u... |
299 |
err = -ENOMEM; |
24f1e32c6 hw-breakpoints: R... |
300 |
memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps)); |
66cb59172 hw-breakpoints: u... |
301 |
|
d3a4f48d4 [PATCH] x86-64 TI... |
302 |
if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) { |
cced40229 x86: Use kmemdup(... |
303 304 |
p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr, IO_BITMAP_BYTES, GFP_KERNEL); |
1da177e4c Linux-2.6.12-rc2 |
305 306 307 308 |
if (!p->thread.io_bitmap_ptr) { p->thread.io_bitmap_max = 0; return -ENOMEM; } |
d3a4f48d4 [PATCH] x86-64 TI... |
309 |
set_tsk_thread_flag(p, TIF_IO_BITMAP); |
6612538ca x86: clean up pro... |
310 |
} |
1da177e4c Linux-2.6.12-rc2 |
311 312 313 314 315 316 317 |
/* * Set a new TLS for the child thread? */ if (clone_flags & CLONE_SETTLS) { #ifdef CONFIG_IA32_EMULATION if (test_thread_flag(TIF_IA32)) |
efd1ca52d x86: TLS cleanup |
318 |
err = do_set_thread_area(p, -1, |
65ea5b034 x86: rename the s... |
319 |
(struct user_desc __user *)childregs->si, 0); |
7de08b4e1 x86: coding style... |
320 321 322 323 |
else #endif err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8); if (err) |
1da177e4c Linux-2.6.12-rc2 |
324 325 326 327 328 329 330 331 |
goto out; } err = 0; out: if (err && p->thread.io_bitmap_ptr) { kfree(p->thread.io_bitmap_ptr); p->thread.io_bitmap_max = 0; } |
66cb59172 hw-breakpoints: u... |
332 |
|
1da177e4c Linux-2.6.12-rc2 |
333 334 |
return err; } |
e634d8fc7 x86-64: merge the... |
335 336 337 338 |
static void start_thread_common(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp, unsigned int _cs, unsigned int _ss, unsigned int _ds) |
513ad84bf x86: de-macro sta... |
339 |
{ |
ada857082 x86: remove open-... |
340 |
loadsegment(fs, 0); |
e634d8fc7 x86-64: merge the... |
341 342 |
loadsegment(es, _ds); loadsegment(ds, _ds); |
513ad84bf x86: de-macro sta... |
343 344 345 |
load_gs_index(0); regs->ip = new_ip; regs->sp = new_sp; |
3d1e42a7c x86-64: Move oldr... |
346 |
percpu_write(old_rsp, new_sp); |
e634d8fc7 x86-64: merge the... |
347 348 |
regs->cs = _cs; regs->ss = _ss; |
a6f05a6a0 x86-64: make comp... |
349 |
regs->flags = X86_EFLAGS_IF; |
aa283f492 x86, fpu: lazy al... |
350 351 352 353 |
/* * Free the old FP and other extended state */ free_thread_xstate(current); |
513ad84bf x86: de-macro sta... |
354 |
} |
e634d8fc7 x86-64: merge the... |
355 356 357 358 359 360 361 |
void start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) { start_thread_common(regs, new_ip, new_sp, __USER_CS, __USER_DS, 0); } |
513ad84bf x86: de-macro sta... |
362 |
|
a6f05a6a0 x86-64: make comp... |
363 364 365 |
#ifdef CONFIG_IA32_EMULATION void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp) { |
e634d8fc7 x86-64: merge the... |
366 367 |
start_thread_common(regs, new_ip, new_sp, __USER32_CS, __USER32_DS, __USER32_DS); |
a6f05a6a0 x86-64: make comp... |
368 369 |
} #endif |
513ad84bf x86: de-macro sta... |
370 |
|
1da177e4c Linux-2.6.12-rc2 |
371 372 373 |
/* * switch_to(x,y) should switch tasks from x to y. * |
6612538ca x86: clean up pro... |
374 |
* This could still be optimized: |
1da177e4c Linux-2.6.12-rc2 |
375 376 |
* - fold all the options into a flag word and test it with a single test. * - could test fs/gs bitsliced |
099f318b8 [PATCH] x86_64: D... |
377 378 |
* * Kprobes not supported here. Set the probe on schedule instead. |
8b96f0119 tracing/function-... |
379 |
* Function graph tracer not supported too. |
1da177e4c Linux-2.6.12-rc2 |
380 |
*/ |
8b96f0119 tracing/function-... |
381 |
__notrace_funcgraph struct task_struct * |
a88cde13b [PATCH] x86_64: F... |
382 |
__switch_to(struct task_struct *prev_p, struct task_struct *next_p) |
1da177e4c Linux-2.6.12-rc2 |
383 |
{ |
87b935a0e x86: clean up for... |
384 385 |
struct thread_struct *prev = &prev_p->thread; struct thread_struct *next = &next_p->thread; |
6612538ca x86: clean up pro... |
386 |
int cpu = smp_processor_id(); |
1da177e4c Linux-2.6.12-rc2 |
387 |
struct tss_struct *tss = &per_cpu(init_tss, cpu); |
478de5a9d x86: save %fs and... |
388 |
unsigned fsindex, gsindex; |
17950c5b2 x86-64: move clts... |
389 390 391 392 393 394 395 396 |
bool preload_fpu; /* * If the task has used fpu the last 5 timeslices, just do a full * restore of the math state immediately to avoid the trap; the * chances of needing FPU soon are obviously high now */ preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5; |
1da177e4c Linux-2.6.12-rc2 |
397 |
|
e07e23e1f [PATCH] non lazy ... |
398 |
/* we're going to use this soon, after a few expensive things */ |
17950c5b2 x86-64: move clts... |
399 |
if (preload_fpu) |
866032833 x86: Introduce 's... |
400 |
prefetch(next->fpu.state); |
e07e23e1f [PATCH] non lazy ... |
401 |
|
1da177e4c Linux-2.6.12-rc2 |
402 403 404 |
/* * Reload esp0, LDT and the page table pointer: */ |
7818a1e02 x86: provide 64-b... |
405 |
load_sp0(tss, next); |
1da177e4c Linux-2.6.12-rc2 |
406 |
|
7de08b4e1 x86: coding style... |
407 |
/* |
1da177e4c Linux-2.6.12-rc2 |
408 409 410 |
* Switch DS and ES. * This won't pick up thread selector changes, but I guess that is ok. */ |
ada857082 x86: remove open-... |
411 |
savesegment(es, prev->es); |
1da177e4c Linux-2.6.12-rc2 |
412 |
if (unlikely(next->es | prev->es)) |
7de08b4e1 x86: coding style... |
413 |
loadsegment(es, next->es); |
ada857082 x86: remove open-... |
414 415 |
savesegment(ds, prev->ds); |
1da177e4c Linux-2.6.12-rc2 |
416 417 |
if (unlikely(next->ds | prev->ds)) loadsegment(ds, next->ds); |
478de5a9d x86: save %fs and... |
418 419 420 421 422 423 424 425 |
/* We must save %fs and %gs before load_TLS() because * %fs and %gs may be cleared by load_TLS(). * * (e.g. xen_load_tls()) */ savesegment(fs, fsindex); savesegment(gs, gsindex); |
1da177e4c Linux-2.6.12-rc2 |
426 |
load_TLS(next, cpu); |
16d9dbf0c x86-64: move unla... |
427 |
/* Must be after DS reload */ |
a4d4fbc77 x86-64, fpu: Disa... |
428 |
__unlazy_fpu(prev_p); |
16d9dbf0c x86-64: move unla... |
429 |
|
17950c5b2 x86-64: move clts... |
430 431 432 |
/* Make sure cpu is ready for new context */ if (preload_fpu) clts(); |
3fe0a63ef x86, 64-bit: __sw... |
433 434 435 436 437 438 439 |
/* * Leave lazy mode, flushing any hypercalls made here. * This must be done before restoring TLS segments so * the GDT and LDT are properly updated, and must be * done before math_state_restore, so the TS bit is up * to date. */ |
224101ed6 x86/paravirt: fin... |
440 |
arch_end_context_switch(next_p); |
3fe0a63ef x86, 64-bit: __sw... |
441 |
|
7de08b4e1 x86: coding style... |
442 |
/* |
1da177e4c Linux-2.6.12-rc2 |
443 |
* Switch FS and GS. |
87b935a0e x86: clean up for... |
444 445 446 447 |
* * Segment register != 0 always requires a reload. Also * reload when it has changed. When prev process used 64bit * base always reload to avoid an information leak. |
1da177e4c Linux-2.6.12-rc2 |
448 |
*/ |
87b935a0e x86: clean up for... |
449 450 |
if (unlikely(fsindex | next->fsindex | prev->fs)) { loadsegment(fs, next->fsindex); |
7de08b4e1 x86: coding style... |
451 |
/* |
87b935a0e x86: clean up for... |
452 453 454 455 456 |
* Check if the user used a selector != 0; if yes * clear 64bit base, since overloaded base is always * mapped to the Null selector */ if (fsindex) |
7de08b4e1 x86: coding style... |
457 |
prev->fs = 0; |
1da177e4c Linux-2.6.12-rc2 |
458 |
} |
87b935a0e x86: clean up for... |
459 460 461 462 463 464 465 466 |
/* when next process has a 64bit base use it */ if (next->fs) wrmsrl(MSR_FS_BASE, next->fs); prev->fsindex = fsindex; if (unlikely(gsindex | next->gsindex | prev->gs)) { load_gs_index(next->gsindex); if (gsindex) |
7de08b4e1 x86: coding style... |
467 |
prev->gs = 0; |
1da177e4c Linux-2.6.12-rc2 |
468 |
} |
87b935a0e x86: clean up for... |
469 470 471 |
if (next->gs) wrmsrl(MSR_KERNEL_GS_BASE, next->gs); prev->gsindex = gsindex; |
1da177e4c Linux-2.6.12-rc2 |
472 |
|
7de08b4e1 x86: coding style... |
473 |
/* |
45948d772 [PATCH] x86_64: s... |
474 |
* Switch the PDA and FPU contexts. |
1da177e4c Linux-2.6.12-rc2 |
475 |
*/ |
3d1e42a7c x86-64: Move oldr... |
476 477 |
prev->usersp = percpu_read(old_rsp); percpu_write(old_rsp, next->usersp); |
c6f5e0acd x86-64: Move curr... |
478 |
percpu_write(current_task, next_p); |
18bd057b1 [PATCH] i386/x86-... |
479 |
|
9af45651f x86-64: Move kern... |
480 |
percpu_write(kernel_stack, |
87b935a0e x86: clean up for... |
481 |
(unsigned long)task_stack_page(next_p) + |
9af45651f x86-64: Move kern... |
482 |
THREAD_SIZE - KERNEL_STACK_OFFSET); |
1da177e4c Linux-2.6.12-rc2 |
483 484 |
/* |
d3a4f48d4 [PATCH] x86-64 TI... |
485 |
* Now maybe reload the debug registers and handle I/O bitmaps |
1da177e4c Linux-2.6.12-rc2 |
486 |
*/ |
eee3af4a2 x86, ptrace: supp... |
487 488 |
if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT || task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV)) |
d3a4f48d4 [PATCH] x86-64 TI... |
489 |
__switch_to_xtra(prev_p, next_p, tss); |
1da177e4c Linux-2.6.12-rc2 |
490 |
|
17950c5b2 x86-64: move clts... |
491 492 493 |
/* * Preload the FPU context, now that we've determined that the * task is likely to be using it. |
e07e23e1f [PATCH] non lazy ... |
494 |
*/ |
17950c5b2 x86-64: move clts... |
495 496 |
if (preload_fpu) __math_state_restore(); |
66cb59172 hw-breakpoints: u... |
497 |
|
1da177e4c Linux-2.6.12-rc2 |
498 499 |
return prev_p; } |
1da177e4c Linux-2.6.12-rc2 |
500 501 502 503 504 |
void set_personality_64bit(void) { /* inherit personality from parent */ /* Make sure to be in 64bit mode */ |
6612538ca x86: clean up pro... |
505 |
clear_thread_flag(TIF_IA32); |
1da177e4c Linux-2.6.12-rc2 |
506 |
|
375906f87 x86: mark associa... |
507 508 509 |
/* Ensure the corresponding mm is not marked. */ if (current->mm) current->mm->context.ia32_compat = 0; |
1da177e4c Linux-2.6.12-rc2 |
510 511 512 |
/* TBD: overwrites user setup. Should have two bits. But 64bit processes have always behaved this way, so it's not too bad. The main problem is just that |
6612538ca x86: clean up pro... |
513 |
32bit childs are affected again. */ |
1da177e4c Linux-2.6.12-rc2 |
514 515 |
current->personality &= ~READ_IMPLIES_EXEC; } |
05d43ed8a x86: get rid of t... |
516 517 518 519 520 521 |
void set_personality_ia32(void) { /* inherit personality from parent */ /* Make sure to be in 32bit mode */ set_thread_flag(TIF_IA32); |
1252f238d x86: set_personal... |
522 |
current->personality |= force_personality32; |
05d43ed8a x86: get rid of t... |
523 |
|
375906f87 x86: mark associa... |
524 525 526 |
/* Mark the associated mm as containing 32-bit tasks. */ if (current->mm) current->mm->context.ia32_compat = 1; |
05d43ed8a x86: get rid of t... |
527 528 529 |
/* Prepare the first "return" to user space */ current_thread_info()->status |= TS_COMPAT; } |
1da177e4c Linux-2.6.12-rc2 |
530 531 532 |
unsigned long get_wchan(struct task_struct *p) { unsigned long stack; |
7de08b4e1 x86: coding style... |
533 |
u64 fp, ip; |
1da177e4c Linux-2.6.12-rc2 |
534 |
int count = 0; |
7de08b4e1 x86: coding style... |
535 536 |
if (!p || p == current || p->state == TASK_RUNNING) return 0; |
57eafdc22 [PATCH] amd64: ta... |
537 |
stack = (unsigned long)task_stack_page(p); |
e1e23bb05 x86: avoid derefe... |
538 |
if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE) |
1da177e4c Linux-2.6.12-rc2 |
539 |
return 0; |
faca62273 x86: use generic ... |
540 |
fp = *(u64 *)(p->thread.sp); |
7de08b4e1 x86: coding style... |
541 |
do { |
a88cde13b [PATCH] x86_64: F... |
542 |
if (fp < (unsigned long)stack || |
e1e23bb05 x86: avoid derefe... |
543 |
fp >= (unsigned long)stack+THREAD_SIZE) |
7de08b4e1 x86: coding style... |
544 |
return 0; |
65ea5b034 x86: rename the s... |
545 546 547 |
ip = *(u64 *)(fp+8); if (!in_sched_functions(ip)) return ip; |
7de08b4e1 x86: coding style... |
548 549 |
fp = *(u64 *)fp; } while (count++ < 16); |
1da177e4c Linux-2.6.12-rc2 |
550 551 552 553 |
return 0; } long do_arch_prctl(struct task_struct *task, int code, unsigned long addr) |
7de08b4e1 x86: coding style... |
554 555 |
{ int ret = 0; |
1da177e4c Linux-2.6.12-rc2 |
556 557 |
int doit = task == current; int cpu; |
7de08b4e1 x86: coding style... |
558 |
switch (code) { |
1da177e4c Linux-2.6.12-rc2 |
559 |
case ARCH_SET_GS: |
84929801e [PATCH] x86_64: T... |
560 |
if (addr >= TASK_SIZE_OF(task)) |
7de08b4e1 x86: coding style... |
561 |
return -EPERM; |
1da177e4c Linux-2.6.12-rc2 |
562 |
cpu = get_cpu(); |
7de08b4e1 x86: coding style... |
563 |
/* handle small bases via the GDT because that's faster to |
1da177e4c Linux-2.6.12-rc2 |
564 |
switch. */ |
7de08b4e1 x86: coding style... |
565 566 567 |
if (addr <= 0xffffffff) { set_32bit_tls(task, GS_TLS, addr); if (doit) { |
1da177e4c Linux-2.6.12-rc2 |
568 |
load_TLS(&task->thread, cpu); |
7de08b4e1 x86: coding style... |
569 |
load_gs_index(GS_TLS_SEL); |
1da177e4c Linux-2.6.12-rc2 |
570 |
} |
7de08b4e1 x86: coding style... |
571 |
task->thread.gsindex = GS_TLS_SEL; |
1da177e4c Linux-2.6.12-rc2 |
572 |
task->thread.gs = 0; |
7de08b4e1 x86: coding style... |
573 |
} else { |
1da177e4c Linux-2.6.12-rc2 |
574 575 576 |
task->thread.gsindex = 0; task->thread.gs = addr; if (doit) { |
a88cde13b [PATCH] x86_64: F... |
577 578 |
load_gs_index(0); ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr); |
7de08b4e1 x86: coding style... |
579 |
} |
1da177e4c Linux-2.6.12-rc2 |
580 581 582 583 584 585 |
} put_cpu(); break; case ARCH_SET_FS: /* Not strictly needed for fs, but do it for symmetry with gs */ |
84929801e [PATCH] x86_64: T... |
586 |
if (addr >= TASK_SIZE_OF(task)) |
6612538ca x86: clean up pro... |
587 |
return -EPERM; |
1da177e4c Linux-2.6.12-rc2 |
588 |
cpu = get_cpu(); |
6612538ca x86: clean up pro... |
589 |
/* handle small bases via the GDT because that's faster to |
1da177e4c Linux-2.6.12-rc2 |
590 |
switch. */ |
6612538ca x86: clean up pro... |
591 |
if (addr <= 0xffffffff) { |
1da177e4c Linux-2.6.12-rc2 |
592 |
set_32bit_tls(task, FS_TLS, addr); |
6612538ca x86: clean up pro... |
593 594 |
if (doit) { load_TLS(&task->thread, cpu); |
ada857082 x86: remove open-... |
595 |
loadsegment(fs, FS_TLS_SEL); |
1da177e4c Linux-2.6.12-rc2 |
596 597 598 |
} task->thread.fsindex = FS_TLS_SEL; task->thread.fs = 0; |
6612538ca x86: clean up pro... |
599 |
} else { |
1da177e4c Linux-2.6.12-rc2 |
600 601 602 603 604 |
task->thread.fsindex = 0; task->thread.fs = addr; if (doit) { /* set the selector to 0 to not confuse __switch_to */ |
ada857082 x86: remove open-... |
605 |
loadsegment(fs, 0); |
a88cde13b [PATCH] x86_64: F... |
606 |
ret = checking_wrmsrl(MSR_FS_BASE, addr); |
1da177e4c Linux-2.6.12-rc2 |
607 608 609 610 |
} } put_cpu(); break; |
6612538ca x86: clean up pro... |
611 612 |
case ARCH_GET_FS: { unsigned long base; |
1da177e4c Linux-2.6.12-rc2 |
613 614 |
if (task->thread.fsindex == FS_TLS_SEL) base = read_32bit_tls(task, FS_TLS); |
a88cde13b [PATCH] x86_64: F... |
615 |
else if (doit) |
1da177e4c Linux-2.6.12-rc2 |
616 |
rdmsrl(MSR_FS_BASE, base); |
a88cde13b [PATCH] x86_64: F... |
617 |
else |
1da177e4c Linux-2.6.12-rc2 |
618 |
base = task->thread.fs; |
6612538ca x86: clean up pro... |
619 620 |
ret = put_user(base, (unsigned long __user *)addr); break; |
1da177e4c Linux-2.6.12-rc2 |
621 |
} |
6612538ca x86: clean up pro... |
622 |
case ARCH_GET_GS: { |
1da177e4c Linux-2.6.12-rc2 |
623 |
unsigned long base; |
97c2803c9 [PATCH] x86_64: P... |
624 |
unsigned gsindex; |
1da177e4c Linux-2.6.12-rc2 |
625 626 |
if (task->thread.gsindex == GS_TLS_SEL) base = read_32bit_tls(task, GS_TLS); |
97c2803c9 [PATCH] x86_64: P... |
627 |
else if (doit) { |
ada857082 x86: remove open-... |
628 |
savesegment(gs, gsindex); |
97c2803c9 [PATCH] x86_64: P... |
629 630 631 632 |
if (gsindex) rdmsrl(MSR_KERNEL_GS_BASE, base); else base = task->thread.gs; |
7de08b4e1 x86: coding style... |
633 |
} else |
1da177e4c Linux-2.6.12-rc2 |
634 |
base = task->thread.gs; |
6612538ca x86: clean up pro... |
635 |
ret = put_user(base, (unsigned long __user *)addr); |
1da177e4c Linux-2.6.12-rc2 |
636 637 638 639 640 641 |
break; } default: ret = -EINVAL; break; |
6612538ca x86: clean up pro... |
642 |
} |
1da177e4c Linux-2.6.12-rc2 |
643 |
|
6612538ca x86: clean up pro... |
644 645 |
return ret; } |
1da177e4c Linux-2.6.12-rc2 |
646 647 648 649 |
long sys_arch_prctl(int code, unsigned long addr) { return do_arch_prctl(current, code, addr); |
1da177e4c Linux-2.6.12-rc2 |
650 |
} |
89240ba05 x86, fs: Fix x86 ... |
651 652 653 654 655 |
unsigned long KSTK_ESP(struct task_struct *task) { return (test_tsk_thread_flag(task, TIF_IA32)) ? (task_pt_regs(task)->sp) : ((task)->thread.usersp); } |