Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 56dd9470d7c8734f055da2a6bac553caf4a468eb

Authored by Frederic Weisbecker 2013-02-24 07:23:25 +0800

Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches

context_tracking: Move exception handling to generic code

Exceptions handling on context tracking should share common
treatment: on entry we exit user mode if the exception triggered
in that context. Then on exception exit we return to that previous
context.

Generalize this to avoid duplication across archs.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Li Zhong <zhong@linux.vnet.ibm.com>
Cc: Kevin Hilman <khilman@linaro.org>
Cc: Mats Liljegren <mats.liljegren@enea.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Namhyung Kim <namhyung.kim@lge.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

Showing 5 changed files with 19 additions and 26 deletions Inline Diff

arch/x86/include/asm/context_tracking.h
arch/x86/kernel/kvm.c
arch/x86/kernel/traps.c
arch/x86/mm/fault.c
include/linux/context_tracking.h

arch/x86/include/asm/context_tracking.h

Diff comments View file @ 56dd947

1	#ifndef _ASM_X86_CONTEXT_TRACKING_H	1	#ifndef _ASM_X86_CONTEXT_TRACKING_H
2	#define _ASM_X86_CONTEXT_TRACKING_H	2	#define _ASM_X86_CONTEXT_TRACKING_H
3		3
4	#ifndef __ASSEMBLY__
5	#include <linux/context_tracking.h>
6	#include <asm/ptrace.h>
7
8	static inline void exception_enter(struct pt_regs *regs)
9	{
10	user_exit();
11	}
12
13	static inline void exception_exit(struct pt_regs *regs)
14	{
15	#ifdef CONFIG_CONTEXT_TRACKING	4	#ifdef CONFIG_CONTEXT_TRACKING
16	if (user_mode(regs))
17	user_enter();
18	#endif
19	}
20
21	#else /* __ASSEMBLY__ */
22
23	#ifdef CONFIG_CONTEXT_TRACKING
24	# define SCHEDULE_USER call schedule_user	5	# define SCHEDULE_USER call schedule_user
25	#else	6	#else
26	# define SCHEDULE_USER call schedule	7	# define SCHEDULE_USER call schedule
27	#endif	8	#endif
28
29	#endif /* !__ASSEMBLY__ */
30		9
31	#endif	10	#endif
32		11

arch/x86/kernel/kvm.c

Diff comments View file @ 56dd947

 /*
  * KVM paravirt_ops implementation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  *
  * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  * Copyright IBM Corporation, 2007
  *   Authors: Anthony Liguori <aliguori@us.ibm.com>
  */
+#include <linux/context_tracking.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/kvm_para.h>
 #include <linux/cpu.h>
 #include <linux/mm.h>
 #include <linux/highmem.h>
 #include <linux/hardirq.h>
 #include <linux/notifier.h>
 #include <linux/reboot.h>
 #include <linux/hash.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/kprobes.h>
 #include <asm/timer.h>
 #include <asm/cpu.h>
 #include <asm/traps.h>
 #include <asm/desc.h>
 #include <asm/tlbflush.h>
 #include <asm/idle.h>
 #include <asm/apic.h>
 #include <asm/apicdef.h>
 #include <asm/hypervisor.h>
 #include <asm/kvm_guest.h>
-#include <asm/context_tracking.h>
 static int kvmapf = 1;
 static int parse_no_kvmapf(char *arg)
 {
         kvmapf = 0;
         return 0;
 }
 early_param("no-kvmapf", parse_no_kvmapf);
 static int steal_acc = 1;
 static int parse_no_stealacc(char *arg)
 {
         steal_acc = 0;
         return 0;
 }
 early_param("no-steal-acc", parse_no_stealacc);
 static int kvmclock_vsyscall = 1;
 static int parse_no_kvmclock_vsyscall(char *arg)
 {
         kvmclock_vsyscall = 0;
         return 0;
 }
 early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall);
 static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
 static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64);
 static int has_steal_clock = 0;
 /*
  * No need for any "IO delay" on KVM
  */
 static void kvm_io_delay(void)
 {
 }
 #define KVM_TASK_SLEEP_HASHBITS 8
 #define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
 struct kvm_task_sleep_node {
 	struct hlist_node link;
 	wait_queue_head_t wq;
 	u32 token;
 	int cpu;
 	bool halted;
 };
 static struct kvm_task_sleep_head {
 	spinlock_t lock;
 	struct hlist_head list;
 } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
 static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
 						  u32 token)
 {
 	struct hlist_node *p;
 	hlist_for_each(p, &b->list) {
 		struct kvm_task_sleep_node *n =
 			hlist_entry(p, typeof(*n), link);
 		if (n->token == token)
 			return n;
 	}
 	return NULL;
 }
 void kvm_async_pf_task_wait(u32 token)
 {
 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
 	struct kvm_task_sleep_node n, *e;
 	DEFINE_WAIT(wait);
 	rcu_irq_enter();
 	spin_lock(&b->lock);
 	e = _find_apf_task(b, token);
 	if (e) {
 		/* dummy entry exist -> wake up was delivered ahead of PF */
 		hlist_del(&e->link);
 		kfree(e);
 		spin_unlock(&b->lock);
 		rcu_irq_exit();
 		return;
 	}
 	n.token = token;
 	n.cpu = smp_processor_id();
 	n.halted = is_idle_task(current) || preempt_count() > 1;
 	init_waitqueue_head(&n.wq);
 	hlist_add_head(&n.link, &b->list);
 	spin_unlock(&b->lock);
 	for (;;) {
 		if (!n.halted)
 			prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
 		if (hlist_unhashed(&n.link))
 			break;
 		if (!n.halted) {
 			local_irq_enable();
 			schedule();
 			local_irq_disable();
 		} else {
 			/*
 			 * We cannot reschedule. So halt.
 			 */
 			rcu_irq_exit();
 			native_safe_halt();
 			rcu_irq_enter();
 			local_irq_disable();
 		}
 	}
 	if (!n.halted)
 		finish_wait(&n.wq, &wait);
 	rcu_irq_exit();
 	return;
 }
 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait);
 static void apf_task_wake_one(struct kvm_task_sleep_node *n)
 {
 	hlist_del_init(&n->link);
 	if (n->halted)
 		smp_send_reschedule(n->cpu);
 	else if (waitqueue_active(&n->wq))
 		wake_up(&n->wq);
 }
 static void apf_task_wake_all(void)
 {
 	int i;
 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
 		struct hlist_node *p, *next;
 		struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
 		spin_lock(&b->lock);
 		hlist_for_each_safe(p, next, &b->list) {
 			struct kvm_task_sleep_node *n =
 				hlist_entry(p, typeof(*n), link);
 			if (n->cpu == smp_processor_id())
 				apf_task_wake_one(n);
 		}
 		spin_unlock(&b->lock);
 	}
 }
 void kvm_async_pf_task_wake(u32 token)
 {
 	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
 	struct kvm_task_sleep_node *n;
 	if (token == ~0) {
 		apf_task_wake_all();
 		return;
 	}
 again:
 	spin_lock(&b->lock);
 	n = _find_apf_task(b, token);
 	if (!n) {
 		/*
 		 * async PF was not yet handled.
 		 * Add dummy entry for the token.
 		 */
 		n = kzalloc(sizeof(*n), GFP_ATOMIC);
 		if (!n) {
 			/*
 			 * Allocation failed! Busy wait while other cpu
 			 * handles async PF.
 			 */
 			spin_unlock(&b->lock);
 			cpu_relax();
 			goto again;
 		}
 		n->token = token;
 		n->cpu = smp_processor_id();
 		init_waitqueue_head(&n->wq);
 		hlist_add_head(&n->link, &b->list);
 	} else
 		apf_task_wake_one(n);
 	spin_unlock(&b->lock);
 	return;
 }
 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
 u32 kvm_read_and_reset_pf_reason(void)
 {
 	u32 reason = 0;
 	if (__get_cpu_var(apf_reason).enabled) {
 		reason = __get_cpu_var(apf_reason).reason;
 		__get_cpu_var(apf_reason).reason = 0;
 	}
 	return reason;
 }
 EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
 dotraplinkage void __kprobes
 do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
 {
 	switch (kvm_read_and_reset_pf_reason()) {
 	default:
 		do_page_fault(regs, error_code);
 		break;
 	case KVM_PV_REASON_PAGE_NOT_PRESENT:
 		/* page is swapped out by the host. */
 		exception_enter(regs);
 		exit_idle();
 		kvm_async_pf_task_wait((u32)read_cr2());
 		exception_exit(regs);
 		break;
 	case KVM_PV_REASON_PAGE_READY:
 		rcu_irq_enter();
 		exit_idle();
 		kvm_async_pf_task_wake((u32)read_cr2());
 		rcu_irq_exit();
 		break;
 	}
 }
 static void __init paravirt_ops_setup(void)
 {
 	pv_info.name = "KVM";
 	pv_info.paravirt_enabled = 1;
 	if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
 		pv_cpu_ops.io_delay = kvm_io_delay;
 #ifdef CONFIG_X86_IO_APIC
 	no_timer_check = 1;
 #endif
 }
 static void kvm_register_steal_time(void)
 {
 	int cpu = smp_processor_id();
 	struct kvm_steal_time *st = &per_cpu(steal_time, cpu);
 	if (!has_steal_clock)
 		return;
 	memset(st, 0, sizeof(*st));
 	wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED));
 	pr_info("kvm-stealtime: cpu %d, msr %llx\n",
 		cpu, (unsigned long long) slow_virt_to_phys(st));
 }
 static DEFINE_PER_CPU(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
 static void kvm_guest_apic_eoi_write(u32 reg, u32 val)
 {
 	/**
 	 * This relies on __test_and_clear_bit to modify the memory
 	 * in a way that is atomic with respect to the local CPU.
 	 * The hypervisor only accesses this memory from the local CPU so
 	 * there's no need for lock or memory barriers.
 	 * An optimization barrier is implied in apic write.
 	 */
 	if (__test_and_clear_bit(KVM_PV_EOI_BIT, &__get_cpu_var(kvm_apic_eoi)))
 		return;
 	apic_write(APIC_EOI, APIC_EOI_ACK);
 }
 void __cpuinit kvm_guest_cpu_init(void)
 {
 	if (!kvm_para_available())
 		return;
 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
 		u64 pa = slow_virt_to_phys(&__get_cpu_var(apf_reason));
 #ifdef CONFIG_PREEMPT
 		pa |= KVM_ASYNC_PF_SEND_ALWAYS;
 #endif
 		wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED);
 		__get_cpu_var(apf_reason).enabled = 1;
 		printk(KERN_INFO"KVM setup async PF for cpu %d\n",
 		       smp_processor_id());
 	}
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) {
 		unsigned long pa;
 		/* Size alignment is implied but just to make it explicit. */
 		BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
 		__get_cpu_var(kvm_apic_eoi) = 0;
 		pa = slow_virt_to_phys(&__get_cpu_var(kvm_apic_eoi))
 			| KVM_MSR_ENABLED;
 		wrmsrl(MSR_KVM_PV_EOI_EN, pa);
 	}
 	if (has_steal_clock)
 		kvm_register_steal_time();
 }
 static void kvm_pv_disable_apf(void)
 {
 	if (!__get_cpu_var(apf_reason).enabled)
 		return;
 	wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
 	__get_cpu_var(apf_reason).enabled = 0;
 	printk(KERN_INFO"Unregister pv shared memory for cpu %d\n",
 	       smp_processor_id());
 }
 static void kvm_pv_guest_cpu_reboot(void *unused)
 {
 	/*
 	 * We disable PV EOI before we load a new kernel by kexec,
 	 * since MSR_KVM_PV_EOI_EN stores a pointer into old kernel's memory.
 	 * New kernel can re-enable when it boots.
 	 */
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
 	kvm_pv_disable_apf();
 	kvm_disable_steal_time();
 }
 static int kvm_pv_reboot_notify(struct notifier_block *nb,
 				unsigned long code, void *unused)
 {
 	if (code == SYS_RESTART)
 		on_each_cpu(kvm_pv_guest_cpu_reboot, NULL, 1);
 	return NOTIFY_DONE;
 }
 static struct notifier_block kvm_pv_reboot_nb = {
 	.notifier_call = kvm_pv_reboot_notify,
 };
 static u64 kvm_steal_clock(int cpu)
 {
 	u64 steal;
 	struct kvm_steal_time *src;
 	int version;
 	src = &per_cpu(steal_time, cpu);
 	do {
 		version = src->version;
 		rmb();
 		steal = src->steal;
 		rmb();
 	} while ((version & 1) || (version != src->version));
 	return steal;
 }
 void kvm_disable_steal_time(void)
 {
 	if (!has_steal_clock)
 		return;
 	wrmsr(MSR_KVM_STEAL_TIME, 0, 0);
 }
 #ifdef CONFIG_SMP
 static void __init kvm_smp_prepare_boot_cpu(void)
 {
 	WARN_ON(kvm_register_clock("primary cpu clock"));
 	kvm_guest_cpu_init();
 	native_smp_prepare_boot_cpu();
 }
 static void __cpuinit kvm_guest_cpu_online(void *dummy)
 {
 	kvm_guest_cpu_init();
 }
 static void kvm_guest_cpu_offline(void *dummy)
 {
 	kvm_disable_steal_time();
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
 	kvm_pv_disable_apf();
 	apf_task_wake_all();
 }
 static int __cpuinit kvm_cpu_notify(struct notifier_block *self,
 				    unsigned long action, void *hcpu)
 {
 	int cpu = (unsigned long)hcpu;
 	switch (action) {
 	case CPU_ONLINE:
 	case CPU_DOWN_FAILED:
 	case CPU_ONLINE_FROZEN:
 		smp_call_function_single(cpu, kvm_guest_cpu_online, NULL, 0);
 		break;
 	case CPU_DOWN_PREPARE:
 	case CPU_DOWN_PREPARE_FROZEN:
 		smp_call_function_single(cpu, kvm_guest_cpu_offline, NULL, 1);
 		break;
 	default:
 		break;
 	}
 	return NOTIFY_OK;
 }
 static struct notifier_block __cpuinitdata kvm_cpu_notifier = {
         .notifier_call  = kvm_cpu_notify,
 };
 #endif
 static void __init kvm_apf_trap_init(void)
 {
 	set_intr_gate(14, &async_page_fault);
 }
 void __init kvm_guest_init(void)
 {
 	int i;
 	if (!kvm_para_available())
 		return;
 	paravirt_ops_setup();
 	register_reboot_notifier(&kvm_pv_reboot_nb);
 	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
 		spin_lock_init(&async_pf_sleepers[i].lock);
 	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
 		x86_init.irqs.trap_init = kvm_apf_trap_init;
 	if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
 		has_steal_clock = 1;
 		pv_time_ops.steal_clock = kvm_steal_clock;
 	}
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		apic_set_eoi_write(kvm_guest_apic_eoi_write);
 	if (kvmclock_vsyscall)
 		kvm_setup_vsyscall_timeinfo();
 #ifdef CONFIG_SMP
 	smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
 	register_cpu_notifier(&kvm_cpu_notifier);
 #else
 	kvm_guest_cpu_init();
 #endif
 }
 static bool __init kvm_detect(void)
 {
 	if (!kvm_para_available())
 		return false;
 	return true;
 }
 const struct hypervisor_x86 x86_hyper_kvm __refconst = {
 	.name			= "KVM",
 	.detect			= kvm_detect,
 	.x2apic_available	= kvm_para_available,
 };
 EXPORT_SYMBOL_GPL(x86_hyper_kvm);
 static __init int activate_jump_labels(void)
 {
 	if (has_steal_clock) {
 		static_key_slow_inc(&paravirt_steal_enabled);
 		if (steal_acc)
 			static_key_slow_inc(&paravirt_steal_rq_enabled);
 	}
 	return 0;
 }
 arch_initcall(activate_jump_labels);

arch/x86/kernel/traps.c

Diff comments View file @ 56dd947

 /*
  *  Copyright (C) 1991, 1992  Linus Torvalds
  *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
  *
  *  Pentium III FXSR, SSE support
  *	Gareth Hughes <gareth@valinux.com>, May 2000
  */
 /*
  * Handle hardware traps and faults.
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/context_tracking.h>
 #include <linux/interrupt.h>
 #include <linux/kallsyms.h>
 #include <linux/spinlock.h>
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
 #include <linux/kdebug.h>
 #include <linux/kgdb.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/ptrace.h>
 #include <linux/string.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/kexec.h>
 #include <linux/sched.h>
 #include <linux/timer.h>
 #include <linux/init.h>
 #include <linux/bug.h>
 #include <linux/nmi.h>
 #include <linux/mm.h>
 #include <linux/smp.h>
 #include <linux/io.h>
 #ifdef CONFIG_EISA
 #include <linux/ioport.h>
 #include <linux/eisa.h>
 #endif
 #if defined(CONFIG_EDAC)
 #include <linux/edac.h>
 #endif
 #include <asm/kmemcheck.h>
 #include <asm/stacktrace.h>
 #include <asm/processor.h>
 #include <asm/debugreg.h>
 #include <linux/atomic.h>
 #include <asm/ftrace.h>
 #include <asm/traps.h>
 #include <asm/desc.h>
 #include <asm/i387.h>
 #include <asm/fpu-internal.h>
 #include <asm/mce.h>
-#include <asm/context_tracking.h>
 #include <asm/mach_traps.h>
 #ifdef CONFIG_X86_64
 #include <asm/x86_init.h>
 #include <asm/pgalloc.h>
 #include <asm/proto.h>
 #else
 #include <asm/processor-flags.h>
 #include <asm/setup.h>
 asmlinkage int system_call(void);
 /*
  * The IDT has to be page-aligned to simplify the Pentium
  * F0 0F bug workaround.
  */
 gate_desc idt_table[NR_VECTORS] __page_aligned_data = { { { { 0, 0 } } }, };
 #endif
 DECLARE_BITMAP(used_vectors, NR_VECTORS);
 EXPORT_SYMBOL_GPL(used_vectors);
 static inline void conditional_sti(struct pt_regs *regs)
 {
 	if (regs->flags & X86_EFLAGS_IF)
 		local_irq_enable();
 }
 static inline void preempt_conditional_sti(struct pt_regs *regs)
 {
 	inc_preempt_count();
 	if (regs->flags & X86_EFLAGS_IF)
 		local_irq_enable();
 }
 static inline void conditional_cli(struct pt_regs *regs)
 {
 	if (regs->flags & X86_EFLAGS_IF)
 		local_irq_disable();
 }
 static inline void preempt_conditional_cli(struct pt_regs *regs)
 {
 	if (regs->flags & X86_EFLAGS_IF)
 		local_irq_disable();
 	dec_preempt_count();
 }
 static int __kprobes
 do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
 		  struct pt_regs *regs,	long error_code)
 {
 #ifdef CONFIG_X86_32
 	if (regs->flags & X86_VM_MASK) {
 		/*
 		 * Traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
 		 * On nmi (interrupt 2), do_trap should not be called.
 		 */
 		if (trapnr < X86_TRAP_UD) {
 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,
 						error_code, trapnr))
 				return 0;
 		}
 		return -1;
 	}
 #endif
 	if (!user_mode(regs)) {
 		if (!fixup_exception(regs)) {
 			tsk->thread.error_code = error_code;
 			tsk->thread.trap_nr = trapnr;
 			die(str, regs, error_code);
 		}
 		return 0;
 	}
 	return -1;
 }
 static void __kprobes
 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
 	long error_code, siginfo_t *info)
 {
 	struct task_struct *tsk = current;
 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))
 		return;
 	/*
 	 * We want error_code and trap_nr set for userspace faults and
 	 * kernelspace faults which result in die(), but not
 	 * kernelspace faults which are fixed up.  die() gives the
 	 * process no chance to handle the signal and notice the
 	 * kernel fault information, so that won't result in polluting
 	 * the information about previously queued, but not yet
 	 * delivered, faults.  See also do_general_protection below.
 	 */
 	tsk->thread.error_code = error_code;
 	tsk->thread.trap_nr = trapnr;
 #ifdef CONFIG_X86_64
 	if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
 	    printk_ratelimit()) {
 		pr_info("%s[%d] trap %s ip:%lx sp:%lx error:%lx",
 			tsk->comm, tsk->pid, str,
 			regs->ip, regs->sp, error_code);
 		print_vma_addr(" in ", regs->ip);
 		pr_cont("\n");
 	}
 #endif
 	if (info)
 		force_sig_info(signr, info, tsk);
 	else
 		force_sig(signr, tsk);
 }
 #define DO_ERROR(trapnr, signr, str, name)				\
 dotraplinkage void do_##name(struct pt_regs *regs, long error_code)	\
 {									\
 	exception_enter(regs);						\
 	if (notify_die(DIE_TRAP, str, regs, error_code,			\
 			trapnr, signr) == NOTIFY_STOP) {		\
 		exception_exit(regs);					\
 		return;							\
 	}								\
 	conditional_sti(regs);						\
 	do_trap(trapnr, signr, str, regs, error_code, NULL);		\
 	exception_exit(regs);						\
 }
 #define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr)		\
 dotraplinkage void do_##name(struct pt_regs *regs, long error_code)	\
 {									\
 	siginfo_t info;							\
 	info.si_signo = signr;						\
 	info.si_errno = 0;						\
 	info.si_code = sicode;						\
 	info.si_addr = (void __user *)siaddr;				\
 	exception_enter(regs);						\
 	if (notify_die(DIE_TRAP, str, regs, error_code,			\
 			trapnr, signr) == NOTIFY_STOP) {		\
 		exception_exit(regs);					\
 		return;							\
 	}								\
 	conditional_sti(regs);						\
 	do_trap(trapnr, signr, str, regs, error_code, &info);		\
 	exception_exit(regs);						\
 }
 DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV,
 		regs->ip)
 DO_ERROR(X86_TRAP_OF, SIGSEGV, "overflow", overflow)
 DO_ERROR(X86_TRAP_BR, SIGSEGV, "bounds", bounds)
 DO_ERROR_INFO(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN,
 		regs->ip)
 DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun",
 		coprocessor_segment_overrun)
 DO_ERROR(X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS)
 DO_ERROR(X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present)
 #ifdef CONFIG_X86_32
 DO_ERROR(X86_TRAP_SS, SIGBUS, "stack segment", stack_segment)
 #endif
 DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check,
 		BUS_ADRALN, 0)
 #ifdef CONFIG_X86_64
 /* Runs on IST stack */
 dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)
 {
 	exception_enter(regs);
 	if (notify_die(DIE_TRAP, "stack segment", regs, error_code,
 		       X86_TRAP_SS, SIGBUS) != NOTIFY_STOP) {
 		preempt_conditional_sti(regs);
 		do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL);
 		preempt_conditional_cli(regs);
 	}
 	exception_exit(regs);
 }
 dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
 {
 	static const char str[] = "double fault";
 	struct task_struct *tsk = current;
 	exception_enter(regs);
 	/* Return not checked because double check cannot be ignored */
 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
 	tsk->thread.error_code = error_code;
 	tsk->thread.trap_nr = X86_TRAP_DF;
 	/*
 	 * This is always a kernel trap and never fixable (and thus must
 	 * never return).
 	 */
 	for (;;)
 		die(str, regs, error_code);
 }
 #endif
 dotraplinkage void __kprobes
 do_general_protection(struct pt_regs *regs, long error_code)
 {
 	struct task_struct *tsk;
 	exception_enter(regs);
 	conditional_sti(regs);
 #ifdef CONFIG_X86_32
 	if (regs->flags & X86_VM_MASK) {
 		local_irq_enable();
 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
 		goto exit;
 	}
 #endif
 	tsk = current;
 	if (!user_mode(regs)) {
 		if (fixup_exception(regs))
 			goto exit;
 		tsk->thread.error_code = error_code;
 		tsk->thread.trap_nr = X86_TRAP_GP;
 		if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
 			       X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)
 			die("general protection fault", regs, error_code);
 		goto exit;
 	}
 	tsk->thread.error_code = error_code;
 	tsk->thread.trap_nr = X86_TRAP_GP;
 	if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
 			printk_ratelimit()) {
 		pr_info("%s[%d] general protection ip:%lx sp:%lx error:%lx",
 			tsk->comm, task_pid_nr(tsk),
 			regs->ip, regs->sp, error_code);
 		print_vma_addr(" in ", regs->ip);
 		pr_cont("\n");
 	}
 	force_sig(SIGSEGV, tsk);
 exit:
 	exception_exit(regs);
 }
 /* May run on IST stack. */
 dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code)
 {
 #ifdef CONFIG_DYNAMIC_FTRACE
 	/*
 	 * ftrace must be first, everything else may cause a recursive crash.
 	 * See note by declaration of modifying_ftrace_code in ftrace.c
 	 */
 	if (unlikely(atomic_read(&modifying_ftrace_code)) &&
 	    ftrace_int3_handler(regs))
 		return;
 #endif
 	exception_enter(regs);
 #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
 				SIGTRAP) == NOTIFY_STOP)
 		goto exit;
 #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
 	if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
 			SIGTRAP) == NOTIFY_STOP)
 		goto exit;
 	/*
 	 * Let others (NMI) know that the debug stack is in use
 	 * as we may switch to the interrupt stack.
 	 */
 	debug_stack_usage_inc();
 	preempt_conditional_sti(regs);
 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
 	preempt_conditional_cli(regs);
 	debug_stack_usage_dec();
 exit:
 	exception_exit(regs);
 }
 #ifdef CONFIG_X86_64
 /*
  * Help handler running on IST stack to switch back to user stack
  * for scheduling or signal handling. The actual stack switch is done in
  * entry.S
  */
 asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
 {
 	struct pt_regs *regs = eregs;
 	/* Did already sync */
 	if (eregs == (struct pt_regs *)eregs->sp)
 		;
 	/* Exception from user space */
 	else if (user_mode(eregs))
 		regs = task_pt_regs(current);
 	/*
 	 * Exception from kernel and interrupts are enabled. Move to
 	 * kernel process stack.
 	 */
 	else if (eregs->flags & X86_EFLAGS_IF)
 		regs = (struct pt_regs *)(eregs->sp -= sizeof(struct pt_regs));
 	if (eregs != regs)
 		*regs = *eregs;
 	return regs;
 }
 #endif
 /*
  * Our handling of the processor debug registers is non-trivial.
  * We do not clear them on entry and exit from the kernel. Therefore
  * it is possible to get a watchpoint trap here from inside the kernel.
  * However, the code in ./ptrace.c has ensured that the user can
  * only set watchpoints on userspace addresses. Therefore the in-kernel
  * watchpoint trap can only occur in code which is reading/writing
  * from user space. Such code must not hold kernel locks (since it
  * can equally take a page fault), therefore it is safe to call
  * force_sig_info even though that claims and releases locks.
  *
  * Code in ./signal.c ensures that the debug control register
  * is restored before we deliver any signal, and therefore that
  * user code runs with the correct debug control register even though
  * we clear it here.
  *
  * Being careful here means that we don't have to be as careful in a
  * lot of more complicated places (task switching can be a bit lazy
  * about restoring all the debug state, and ptrace doesn't have to
  * find every occurrence of the TF bit that could be saved away even
  * by user code)
  *
  * May run on IST stack.
  */
 dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
 {
 	struct task_struct *tsk = current;
 	int user_icebp = 0;
 	unsigned long dr6;
 	int si_code;
 	exception_enter(regs);
 	get_debugreg(dr6, 6);
 	/* Filter out all the reserved bits which are preset to 1 */
 	dr6 &= ~DR6_RESERVED;
 	/*
 	 * If dr6 has no reason to give us about the origin of this trap,
 	 * then it's very likely the result of an icebp/int01 trap.
 	 * User wants a sigtrap for that.
 	 */
 	if (!dr6 && user_mode(regs))
 		user_icebp = 1;
 	/* Catch kmemcheck conditions first of all! */
 	if ((dr6 & DR_STEP) && kmemcheck_trap(regs))
 		goto exit;
 	/* DR6 may or may not be cleared by the CPU */
 	set_debugreg(0, 6);
 	/*
 	 * The processor cleared BTF, so don't mark that we need it set.
 	 */
 	clear_tsk_thread_flag(tsk, TIF_BLOCKSTEP);
 	/* Store the virtualized DR6 value */
 	tsk->thread.debugreg6 = dr6;
 	if (notify_die(DIE_DEBUG, "debug", regs, PTR_ERR(&dr6), error_code,
 							SIGTRAP) == NOTIFY_STOP)
 		goto exit;
 	/*
 	 * Let others (NMI) know that the debug stack is in use
 	 * as we may switch to the interrupt stack.
 	 */
 	debug_stack_usage_inc();
 	/* It's safe to allow irq's after DR6 has been saved */
 	preempt_conditional_sti(regs);
 	if (regs->flags & X86_VM_MASK) {
 		handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,
 					X86_TRAP_DB);
 		preempt_conditional_cli(regs);
 		debug_stack_usage_dec();
 		goto exit;
 	}
 	/*
 	 * Single-stepping through system calls: ignore any exceptions in
 	 * kernel space, but re-enable TF when returning to user mode.
 	 *
 	 * We already checked v86 mode above, so we can check for kernel mode
 	 * by just checking the CPL of CS.
 	 */
 	if ((dr6 & DR_STEP) && !user_mode(regs)) {
 		tsk->thread.debugreg6 &= ~DR_STEP;
 		set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
 		regs->flags &= ~X86_EFLAGS_TF;
 	}
 	si_code = get_si_code(tsk->thread.debugreg6);
 	if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp)
 		send_sigtrap(tsk, regs, error_code, si_code);
 	preempt_conditional_cli(regs);
 	debug_stack_usage_dec();
 exit:
 	exception_exit(regs);
 }
 /*
  * Note that we play around with the 'TS' bit in an attempt to get
  * the correct behaviour even in the presence of the asynchronous
  * IRQ13 behaviour
  */
 void math_error(struct pt_regs *regs, int error_code, int trapnr)
 {
 	struct task_struct *task = current;
 	siginfo_t info;
 	unsigned short err;
 	char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
 						"simd exception";
 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP)
 		return;
 	conditional_sti(regs);
 	if (!user_mode_vm(regs))
 	{
 		if (!fixup_exception(regs)) {
 			task->thread.error_code = error_code;
 			task->thread.trap_nr = trapnr;
 			die(str, regs, error_code);
 		}
 		return;
 	}
 	/*
 	 * Save the info for the exception handler and clear the error.
 	 */
 	save_init_fpu(task);
 	task->thread.trap_nr = trapnr;
 	task->thread.error_code = error_code;
 	info.si_signo = SIGFPE;
 	info.si_errno = 0;
 	info.si_addr = (void __user *)regs->ip;
 	if (trapnr == X86_TRAP_MF) {
 		unsigned short cwd, swd;
 		/*
 		 * (~cwd & swd) will mask out exceptions that are not set to unmasked
 		 * status.  0x3f is the exception bits in these regs, 0x200 is the
 		 * C1 reg you need in case of a stack fault, 0x040 is the stack
 		 * fault bit.  We should only be taking one exception at a time,
 		 * so if this combination doesn't produce any single exception,
 		 * then we have a bad program that isn't synchronizing its FPU usage
 		 * and it will suffer the consequences since we won't be able to
 		 * fully reproduce the context of the exception
 		 */
 		cwd = get_fpu_cwd(task);
 		swd = get_fpu_swd(task);
 		err = swd & ~cwd;
 	} else {
 		/*
 		 * The SIMD FPU exceptions are handled a little differently, as there
 		 * is only a single status/control register.  Thus, to determine which
 		 * unmasked exception was caught we must mask the exception mask bits
 		 * at 0x1f80, and then use these to mask the exception bits at 0x3f.
 		 */
 		unsigned short mxcsr = get_fpu_mxcsr(task);
 		err = ~(mxcsr >> 7) & mxcsr;
 	}
 	if (err & 0x001) {	/* Invalid op */
 		/*
 		 * swd & 0x240 == 0x040: Stack Underflow
 		 * swd & 0x240 == 0x240: Stack Overflow
 		 * User must clear the SF bit (0x40) if set
 		 */
 		info.si_code = FPE_FLTINV;
 	} else if (err & 0x004) { /* Divide by Zero */
 		info.si_code = FPE_FLTDIV;
 	} else if (err & 0x008) { /* Overflow */
 		info.si_code = FPE_FLTOVF;
 	} else if (err & 0x012) { /* Denormal, Underflow */
 		info.si_code = FPE_FLTUND;
 	} else if (err & 0x020) { /* Precision */
 		info.si_code = FPE_FLTRES;
 	} else {
 		/*
 		 * If we're using IRQ 13, or supposedly even some trap
 		 * X86_TRAP_MF implementations, it's possible
 		 * we get a spurious trap, which is not an error.
 		 */
 		return;
 	}
 	force_sig_info(SIGFPE, &info, task);
 }
 dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
 {
 	exception_enter(regs);
 	math_error(regs, error_code, X86_TRAP_MF);
 	exception_exit(regs);
 }
 dotraplinkage void
 do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
 {
 	exception_enter(regs);
 	math_error(regs, error_code, X86_TRAP_XF);
 	exception_exit(regs);
 }
 dotraplinkage void
 do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
 {
 	conditional_sti(regs);
 #if 0
 	/* No need to warn about this any longer. */
 	pr_info("Ignoring P6 Local APIC Spurious Interrupt Bug...\n");
 #endif
 }
 asmlinkage void __attribute__((weak)) smp_thermal_interrupt(void)
 {
 }
 asmlinkage void __attribute__((weak)) smp_threshold_interrupt(void)
 {
 }
 /*
  * 'math_state_restore()' saves the current math information in the
  * old math state array, and gets the new ones from the current task
  *
  * Careful.. There are problems with IBM-designed IRQ13 behaviour.
  * Don't touch unless you *really* know how it works.
  *
  * Must be called with kernel preemption disabled (eg with local
  * local interrupts as in the case of do_device_not_available).
  */
 void math_state_restore(void)
 {
 	struct task_struct *tsk = current;
 	if (!tsk_used_math(tsk)) {
 		local_irq_enable();
 		/*
 		 * does a slab alloc which can sleep
 		 */
 		if (init_fpu(tsk)) {
 			/*
 			 * ran out of memory!
 			 */
 			do_group_exit(SIGKILL);
 			return;
 		}
 		local_irq_disable();
 	}
 	__thread_fpu_begin(tsk);
 	/*
 	 * Paranoid restore. send a SIGSEGV if we fail to restore the state.
 	 */
 	if (unlikely(restore_fpu_checking(tsk))) {
 		drop_init_fpu(tsk);
 		force_sig(SIGSEGV, tsk);
 		return;
 	}
 	tsk->fpu_counter++;
 }
 EXPORT_SYMBOL_GPL(math_state_restore);
 dotraplinkage void __kprobes
 do_device_not_available(struct pt_regs *regs, long error_code)
 {
 	exception_enter(regs);
 	BUG_ON(use_eager_fpu());
 #ifdef CONFIG_MATH_EMULATION
 	if (read_cr0() & X86_CR0_EM) {
 		struct math_emu_info info = { };
 		conditional_sti(regs);
 		info.regs = regs;
 		math_emulate(&info);
 		exception_exit(regs);
 		return;
 	}
 #endif
 	math_state_restore(); /* interrupts still off */
 #ifdef CONFIG_X86_32
 	conditional_sti(regs);
 #endif
 	exception_exit(regs);
 }
 #ifdef CONFIG_X86_32
 dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
 {
 	siginfo_t info;
 	exception_enter(regs);
 	local_irq_enable();
 	info.si_signo = SIGILL;
 	info.si_errno = 0;
 	info.si_code = ILL_BADSTK;
 	info.si_addr = NULL;
 	if (notify_die(DIE_TRAP, "iret exception", regs, error_code,
 			X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) {
 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
 			&info);
 	}
 	exception_exit(regs);
 }
 #endif
 /* Set of traps needed for early debugging. */
 void __init early_trap_init(void)
 {
 	set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
 	/* int3 can be called from all */
 	set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
 #ifdef CONFIG_X86_32
 	set_intr_gate(X86_TRAP_PF, &page_fault);
 #endif
 	load_idt(&idt_descr);
 }
 void __init early_trap_pf_init(void)
 {
 #ifdef CONFIG_X86_64
 	set_intr_gate(X86_TRAP_PF, &page_fault);
 #endif
 }
 void __init trap_init(void)
 {
 	int i;
 #ifdef CONFIG_EISA
 	void __iomem *p = early_ioremap(0x0FFFD9, 4);
 	if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
 		EISA_bus = 1;
 	early_iounmap(p, 4);
 #endif
 	set_intr_gate(X86_TRAP_DE, &divide_error);
 	set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK);
 	/* int4 can be called from all */
 	set_system_intr_gate(X86_TRAP_OF, &overflow);
 	set_intr_gate(X86_TRAP_BR, &bounds);
 	set_intr_gate(X86_TRAP_UD, &invalid_op);
 	set_intr_gate(X86_TRAP_NM, &device_not_available);
 #ifdef CONFIG_X86_32
 	set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS);
 #else
 	set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK);
 #endif
 	set_intr_gate(X86_TRAP_OLD_MF, &coprocessor_segment_overrun);
 	set_intr_gate(X86_TRAP_TS, &invalid_TSS);
 	set_intr_gate(X86_TRAP_NP, &segment_not_present);
 	set_intr_gate_ist(X86_TRAP_SS, &stack_segment, STACKFAULT_STACK);
 	set_intr_gate(X86_TRAP_GP, &general_protection);
 	set_intr_gate(X86_TRAP_SPURIOUS, &spurious_interrupt_bug);
 	set_intr_gate(X86_TRAP_MF, &coprocessor_error);
 	set_intr_gate(X86_TRAP_AC, &alignment_check);
 #ifdef CONFIG_X86_MCE
 	set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK);
 #endif
 	set_intr_gate(X86_TRAP_XF, &simd_coprocessor_error);
 	/* Reserve all the builtin and the syscall vector: */
 	for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
 		set_bit(i, used_vectors);
 #ifdef CONFIG_IA32_EMULATION
 	set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
 	set_bit(IA32_SYSCALL_VECTOR, used_vectors);
 #endif
 #ifdef CONFIG_X86_32
 	set_system_trap_gate(SYSCALL_VECTOR, &system_call);
 	set_bit(SYSCALL_VECTOR, used_vectors);
 #endif
 	/*
 	 * Should be a barrier for any external CPU state:
 	 */
 	cpu_init();
 	x86_init.irqs.trap_init();
 #ifdef CONFIG_X86_64
 	memcpy(&nmi_idt_table, &idt_table, IDT_ENTRIES * 16);
 	set_nmi_gate(X86_TRAP_DB, &debug);
 	set_nmi_gate(X86_TRAP_BP, &int3);
 #endif
 }

arch/x86/mm/fault.c

Diff comments View file @ 56dd947

 /*
  *  Copyright (C) 1995  Linus Torvalds
  *  Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
  *  Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
  */
 #include <linux/magic.h>		/* STACK_END_MAGIC		*/
 #include <linux/sched.h>		/* test_thread_flag(), ...	*/
 #include <linux/kdebug.h>		/* oops_begin/end, ...		*/
 #include <linux/module.h>		/* search_exception_table	*/
 #include <linux/bootmem.h>		/* max_low_pfn			*/
 #include <linux/kprobes.h>		/* __kprobes, ...		*/
 #include <linux/mmiotrace.h>		/* kmmio_handler, ...		*/
 #include <linux/perf_event.h>		/* perf_sw_event		*/
 #include <linux/hugetlb.h>		/* hstate_index_to_shift	*/
 #include <linux/prefetch.h>		/* prefetchw			*/
+#include <linux/context_tracking.h>	/* exception_enter(), ...	*/
 #include <asm/traps.h>			/* dotraplinkage, ...		*/
 #include <asm/pgalloc.h>		/* pgd_*(), ...			*/
 #include <asm/kmemcheck.h>		/* kmemcheck_*(), ...		*/
 #include <asm/fixmap.h>			/* VSYSCALL_START		*/
-#include <asm/context_tracking.h>	/* exception_enter(), ...	*/
 /*
  * Page fault error code bits:
  *
  *   bit 0 ==	 0: no page found	1: protection fault
  *   bit 1 ==	 0: read access		1: write access
  *   bit 2 ==	 0: kernel-mode access	1: user-mode access
  *   bit 3 ==				1: use of reserved bit detected
  *   bit 4 ==				1: fault was an instruction fetch
  */
 enum x86_pf_error_code {
 	PF_PROT		=		1 << 0,
 	PF_WRITE	=		1 << 1,
 	PF_USER		=		1 << 2,
 	PF_RSVD		=		1 << 3,
 	PF_INSTR	=		1 << 4,
 };
 /*
  * Returns 0 if mmiotrace is disabled, or if the fault is not
  * handled by mmiotrace:
  */
 static inline int __kprobes
 kmmio_fault(struct pt_regs *regs, unsigned long addr)
 {
 	if (unlikely(is_kmmio_active()))
 		if (kmmio_handler(regs, addr) == 1)
 			return -1;
 	return 0;
 }
 static inline int __kprobes notify_page_fault(struct pt_regs *regs)
 {
 	int ret = 0;
 	/* kprobe_running() needs smp_processor_id() */
 	if (kprobes_built_in() && !user_mode_vm(regs)) {
 		preempt_disable();
 		if (kprobe_running() && kprobe_fault_handler(regs, 14))
 			ret = 1;
 		preempt_enable();
 	}
 	return ret;
 }
 /*
  * Prefetch quirks:
  *
  * 32-bit mode:
  *
  *   Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
  *   Check that here and ignore it.
  *
  * 64-bit mode:
  *
  *   Sometimes the CPU reports invalid exceptions on prefetch.
  *   Check that here and ignore it.
  *
  * Opcode checker based on code by Richard Brunner.
  */
 static inline int
 check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
 		      unsigned char opcode, int *prefetch)
 {
 	unsigned char instr_hi = opcode & 0xf0;
 	unsigned char instr_lo = opcode & 0x0f;
 	switch (instr_hi) {
 	case 0x20:
 	case 0x30:
 		/*
 		 * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
 		 * In X86_64 long mode, the CPU will signal invalid
 		 * opcode if some of these prefixes are present so
 		 * X86_64 will never get here anyway
 		 */
 		return ((instr_lo & 7) == 0x6);
 #ifdef CONFIG_X86_64
 	case 0x40:
 		/*
 		 * In AMD64 long mode 0x40..0x4F are valid REX prefixes
 		 * Need to figure out under what instruction mode the
 		 * instruction was issued. Could check the LDT for lm,
 		 * but for now it's good enough to assume that long
 		 * mode only uses well known segments or kernel.
 		 */
 		return (!user_mode(regs) || user_64bit_mode(regs));
 #endif
 	case 0x60:
 		/* 0x64 thru 0x67 are valid prefixes in all modes. */
 		return (instr_lo & 0xC) == 0x4;
 	case 0xF0:
 		/* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
 		return !instr_lo || (instr_lo>>1) == 1;
 	case 0x00:
 		/* Prefetch instruction is 0x0F0D or 0x0F18 */
 		if (probe_kernel_address(instr, opcode))
 			return 0;
 		*prefetch = (instr_lo == 0xF) &&
 			(opcode == 0x0D || opcode == 0x18);
 		return 0;
 	default:
 		return 0;
 	}
 }
 static int
 is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
 {
 	unsigned char *max_instr;
 	unsigned char *instr;
 	int prefetch = 0;
 	/*
 	 * If it was a exec (instruction fetch) fault on NX page, then
 	 * do not ignore the fault:
 	 */
 	if (error_code & PF_INSTR)
 		return 0;
 	instr = (void *)convert_ip_to_linear(current, regs);
 	max_instr = instr + 15;
 	if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
 		return 0;
 	while (instr < max_instr) {
 		unsigned char opcode;
 		if (probe_kernel_address(instr, opcode))
 			break;
 		instr++;
 		if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
 			break;
 	}
 	return prefetch;
 }
 static void
 force_sig_info_fault(int si_signo, int si_code, unsigned long address,
 		     struct task_struct *tsk, int fault)
 {
 	unsigned lsb = 0;
 	siginfo_t info;
 	info.si_signo	= si_signo;
 	info.si_errno	= 0;
 	info.si_code	= si_code;
 	info.si_addr	= (void __user *)address;
 	if (fault & VM_FAULT_HWPOISON_LARGE)
 		lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
 	if (fault & VM_FAULT_HWPOISON)
 		lsb = PAGE_SHIFT;
 	info.si_addr_lsb = lsb;
 	force_sig_info(si_signo, &info, tsk);
 }
 DEFINE_SPINLOCK(pgd_lock);
 LIST_HEAD(pgd_list);
 #ifdef CONFIG_X86_32
 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
 {
 	unsigned index = pgd_index(address);
 	pgd_t *pgd_k;
 	pud_t *pud, *pud_k;
 	pmd_t *pmd, *pmd_k;
 	pgd += index;
 	pgd_k = init_mm.pgd + index;
 	if (!pgd_present(*pgd_k))
 		return NULL;
 	/*
 	 * set_pgd(pgd, *pgd_k); here would be useless on PAE
 	 * and redundant with the set_pmd() on non-PAE. As would
 	 * set_pud.
 	 */
 	pud = pud_offset(pgd, address);
 	pud_k = pud_offset(pgd_k, address);
 	if (!pud_present(*pud_k))
 		return NULL;
 	pmd = pmd_offset(pud, address);
 	pmd_k = pmd_offset(pud_k, address);
 	if (!pmd_present(*pmd_k))
 		return NULL;
 	if (!pmd_present(*pmd))
 		set_pmd(pmd, *pmd_k);
 	else
 		BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
 	return pmd_k;
 }
 void vmalloc_sync_all(void)
 {
 	unsigned long address;
 	if (SHARED_KERNEL_PMD)
 		return;
 	for (address = VMALLOC_START & PMD_MASK;
 	     address >= TASK_SIZE && address < FIXADDR_TOP;
 	     address += PMD_SIZE) {
 		struct page *page;
 		spin_lock(&pgd_lock);
 		list_for_each_entry(page, &pgd_list, lru) {
 			spinlock_t *pgt_lock;
 			pmd_t *ret;
 			/* the pgt_lock only for Xen */
 			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
 			spin_lock(pgt_lock);
 			ret = vmalloc_sync_one(page_address(page), address);
 			spin_unlock(pgt_lock);
 			if (!ret)
 				break;
 		}
 		spin_unlock(&pgd_lock);
 	}
 }
 /*
  * 32-bit:
  *
  *   Handle a fault on the vmalloc or module mapping area
  */
 static noinline __kprobes int vmalloc_fault(unsigned long address)
 {
 	unsigned long pgd_paddr;
 	pmd_t *pmd_k;
 	pte_t *pte_k;
 	/* Make sure we are in vmalloc area: */
 	if (!(address >= VMALLOC_START && address < VMALLOC_END))
 		return -1;
 	WARN_ON_ONCE(in_nmi());
 	/*
 	 * Synchronize this task's top level page-table
 	 * with the 'reference' page table.
 	 *
 	 * Do _not_ use "current" here. We might be inside
 	 * an interrupt in the middle of a task switch..
 	 */
 	pgd_paddr = read_cr3();
 	pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
 	if (!pmd_k)
 		return -1;
 	pte_k = pte_offset_kernel(pmd_k, address);
 	if (!pte_present(*pte_k))
 		return -1;
 	return 0;
 }
 /*
  * Did it hit the DOS screen memory VA from vm86 mode?
  */
 static inline void
 check_v8086_mode(struct pt_regs *regs, unsigned long address,
 		 struct task_struct *tsk)
 {
 	unsigned long bit;
 	if (!v8086_mode(regs))
 		return;
 	bit = (address - 0xA0000) >> PAGE_SHIFT;
 	if (bit < 32)
 		tsk->thread.screen_bitmap |= 1 << bit;
 }
 static bool low_pfn(unsigned long pfn)
 {
 	return pfn < max_low_pfn;
 }
 static void dump_pagetable(unsigned long address)
 {
 	pgd_t *base = __va(read_cr3());
 	pgd_t *pgd = &base[pgd_index(address)];
 	pmd_t *pmd;
 	pte_t *pte;
 #ifdef CONFIG_X86_PAE
 	printk("*pdpt = %016Lx ", pgd_val(*pgd));
 	if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
 		goto out;
 #endif
 	pmd = pmd_offset(pud_offset(pgd, address), address);
 	printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
 	/*
 	 * We must not directly access the pte in the highpte
 	 * case if the page table is located in highmem.
 	 * And let's rather not kmap-atomic the pte, just in case
 	 * it's allocated already:
 	 */
 	if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
 		goto out;
 	pte = pte_offset_kernel(pmd, address);
 	printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
 out:
 	printk("\n");
 }
 #else /* CONFIG_X86_64: */
 void vmalloc_sync_all(void)
 {
 	sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
 }
 /*
  * 64-bit:
  *
  *   Handle a fault on the vmalloc area
  *
  * This assumes no large pages in there.
  */
 static noinline __kprobes int vmalloc_fault(unsigned long address)
 {
 	pgd_t *pgd, *pgd_ref;
 	pud_t *pud, *pud_ref;
 	pmd_t *pmd, *pmd_ref;
 	pte_t *pte, *pte_ref;
 	/* Make sure we are in vmalloc area: */
 	if (!(address >= VMALLOC_START && address < VMALLOC_END))
 		return -1;
 	WARN_ON_ONCE(in_nmi());
 	/*
 	 * Copy kernel mappings over when needed. This can also
 	 * happen within a race in page table update. In the later
 	 * case just flush:
 	 */
 	pgd = pgd_offset(current->active_mm, address);
 	pgd_ref = pgd_offset_k(address);
 	if (pgd_none(*pgd_ref))
 		return -1;
 	if (pgd_none(*pgd))
 		set_pgd(pgd, *pgd_ref);
 	else
 		BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
 	/*
 	 * Below here mismatches are bugs because these lower tables
 	 * are shared:
 	 */
 	pud = pud_offset(pgd, address);
 	pud_ref = pud_offset(pgd_ref, address);
 	if (pud_none(*pud_ref))
 		return -1;
 	if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
 		BUG();
 	pmd = pmd_offset(pud, address);
 	pmd_ref = pmd_offset(pud_ref, address);
 	if (pmd_none(*pmd_ref))
 		return -1;
 	if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
 		BUG();
 	pte_ref = pte_offset_kernel(pmd_ref, address);
 	if (!pte_present(*pte_ref))
 		return -1;
 	pte = pte_offset_kernel(pmd, address);
 	/*
 	 * Don't use pte_page here, because the mappings can point
 	 * outside mem_map, and the NUMA hash lookup cannot handle
 	 * that:
 	 */
 	if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
 		BUG();
 	return 0;
 }
 #ifdef CONFIG_CPU_SUP_AMD
 static const char errata93_warning[] =
 KERN_ERR
 "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
 "******* Working around it, but it may cause SEGVs or burn power.\n"
 "******* Please consider a BIOS update.\n"
 "******* Disabling USB legacy in the BIOS may also help.\n";
 #endif
 /*
  * No vm86 mode in 64-bit mode:
  */
 static inline void
 check_v8086_mode(struct pt_regs *regs, unsigned long address,
 		 struct task_struct *tsk)
 {
 }
 static int bad_address(void *p)
 {
 	unsigned long dummy;
 	return probe_kernel_address((unsigned long *)p, dummy);
 }
 static void dump_pagetable(unsigned long address)
 {
 	pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
 	pgd_t *pgd = base + pgd_index(address);
 	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 	if (bad_address(pgd))
 		goto bad;
 	printk("PGD %lx ", pgd_val(*pgd));
 	if (!pgd_present(*pgd))
 		goto out;
 	pud = pud_offset(pgd, address);
 	if (bad_address(pud))
 		goto bad;
 	printk("PUD %lx ", pud_val(*pud));
 	if (!pud_present(*pud) || pud_large(*pud))
 		goto out;
 	pmd = pmd_offset(pud, address);
 	if (bad_address(pmd))
 		goto bad;
 	printk("PMD %lx ", pmd_val(*pmd));
 	if (!pmd_present(*pmd) || pmd_large(*pmd))
 		goto out;
 	pte = pte_offset_kernel(pmd, address);
 	if (bad_address(pte))
 		goto bad;
 	printk("PTE %lx", pte_val(*pte));
 out:
 	printk("\n");
 	return;
 bad:
 	printk("BAD\n");
 }
 #endif /* CONFIG_X86_64 */
 /*
  * Workaround for K8 erratum #93 & buggy BIOS.
  *
  * BIOS SMM functions are required to use a specific workaround
  * to avoid corruption of the 64bit RIP register on C stepping K8.
  *
  * A lot of BIOS that didn't get tested properly miss this.
  *
  * The OS sees this as a page fault with the upper 32bits of RIP cleared.
  * Try to work around it here.
  *
  * Note we only handle faults in kernel here.
  * Does nothing on 32-bit.
  */
 static int is_errata93(struct pt_regs *regs, unsigned long address)
 {
 #if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD)
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD
 	    || boot_cpu_data.x86 != 0xf)
 		return 0;
 	if (address != regs->ip)
 		return 0;
 	if ((address >> 32) != 0)
 		return 0;
 	address |= 0xffffffffUL << 32;
 	if ((address >= (u64)_stext && address <= (u64)_etext) ||
 	    (address >= MODULES_VADDR && address <= MODULES_END)) {
 		printk_once(errata93_warning);
 		regs->ip = address;
 		return 1;
 	}
 #endif
 	return 0;
 }
 /*
  * Work around K8 erratum #100 K8 in compat mode occasionally jumps
  * to illegal addresses >4GB.
  *
  * We catch this in the page fault handler because these addresses
  * are not reachable. Just detect this case and return.  Any code
  * segment in LDT is compatibility mode.
  */
 static int is_errata100(struct pt_regs *regs, unsigned long address)
 {
 #ifdef CONFIG_X86_64
 	if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
 		return 1;
 #endif
 	return 0;
 }
 static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
 {
 #ifdef CONFIG_X86_F00F_BUG
 	unsigned long nr;
 	/*
 	 * Pentium F0 0F C7 C8 bug workaround:
 	 */
 	if (boot_cpu_data.f00f_bug) {
 		nr = (address - idt_descr.address) >> 3;
 		if (nr == 6) {
 			do_invalid_op(regs, 0);
 			return 1;
 		}
 	}
 #endif
 	return 0;
 }
 static const char nx_warning[] = KERN_CRIT
 "kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
 static void
 show_fault_oops(struct pt_regs *regs, unsigned long error_code,
 		unsigned long address)
 {
 	if (!oops_may_print())
 		return;
 	if (error_code & PF_INSTR) {
 		unsigned int level;
 		pte_t *pte = lookup_address(address, &level);
 		if (pte && pte_present(*pte) && !pte_exec(*pte))
 			printk(nx_warning, from_kuid(&init_user_ns, current_uid()));
 	}
 	printk(KERN_ALERT "BUG: unable to handle kernel ");
 	if (address < PAGE_SIZE)
 		printk(KERN_CONT "NULL pointer dereference");
 	else
 		printk(KERN_CONT "paging request");
 	printk(KERN_CONT " at %p\n", (void *) address);
 	printk(KERN_ALERT "IP:");
 	printk_address(regs->ip, 1);
 	dump_pagetable(address);
 }
 static noinline void
 pgtable_bad(struct pt_regs *regs, unsigned long error_code,
 	    unsigned long address)
 {
 	struct task_struct *tsk;
 	unsigned long flags;
 	int sig;
 	flags = oops_begin();
 	tsk = current;
 	sig = SIGKILL;
 	printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
 	       tsk->comm, address);
 	dump_pagetable(address);
 	tsk->thread.cr2		= address;
 	tsk->thread.trap_nr	= X86_TRAP_PF;
 	tsk->thread.error_code	= error_code;
 	if (__die("Bad pagetable", regs, error_code))
 		sig = 0;
 	oops_end(flags, regs, sig);
 }
 static noinline void
 no_context(struct pt_regs *regs, unsigned long error_code,
 	   unsigned long address, int signal, int si_code)
 {
 	struct task_struct *tsk = current;
 	unsigned long *stackend;
 	unsigned long flags;
 	int sig;
 	/* Are we prepared to handle this kernel fault? */
 	if (fixup_exception(regs)) {
 		if (current_thread_info()->sig_on_uaccess_error && signal) {
 			tsk->thread.trap_nr = X86_TRAP_PF;
 			tsk->thread.error_code = error_code | PF_USER;
 			tsk->thread.cr2 = address;
 			/* XXX: hwpoison faults will set the wrong code. */
 			force_sig_info_fault(signal, si_code, address, tsk, 0);
 		}
 		return;
 	}
 	/*
 	 * 32-bit:
 	 *
 	 *   Valid to do another page fault here, because if this fault
 	 *   had been triggered by is_prefetch fixup_exception would have
 	 *   handled it.
 	 *
 	 * 64-bit:
 	 *
 	 *   Hall of shame of CPU/BIOS bugs.
 	 */
 	if (is_prefetch(regs, error_code, address))
 		return;
 	if (is_errata93(regs, address))
 		return;
 	/*
 	 * Oops. The kernel tried to access some bad page. We'll have to
 	 * terminate things with extreme prejudice:
 	 */
 	flags = oops_begin();
 	show_fault_oops(regs, error_code, address);
 	stackend = end_of_stack(tsk);
 	if (tsk != &init_task && *stackend != STACK_END_MAGIC)
 		printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
 	tsk->thread.cr2		= address;
 	tsk->thread.trap_nr	= X86_TRAP_PF;
 	tsk->thread.error_code	= error_code;
 	sig = SIGKILL;
 	if (__die("Oops", regs, error_code))
 		sig = 0;
 	/* Executive summary in case the body of the oops scrolled away */
 	printk(KERN_DEFAULT "CR2: %016lx\n", address);
 	oops_end(flags, regs, sig);
 }
 /*
  * Print out info about fatal segfaults, if the show_unhandled_signals
  * sysctl is set:
  */
 static inline void
 show_signal_msg(struct pt_regs *regs, unsigned long error_code,
 		unsigned long address, struct task_struct *tsk)
 {
 	if (!unhandled_signal(tsk, SIGSEGV))
 		return;
 	if (!printk_ratelimit())
 		return;
 	printk("%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
 		task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
 		tsk->comm, task_pid_nr(tsk), address,
 		(void *)regs->ip, (void *)regs->sp, error_code);
 	print_vma_addr(KERN_CONT " in ", regs->ip);
 	printk(KERN_CONT "\n");
 }
 static void
 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 		       unsigned long address, int si_code)
 {
 	struct task_struct *tsk = current;
 	/* User mode accesses just cause a SIGSEGV */
 	if (error_code & PF_USER) {
 		/*
 		 * It's possible to have interrupts off here:
 		 */
 		local_irq_enable();
 		/*
 		 * Valid to do another page fault here because this one came
 		 * from user space:
 		 */
 		if (is_prefetch(regs, error_code, address))
 			return;
 		if (is_errata100(regs, address))
 			return;
 #ifdef CONFIG_X86_64
 		/*
 		 * Instruction fetch faults in the vsyscall page might need
 		 * emulation.
 		 */
 		if (unlikely((error_code & PF_INSTR) &&
 			     ((address & ~0xfff) == VSYSCALL_START))) {
 			if (emulate_vsyscall(regs, address))
 				return;
 		}
 #endif
 		/* Kernel addresses are always protection faults: */
 		if (address >= TASK_SIZE)
 			error_code |= PF_PROT;
 		if (likely(show_unhandled_signals))
 			show_signal_msg(regs, error_code, address, tsk);
 		tsk->thread.cr2		= address;
 		tsk->thread.error_code	= error_code;
 		tsk->thread.trap_nr	= X86_TRAP_PF;
 		force_sig_info_fault(SIGSEGV, si_code, address, tsk, 0);
 		return;
 	}
 	if (is_f00f_bug(regs, address))
 		return;
 	no_context(regs, error_code, address, SIGSEGV, si_code);
 }
 static noinline void
 bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 		     unsigned long address)
 {
 	__bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
 }
 static void
 __bad_area(struct pt_regs *regs, unsigned long error_code,
 	   unsigned long address, int si_code)
 {
 	struct mm_struct *mm = current->mm;
 	/*
 	 * Something tried to access memory that isn't in our memory map..
 	 * Fix it, but check if it's kernel or user first..
 	 */
 	up_read(&mm->mmap_sem);
 	__bad_area_nosemaphore(regs, error_code, address, si_code);
 }
 static noinline void
 bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
 {
 	__bad_area(regs, error_code, address, SEGV_MAPERR);
 }
 static noinline void
 bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
 		      unsigned long address)
 {
 	__bad_area(regs, error_code, address, SEGV_ACCERR);
 }
 static void
 do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
 	  unsigned int fault)
 {
 	struct task_struct *tsk = current;
 	struct mm_struct *mm = tsk->mm;
 	int code = BUS_ADRERR;
 	up_read(&mm->mmap_sem);
 	/* Kernel mode? Handle exceptions or die: */
 	if (!(error_code & PF_USER)) {
 		no_context(regs, error_code, address, SIGBUS, BUS_ADRERR);
 		return;
 	}
 	/* User-space => ok to do another page fault: */
 	if (is_prefetch(regs, error_code, address))
 		return;
 	tsk->thread.cr2		= address;
 	tsk->thread.error_code	= error_code;
 	tsk->thread.trap_nr	= X86_TRAP_PF;
 #ifdef CONFIG_MEMORY_FAILURE
 	if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
 		printk(KERN_ERR
 	"MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
 			tsk->comm, tsk->pid, address);
 		code = BUS_MCEERR_AR;
 	}
 #endif
 	force_sig_info_fault(SIGBUS, code, address, tsk, fault);
 }
 static noinline int
 mm_fault_error(struct pt_regs *regs, unsigned long error_code,
 	       unsigned long address, unsigned int fault)
 {
 	/*
 	 * Pagefault was interrupted by SIGKILL. We have no reason to
 	 * continue pagefault.
 	 */
 	if (fatal_signal_pending(current)) {
 		if (!(fault & VM_FAULT_RETRY))
 			up_read(&current->mm->mmap_sem);
 		if (!(error_code & PF_USER))
 			no_context(regs, error_code, address, 0, 0);
 		return 1;
 	}
 	if (!(fault & VM_FAULT_ERROR))
 		return 0;
 	if (fault & VM_FAULT_OOM) {
 		/* Kernel mode? Handle exceptions or die: */
 		if (!(error_code & PF_USER)) {
 			up_read(&current->mm->mmap_sem);
 			no_context(regs, error_code, address,
 				   SIGSEGV, SEGV_MAPERR);
 			return 1;
 		}
 		up_read(&current->mm->mmap_sem);
 		/*
 		 * We ran out of memory, call the OOM killer, and return the
 		 * userspace (which will retry the fault, or kill us if we got
 		 * oom-killed):
 		 */
 		pagefault_out_of_memory();
 	} else {
 		if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
 			     VM_FAULT_HWPOISON_LARGE))
 			do_sigbus(regs, error_code, address, fault);
 		else
 			BUG();
 	}
 	return 1;
 }
 static int spurious_fault_check(unsigned long error_code, pte_t *pte)
 {
 	if ((error_code & PF_WRITE) && !pte_write(*pte))
 		return 0;
 	if ((error_code & PF_INSTR) && !pte_exec(*pte))
 		return 0;
 	return 1;
 }
 /*
  * Handle a spurious fault caused by a stale TLB entry.
  *
  * This allows us to lazily refresh the TLB when increasing the
  * permissions of a kernel page (RO -> RW or NX -> X).  Doing it
  * eagerly is very expensive since that implies doing a full
  * cross-processor TLB flush, even if no stale TLB entries exist
  * on other processors.
  *
  * There are no security implications to leaving a stale TLB when
  * increasing the permissions on a page.
  */
 static noinline __kprobes int
 spurious_fault(unsigned long error_code, unsigned long address)
 {
 	pgd_t *pgd;
 	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 	int ret;
 	/* Reserved-bit violation or user access to kernel space? */
 	if (error_code & (PF_USER | PF_RSVD))
 		return 0;
 	pgd = init_mm.pgd + pgd_index(address);
 	if (!pgd_present(*pgd))
 		return 0;
 	pud = pud_offset(pgd, address);
 	if (!pud_present(*pud))
 		return 0;
 	if (pud_large(*pud))
 		return spurious_fault_check(error_code, (pte_t *) pud);
 	pmd = pmd_offset(pud, address);
 	if (!pmd_present(*pmd))
 		return 0;
 	if (pmd_large(*pmd))
 		return spurious_fault_check(error_code, (pte_t *) pmd);
 	pte = pte_offset_kernel(pmd, address);
 	if (!pte_present(*pte))
 		return 0;
 	ret = spurious_fault_check(error_code, pte);
 	if (!ret)
 		return 0;
 	/*
 	 * Make sure we have permissions in PMD.
 	 * If not, then there's a bug in the page tables:
 	 */
 	ret = spurious_fault_check(error_code, (pte_t *) pmd);
 	WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
 	return ret;
 }
 int show_unhandled_signals = 1;
 static inline int
 access_error(unsigned long error_code, struct vm_area_struct *vma)
 {
 	if (error_code & PF_WRITE) {
 		/* write, present and write, not present: */
 		if (unlikely(!(vma->vm_flags & VM_WRITE)))
 			return 1;
 		return 0;
 	}
 	/* read, present: */
 	if (unlikely(error_code & PF_PROT))
 		return 1;
 	/* read, not present: */
 	if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
 		return 1;
 	return 0;
 }
 static int fault_in_kernel_space(unsigned long address)
 {
 	return address >= TASK_SIZE_MAX;
 }
 static inline bool smap_violation(int error_code, struct pt_regs *regs)
 {
 	if (error_code & PF_USER)
 		return false;
 	if (!user_mode_vm(regs) && (regs->flags & X86_EFLAGS_AC))
 		return false;
 	return true;
 }
 /*
  * This routine handles page faults.  It determines the address,
  * and the problem, and then passes it off to one of the appropriate
  * routines.
  */
 static void __kprobes
 __do_page_fault(struct pt_regs *regs, unsigned long error_code)
 {
 	struct vm_area_struct *vma;
 	struct task_struct *tsk;
 	unsigned long address;
 	struct mm_struct *mm;
 	int fault;
 	int write = error_code & PF_WRITE;
 	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
 					(write ? FAULT_FLAG_WRITE : 0);
 	tsk = current;
 	mm = tsk->mm;
 	/* Get the faulting address: */
 	address = read_cr2();
 	/*
 	 * Detect and handle instructions that would cause a page fault for
 	 * both a tracked kernel page and a userspace page.
 	 */
 	if (kmemcheck_active(regs))
 		kmemcheck_hide(regs);
 	prefetchw(&mm->mmap_sem);
 	if (unlikely(kmmio_fault(regs, address)))
 		return;
 	/*
 	 * We fault-in kernel-space virtual memory on-demand. The
 	 * 'reference' page table is init_mm.pgd.
 	 *
 	 * NOTE! We MUST NOT take any locks for this case. We may
 	 * be in an interrupt or a critical region, and should
 	 * only copy the information from the master page table,
 	 * nothing more.
 	 *
 	 * This verifies that the fault happens in kernel space
 	 * (error_code & 4) == 0, and that the fault was not a
 	 * protection error (error_code & 9) == 0.
 	 */
 	if (unlikely(fault_in_kernel_space(address))) {
 		if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) {
 			if (vmalloc_fault(address) >= 0)
 				return;
 			if (kmemcheck_fault(regs, address, error_code))
 				return;
 		}
 		/* Can handle a stale RO->RW TLB: */
 		if (spurious_fault(error_code, address))
 			return;
 		/* kprobes don't want to hook the spurious faults: */
 		if (notify_page_fault(regs))
 			return;
 		/*
 		 * Don't take the mm semaphore here. If we fixup a prefetch
 		 * fault we could otherwise deadlock:
 		 */
 		bad_area_nosemaphore(regs, error_code, address);
 		return;
 	}
 	/* kprobes don't want to hook the spurious faults: */
 	if (unlikely(notify_page_fault(regs)))
 		return;
 	/*
 	 * It's safe to allow irq's after cr2 has been saved and the
 	 * vmalloc fault has been handled.
 	 *
 	 * User-mode registers count as a user access even for any
 	 * potential system fault or CPU buglet:
 	 */
 	if (user_mode_vm(regs)) {
 		local_irq_enable();
 		error_code |= PF_USER;
 	} else {
 		if (regs->flags & X86_EFLAGS_IF)
 			local_irq_enable();
 	}
 	if (unlikely(error_code & PF_RSVD))
 		pgtable_bad(regs, error_code, address);
 	if (static_cpu_has(X86_FEATURE_SMAP)) {
 		if (unlikely(smap_violation(error_code, regs))) {
 			bad_area_nosemaphore(regs, error_code, address);
 			return;
 		}
 	}
 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 	/*
 	 * If we're in an interrupt, have no user context or are running
 	 * in an atomic region then we must not take the fault:
 	 */
 	if (unlikely(in_atomic() || !mm)) {
 		bad_area_nosemaphore(regs, error_code, address);
 		return;
 	}
 	/*
 	 * When running in the kernel we expect faults to occur only to
 	 * addresses in user space.  All other faults represent errors in
 	 * the kernel and should generate an OOPS.  Unfortunately, in the
 	 * case of an erroneous fault occurring in a code path which already
 	 * holds mmap_sem we will deadlock attempting to validate the fault
 	 * against the address space.  Luckily the kernel only validly
 	 * references user space from well defined areas of code, which are
 	 * listed in the exceptions table.
 	 *
 	 * As the vast majority of faults will be valid we will only perform
 	 * the source reference check when there is a possibility of a
 	 * deadlock. Attempt to lock the address space, if we cannot we then
 	 * validate the source. If this is invalid we can skip the address
 	 * space check, thus avoiding the deadlock:
 	 */
 	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
 		if ((error_code & PF_USER) == 0 &&
 		    !search_exception_tables(regs->ip)) {
 			bad_area_nosemaphore(regs, error_code, address);
 			return;
 		}
 retry:
 		down_read(&mm->mmap_sem);
 	} else {
 		/*
 		 * The above down_read_trylock() might have succeeded in
 		 * which case we'll have missed the might_sleep() from
 		 * down_read():
 		 */
 		might_sleep();
 	}
 	vma = find_vma(mm, address);
 	if (unlikely(!vma)) {
 		bad_area(regs, error_code, address);
 		return;
 	}
 	if (likely(vma->vm_start <= address))
 		goto good_area;
 	if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
 		bad_area(regs, error_code, address);
 		return;
 	}
 	if (error_code & PF_USER) {
 		/*
 		 * Accessing the stack below %sp is always a bug.
 		 * The large cushion allows instructions like enter
 		 * and pusha to work. ("enter $65535, $31" pushes
 		 * 32 pointers and then decrements %sp by 65535.)
 		 */
 		if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
 			bad_area(regs, error_code, address);
 			return;
 		}
 	}
 	if (unlikely(expand_stack(vma, address))) {
 		bad_area(regs, error_code, address);
 		return;
 	}
 	/*
 	 * Ok, we have a good vm_area for this memory access, so
 	 * we can handle it..
 	 */
 good_area:
 	if (unlikely(access_error(error_code, vma))) {
 		bad_area_access_error(regs, error_code, address);
 		return;
 	}
 	/*
 	 * If for any reason at all we couldn't handle the fault,
 	 * make sure we exit gracefully rather than endlessly redo
 	 * the fault:
 	 */
 	fault = handle_mm_fault(mm, vma, address, flags);
 	if (unlikely(fault & (VM_FAULT_RETRY|VM_FAULT_ERROR))) {
 		if (mm_fault_error(regs, error_code, address, fault))
 			return;
 	}
 	/*
 	 * Major/minor page fault accounting is only done on the
 	 * initial attempt. If we go through a retry, it is extremely
 	 * likely that the page will be found in page cache at that point.
 	 */
 	if (flags & FAULT_FLAG_ALLOW_RETRY) {
 		if (fault & VM_FAULT_MAJOR) {
 			tsk->maj_flt++;
 			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
 				      regs, address);
 		} else {
 			tsk->min_flt++;
 			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
 				      regs, address);
 		}
 		if (fault & VM_FAULT_RETRY) {
 			/* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
 			 * of starvation. */
 			flags &= ~FAULT_FLAG_ALLOW_RETRY;
 			flags |= FAULT_FLAG_TRIED;
 			goto retry;
 		}
 	}
 	check_v8086_mode(regs, address, tsk);
 	up_read(&mm->mmap_sem);
 }
 dotraplinkage void __kprobes
 do_page_fault(struct pt_regs *regs, unsigned long error_code)
 {
 	exception_enter(regs);
 	__do_page_fault(regs, error_code);
 	exception_exit(regs);
 }

include/linux/context_tracking.h

Diff comments View file @ 56dd947

 #ifndef _LINUX_CONTEXT_TRACKING_H
 #define _LINUX_CONTEXT_TRACKING_H
-#ifdef CONFIG_CONTEXT_TRACKING
 #include <linux/sched.h>
 #include <linux/percpu.h>
+#include <asm/ptrace.h>
+#ifdef CONFIG_CONTEXT_TRACKING
 struct context_tracking {
 	/*
 	 * When active is false, probes are unset in order
 	 * to minimize overhead: TIF flags are cleared
 	 * and calls to user_enter/exit are ignored. This
 	 * may be further optimized using static keys.
 	 */
 	bool active;
 	enum {
 		IN_KERNEL = 0,
 		IN_USER,
 	} state;
 };
 DECLARE_PER_CPU(struct context_tracking, context_tracking);
 static inline bool context_tracking_in_user(void)
 {
 	return __this_cpu_read(context_tracking.state) == IN_USER;
 }
 static inline bool context_tracking_active(void)
 {
 	return __this_cpu_read(context_tracking.active);
 }
 extern void user_enter(void);
 extern void user_exit(void);
+static inline void exception_enter(struct pt_regs *regs)
+{
+	user_exit();
+}
+static inline void exception_exit(struct pt_regs *regs)
+{
+	if (user_mode(regs))
+		user_enter();
+}
 extern void context_tracking_task_switch(struct task_struct *prev,
 					 struct task_struct *next);
 #else
 static inline bool context_tracking_in_user(void) { return false; }
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void exception_enter(struct pt_regs *regs) { }
+static inline void exception_exit(struct pt_regs *regs) { }
 static inline void context_tracking_task_switch(struct task_struct *prev,
 						struct task_struct *next) { }
 #endif /* !CONFIG_CONTEXT_TRACKING */
 #endif