Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (39 commits) perf tools: Fix compile error on x86_64 Ubuntu perf report: Fix --stdio output alignment when --showcpuutilization used perf annotate: Get rid of field_sep check perf annotate: Fix usage string perf kmem: Fix a memory leak perf kmem: Add missing closedir() calls perf top: Add error message for EMFILE perf test: Change type of '-v' option to INCR perf script: Add missing closedir() calls tracing: Fix compile error when static ftrace is enabled recordmcount: Fix handling of elf64 big-endian objects. perf tools: Add const.h to MANIFEST to make perf-tar-src-pkg work again perf tools: Add support for guest/host-only profiling perf kvm: Do guest-only counting by default perf top: Don't update total_period on process_sample perf hists: Stop using 'self' for struct hist_entry perf hists: Rename total_session to total_period x86: Add counter when debug stack is used with interrupts enabled x86: Allow NMIs to hit breakpoints in i386 x86: Keep current stack in NMI breakpoints ...

Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (39 commits) perf tools: Fix compile error on x86_64 Ubuntu perf report: Fix --stdio output alignment when --showcpuutilization used perf annotate: Get rid of field_sep check perf annotate: Fix usage string perf kmem: Fix a memory leak perf kmem: Add missing closedir() calls perf top: Add error message for EMFILE perf test: Change type of '-v' option to INCR perf script: Add missing closedir() calls tracing: Fix compile error when static ftrace is enabled recordmcount: Fix handling of elf64 big-endian objects. perf tools: Add const.h to MANIFEST to make perf-tar-src-pkg work again perf tools: Add support for guest/host-only profiling perf kvm: Do guest-only counting by default perf top: Don't update total_period on process_sample perf hists: Stop using 'self' for struct hist_entry perf hists: Rename total_session to total_period x86: Add counter when debug stack is used with interrupts enabled x86: Allow NMIs to hit breakpoints in i386 x86: Keep current stack in NMI breakpoints ...
Linus Torvalds
2 parents f0ed5b9a28 172d1b0b73
Showing 29 changed files Side-by-side Diff
Documentation/kernel-parameters.txt
arch/x86/include/asm/debugreg.h
arch/x86/include/asm/desc.h
arch/x86/kernel/cpu/common.c
arch/x86/kernel/entry_64.S
arch/x86/kernel/head_64.S
arch/x86/kernel/nmi.c
arch/x86/kernel/traps.c
include/linux/compiler-gcc.h
include/linux/ftrace.h
kernel/trace/ftrace.c
kernel/trace/trace_events_filter.c
kernel/trace/trace_stack.c
scripts/recordmcount.h
tools/perf/Documentation/perf-list.txt
tools/perf/MANIFEST
tools/perf/builtin-annotate.c
tools/perf/builtin-kmem.c
tools/perf/builtin-kvm.c
tools/perf/builtin-script.c
@@ -2475,6 +2475,14 @@
 	stacktrace	[FTRACE]
 			Enabled the stack tracer on boot up.
  
+	stacktrace_filter=[function-list]
+			[FTRACE] Limit the functions that the stack tracer
+			will trace at boot up. function-list is a comma separated
+			list of functions. This list can be changed at run
+			time by the stack_trace_filter file in the debugfs
+			tracing directory. Note, this enables stack tracing
+			and the stacktrace above is not needed.
+
 	sti=		[PARISC,HW]
 			Format: <num>
 			Set the STI (builtin display/keyboard on the HP-PARISC
@@ -101,6 +101,28 @@
  
 extern void hw_breakpoint_restore(void);
  
+#ifdef CONFIG_X86_64
+DECLARE_PER_CPU(int, debug_stack_usage);
+static inline void debug_stack_usage_inc(void)
+{
+	__get_cpu_var(debug_stack_usage)++;
+}
+static inline void debug_stack_usage_dec(void)
+{
+	__get_cpu_var(debug_stack_usage)--;
+}
+int is_debug_stack(unsigned long addr);
+void debug_stack_set_zero(void);
+void debug_stack_reset(void);
+#else /* !X86_64 */
+static inline int is_debug_stack(unsigned long addr) { return 0; }
+static inline void debug_stack_set_zero(void) { }
+static inline void debug_stack_reset(void) { }
+static inline void debug_stack_usage_inc(void) { }
+static inline void debug_stack_usage_dec(void) { }
+#endif /* X86_64 */
+
+
 #endif	/* __KERNEL__ */
  
 #endif /* _ASM_X86_DEBUGREG_H */
@@ -35,6 +35,8 @@
  
 extern struct desc_ptr idt_descr;
 extern gate_desc idt_table[];
+extern struct desc_ptr nmi_idt_descr;
+extern gate_desc nmi_idt_table[];
  
 struct gdt_page {
 	struct desc_struct gdt[GDT_ENTRIES];
@@ -306,6 +308,16 @@
 	desc->limit0 = limit & 0xffff;
 	desc->limit = (limit >> 16) & 0xf;
 }
+
+#ifdef CONFIG_X86_64
+static inline void set_nmi_gate(int gate, void *addr)
+{
+	gate_desc s;
+
+	pack_gate(&s, GATE_INTERRUPT, (unsigned long)addr, 0, 0, __KERNEL_CS);
+	write_idt_entry(nmi_idt_table, gate, &s);
+}
+#endif
  
 static inline void _set_gate(int gate, unsigned type, void *addr,
 			     unsigned dpl, unsigned ist, unsigned seg)
@@ -1021,6 +1021,8 @@
  
 #ifdef CONFIG_X86_64
 struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
+struct desc_ptr nmi_idt_descr = { NR_VECTORS * 16 - 1,
+				    (unsigned long) nmi_idt_table };
  
 DEFINE_PER_CPU_FIRST(union irq_stack_union,
 		     irq_stack_union) __aligned(PAGE_SIZE);
@@ -1085,6 +1087,26 @@
  */
 DEFINE_PER_CPU(struct orig_ist, orig_ist);
  
+static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
+DEFINE_PER_CPU(int, debug_stack_usage);
+
+int is_debug_stack(unsigned long addr)
+{
+	return __get_cpu_var(debug_stack_usage) ||
+		(addr <= __get_cpu_var(debug_stack_addr) &&
+		 addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ));
+}
+
+void debug_stack_set_zero(void)
+{
+	load_idt((const struct desc_ptr *)&nmi_idt_descr);
+}
+
+void debug_stack_reset(void)
+{
+	load_idt((const struct desc_ptr *)&idt_descr);
+}
+
 #else	/* CONFIG_X86_64 */
  
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
@@ -1212,6 +1234,8 @@
 			estacks += exception_stack_sizes[v];
 			oist->ist[v] = t->x86_tss.ist[v] =
 					(unsigned long)estacks;
+			if (v == DEBUG_STACK-1)
+				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
 		}
 	}
  
@@ -1480,61 +1480,213 @@
 	CFI_ENDPROC
 END(error_exit)
  
+/*
+ * Test if a given stack is an NMI stack or not.
+ */
+	.macro test_in_nmi reg stack nmi_ret normal_ret
+	cmpq %\reg, \stack
+	ja \normal_ret
+	subq $EXCEPTION_STKSZ, %\reg
+	cmpq %\reg, \stack
+	jb \normal_ret
+	jmp \nmi_ret
+	.endm
  
 	/* runs on exception stack */
 ENTRY(nmi)
 	INTR_FRAME
 	PARAVIRT_ADJUST_EXCEPTION_FRAME
-	pushq_cfi $-1
+	/*
+	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
+	 * the iretq it performs will take us out of NMI context.
+	 * This means that we can have nested NMIs where the next
+	 * NMI is using the top of the stack of the previous NMI. We
+	 * can't let it execute because the nested NMI will corrupt the
+	 * stack of the previous NMI. NMI handlers are not re-entrant
+	 * anyway.
+	 *
+	 * To handle this case we do the following:
+	 *  Check the a special location on the stack that contains
+	 *  a variable that is set when NMIs are executing.
+	 *  The interrupted task's stack is also checked to see if it
+	 *  is an NMI stack.
+	 *  If the variable is not set and the stack is not the NMI
+	 *  stack then:
+	 *    o Set the special variable on the stack
+	 *    o Copy the interrupt frame into a "saved" location on the stack
+	 *    o Copy the interrupt frame into a "copy" location on the stack
+	 *    o Continue processing the NMI
+	 *  If the variable is set or the previous stack is the NMI stack:
+	 *    o Modify the "copy" location to jump to the repeate_nmi
+	 *    o return back to the first NMI
+	 *
+	 * Now on exit of the first NMI, we first clear the stack variable
+	 * The NMI stack will tell any nested NMIs at that point that it is
+	 * nested. Then we pop the stack normally with iret, and if there was
+	 * a nested NMI that updated the copy interrupt stack frame, a
+	 * jump will be made to the repeat_nmi code that will handle the second
+	 * NMI.
+	 */
+
+	/* Use %rdx as out temp variable throughout */
+	pushq_cfi %rdx
+
+	/*
+	 * Check the special variable on the stack to see if NMIs are
+	 * executing.
+	 */
+	cmp $1, -8(%rsp)
+	je nested_nmi
+
+	/*
+	 * Now test if the previous stack was an NMI stack.
+	 * We need the double check. We check the NMI stack to satisfy the
+	 * race when the first NMI clears the variable before returning.
+	 * We check the variable because the first NMI could be in a
+	 * breakpoint routine using a breakpoint stack.
+	 */
+	lea 6*8(%rsp), %rdx
+	test_in_nmi rdx, 4*8(%rsp), nested_nmi, first_nmi
+
+nested_nmi:
+	/*
+	 * Do nothing if we interrupted the fixup in repeat_nmi.
+	 * It's about to repeat the NMI handler, so we are fine
+	 * with ignoring this one.
+	 */
+	movq $repeat_nmi, %rdx
+	cmpq 8(%rsp), %rdx
+	ja 1f
+	movq $end_repeat_nmi, %rdx
+	cmpq 8(%rsp), %rdx
+	ja nested_nmi_out
+
+1:
+	/* Set up the interrupted NMIs stack to jump to repeat_nmi */
+	leaq -6*8(%rsp), %rdx
+	movq %rdx, %rsp
+	CFI_ADJUST_CFA_OFFSET 6*8
+	pushq_cfi $__KERNEL_DS
+	pushq_cfi %rdx
+	pushfq_cfi
+	pushq_cfi $__KERNEL_CS
+	pushq_cfi $repeat_nmi
+
+	/* Put stack back */
+	addq $(11*8), %rsp
+	CFI_ADJUST_CFA_OFFSET -11*8
+
+nested_nmi_out:
+	popq_cfi %rdx
+
+	/* No need to check faults here */
+	INTERRUPT_RETURN
+
+first_nmi:
+	/*
+	 * Because nested NMIs will use the pushed location that we
+	 * stored in rdx, we must keep that space available.
+	 * Here's what our stack frame will look like:
+	 * +-------------------------+
+	 * | original SS             |
+	 * | original Return RSP     |
+	 * | original RFLAGS         |
+	 * | original CS             |
+	 * | original RIP            |
+	 * +-------------------------+
+	 * | temp storage for rdx    |
+	 * +-------------------------+
+	 * | NMI executing variable  |
+	 * +-------------------------+
+	 * | Saved SS                |
+	 * | Saved Return RSP        |
+	 * | Saved RFLAGS            |
+	 * | Saved CS                |
+	 * | Saved RIP               |
+	 * +-------------------------+
+	 * | copied SS               |
+	 * | copied Return RSP       |
+	 * | copied RFLAGS           |
+	 * | copied CS               |
+	 * | copied RIP              |
+	 * +-------------------------+
+	 * | pt_regs                 |
+	 * +-------------------------+
+	 *
+	 * The saved RIP is used to fix up the copied RIP that a nested
+	 * NMI may zero out. The original stack frame and the temp storage
+	 * is also used by nested NMIs and can not be trusted on exit.
+	 */
+	/* Set the NMI executing variable on the stack. */
+	pushq_cfi $1
+
+	/* Copy the stack frame to the Saved frame */
+	.rept 5
+	pushq_cfi 6*8(%rsp)
+	.endr
+
+	/* Make another copy, this one may be modified by nested NMIs */
+	.rept 5
+	pushq_cfi 4*8(%rsp)
+	.endr
+
+	/* Do not pop rdx, nested NMIs will corrupt it */
+	movq 11*8(%rsp), %rdx
+
+	/*
+	 * Everything below this point can be preempted by a nested
+	 * NMI if the first NMI took an exception. Repeated NMIs
+	 * caused by an exception and nested NMI will start here, and
+	 * can still be preempted by another NMI.
+	 */
+restart_nmi:
+	pushq_cfi $-1		/* ORIG_RAX: no syscall to restart */
 	subq $ORIG_RAX-R15, %rsp
 	CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
+	/*
+	 * Use save_paranoid to handle SWAPGS, but no need to use paranoid_exit
+	 * as we should not be calling schedule in NMI context.
+	 * Even with normal interrupts enabled. An NMI should not be
+	 * setting NEED_RESCHED or anything that normal interrupts and
+	 * exceptions might do.
+	 */
 	call save_paranoid
 	DEFAULT_FRAME 0
 	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
 	movq %rsp,%rdi
 	movq $-1,%rsi
 	call do_nmi
-#ifdef CONFIG_TRACE_IRQFLAGS
-	/* paranoidexit; without TRACE_IRQS_OFF */
-	/* ebx:	no swapgs flag */
-	DISABLE_INTERRUPTS(CLBR_NONE)
 	testl %ebx,%ebx				/* swapgs needed? */
 	jnz nmi_restore
-	testl $3,CS(%rsp)
-	jnz nmi_userspace
 nmi_swapgs:
 	SWAPGS_UNSAFE_STACK
 nmi_restore:
 	RESTORE_ALL 8
+	/* Clear the NMI executing stack variable */
+	movq $0, 10*8(%rsp)
 	jmp irq_return
-nmi_userspace:
-	GET_THREAD_INFO(%rcx)
-	movl TI_flags(%rcx),%ebx
-	andl $_TIF_WORK_MASK,%ebx
-	jz nmi_swapgs
-	movq %rsp,%rdi			/* &pt_regs */
-	call sync_regs
-	movq %rax,%rsp			/* switch stack for scheduling */
-	testl $_TIF_NEED_RESCHED,%ebx
-	jnz nmi_schedule
-	movl %ebx,%edx			/* arg3: thread flags */
-	ENABLE_INTERRUPTS(CLBR_NONE)
-	xorl %esi,%esi 			/* arg2: oldset */
-	movq %rsp,%rdi 			/* arg1: &pt_regs */
-	call do_notify_resume
-	DISABLE_INTERRUPTS(CLBR_NONE)
-	jmp nmi_userspace
-nmi_schedule:
-	ENABLE_INTERRUPTS(CLBR_ANY)
-	call schedule
-	DISABLE_INTERRUPTS(CLBR_ANY)
-	jmp nmi_userspace
 	CFI_ENDPROC
-#else
-	jmp paranoid_exit
-	CFI_ENDPROC
-#endif
 END(nmi)
+
+	/*
+	 * If an NMI hit an iret because of an exception or breakpoint,
+	 * it can lose its NMI context, and a nested NMI may come in.
+	 * In that case, the nested NMI will change the preempted NMI's
+	 * stack to jump to here when it does the final iret.
+	 */
+repeat_nmi:
+	INTR_FRAME
+	/* Update the stack variable to say we are still in NMI */
+	movq $1, 5*8(%rsp)
+
+	/* copy the saved stack back to copy stack */
+	.rept 5
+	pushq_cfi 4*8(%rsp)
+	.endr
+
+	jmp restart_nmi
+	CFI_ENDPROC
+end_repeat_nmi:
  
 ENTRY(ignore_sysret)
 	CFI_STARTPROC
@@ -417,6 +417,10 @@
 ENTRY(idt_table)
 	.skip IDT_ENTRIES * 16
  
+	.align L1_CACHE_BYTES
+ENTRY(nmi_idt_table)
+	.skip IDT_ENTRIES * 16
+
 	__PAGE_ALIGNED_BSS
 	.align PAGE_SIZE
 ENTRY(empty_zero_page)
@@ -405,9 +405,108 @@
 		unknown_nmi_error(reason, regs);
 }
  
+/*
+ * NMIs can hit breakpoints which will cause it to lose its
+ * NMI context with the CPU when the breakpoint does an iret.
+ */
+#ifdef CONFIG_X86_32
+/*
+ * For i386, NMIs use the same stack as the kernel, and we can
+ * add a workaround to the iret problem in C. Simply have 3 states
+ * the NMI can be in.
+ *
+ *  1) not running
+ *  2) executing
+ *  3) latched
+ *
+ * When no NMI is in progress, it is in the "not running" state.
+ * When an NMI comes in, it goes into the "executing" state.
+ * Normally, if another NMI is triggered, it does not interrupt
+ * the running NMI and the HW will simply latch it so that when
+ * the first NMI finishes, it will restart the second NMI.
+ * (Note, the latch is binary, thus multiple NMIs triggering,
+ *  when one is running, are ignored. Only one NMI is restarted.)
+ *
+ * If an NMI hits a breakpoint that executes an iret, another
+ * NMI can preempt it. We do not want to allow this new NMI
+ * to run, but we want to execute it when the first one finishes.
+ * We set the state to "latched", and the first NMI will perform
+ * an cmpxchg on the state, and if it doesn't successfully
+ * reset the state to "not running" it will restart the next
+ * NMI.
+ */
+enum nmi_states {
+	NMI_NOT_RUNNING,
+	NMI_EXECUTING,
+	NMI_LATCHED,
+};
+static DEFINE_PER_CPU(enum nmi_states, nmi_state);
+
+#define nmi_nesting_preprocess(regs)					\
+	do {								\
+		if (__get_cpu_var(nmi_state) != NMI_NOT_RUNNING) {	\
+			__get_cpu_var(nmi_state) = NMI_LATCHED;		\
+			return;						\
+		}							\
+	nmi_restart:							\
+		__get_cpu_var(nmi_state) = NMI_EXECUTING;		\
+	} while (0)
+
+#define nmi_nesting_postprocess()					\
+	do {								\
+		if (cmpxchg(&__get_cpu_var(nmi_state),			\
+		    NMI_EXECUTING, NMI_NOT_RUNNING) != NMI_EXECUTING)	\
+			goto nmi_restart;				\
+	} while (0)
+#else /* x86_64 */
+/*
+ * In x86_64 things are a bit more difficult. This has the same problem
+ * where an NMI hitting a breakpoint that calls iret will remove the
+ * NMI context, allowing a nested NMI to enter. What makes this more
+ * difficult is that both NMIs and breakpoints have their own stack.
+ * When a new NMI or breakpoint is executed, the stack is set to a fixed
+ * point. If an NMI is nested, it will have its stack set at that same
+ * fixed address that the first NMI had, and will start corrupting the
+ * stack. This is handled in entry_64.S, but the same problem exists with
+ * the breakpoint stack.
+ *
+ * If a breakpoint is being processed, and the debug stack is being used,
+ * if an NMI comes in and also hits a breakpoint, the stack pointer
+ * will be set to the same fixed address as the breakpoint that was
+ * interrupted, causing that stack to be corrupted. To handle this case,
+ * check if the stack that was interrupted is the debug stack, and if
+ * so, change the IDT so that new breakpoints will use the current stack
+ * and not switch to the fixed address. On return of the NMI, switch back
+ * to the original IDT.
+ */
+static DEFINE_PER_CPU(int, update_debug_stack);
+
+static inline void nmi_nesting_preprocess(struct pt_regs *regs)
+{
+	/*
+	 * If we interrupted a breakpoint, it is possible that
+	 * the nmi handler will have breakpoints too. We need to
+	 * change the IDT such that breakpoints that happen here
+	 * continue to use the NMI stack.
+	 */
+	if (unlikely(is_debug_stack(regs->sp))) {
+		debug_stack_set_zero();
+		__get_cpu_var(update_debug_stack) = 1;
+	}
+}
+
+static inline void nmi_nesting_postprocess(void)
+{
+	if (unlikely(__get_cpu_var(update_debug_stack)))
+		debug_stack_reset();
+}
+#endif
+
 dotraplinkage notrace __kprobes void
 do_nmi(struct pt_regs *regs, long error_code)
 {
+	nmi_nesting_preprocess(regs);
+
 	nmi_enter();
  
 	inc_irq_stat(__nmi_count);
@@ -416,6 +515,9 @@
 		default_do_nmi(regs);
  
 	nmi_exit();
+
+	/* On i386, may loop back to preprocess */
+	nmi_nesting_postprocess();
 }
  
 void stop_nmi(void)
@@ -311,9 +311,15 @@
 			== NOTIFY_STOP)
 		return;
  
+	/*
+	 * Let others (NMI) know that the debug stack is in use
+	 * as we may switch to the interrupt stack.
+	 */
+	debug_stack_usage_inc();
 	preempt_conditional_sti(regs);
 	do_trap(3, SIGTRAP, "int3", regs, error_code, NULL);
 	preempt_conditional_cli(regs);
+	debug_stack_usage_dec();
 }
  
 #ifdef CONFIG_X86_64
@@ -406,6 +412,12 @@
 							SIGTRAP) == NOTIFY_STOP)
 		return;
  
+	/*
+	 * Let others (NMI) know that the debug stack is in use
+	 * as we may switch to the interrupt stack.
+	 */
+	debug_stack_usage_inc();
+
 	/* It's safe to allow irq's after DR6 has been saved */
 	preempt_conditional_sti(regs);
  
@@ -413,6 +425,7 @@
 		handle_vm86_trap((struct kernel_vm86_regs *) regs,
 				error_code, 1);
 		preempt_conditional_cli(regs);
+		debug_stack_usage_dec();
 		return;
 	}
  
@@ -432,6 +445,7 @@
 	if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp)
 		send_sigtrap(tsk, regs, error_code, si_code);
 	preempt_conditional_cli(regs);
+	debug_stack_usage_dec();
  
 	return;
 }
@@ -718,5 +732,11 @@
 	cpu_init();
  
 	x86_init.irqs.trap_init();
+
+#ifdef CONFIG_X86_64
+	memcpy(&nmi_idt_table, &idt_table, IDT_ENTRIES * 16);
+	set_nmi_gate(1, &debug);
+	set_nmi_gate(3, &int3);
+#endif
 }
@@ -50,6 +50,11 @@
 # define inline		inline		__attribute__((always_inline))
 # define __inline__	__inline__	__attribute__((always_inline))
 # define __inline	__inline	__attribute__((always_inline))
+#else
+/* A lot of inline functions can cause havoc with function tracing */
+# define inline		inline		notrace
+# define __inline__	__inline__	notrace
+# define __inline	__inline	notrace
 #endif
  
 #define __deprecated			__attribute__((deprecated))
@@ -133,6 +133,8 @@
 int ftrace_arch_code_modify_prepare(void);
 int ftrace_arch_code_modify_post_process(void);
  
+void ftrace_bug(int err, unsigned long ip);
+
 struct seq_file;
  
 struct ftrace_probe_ops {
@@ -161,7 +163,6 @@
  
 enum {
 	FTRACE_FL_ENABLED	= (1 << 30),
-	FTRACE_FL_FREE		= (1 << 31),
 };
  
 #define FTRACE_FL_MASK		(0x3UL << 30)
@@ -172,10 +173,7 @@
 		unsigned long		ip; /* address of mcount call-site */
 		struct dyn_ftrace	*freelist;
 	};
-	union {
-		unsigned long		flags;
-		struct dyn_ftrace	*newlist;
-	};
+	unsigned long		flags;
 	struct dyn_arch_ftrace		arch;
 };
  
@@ -190,6 +188,56 @@
 int register_ftrace_command(struct ftrace_func_command *cmd);
 int unregister_ftrace_command(struct ftrace_func_command *cmd);
  
+enum {
+	FTRACE_UPDATE_CALLS		= (1 << 0),
+	FTRACE_DISABLE_CALLS		= (1 << 1),
+	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
+	FTRACE_START_FUNC_RET		= (1 << 3),
+	FTRACE_STOP_FUNC_RET		= (1 << 4),
+};
+
+enum {
+	FTRACE_UPDATE_IGNORE,
+	FTRACE_UPDATE_MAKE_CALL,
+	FTRACE_UPDATE_MAKE_NOP,
+};
+
+enum {
+	FTRACE_ITER_FILTER	= (1 << 0),
+	FTRACE_ITER_NOTRACE	= (1 << 1),
+	FTRACE_ITER_PRINTALL	= (1 << 2),
+	FTRACE_ITER_DO_HASH	= (1 << 3),
+	FTRACE_ITER_HASH	= (1 << 4),
+	FTRACE_ITER_ENABLED	= (1 << 5),
+};
+
+void arch_ftrace_update_code(int command);
+
+struct ftrace_rec_iter;
+
+struct ftrace_rec_iter *ftrace_rec_iter_start(void);
+struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter);
+struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter);
+
+int ftrace_update_record(struct dyn_ftrace *rec, int enable);
+int ftrace_test_record(struct dyn_ftrace *rec, int enable);
+void ftrace_run_stop_machine(int command);
+int ftrace_location(unsigned long ip);
+
+extern ftrace_func_t ftrace_trace_function;
+
+int ftrace_regex_open(struct ftrace_ops *ops, int flag,
+		  struct inode *inode, struct file *file);
+ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
+			    size_t cnt, loff_t *ppos);
+ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
+			     size_t cnt, loff_t *ppos);
+loff_t ftrace_regex_lseek(struct file *file, loff_t offset, int origin);
+int ftrace_regex_release(struct inode *inode, struct file *file);
+
+void __init
+ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable);
+
 /* defined in arch */
 extern int ftrace_ip_converted(unsigned long ip);
 extern int ftrace_dyn_arch_init(void *data);
@@ -284,6 +332,25 @@
 {
 	return 0;
 }
+
+/*
+ * Again users of functions that have ftrace_ops may not
+ * have them defined when ftrace is not enabled, but these
+ * functions may still be called. Use a macro instead of inline.
+ */
+#define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; })
+#define ftrace_set_early_filter(ops, buf, enable) do { } while (0)
+
+static inline ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
+			    size_t cnt, loff_t *ppos) { return -ENODEV; }
+static inline ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
+			     size_t cnt, loff_t *ppos) { return -ENODEV; }
+static inline loff_t ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
+{
+	return -ENODEV;
+}
+static inline int
+ftrace_regex_release(struct inode *inode, struct file *file) { return -ENODEV; }
 #endif /* CONFIG_DYNAMIC_FTRACE */
  
 /* totally disable ftrace - can not re-enable after this */
@@ -22,11 +22,13 @@
 #include <linux/hardirq.h>
 #include <linux/kthread.h>
 #include <linux/uaccess.h>
+#include <linux/bsearch.h>
 #include <linux/module.h>
 #include <linux/ftrace.h>
 #include <linux/sysctl.h>
 #include <linux/slab.h>
 #include <linux/ctype.h>
+#include <linux/sort.h>
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/rcupdate.h>
@@ -947,13 +949,6 @@
 	struct rcu_head		rcu;
 };
  
-enum {
-	FTRACE_ENABLE_CALLS		= (1 << 0),
-	FTRACE_DISABLE_CALLS		= (1 << 1),
-	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
-	FTRACE_START_FUNC_RET		= (1 << 3),
-	FTRACE_STOP_FUNC_RET		= (1 << 4),
-};
 struct ftrace_func_entry {
 	struct hlist_node hlist;
 	unsigned long ip;
  
  
  
  
  
@@ -984,26 +979,30 @@
 	.filter_hash		= EMPTY_HASH,
 };
  
-static struct dyn_ftrace *ftrace_new_addrs;
-
 static DEFINE_MUTEX(ftrace_regex_lock);
  
 struct ftrace_page {
 	struct ftrace_page	*next;
+	struct dyn_ftrace	*records;
 	int			index;
-	struct dyn_ftrace	records[];
+	int			size;
 };
  
-#define ENTRIES_PER_PAGE \
-  ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
+static struct ftrace_page *ftrace_new_pgs;
  
+#define ENTRY_SIZE sizeof(struct dyn_ftrace)
+#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
+
 /* estimate from running different kernels */
 #define NR_TO_INIT		10000
  
 static struct ftrace_page	*ftrace_pages_start;
 static struct ftrace_page	*ftrace_pages;
  
-static struct dyn_ftrace *ftrace_free_records;
+static bool ftrace_hash_empty(struct ftrace_hash *hash)
+{
+	return !hash || !hash->count;
+}
  
 static struct ftrace_func_entry *
 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
@@ -1013,7 +1012,7 @@
 	struct hlist_head *hhd;
 	struct hlist_node *n;
  
-	if (!hash->count)
+	if (ftrace_hash_empty(hash))
 		return NULL;
  
 	if (hash->size_bits > 0)
@@ -1157,7 +1156,7 @@
 		return NULL;
  
 	/* Empty hash? */
-	if (!hash || !hash->count)
+	if (ftrace_hash_empty(hash))
 		return new_hash;
  
 	size = 1 << hash->size_bits;
  
@@ -1282,9 +1281,9 @@
 	filter_hash = rcu_dereference_raw(ops->filter_hash);
 	notrace_hash = rcu_dereference_raw(ops->notrace_hash);
  
-	if ((!filter_hash || !filter_hash->count ||
+	if ((ftrace_hash_empty(filter_hash) ||
 	     ftrace_lookup_ip(filter_hash, ip)) &&
-	    (!notrace_hash || !notrace_hash->count ||
+	    (ftrace_hash_empty(notrace_hash) ||
 	     !ftrace_lookup_ip(notrace_hash, ip)))
 		ret = 1;
 	else
@@ -1307,6 +1306,47 @@
 		}				\
 	}
  
+
+static int ftrace_cmp_recs(const void *a, const void *b)
+{
+	const struct dyn_ftrace *reca = a;
+	const struct dyn_ftrace *recb = b;
+
+	if (reca->ip > recb->ip)
+		return 1;
+	if (reca->ip < recb->ip)
+		return -1;
+	return 0;
+}
+
+/**
+ * ftrace_location - return true if the ip giving is a traced location
+ * @ip: the instruction pointer to check
+ *
+ * Returns 1 if @ip given is a pointer to a ftrace location.
+ * That is, the instruction that is either a NOP or call to
+ * the function tracer. It checks the ftrace internal tables to
+ * determine if the address belongs or not.
+ */
+int ftrace_location(unsigned long ip)
+{
+	struct ftrace_page *pg;
+	struct dyn_ftrace *rec;
+	struct dyn_ftrace key;
+
+	key.ip = ip;
+
+	for (pg = ftrace_pages_start; pg; pg = pg->next) {
+		rec = bsearch(&key, pg->records, pg->index,
+			      sizeof(struct dyn_ftrace),
+			      ftrace_cmp_recs);
+		if (rec)
+			return 1;
+	}
+
+	return 0;
+}
+
 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
 				     int filter_hash,
 				     bool inc)
@@ -1336,7 +1376,7 @@
 	if (filter_hash) {
 		hash = ops->filter_hash;
 		other_hash = ops->notrace_hash;
-		if (!hash || !hash->count)
+		if (ftrace_hash_empty(hash))
 			all = 1;
 	} else {
 		inc = !inc;
@@ -1346,7 +1386,7 @@
 		 * If the notrace hash has no items,
 		 * then there's nothing to do.
 		 */
-		if (hash && !hash->count)
+		if (ftrace_hash_empty(hash))
 			return;
 	}
  
@@ -1363,8 +1403,8 @@
 			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
 				match = 1;
 		} else {
-			in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip);
-			in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip);
+			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
+			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
  
 			/*
 			 *
@@ -1372,7 +1412,7 @@
 			if (filter_hash && in_hash && !in_other_hash)
 				match = 1;
 			else if (!filter_hash && in_hash &&
-				 (in_other_hash || !other_hash->count))
+				 (in_other_hash || ftrace_hash_empty(other_hash)))
 				match = 1;
 		}
 		if (!match)
  
  
@@ -1406,40 +1446,12 @@
 	__ftrace_hash_rec_update(ops, filter_hash, 1);
 }
  
-static void ftrace_free_rec(struct dyn_ftrace *rec)
-{
-	rec->freelist = ftrace_free_records;
-	ftrace_free_records = rec;
-	rec->flags |= FTRACE_FL_FREE;
-}
-
 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 {
-	struct dyn_ftrace *rec;
-
-	/* First check for freed records */
-	if (ftrace_free_records) {
-		rec = ftrace_free_records;
-
-		if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
-			FTRACE_WARN_ON_ONCE(1);
-			ftrace_free_records = NULL;
+	if (ftrace_pages->index == ftrace_pages->size) {
+		/* We should have allocated enough */
+		if (WARN_ON(!ftrace_pages->next))
 			return NULL;
-		}
-
-		ftrace_free_records = rec->freelist;
-		memset(rec, 0, sizeof(*rec));
-		return rec;
-	}
-
-	if (ftrace_pages->index == ENTRIES_PER_PAGE) {
-		if (!ftrace_pages->next) {
-			/* allocate another page */
-			ftrace_pages->next =
-				(void *)get_zeroed_page(GFP_KERNEL);
-			if (!ftrace_pages->next)
-				return NULL;
-		}
 		ftrace_pages = ftrace_pages->next;
 	}
  
@@ -1459,8 +1471,6 @@
 		return NULL;
  
 	rec->ip = ip;
-	rec->newlist = ftrace_new_addrs;
-	ftrace_new_addrs = rec;
  
 	return rec;
 }
@@ -1475,7 +1485,19 @@
 		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 }
  
-static void ftrace_bug(int failed, unsigned long ip)
+/**
+ * ftrace_bug - report and shutdown function tracer
+ * @failed: The failed type (EFAULT, EINVAL, EPERM)
+ * @ip: The address that failed
+ *
+ * The arch code that enables or disables the function tracing
+ * can call ftrace_bug() when it has detected a problem in
+ * modifying the code. @failed should be one of either:
+ * EFAULT - if the problem happens on reading the @ip address
+ * EINVAL - if what is read at @ip is not what was expected
+ * EPERM - if the problem happens on writting to the @ip address
+ */
+void ftrace_bug(int failed, unsigned long ip)
 {
 	switch (failed) {
 	case -EFAULT:
  
  
  
  
@@ -1517,24 +1539,19 @@
 	return 0;
 }
  
-
-static int
-__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
+static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
 {
-	unsigned long ftrace_addr;
 	unsigned long flag = 0UL;
  
-	ftrace_addr = (unsigned long)FTRACE_ADDR;
-
 	/*
-	 * If we are enabling tracing:
+	 * If we are updating calls:
 	 *
 	 *   If the record has a ref count, then we need to enable it
 	 *   because someone is using it.
 	 *
 	 *   Otherwise we make sure its disabled.
 	 *
-	 * If we are disabling tracing, then disable all records that
+	 * If we are disabling calls, then disable all records that
 	 * are enabled.
 	 */
 	if (enable && (rec->flags & ~FTRACE_FL_MASK))
  
  
  
  
@@ -1542,18 +1559,72 @@
  
 	/* If the state of this record hasn't changed, then do nothing */
 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
-		return 0;
+		return FTRACE_UPDATE_IGNORE;
  
 	if (flag) {
-		rec->flags |= FTRACE_FL_ENABLED;
+		if (update)
+			rec->flags |= FTRACE_FL_ENABLED;
+		return FTRACE_UPDATE_MAKE_CALL;
+	}
+
+	if (update)
+		rec->flags &= ~FTRACE_FL_ENABLED;
+
+	return FTRACE_UPDATE_MAKE_NOP;
+}
+
+/**
+ * ftrace_update_record, set a record that now is tracing or not
+ * @rec: the record to update
+ * @enable: set to 1 if the record is tracing, zero to force disable
+ *
+ * The records that represent all functions that can be traced need
+ * to be updated when tracing has been enabled.
+ */
+int ftrace_update_record(struct dyn_ftrace *rec, int enable)
+{
+	return ftrace_check_record(rec, enable, 1);
+}
+
+/**
+ * ftrace_test_record, check if the record has been enabled or not
+ * @rec: the record to test
+ * @enable: set to 1 to check if enabled, 0 if it is disabled
+ *
+ * The arch code may need to test if a record is already set to
+ * tracing to determine how to modify the function code that it
+ * represents.
+ */
+int ftrace_test_record(struct dyn_ftrace *rec, int enable)
+{
+	return ftrace_check_record(rec, enable, 0);
+}
+
+static int
+__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
+{
+	unsigned long ftrace_addr;
+	int ret;
+
+	ftrace_addr = (unsigned long)FTRACE_ADDR;
+
+	ret = ftrace_update_record(rec, enable);
+
+	switch (ret) {
+	case FTRACE_UPDATE_IGNORE:
+		return 0;
+
+	case FTRACE_UPDATE_MAKE_CALL:
 		return ftrace_make_call(rec, ftrace_addr);
+
+	case FTRACE_UPDATE_MAKE_NOP:
+		return ftrace_make_nop(NULL, rec, ftrace_addr);
 	}
  
-	rec->flags &= ~FTRACE_FL_ENABLED;
-	return ftrace_make_nop(NULL, rec, ftrace_addr);
+	return -1; /* unknow ftrace bug */
 }
  
-static void ftrace_replace_code(int enable)
+static void ftrace_replace_code(int update)
 {
 	struct dyn_ftrace *rec;
 	struct ftrace_page *pg;
@@ -1563,11 +1634,7 @@
 		return;
  
 	do_for_each_ftrace_rec(pg, rec) {
-		/* Skip over free records */
-		if (rec->flags & FTRACE_FL_FREE)
-			continue;
-
-		failed = __ftrace_replace_code(rec, enable);
+		failed = __ftrace_replace_code(rec, update);
 		if (failed) {
...	...	@@ -2475,6 +2475,14 @@
2475	2475	stacktrace [FTRACE]
2476	2476	Enabled the stack tracer on boot up.
2477	2477
	2478	+ stacktrace_filter=[function-list]
	2479	+ [FTRACE] Limit the functions that the stack tracer
	2480	+ will trace at boot up. function-list is a comma separated
	2481	+ list of functions. This list can be changed at run
	2482	+ time by the stack_trace_filter file in the debugfs
	2483	+ tracing directory. Note, this enables stack tracing
	2484	+ and the stacktrace above is not needed.
	2485	+
2478	2486	sti= [PARISC,HW]
2479	2487	Format: <num>
2480	2488	Set the STI (builtin display/keyboard on the HP-PARISC
...	...	@@ -101,6 +101,28 @@
101	101
102	102	extern void hw_breakpoint_restore(void);
103	103
	104	+#ifdef CONFIG_X86_64
	105	+DECLARE_PER_CPU(int, debug_stack_usage);
	106	+static inline void debug_stack_usage_inc(void)
	107	+{
	108	+ __get_cpu_var(debug_stack_usage)++;
	109	+}
	110	+static inline void debug_stack_usage_dec(void)
	111	+{
	112	+ __get_cpu_var(debug_stack_usage)--;
	113	+}
	114	+int is_debug_stack(unsigned long addr);
	115	+void debug_stack_set_zero(void);
	116	+void debug_stack_reset(void);
	117	+#else /* !X86_64 */
	118	+static inline int is_debug_stack(unsigned long addr) { return 0; }
	119	+static inline void debug_stack_set_zero(void) { }
	120	+static inline void debug_stack_reset(void) { }
	121	+static inline void debug_stack_usage_inc(void) { }
	122	+static inline void debug_stack_usage_dec(void) { }
	123	+#endif /* X86_64 */
	124	+
	125	+
104	126	#endif /* __KERNEL__ */
105	127
106	128	#endif /* _ASM_X86_DEBUGREG_H */
...	...	@@ -35,6 +35,8 @@
35	35
36	36	extern struct desc_ptr idt_descr;
37	37	extern gate_desc idt_table[];
	38	+extern struct desc_ptr nmi_idt_descr;
	39	+extern gate_desc nmi_idt_table[];
38	40
39	41	struct gdt_page {
40	42	struct desc_struct gdt[GDT_ENTRIES];
...	...	@@ -306,6 +308,16 @@
306	308	desc->limit0 = limit & 0xffff;
307	309	desc->limit = (limit >> 16) & 0xf;
308	310	}
	311	+
	312	+#ifdef CONFIG_X86_64
	313	+static inline void set_nmi_gate(int gate, void *addr)
	314	+{
	315	+ gate_desc s;
	316	+
	317	+ pack_gate(&s, GATE_INTERRUPT, (unsigned long)addr, 0, 0, __KERNEL_CS);
	318	+ write_idt_entry(nmi_idt_table, gate, &s);
	319	+}
	320	+#endif
309	321
310	322	static inline void _set_gate(int gate, unsigned type, void *addr,
311	323	unsigned dpl, unsigned ist, unsigned seg)
...	...	@@ -1021,6 +1021,8 @@
1021	1021
1022	1022	#ifdef CONFIG_X86_64
1023	1023	struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
	1024	+struct desc_ptr nmi_idt_descr = { NR_VECTORS * 16 - 1,
	1025	+ (unsigned long) nmi_idt_table };
1024	1026
1025	1027	DEFINE_PER_CPU_FIRST(union irq_stack_union,
1026	1028	irq_stack_union) __aligned(PAGE_SIZE);
...	...	@@ -1085,6 +1087,26 @@
1085	1087	*/
1086	1088	DEFINE_PER_CPU(struct orig_ist, orig_ist);
1087	1089
	1090	+static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
	1091	+DEFINE_PER_CPU(int, debug_stack_usage);
	1092	+
	1093	+int is_debug_stack(unsigned long addr)
	1094	+{
	1095	+ return __get_cpu_var(debug_stack_usage) \|\|
	1096	+ (addr <= __get_cpu_var(debug_stack_addr) &&
	1097	+ addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ));
	1098	+}
	1099	+
	1100	+void debug_stack_set_zero(void)
	1101	+{
	1102	+ load_idt((const struct desc_ptr *)&nmi_idt_descr);
	1103	+}
	1104	+
	1105	+void debug_stack_reset(void)
	1106	+{
	1107	+ load_idt((const struct desc_ptr *)&idt_descr);
	1108	+}
	1109	+
1088	1110	#else /* CONFIG_X86_64 */
1089	1111
1090	1112	DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
...	...	@@ -1212,6 +1234,8 @@
1212	1234	estacks += exception_stack_sizes[v];
1213	1235	oist->ist[v] = t->x86_tss.ist[v] =
1214	1236	(unsigned long)estacks;
	1237	+ if (v == DEBUG_STACK-1)
	1238	+ per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1215	1239	}
1216	1240	}
1217	1241
...	...	@@ -1480,61 +1480,213 @@
1480	1480	CFI_ENDPROC
1481	1481	END(error_exit)
1482	1482
	1483	+/*
	1484	+ * Test if a given stack is an NMI stack or not.
	1485	+ */
	1486	+ .macro test_in_nmi reg stack nmi_ret normal_ret
	1487	+ cmpq %\reg, \stack
	1488	+ ja \normal_ret
	1489	+ subq $EXCEPTION_STKSZ, %\reg
	1490	+ cmpq %\reg, \stack
	1491	+ jb \normal_ret
	1492	+ jmp \nmi_ret
	1493	+ .endm
1483	1494
1484	1495	/* runs on exception stack */
1485	1496	ENTRY(nmi)
1486	1497	INTR_FRAME
1487	1498	PARAVIRT_ADJUST_EXCEPTION_FRAME
1488		- pushq_cfi $-1
	1499	+ /*
	1500	+ * We allow breakpoints in NMIs. If a breakpoint occurs, then
	1501	+ * the iretq it performs will take us out of NMI context.
	1502	+ * This means that we can have nested NMIs where the next
	1503	+ * NMI is using the top of the stack of the previous NMI. We
	1504	+ * can't let it execute because the nested NMI will corrupt the
	1505	+ * stack of the previous NMI. NMI handlers are not re-entrant
	1506	+ * anyway.
	1507	+ *
	1508	+ * To handle this case we do the following:
	1509	+ * Check the a special location on the stack that contains
	1510	+ * a variable that is set when NMIs are executing.
	1511	+ * The interrupted task's stack is also checked to see if it
	1512	+ * is an NMI stack.
	1513	+ * If the variable is not set and the stack is not the NMI
	1514	+ * stack then:
	1515	+ * o Set the special variable on the stack
	1516	+ * o Copy the interrupt frame into a "saved" location on the stack
	1517	+ * o Copy the interrupt frame into a "copy" location on the stack
	1518	+ * o Continue processing the NMI
	1519	+ * If the variable is set or the previous stack is the NMI stack:
	1520	+ * o Modify the "copy" location to jump to the repeate_nmi
	1521	+ * o return back to the first NMI
	1522	+ *
	1523	+ * Now on exit of the first NMI, we first clear the stack variable
	1524	+ * The NMI stack will tell any nested NMIs at that point that it is
	1525	+ * nested. Then we pop the stack normally with iret, and if there was
	1526	+ * a nested NMI that updated the copy interrupt stack frame, a
	1527	+ * jump will be made to the repeat_nmi code that will handle the second
	1528	+ * NMI.
	1529	+ */
	1530	+
	1531	+ /* Use %rdx as out temp variable throughout */
	1532	+ pushq_cfi %rdx
	1533	+
	1534	+ /*
	1535	+ * Check the special variable on the stack to see if NMIs are
	1536	+ * executing.
	1537	+ */
	1538	+ cmp $1, -8(%rsp)
	1539	+ je nested_nmi
	1540	+
	1541	+ /*
	1542	+ * Now test if the previous stack was an NMI stack.
	1543	+ * We need the double check. We check the NMI stack to satisfy the
	1544	+ * race when the first NMI clears the variable before returning.
	1545	+ * We check the variable because the first NMI could be in a
	1546	+ * breakpoint routine using a breakpoint stack.
	1547	+ */
	1548	+ lea 6*8(%rsp), %rdx
	1549	+ test_in_nmi rdx, 4*8(%rsp), nested_nmi, first_nmi
	1550	+
	1551	+nested_nmi:
	1552	+ /*
	1553	+ * Do nothing if we interrupted the fixup in repeat_nmi.
	1554	+ * It's about to repeat the NMI handler, so we are fine
	1555	+ * with ignoring this one.
	1556	+ */
	1557	+ movq $repeat_nmi, %rdx
	1558	+ cmpq 8(%rsp), %rdx
	1559	+ ja 1f
	1560	+ movq $end_repeat_nmi, %rdx
	1561	+ cmpq 8(%rsp), %rdx
	1562	+ ja nested_nmi_out
	1563	+
	1564	+1:
	1565	+ /* Set up the interrupted NMIs stack to jump to repeat_nmi */
	1566	+ leaq -6*8(%rsp), %rdx
	1567	+ movq %rdx, %rsp
	1568	+ CFI_ADJUST_CFA_OFFSET 6*8
	1569	+ pushq_cfi $__KERNEL_DS
	1570	+ pushq_cfi %rdx
	1571	+ pushfq_cfi
	1572	+ pushq_cfi $__KERNEL_CS
	1573	+ pushq_cfi $repeat_nmi
	1574	+
	1575	+ /* Put stack back */
	1576	+ addq $(11*8), %rsp
	1577	+ CFI_ADJUST_CFA_OFFSET -11*8
	1578	+
	1579	+nested_nmi_out:
	1580	+ popq_cfi %rdx
	1581	+
	1582	+ /* No need to check faults here */
	1583	+ INTERRUPT_RETURN
	1584	+
	1585	+first_nmi:
	1586	+ /*
	1587	+ * Because nested NMIs will use the pushed location that we
	1588	+ * stored in rdx, we must keep that space available.
	1589	+ * Here's what our stack frame will look like:
	1590	+ * +-------------------------+
	1591	+ * \| original SS \|
	1592	+ * \| original Return RSP \|
	1593	+ * \| original RFLAGS \|
	1594	+ * \| original CS \|
	1595	+ * \| original RIP \|
	1596	+ * +-------------------------+
	1597	+ * \| temp storage for rdx \|
	1598	+ * +-------------------------+
	1599	+ * \| NMI executing variable \|
	1600	+ * +-------------------------+
	1601	+ * \| Saved SS \|
	1602	+ * \| Saved Return RSP \|
	1603	+ * \| Saved RFLAGS \|
	1604	+ * \| Saved CS \|
	1605	+ * \| Saved RIP \|
	1606	+ * +-------------------------+
	1607	+ * \| copied SS \|
	1608	+ * \| copied Return RSP \|
	1609	+ * \| copied RFLAGS \|
	1610	+ * \| copied CS \|
	1611	+ * \| copied RIP \|
	1612	+ * +-------------------------+
	1613	+ * \| pt_regs \|
	1614	+ * +-------------------------+
	1615	+ *
	1616	+ * The saved RIP is used to fix up the copied RIP that a nested
	1617	+ * NMI may zero out. The original stack frame and the temp storage
	1618	+ * is also used by nested NMIs and can not be trusted on exit.
	1619	+ */
	1620	+ /* Set the NMI executing variable on the stack. */
	1621	+ pushq_cfi $1
	1622	+
	1623	+ /* Copy the stack frame to the Saved frame */
	1624	+ .rept 5
	1625	+ pushq_cfi 6*8(%rsp)
	1626	+ .endr
	1627	+
	1628	+ /* Make another copy, this one may be modified by nested NMIs */
	1629	+ .rept 5
	1630	+ pushq_cfi 4*8(%rsp)
	1631	+ .endr
	1632	+
	1633	+ /* Do not pop rdx, nested NMIs will corrupt it */
	1634	+ movq 11*8(%rsp), %rdx
	1635	+
	1636	+ /*
	1637	+ * Everything below this point can be preempted by a nested
	1638	+ * NMI if the first NMI took an exception. Repeated NMIs
	1639	+ * caused by an exception and nested NMI will start here, and
	1640	+ * can still be preempted by another NMI.
	1641	+ */
	1642	+restart_nmi:
	1643	+ pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
1489	1644	subq $ORIG_RAX-R15, %rsp
1490	1645	CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
	1646	+ /*
	1647	+ * Use save_paranoid to handle SWAPGS, but no need to use paranoid_exit
	1648	+ * as we should not be calling schedule in NMI context.
	1649	+ * Even with normal interrupts enabled. An NMI should not be
	1650	+ * setting NEED_RESCHED or anything that normal interrupts and
	1651	+ * exceptions might do.
	1652	+ */
1491	1653	call save_paranoid
1492	1654	DEFAULT_FRAME 0
1493	1655	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1494	1656	movq %rsp,%rdi
1495	1657	movq $-1,%rsi
1496	1658	call do_nmi
1497		-#ifdef CONFIG_TRACE_IRQFLAGS
1498		- /* paranoidexit; without TRACE_IRQS_OFF */
1499		- /* ebx: no swapgs flag */
1500		- DISABLE_INTERRUPTS(CLBR_NONE)
1501	1659	testl %ebx,%ebx /* swapgs needed? */
1502	1660	jnz nmi_restore
1503		- testl $3,CS(%rsp)
1504		- jnz nmi_userspace
1505	1661	nmi_swapgs:
1506	1662	SWAPGS_UNSAFE_STACK
1507	1663	nmi_restore:
1508	1664	RESTORE_ALL 8
	1665	+ /* Clear the NMI executing stack variable */
	1666	+ movq $0, 10*8(%rsp)
1509	1667	jmp irq_return
1510		-nmi_userspace:
1511		- GET_THREAD_INFO(%rcx)
1512		- movl TI_flags(%rcx),%ebx
1513		- andl $_TIF_WORK_MASK,%ebx
1514		- jz nmi_swapgs
1515		- movq %rsp,%rdi /* &pt_regs */
1516		- call sync_regs
1517		- movq %rax,%rsp /* switch stack for scheduling */
1518		- testl $_TIF_NEED_RESCHED,%ebx
1519		- jnz nmi_schedule
1520		- movl %ebx,%edx /* arg3: thread flags */
1521		- ENABLE_INTERRUPTS(CLBR_NONE)
1522		- xorl %esi,%esi /* arg2: oldset */
1523		- movq %rsp,%rdi /* arg1: &pt_regs */
1524		- call do_notify_resume
1525		- DISABLE_INTERRUPTS(CLBR_NONE)
1526		- jmp nmi_userspace
1527		-nmi_schedule:
1528		- ENABLE_INTERRUPTS(CLBR_ANY)
1529		- call schedule
1530		- DISABLE_INTERRUPTS(CLBR_ANY)
1531		- jmp nmi_userspace
1532	1668	CFI_ENDPROC
1533		-#else
1534		- jmp paranoid_exit
1535		- CFI_ENDPROC
1536		-#endif
1537	1669	END(nmi)
	1670	+
	1671	+ /*
	1672	+ * If an NMI hit an iret because of an exception or breakpoint,
	1673	+ * it can lose its NMI context, and a nested NMI may come in.
	1674	+ * In that case, the nested NMI will change the preempted NMI's
	1675	+ * stack to jump to here when it does the final iret.
	1676	+ */
	1677	+repeat_nmi:
	1678	+ INTR_FRAME
	1679	+ /* Update the stack variable to say we are still in NMI */
	1680	+ movq $1, 5*8(%rsp)
	1681	+
	1682	+ /* copy the saved stack back to copy stack */
	1683	+ .rept 5
	1684	+ pushq_cfi 4*8(%rsp)
	1685	+ .endr
	1686	+
	1687	+ jmp restart_nmi
	1688	+ CFI_ENDPROC
	1689	+end_repeat_nmi:
1538	1690
1539	1691	ENTRY(ignore_sysret)
1540	1692	CFI_STARTPROC
...	...	@@ -417,6 +417,10 @@
417	417	ENTRY(idt_table)
418	418	.skip IDT_ENTRIES * 16
419	419
	420	+ .align L1_CACHE_BYTES
	421	+ENTRY(nmi_idt_table)
	422	+ .skip IDT_ENTRIES * 16
	423	+
420	424	__PAGE_ALIGNED_BSS
421	425	.align PAGE_SIZE
422	426	ENTRY(empty_zero_page)
...	...	@@ -405,9 +405,108 @@
405	405	unknown_nmi_error(reason, regs);
406	406	}
407	407
	408	+/*
	409	+ * NMIs can hit breakpoints which will cause it to lose its
	410	+ * NMI context with the CPU when the breakpoint does an iret.
	411	+ */
	412	+#ifdef CONFIG_X86_32
	413	+/*
	414	+ * For i386, NMIs use the same stack as the kernel, and we can
	415	+ * add a workaround to the iret problem in C. Simply have 3 states
	416	+ * the NMI can be in.
	417	+ *
	418	+ * 1) not running
	419	+ * 2) executing
	420	+ * 3) latched
	421	+ *
	422	+ * When no NMI is in progress, it is in the "not running" state.
	423	+ * When an NMI comes in, it goes into the "executing" state.
	424	+ * Normally, if another NMI is triggered, it does not interrupt
	425	+ * the running NMI and the HW will simply latch it so that when
	426	+ * the first NMI finishes, it will restart the second NMI.
	427	+ * (Note, the latch is binary, thus multiple NMIs triggering,
	428	+ * when one is running, are ignored. Only one NMI is restarted.)
	429	+ *
	430	+ * If an NMI hits a breakpoint that executes an iret, another
	431	+ * NMI can preempt it. We do not want to allow this new NMI
	432	+ * to run, but we want to execute it when the first one finishes.
	433	+ * We set the state to "latched", and the first NMI will perform
	434	+ * an cmpxchg on the state, and if it doesn't successfully
	435	+ * reset the state to "not running" it will restart the next
	436	+ * NMI.
	437	+ */
	438	+enum nmi_states {
	439	+ NMI_NOT_RUNNING,
	440	+ NMI_EXECUTING,
	441	+ NMI_LATCHED,
	442	+};
	443	+static DEFINE_PER_CPU(enum nmi_states, nmi_state);
	444	+
	445	+#define nmi_nesting_preprocess(regs) \
	446	+ do { \
	447	+ if (__get_cpu_var(nmi_state) != NMI_NOT_RUNNING) { \
	448	+ __get_cpu_var(nmi_state) = NMI_LATCHED; \
	449	+ return; \
	450	+ } \
	451	+ nmi_restart: \
	452	+ __get_cpu_var(nmi_state) = NMI_EXECUTING; \
	453	+ } while (0)
	454	+
	455	+#define nmi_nesting_postprocess() \
	456	+ do { \
	457	+ if (cmpxchg(&__get_cpu_var(nmi_state), \
	458	+ NMI_EXECUTING, NMI_NOT_RUNNING) != NMI_EXECUTING) \
	459	+ goto nmi_restart; \
	460	+ } while (0)
	461	+#else /* x86_64 */
	462	+/*
	463	+ * In x86_64 things are a bit more difficult. This has the same problem
	464	+ * where an NMI hitting a breakpoint that calls iret will remove the
	465	+ * NMI context, allowing a nested NMI to enter. What makes this more
	466	+ * difficult is that both NMIs and breakpoints have their own stack.
	467	+ * When a new NMI or breakpoint is executed, the stack is set to a fixed
	468	+ * point. If an NMI is nested, it will have its stack set at that same
	469	+ * fixed address that the first NMI had, and will start corrupting the
	470	+ * stack. This is handled in entry_64.S, but the same problem exists with
	471	+ * the breakpoint stack.
	472	+ *
	473	+ * If a breakpoint is being processed, and the debug stack is being used,
	474	+ * if an NMI comes in and also hits a breakpoint, the stack pointer
	475	+ * will be set to the same fixed address as the breakpoint that was
	476	+ * interrupted, causing that stack to be corrupted. To handle this case,
	477	+ * check if the stack that was interrupted is the debug stack, and if
	478	+ * so, change the IDT so that new breakpoints will use the current stack
	479	+ * and not switch to the fixed address. On return of the NMI, switch back
	480	+ * to the original IDT.
	481	+ */
	482	+static DEFINE_PER_CPU(int, update_debug_stack);
	483	+
	484	+static inline void nmi_nesting_preprocess(struct pt_regs *regs)
	485	+{
	486	+ /*
	487	+ * If we interrupted a breakpoint, it is possible that
	488	+ * the nmi handler will have breakpoints too. We need to
	489	+ * change the IDT such that breakpoints that happen here
	490	+ * continue to use the NMI stack.
	491	+ */
	492	+ if (unlikely(is_debug_stack(regs->sp))) {
	493	+ debug_stack_set_zero();
	494	+ __get_cpu_var(update_debug_stack) = 1;
	495	+ }
	496	+}
	497	+
	498	+static inline void nmi_nesting_postprocess(void)
	499	+{
	500	+ if (unlikely(__get_cpu_var(update_debug_stack)))
	501	+ debug_stack_reset();
	502	+}
	503	+#endif
	504	+
408	505	dotraplinkage notrace __kprobes void
409	506	do_nmi(struct pt_regs *regs, long error_code)
410	507	{
	508	+ nmi_nesting_preprocess(regs);
	509	+
411	510	nmi_enter();
412	511
413	512	inc_irq_stat(__nmi_count);
...	...	@@ -416,6 +515,9 @@
416	515	default_do_nmi(regs);
417	516
418	517	nmi_exit();
	518	+
	519	+ /* On i386, may loop back to preprocess */
	520	+ nmi_nesting_postprocess();
419	521	}
420	522
421	523	void stop_nmi(void)
...	...	@@ -311,9 +311,15 @@
311	311	== NOTIFY_STOP)
312	312	return;
313	313
	314	+ /*
	315	+ * Let others (NMI) know that the debug stack is in use
	316	+ * as we may switch to the interrupt stack.
	317	+ */
	318	+ debug_stack_usage_inc();
314	319	preempt_conditional_sti(regs);
315	320	do_trap(3, SIGTRAP, "int3", regs, error_code, NULL);
316	321	preempt_conditional_cli(regs);
	322	+ debug_stack_usage_dec();
317	323	}
318	324
319	325	#ifdef CONFIG_X86_64
...	...	@@ -406,6 +412,12 @@
406	412	SIGTRAP) == NOTIFY_STOP)
407	413	return;
408	414
	415	+ /*
	416	+ * Let others (NMI) know that the debug stack is in use
	417	+ * as we may switch to the interrupt stack.
	418	+ */
	419	+ debug_stack_usage_inc();
	420	+
409	421	/* It's safe to allow irq's after DR6 has been saved */
410	422	preempt_conditional_sti(regs);
411	423
...	...	@@ -413,6 +425,7 @@
413	425	handle_vm86_trap((struct kernel_vm86_regs *) regs,
414	426	error_code, 1);
415	427	preempt_conditional_cli(regs);
	428	+ debug_stack_usage_dec();
416	429	return;
417	430	}
418	431
...	...	@@ -432,6 +445,7 @@
432	445	if (tsk->thread.debugreg6 & (DR_STEP \| DR_TRAP_BITS) \|\| user_icebp)
433	446	send_sigtrap(tsk, regs, error_code, si_code);
434	447	preempt_conditional_cli(regs);
	448	+ debug_stack_usage_dec();
435	449
436	450	return;
437	451	}
...	...	@@ -718,5 +732,11 @@
718	732	cpu_init();
719	733
720	734	x86_init.irqs.trap_init();
	735	+
	736	+#ifdef CONFIG_X86_64
	737	+ memcpy(&nmi_idt_table, &idt_table, IDT_ENTRIES * 16);
	738	+ set_nmi_gate(1, &debug);
	739	+ set_nmi_gate(3, &int3);
	740	+#endif
721	741	}
...	...	@@ -50,6 +50,11 @@
50	50	# define inline inline __attribute__((always_inline))
51	51	# define __inline__ __inline__ __attribute__((always_inline))
52	52	# define __inline __inline __attribute__((always_inline))
	53	+#else
	54	+/* A lot of inline functions can cause havoc with function tracing */
	55	+# define inline inline notrace
	56	+# define __inline__ __inline__ notrace
	57	+# define __inline __inline notrace
53	58	#endif
54	59
55	60	#define __deprecated __attribute__((deprecated))
...	...	@@ -133,6 +133,8 @@
133	133	int ftrace_arch_code_modify_prepare(void);
134	134	int ftrace_arch_code_modify_post_process(void);
135	135
	136	+void ftrace_bug(int err, unsigned long ip);
	137	+
136	138	struct seq_file;
137	139
138	140	struct ftrace_probe_ops {
...	...	@@ -161,7 +163,6 @@
161	163
162	164	enum {
163	165	FTRACE_FL_ENABLED = (1 << 30),
164		- FTRACE_FL_FREE = (1 << 31),
165	166	};
166	167
167	168	#define FTRACE_FL_MASK (0x3UL << 30)
...	...	@@ -172,10 +173,7 @@
172	173	unsigned long ip; /* address of mcount call-site */
173	174	struct dyn_ftrace *freelist;
174	175	};
175		- union {
176		- unsigned long flags;
177		- struct dyn_ftrace *newlist;
178		- };
	176	+ unsigned long flags;
179	177	struct dyn_arch_ftrace arch;
180	178	};
181	179
...	...	@@ -190,6 +188,56 @@
190	188	int register_ftrace_command(struct ftrace_func_command *cmd);
191	189	int unregister_ftrace_command(struct ftrace_func_command *cmd);
192	190
	191	+enum {
	192	+ FTRACE_UPDATE_CALLS = (1 << 0),
	193	+ FTRACE_DISABLE_CALLS = (1 << 1),
	194	+ FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
	195	+ FTRACE_START_FUNC_RET = (1 << 3),
	196	+ FTRACE_STOP_FUNC_RET = (1 << 4),
	197	+};
	198	+
	199	+enum {
	200	+ FTRACE_UPDATE_IGNORE,
	201	+ FTRACE_UPDATE_MAKE_CALL,
	202	+ FTRACE_UPDATE_MAKE_NOP,
	203	+};
	204	+
	205	+enum {
	206	+ FTRACE_ITER_FILTER = (1 << 0),
	207	+ FTRACE_ITER_NOTRACE = (1 << 1),
	208	+ FTRACE_ITER_PRINTALL = (1 << 2),
	209	+ FTRACE_ITER_DO_HASH = (1 << 3),
	210	+ FTRACE_ITER_HASH = (1 << 4),
	211	+ FTRACE_ITER_ENABLED = (1 << 5),
	212	+};
	213	+
	214	+void arch_ftrace_update_code(int command);
	215	+
	216	+struct ftrace_rec_iter;
	217	+
	218	+struct ftrace_rec_iter *ftrace_rec_iter_start(void);
	219	+struct ftrace_rec_iter ftrace_rec_iter_next(struct ftrace_rec_iter iter);
	220	+struct dyn_ftrace ftrace_rec_iter_record(struct ftrace_rec_iter iter);
	221	+
	222	+int ftrace_update_record(struct dyn_ftrace *rec, int enable);
	223	+int ftrace_test_record(struct dyn_ftrace *rec, int enable);
	224	+void ftrace_run_stop_machine(int command);
	225	+int ftrace_location(unsigned long ip);
	226	+
	227	+extern ftrace_func_t ftrace_trace_function;
	228	+
	229	+int ftrace_regex_open(struct ftrace_ops *ops, int flag,
	230	+ struct inode inode, struct file file);
	231	+ssize_t ftrace_filter_write(struct file file, const char __user ubuf,
	232	+ size_t cnt, loff_t *ppos);
	233	+ssize_t ftrace_notrace_write(struct file file, const char __user ubuf,
	234	+ size_t cnt, loff_t *ppos);
	235	+loff_t ftrace_regex_lseek(struct file *file, loff_t offset, int origin);
	236	+int ftrace_regex_release(struct inode inode, struct file file);
	237	+
	238	+void __init
	239	+ftrace_set_early_filter(struct ftrace_ops ops, char buf, int enable);
	240	+
193	241	/* defined in arch */
194	242	extern int ftrace_ip_converted(unsigned long ip);
195	243	extern int ftrace_dyn_arch_init(void *data);
...	...	@@ -284,6 +332,25 @@
284	332	{
285	333	return 0;
286	334	}
	335	+
	336	+/*
	337	+ * Again users of functions that have ftrace_ops may not
	338	+ * have them defined when ftrace is not enabled, but these
	339	+ * functions may still be called. Use a macro instead of inline.
	340	+ */
	341	+#define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; })
	342	+#define ftrace_set_early_filter(ops, buf, enable) do { } while (0)
	343	+
	344	+static inline ssize_t ftrace_filter_write(struct file file, const char __user ubuf,
	345	+ size_t cnt, loff_t *ppos) { return -ENODEV; }
	346	+static inline ssize_t ftrace_notrace_write(struct file file, const char __user ubuf,
	347	+ size_t cnt, loff_t *ppos) { return -ENODEV; }
	348	+static inline loff_t ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
	349	+{
	350	+ return -ENODEV;
	351	+}
	352	+static inline int
	353	+ftrace_regex_release(struct inode inode, struct file file) { return -ENODEV; }
287	354	#endif /* CONFIG_DYNAMIC_FTRACE */
288	355
289	356	/* totally disable ftrace - can not re-enable after this */