Merge branch 'core' of git://git.kernel.org/pub/scm/linux/kernel/git/rric/oprofile into perf/core

Ingo Molnar
2 parents 729aa21ab8 a406ab6d77
Showing 8 changed files Side-by-side Diff
arch/mips/include/asm/stacktrace.h
arch/mips/kernel/process.c
arch/mips/oprofile/Makefile
arch/mips/oprofile/backtrace.c
arch/mips/oprofile/common.c
arch/mips/oprofile/op_impl.h
arch/x86/oprofile/backtrace.c
arch/x86/oprofile/nmi_int.c
@@ -7,6 +7,10 @@
 extern int raw_show_trace;
 extern unsigned long unwind_stack(struct task_struct *task, unsigned long *sp,
 				  unsigned long pc, unsigned long *ra);
+extern unsigned long unwind_stack_by_address(unsigned long stack_page,
+					     unsigned long *sp,
+					     unsigned long pc,
+					     unsigned long *ra);
 #else
 #define raw_show_trace 1
 static inline unsigned long unwind_stack(struct task_struct *task,
@@ -373,18 +373,18 @@
  
  
 #ifdef CONFIG_KALLSYMS
-/* used by show_backtrace() */
-unsigned long unwind_stack(struct task_struct *task, unsigned long *sp,
-			   unsigned long pc, unsigned long *ra)
+/* generic stack unwinding function */
+unsigned long notrace unwind_stack_by_address(unsigned long stack_page,
+					      unsigned long *sp,
+					      unsigned long pc,
+					      unsigned long *ra)
 {
-	unsigned long stack_page;
 	struct mips_frame_info info;
 	unsigned long size, ofs;
 	int leaf;
 	extern void ret_from_irq(void);
 	extern void ret_from_exception(void);
  
-	stack_page = (unsigned long)task_stack_page(task);
 	if (!stack_page)
 		return 0;
  
@@ -442,6 +442,15 @@
 	*sp += info.frame_size;
 	*ra = 0;
 	return __kernel_text_address(pc) ? pc : 0;
+}
+EXPORT_SYMBOL(unwind_stack_by_address);
+
+/* used by show_backtrace() */
+unsigned long unwind_stack(struct task_struct *task, unsigned long *sp,
+			   unsigned long pc, unsigned long *ra)
+{
+	unsigned long stack_page = (unsigned long)task_stack_page(task);
+	return unwind_stack_by_address(stack_page, sp, pc, ra);
 }
 #endif
  
@@ -8,7 +8,7 @@
 		oprofilefs.o oprofile_stats.o \
 		timer_int.o )
  
-oprofile-y				:= $(DRIVER_OBJS) common.o
+oprofile-y				:= $(DRIVER_OBJS) common.o backtrace.o
  
 oprofile-$(CONFIG_CPU_MIPS32)		+= op_model_mipsxx.o
 oprofile-$(CONFIG_CPU_MIPS64)		+= op_model_mipsxx.o
+#include <linux/oprofile.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/uaccess.h>
+#include <asm/ptrace.h>
+#include <asm/stacktrace.h>
+#include <linux/stacktrace.h>
+#include <linux/kernel.h>
+#include <asm/sections.h>
+#include <asm/inst.h>
+
+struct stackframe {
+	unsigned long sp;
+	unsigned long pc;
+	unsigned long ra;
+};
+
+static inline int get_mem(unsigned long addr, unsigned long *result)
+{
+	unsigned long *address = (unsigned long *) addr;
+	if (!access_ok(VERIFY_READ, addr, sizeof(unsigned long)))
+		return -1;
+	if (__copy_from_user_inatomic(result, address, sizeof(unsigned long)))
+		return -3;
+	return 0;
+}
+
+/*
+ * These two instruction helpers were taken from process.c
+ */
+static inline int is_ra_save_ins(union mips_instruction *ip)
+{
+	/* sw / sd $ra, offset($sp) */
+	return (ip->i_format.opcode == sw_op || ip->i_format.opcode == sd_op)
+		&& ip->i_format.rs == 29 && ip->i_format.rt == 31;
+}
+
+static inline int is_sp_move_ins(union mips_instruction *ip)
+{
+	/* addiu/daddiu sp,sp,-imm */
+	if (ip->i_format.rs != 29 || ip->i_format.rt != 29)
+		return 0;
+	if (ip->i_format.opcode == addiu_op || ip->i_format.opcode == daddiu_op)
+		return 1;
+	return 0;
+}
+
+/*
+ * Looks for specific instructions that mark the end of a function.
+ * This usually means we ran into the code area of the previous function.
+ */
+static inline int is_end_of_function_marker(union mips_instruction *ip)
+{
+	/* jr ra */
+	if (ip->r_format.func == jr_op && ip->r_format.rs == 31)
+		return 1;
+	/* lui gp */
+	if (ip->i_format.opcode == lui_op && ip->i_format.rt == 28)
+		return 1;
+	return 0;
+}
+
+/*
+ * TODO for userspace stack unwinding:
+ * - handle cases where the stack is adjusted inside a function
+ *     (generally doesn't happen)
+ * - find optimal value for max_instr_check
+ * - try to find a way to handle leaf functions
+ */
+
+static inline int unwind_user_frame(struct stackframe *old_frame,
+				    const unsigned int max_instr_check)
+{
+	struct stackframe new_frame = *old_frame;
+	off_t ra_offset = 0;
+	size_t stack_size = 0;
+	unsigned long addr;
+
+	if (old_frame->pc == 0 || old_frame->sp == 0 || old_frame->ra == 0)
+		return -9;
+
+	for (addr = new_frame.pc; (addr + max_instr_check > new_frame.pc)
+		&& (!ra_offset || !stack_size); --addr) {
+		union mips_instruction ip;
+
+		if (get_mem(addr, (unsigned long *) &ip))
+			return -11;
+
+		if (is_sp_move_ins(&ip)) {
+			int stack_adjustment = ip.i_format.simmediate;
+			if (stack_adjustment > 0)
+				/* This marks the end of the previous function,
+				   which means we overran. */
+				break;
+			stack_size = (unsigned) stack_adjustment;
+		} else if (is_ra_save_ins(&ip)) {
+			int ra_slot = ip.i_format.simmediate;
+			if (ra_slot < 0)
+				/* This shouldn't happen. */
+				break;
+			ra_offset = ra_slot;
+		} else if (is_end_of_function_marker(&ip))
+			break;
+	}
+
+	if (!ra_offset || !stack_size)
+		return -1;
+
+	if (ra_offset) {
+		new_frame.ra = old_frame->sp + ra_offset;
+		if (get_mem(new_frame.ra, &(new_frame.ra)))
+			return -13;
+	}
+
+	if (stack_size) {
+		new_frame.sp = old_frame->sp + stack_size;
+		if (get_mem(new_frame.sp, &(new_frame.sp)))
+			return -14;
+	}
+
+	if (new_frame.sp > old_frame->sp)
+		return -2;
+
+	new_frame.pc = old_frame->ra;
+	*old_frame = new_frame;
+
+	return 0;
+}
+
+static inline void do_user_backtrace(unsigned long low_addr,
+				     struct stackframe *frame,
+				     unsigned int depth)
+{
+	const unsigned int max_instr_check = 512;
+	const unsigned long high_addr = low_addr + THREAD_SIZE;
+
+	while (depth-- && !unwind_user_frame(frame, max_instr_check)) {
+		oprofile_add_trace(frame->ra);
+		if (frame->sp < low_addr || frame->sp > high_addr)
+			break;
+	}
+}
+
+#ifndef CONFIG_KALLSYMS
+static inline void do_kernel_backtrace(unsigned long low_addr,
+				       struct stackframe *frame,
+				       unsigned int depth) { }
+#else
+static inline void do_kernel_backtrace(unsigned long low_addr,
+				       struct stackframe *frame,
+				       unsigned int depth)
+{
+	while (depth-- && frame->pc) {
+		frame->pc = unwind_stack_by_address(low_addr,
+						    &(frame->sp),
+						    frame->pc,
+						    &(frame->ra));
+		oprofile_add_trace(frame->ra);
+	}
+}
+#endif
+
+void notrace op_mips_backtrace(struct pt_regs *const regs, unsigned int depth)
+{
+	struct stackframe frame = { .sp = regs->regs[29],
+				    .pc = regs->cp0_epc,
+				    .ra = regs->regs[31] };
+	const int userspace = user_mode(regs);
+	const unsigned long low_addr = ALIGN(frame.sp, THREAD_SIZE);
+
+	if (userspace)
+		do_user_backtrace(low_addr, &frame, depth);
+	else
+		do_kernel_backtrace(low_addr, &frame, depth);
+}
@@ -115,6 +115,7 @@
 	ops->start		= op_mips_start;
 	ops->stop		= op_mips_stop;
 	ops->cpu_type		= lmodel->cpu_type;
+	ops->backtrace		= op_mips_backtrace;
  
 	printk(KERN_INFO "oprofile: using %s performance monitoring.\n",
 	       lmodel->cpu_type);
@@ -36,5 +36,7 @@
 	unsigned char num_counters;
 };
  
+void op_mips_backtrace(struct pt_regs * const regs, unsigned int depth);
+
 #endif
@@ -11,10 +11,12 @@
 #include <linux/oprofile.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
+#include <linux/compat.h>
+#include <linux/highmem.h>
+
 #include <asm/ptrace.h>
 #include <asm/uaccess.h>
 #include <asm/stacktrace.h>
-#include <linux/compat.h>
  
 static int backtrace_stack(void *data, char *name)
 {
  
  
  
  
@@ -36,18 +38,54 @@
 	.walk_stack	= print_context_stack,
 };
  
+/* from arch/x86/kernel/cpu/perf_event.c: */
+
+/*
+ * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
+ */
+static unsigned long
+copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
+{
+	unsigned long offset, addr = (unsigned long)from;
+	unsigned long size, len = 0;
+	struct page *page;
+	void *map;
+	int ret;
+
+	do {
+		ret = __get_user_pages_fast(addr, 1, 0, &page);
+		if (!ret)
+			break;
+
+		offset = addr & (PAGE_SIZE - 1);
+		size = min(PAGE_SIZE - offset, n - len);
+
+		map = kmap_atomic(page);
+		memcpy(to, map+offset, size);
+		kunmap_atomic(map);
+		put_page(page);
+
+		len  += size;
+		to   += size;
+		addr += size;
+
+	} while (len < n);
+
+	return len;
+}
+
 #ifdef CONFIG_COMPAT
 static struct stack_frame_ia32 *
 dump_user_backtrace_32(struct stack_frame_ia32 *head)
 {
+	/* Also check accessibility of one struct frame_head beyond: */
 	struct stack_frame_ia32 bufhead[2];
 	struct stack_frame_ia32 *fp;
+	unsigned long bytes;
  
-	/* Also check accessibility of one struct frame_head beyond */
-	if (!access_ok(VERIFY_READ, head, sizeof(bufhead)))
+	bytes = copy_from_user_nmi(bufhead, head, sizeof(bufhead));
+	if (bytes != sizeof(bufhead))
 		return NULL;
-	if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead)))
-		return NULL;
  
 	fp = (struct stack_frame_ia32 *) compat_ptr(bufhead[0].next_frame);
  
  
  
@@ -87,12 +125,12 @@
  
 static struct stack_frame *dump_user_backtrace(struct stack_frame *head)
 {
+	/* Also check accessibility of one struct frame_head beyond: */
 	struct stack_frame bufhead[2];
+	unsigned long bytes;
  
-	/* Also check accessibility of one struct stack_frame beyond */
-	if (!access_ok(VERIFY_READ, head, sizeof(bufhead)))
-		return NULL;
-	if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead)))
+	bytes = copy_from_user_nmi(bufhead, head, sizeof(bufhead));
+	if (bytes != sizeof(bufhead))
 		return NULL;
  
 	oprofile_add_trace(bufhead[0].return_address);
@@ -112,8 +112,10 @@
 static int nmi_start(void)
 {
 	get_online_cpus();
-	on_each_cpu(nmi_cpu_start, NULL, 1);
 	ctr_running = 1;
+	/* make ctr_running visible to the nmi handler: */
+	smp_mb();
+	on_each_cpu(nmi_cpu_start, NULL, 1);
 	put_online_cpus();
 	return 0;
 }
  
  
@@ -504,15 +506,18 @@
  
 	nmi_enabled = 0;
 	ctr_running = 0;
-	barrier();
+	/* make variables visible to the nmi handler: */
+	smp_mb();
 	err = register_die_notifier(&profile_exceptions_nb);
 	if (err)
 		goto fail;
  
 	get_online_cpus();
 	register_cpu_notifier(&oprofile_cpu_nb);
-	on_each_cpu(nmi_cpu_setup, NULL, 1);
 	nmi_enabled = 1;
+	/* make nmi_enabled visible to the nmi handler: */
+	smp_mb();
+	on_each_cpu(nmi_cpu_setup, NULL, 1);
 	put_online_cpus();
  
 	return 0;
@@ -531,7 +536,8 @@
 	nmi_enabled = 0;
 	ctr_running = 0;
 	put_online_cpus();
-	barrier();
+	/* make variables visible to the nmi handler: */
+	smp_mb();
 	unregister_die_notifier(&profile_exceptions_nb);
 	msrs = &get_cpu_var(cpu_msrs);
 	model->shutdown(msrs);
...	...	@@ -7,6 +7,10 @@
7	7	extern int raw_show_trace;
8	8	extern unsigned long unwind_stack(struct task_struct task, unsigned long sp,
9	9	unsigned long pc, unsigned long *ra);
	10	+extern unsigned long unwind_stack_by_address(unsigned long stack_page,
	11	+ unsigned long *sp,
	12	+ unsigned long pc,
	13	+ unsigned long *ra);
10	14	#else
11	15	#define raw_show_trace 1
12	16	static inline unsigned long unwind_stack(struct task_struct *task,
...	...	@@ -373,18 +373,18 @@
373	373
374	374
375	375	#ifdef CONFIG_KALLSYMS
376		-/* used by show_backtrace() */
377		-unsigned long unwind_stack(struct task_struct task, unsigned long sp,
378		- unsigned long pc, unsigned long *ra)
	376	+/* generic stack unwinding function */
	377	+unsigned long notrace unwind_stack_by_address(unsigned long stack_page,
	378	+ unsigned long *sp,
	379	+ unsigned long pc,
	380	+ unsigned long *ra)
379	381	{
380		- unsigned long stack_page;
381	382	struct mips_frame_info info;
382	383	unsigned long size, ofs;
383	384	int leaf;
384	385	extern void ret_from_irq(void);
385	386	extern void ret_from_exception(void);
386	387
387		- stack_page = (unsigned long)task_stack_page(task);
388	388	if (!stack_page)
389	389	return 0;
390	390
...	...	@@ -442,6 +442,15 @@
442	442	*sp += info.frame_size;
443	443	*ra = 0;
444	444	return __kernel_text_address(pc) ? pc : 0;
	445	+}
	446	+EXPORT_SYMBOL(unwind_stack_by_address);
	447	+
	448	+/* used by show_backtrace() */
	449	+unsigned long unwind_stack(struct task_struct task, unsigned long sp,
	450	+ unsigned long pc, unsigned long *ra)
	451	+{
	452	+ unsigned long stack_page = (unsigned long)task_stack_page(task);
	453	+ return unwind_stack_by_address(stack_page, sp, pc, ra);
445	454	}
446	455	#endif
447	456
...	...	@@ -8,7 +8,7 @@
8	8	oprofilefs.o oprofile_stats.o \
9	9	timer_int.o )
10	10
11		-oprofile-y := $(DRIVER_OBJS) common.o
	11	+oprofile-y := $(DRIVER_OBJS) common.o backtrace.o
12	12
13	13	oprofile-$(CONFIG_CPU_MIPS32) += op_model_mipsxx.o
14	14	oprofile-$(CONFIG_CPU_MIPS64) += op_model_mipsxx.o
	1	+#include <linux/oprofile.h>
	2	+#include <linux/sched.h>
	3	+#include <linux/mm.h>
	4	+#include <linux/uaccess.h>
	5	+#include <asm/ptrace.h>
	6	+#include <asm/stacktrace.h>
	7	+#include <linux/stacktrace.h>
	8	+#include <linux/kernel.h>
	9	+#include <asm/sections.h>
	10	+#include <asm/inst.h>
	11	+
	12	+struct stackframe {
	13	+ unsigned long sp;
	14	+ unsigned long pc;
	15	+ unsigned long ra;
	16	+};
	17	+
	18	+static inline int get_mem(unsigned long addr, unsigned long *result)
	19	+{
	20	+ unsigned long address = (unsigned long ) addr;
	21	+ if (!access_ok(VERIFY_READ, addr, sizeof(unsigned long)))
	22	+ return -1;
	23	+ if (__copy_from_user_inatomic(result, address, sizeof(unsigned long)))
	24	+ return -3;
	25	+ return 0;
	26	+}
	27	+
	28	+/*
	29	+ * These two instruction helpers were taken from process.c
	30	+ */
	31	+static inline int is_ra_save_ins(union mips_instruction *ip)
	32	+{
	33	+ /* sw / sd $ra, offset($sp) */
	34	+ return (ip->i_format.opcode == sw_op \|\| ip->i_format.opcode == sd_op)
	35	+ && ip->i_format.rs == 29 && ip->i_format.rt == 31;
	36	+}
	37	+
	38	+static inline int is_sp_move_ins(union mips_instruction *ip)
	39	+{
	40	+ /* addiu/daddiu sp,sp,-imm */
	41	+ if (ip->i_format.rs != 29 \|\| ip->i_format.rt != 29)
	42	+ return 0;
	43	+ if (ip->i_format.opcode == addiu_op \|\| ip->i_format.opcode == daddiu_op)
	44	+ return 1;
	45	+ return 0;
	46	+}
	47	+
	48	+/*
	49	+ * Looks for specific instructions that mark the end of a function.
	50	+ * This usually means we ran into the code area of the previous function.
	51	+ */
	52	+static inline int is_end_of_function_marker(union mips_instruction *ip)
	53	+{
	54	+ /* jr ra */
	55	+ if (ip->r_format.func == jr_op && ip->r_format.rs == 31)
	56	+ return 1;
	57	+ /* lui gp */
	58	+ if (ip->i_format.opcode == lui_op && ip->i_format.rt == 28)
	59	+ return 1;
	60	+ return 0;
	61	+}
	62	+
	63	+/*
	64	+ * TODO for userspace stack unwinding:
	65	+ * - handle cases where the stack is adjusted inside a function
	66	+ * (generally doesn't happen)
	67	+ * - find optimal value for max_instr_check
	68	+ * - try to find a way to handle leaf functions
	69	+ */
	70	+
	71	+static inline int unwind_user_frame(struct stackframe *old_frame,
	72	+ const unsigned int max_instr_check)
	73	+{
	74	+ struct stackframe new_frame = *old_frame;
	75	+ off_t ra_offset = 0;
	76	+ size_t stack_size = 0;
	77	+ unsigned long addr;
	78	+
	79	+ if (old_frame->pc == 0 \|\| old_frame->sp == 0 \|\| old_frame->ra == 0)
	80	+ return -9;
	81	+
	82	+ for (addr = new_frame.pc; (addr + max_instr_check > new_frame.pc)
	83	+ && (!ra_offset \|\| !stack_size); --addr) {
	84	+ union mips_instruction ip;
	85	+
	86	+ if (get_mem(addr, (unsigned long *) &ip))
	87	+ return -11;
	88	+
	89	+ if (is_sp_move_ins(&ip)) {
	90	+ int stack_adjustment = ip.i_format.simmediate;
	91	+ if (stack_adjustment > 0)
	92	+ /* This marks the end of the previous function,
	93	+ which means we overran. */
	94	+ break;
	95	+ stack_size = (unsigned) stack_adjustment;
	96	+ } else if (is_ra_save_ins(&ip)) {
	97	+ int ra_slot = ip.i_format.simmediate;
	98	+ if (ra_slot < 0)
	99	+ /* This shouldn't happen. */
	100	+ break;
	101	+ ra_offset = ra_slot;
	102	+ } else if (is_end_of_function_marker(&ip))
	103	+ break;
	104	+ }
	105	+
	106	+ if (!ra_offset \|\| !stack_size)
	107	+ return -1;
	108	+
	109	+ if (ra_offset) {
	110	+ new_frame.ra = old_frame->sp + ra_offset;
	111	+ if (get_mem(new_frame.ra, &(new_frame.ra)))
	112	+ return -13;
	113	+ }
	114	+
	115	+ if (stack_size) {
	116	+ new_frame.sp = old_frame->sp + stack_size;
	117	+ if (get_mem(new_frame.sp, &(new_frame.sp)))
	118	+ return -14;
	119	+ }
	120	+
	121	+ if (new_frame.sp > old_frame->sp)
	122	+ return -2;
	123	+
	124	+ new_frame.pc = old_frame->ra;
	125	+ *old_frame = new_frame;
	126	+
	127	+ return 0;
	128	+}
	129	+
	130	+static inline void do_user_backtrace(unsigned long low_addr,
	131	+ struct stackframe *frame,
	132	+ unsigned int depth)
	133	+{
	134	+ const unsigned int max_instr_check = 512;
	135	+ const unsigned long high_addr = low_addr + THREAD_SIZE;
	136	+
	137	+ while (depth-- && !unwind_user_frame(frame, max_instr_check)) {
	138	+ oprofile_add_trace(frame->ra);
	139	+ if (frame->sp < low_addr \|\| frame->sp > high_addr)
	140	+ break;
	141	+ }
	142	+}
	143	+
	144	+#ifndef CONFIG_KALLSYMS
	145	+static inline void do_kernel_backtrace(unsigned long low_addr,
	146	+ struct stackframe *frame,
	147	+ unsigned int depth) { }
	148	+#else
	149	+static inline void do_kernel_backtrace(unsigned long low_addr,
	150	+ struct stackframe *frame,
	151	+ unsigned int depth)
	152	+{
	153	+ while (depth-- && frame->pc) {
	154	+ frame->pc = unwind_stack_by_address(low_addr,
	155	+ &(frame->sp),
	156	+ frame->pc,
	157	+ &(frame->ra));
	158	+ oprofile_add_trace(frame->ra);
	159	+ }
	160	+}
	161	+#endif
	162	+
	163	+void notrace op_mips_backtrace(struct pt_regs *const regs, unsigned int depth)
	164	+{
	165	+ struct stackframe frame = { .sp = regs->regs[29],
	166	+ .pc = regs->cp0_epc,
	167	+ .ra = regs->regs[31] };
	168	+ const int userspace = user_mode(regs);
	169	+ const unsigned long low_addr = ALIGN(frame.sp, THREAD_SIZE);
	170	+
	171	+ if (userspace)
	172	+ do_user_backtrace(low_addr, &frame, depth);
	173	+ else
	174	+ do_kernel_backtrace(low_addr, &frame, depth);
	175	+}
...	...	@@ -115,6 +115,7 @@
115	115	ops->start = op_mips_start;
116	116	ops->stop = op_mips_stop;
117	117	ops->cpu_type = lmodel->cpu_type;
	118	+ ops->backtrace = op_mips_backtrace;
118	119
119	120	printk(KERN_INFO "oprofile: using %s performance monitoring.\n",
120	121	lmodel->cpu_type);
...	...	@@ -36,5 +36,7 @@
36	36	unsigned char num_counters;
37	37	};
38	38
	39	+void op_mips_backtrace(struct pt_regs * const regs, unsigned int depth);
	40	+
39	41	#endif
...	...	@@ -11,10 +11,12 @@
11	11	#include <linux/oprofile.h>
12	12	#include <linux/sched.h>
13	13	#include <linux/mm.h>
	14	+#include <linux/compat.h>
	15	+#include <linux/highmem.h>
	16	+
14	17	#include <asm/ptrace.h>
15	18	#include <asm/uaccess.h>
16	19	#include <asm/stacktrace.h>
17		-#include <linux/compat.h>
18	20
19	21	static int backtrace_stack(void data, char name)
20	22	{
21	23
22	24
23	25
24	26
...	...	@@ -36,18 +38,54 @@
36	38	.walk_stack = print_context_stack,
37	39	};
38	40
	41	+/* from arch/x86/kernel/cpu/perf_event.c: */
	42	+
	43	+/*
	44	+ * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
	45	+ */
	46	+static unsigned long
	47	+copy_from_user_nmi(void to, const void __user from, unsigned long n)
	48	+{
	49	+ unsigned long offset, addr = (unsigned long)from;
	50	+ unsigned long size, len = 0;
	51	+ struct page *page;
	52	+ void *map;
	53	+ int ret;
	54	+
	55	+ do {
	56	+ ret = __get_user_pages_fast(addr, 1, 0, &page);
	57	+ if (!ret)
	58	+ break;
	59	+
	60	+ offset = addr & (PAGE_SIZE - 1);
	61	+ size = min(PAGE_SIZE - offset, n - len);
	62	+
	63	+ map = kmap_atomic(page);
	64	+ memcpy(to, map+offset, size);
	65	+ kunmap_atomic(map);
	66	+ put_page(page);
	67	+
	68	+ len += size;
	69	+ to += size;
	70	+ addr += size;
	71	+
	72	+ } while (len < n);
	73	+
	74	+ return len;
	75	+}
	76	+
39	77	#ifdef CONFIG_COMPAT
40	78	static struct stack_frame_ia32 *
41	79	dump_user_backtrace_32(struct stack_frame_ia32 *head)
42	80	{
	81	+ /* Also check accessibility of one struct frame_head beyond: */
43	82	struct stack_frame_ia32 bufhead[2];
44	83	struct stack_frame_ia32 *fp;
	84	+ unsigned long bytes;
45	85
46		- /* Also check accessibility of one struct frame_head beyond */
47		- if (!access_ok(VERIFY_READ, head, sizeof(bufhead)))
	86	+ bytes = copy_from_user_nmi(bufhead, head, sizeof(bufhead));
	87	+ if (bytes != sizeof(bufhead))
48	88	return NULL;
49		- if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead)))
50		- return NULL;
51	89
52	90	fp = (struct stack_frame_ia32 *) compat_ptr(bufhead[0].next_frame);
53	91
54	92
55	93
...	...	@@ -87,12 +125,12 @@
87	125
88	126	static struct stack_frame dump_user_backtrace(struct stack_frame head)
89	127	{
	128	+ /* Also check accessibility of one struct frame_head beyond: */
90	129	struct stack_frame bufhead[2];
	130	+ unsigned long bytes;
91	131
92		- /* Also check accessibility of one struct stack_frame beyond */
93		- if (!access_ok(VERIFY_READ, head, sizeof(bufhead)))
94		- return NULL;
95		- if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead)))
	132	+ bytes = copy_from_user_nmi(bufhead, head, sizeof(bufhead));
	133	+ if (bytes != sizeof(bufhead))
96	134	return NULL;
97	135
98	136	oprofile_add_trace(bufhead[0].return_address);
...	...	@@ -112,8 +112,10 @@
112	112	static int nmi_start(void)
113	113	{
114	114	get_online_cpus();
115		- on_each_cpu(nmi_cpu_start, NULL, 1);
116	115	ctr_running = 1;
	116	+ /* make ctr_running visible to the nmi handler: */
	117	+ smp_mb();
	118	+ on_each_cpu(nmi_cpu_start, NULL, 1);
117	119	put_online_cpus();
118	120	return 0;
119	121	}
120	122
121	123
...	...	@@ -504,15 +506,18 @@
504	506
505	507	nmi_enabled = 0;
506	508	ctr_running = 0;
507		- barrier();
	509	+ /* make variables visible to the nmi handler: */
	510	+ smp_mb();
508	511	err = register_die_notifier(&profile_exceptions_nb);
509	512	if (err)
510	513	goto fail;
511	514
512	515	get_online_cpus();
513	516	register_cpu_notifier(&oprofile_cpu_nb);
514		- on_each_cpu(nmi_cpu_setup, NULL, 1);
515	517	nmi_enabled = 1;
	518	+ /* make nmi_enabled visible to the nmi handler: */
	519	+ smp_mb();
	520	+ on_each_cpu(nmi_cpu_setup, NULL, 1);
516	521	put_online_cpus();
517	522
518	523	return 0;
...	...	@@ -531,7 +536,8 @@
531	536	nmi_enabled = 0;
532	537	ctr_running = 0;
533	538	put_online_cpus();
534		- barrier();
	539	+ /* make variables visible to the nmi handler: */
	540	+ smp_mb();
535	541	unregister_die_notifier(&profile_exceptions_nb);
536	542	msrs = &get_cpu_var(cpu_msrs);
537	543	model->shutdown(msrs);