Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 7eb69529cbaf4229baf5559a400a7a46352c6e52

Authored by Linus Torvalds 2013-09-10 05:42:15 +0800

Exists in smarc-imx_3.14.28_1.0.0_ga and in 1 other branch

Merge tag 'trace-3.12' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace

Pull tracing updates from Steven Rostedt:
 "Not much changes for the 3.12 merge window.  The major tracing changes
  are still in flux, and will have to wait for 3.13.

  The changes for 3.12 are mostly clean ups and minor fixes.

  H Peter Anvin added a check to x86_32 static function tracing that
  helps a small segment of the kernel community.

  Oleg Nesterov had a few changes from 3.11, but were mostly clean ups
  and not worth pushing in the -rc time frame.

  Li Zefan had small clean up with annotating a raw_init with __init.

  I fixed a slight race in updating function callbacks, but the race is
  so small and the bug that happens when it occurs is so minor it's not
  even worth pushing to stable.

  The only real enhancement is from Alexander Z Lam that made the
  tracing_cpumask work for trace buffer instances, instead of them all
  sharing a global cpumask"

* tag 'trace-3.12' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace:
  ftrace/rcu: Do not trace debug_lockdep_rcu_enabled()
  x86-32, ftrace: Fix static ftrace when early microcode is enabled
  ftrace: Fix a slight race in modifying what function callback gets traced
  tracing: Make tracing_cpumask available for all instances
  tracing: Kill the !CONFIG_MODULES code in trace_events.c
  tracing: Don't pass file_operations array to event_create_dir()
  tracing: Kill trace_create_file_ops() and friends
  tracing/syscalls: Annotate raw_init function with __init

Showing 7 changed files Inline Diff

arch/x86/kernel/entry_32.S
kernel/rcupdate.c
kernel/trace/ftrace.c
kernel/trace/trace.c
kernel/trace/trace.h
kernel/trace/trace_events.c
kernel/trace/trace_syscalls.c

arch/x86/kernel/entry_32.S

Diff comments View file @ 7eb6952

 /*
  *
  *  Copyright (C) 1991, 1992  Linus Torvalds
  */
 /*
  * entry.S contains the system-call and fault low-level handling routines.
  * This also contains the timer-interrupt handler, as well as all interrupts
  * and faults that can result in a task-switch.
  *
  * NOTE: This code handles signal-recognition, which happens every time
  * after a timer-interrupt and after each system call.
  *
  * I changed all the .align's to 4 (16 byte alignment), as that's faster
  * on a 486.
  *
  * Stack layout in 'syscall_exit':
  * 	ptrace needs to have all regs on the stack.
  *	if the order here is changed, it needs to be
  *	updated in fork.c:copy_process, signal.c:do_signal,
  *	ptrace.c and ptrace.h
  *
  *	 0(%esp) - %ebx
  *	 4(%esp) - %ecx
  *	 8(%esp) - %edx
  *       C(%esp) - %esi
  *	10(%esp) - %edi
  *	14(%esp) - %ebp
  *	18(%esp) - %eax
  *	1C(%esp) - %ds
  *	20(%esp) - %es
  *	24(%esp) - %fs
  *	28(%esp) - %gs		saved iff !CONFIG_X86_32_LAZY_GS
  *	2C(%esp) - orig_eax
  *	30(%esp) - %eip
  *	34(%esp) - %cs
  *	38(%esp) - %eflags
  *	3C(%esp) - %oldesp
  *	40(%esp) - %oldss
  *
  * "current" is in register %ebx during any slow entries.
  */
 #include <linux/linkage.h>
 #include <linux/err.h>
 #include <asm/thread_info.h>
 #include <asm/irqflags.h>
 #include <asm/errno.h>
 #include <asm/segment.h>
 #include <asm/smp.h>
 #include <asm/page_types.h>
 #include <asm/percpu.h>
 #include <asm/dwarf2.h>
 #include <asm/processor-flags.h>
 #include <asm/ftrace.h>
 #include <asm/irq_vectors.h>
 #include <asm/cpufeature.h>
 #include <asm/alternative-asm.h>
 #include <asm/asm.h>
 #include <asm/smap.h>
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
 #define AUDIT_ARCH_I386		(EM_386|__AUDIT_ARCH_LE)
 #define __AUDIT_ARCH_LE	   0x40000000
 #ifndef CONFIG_AUDITSYSCALL
 #define sysenter_audit	syscall_trace_entry
 #define sysexit_audit	syscall_exit_work
 #endif
 	.section .entry.text, "ax"
 /*
  * We use macros for low-level operations which need to be overridden
  * for paravirtualization.  The following will never clobber any registers:
  *   INTERRUPT_RETURN (aka. "iret")
  *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
  *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
  *
  * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
  * Allowing a register to be clobbered can shrink the paravirt replacement
  * enough to patch inline, increasing performance.
  */
 #ifdef CONFIG_PREEMPT
 #define preempt_stop(clobbers)	DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
 #else
 #define preempt_stop(clobbers)
 #define resume_kernel		restore_all
 #endif
 .macro TRACE_IRQS_IRET
 #ifdef CONFIG_TRACE_IRQFLAGS
 	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
 	jz 1f
 	TRACE_IRQS_ON
 1:
 #endif
 .endm
 /*
  * User gs save/restore
  *
  * %gs is used for userland TLS and kernel only uses it for stack
  * canary which is required to be at %gs:20 by gcc.  Read the comment
  * at the top of stackprotector.h for more info.
  *
  * Local labels 98 and 99 are used.
  */
 #ifdef CONFIG_X86_32_LAZY_GS
  /* unfortunately push/pop can't be no-op */
 .macro PUSH_GS
 	pushl_cfi $0
 .endm
 .macro POP_GS pop=0
 	addl $(4 + \pop), %esp
 	CFI_ADJUST_CFA_OFFSET -(4 + \pop)
 .endm
 .macro POP_GS_EX
 .endm
  /* all the rest are no-op */
 .macro PTGS_TO_GS
 .endm
 .macro PTGS_TO_GS_EX
 .endm
 .macro GS_TO_REG reg
 .endm
 .macro REG_TO_PTGS reg
 .endm
 .macro SET_KERNEL_GS reg
 .endm
 #else	/* CONFIG_X86_32_LAZY_GS */
 .macro PUSH_GS
 	pushl_cfi %gs
 	/*CFI_REL_OFFSET gs, 0*/
 .endm
 .macro POP_GS pop=0
 98:	popl_cfi %gs
 	/*CFI_RESTORE gs*/
   .if \pop <> 0
 	add $\pop, %esp
 	CFI_ADJUST_CFA_OFFSET -\pop
   .endif
 .endm
 .macro POP_GS_EX
 .pushsection .fixup, "ax"
 99:	movl $0, (%esp)
 	jmp 98b
 .popsection
 	_ASM_EXTABLE(98b,99b)
 .endm
 .macro PTGS_TO_GS
 98:	mov PT_GS(%esp), %gs
 .endm
 .macro PTGS_TO_GS_EX
 .pushsection .fixup, "ax"
 99:	movl $0, PT_GS(%esp)
 	jmp 98b
 .popsection
 	_ASM_EXTABLE(98b,99b)
 .endm
 .macro GS_TO_REG reg
 	movl %gs, \reg
 	/*CFI_REGISTER gs, \reg*/
 .endm
 .macro REG_TO_PTGS reg
 	movl \reg, PT_GS(%esp)
 	/*CFI_REL_OFFSET gs, PT_GS*/
 .endm
 .macro SET_KERNEL_GS reg
 	movl $(__KERNEL_STACK_CANARY), \reg
 	movl \reg, %gs
 .endm
 #endif	/* CONFIG_X86_32_LAZY_GS */
 .macro SAVE_ALL
 	cld
 	PUSH_GS
 	pushl_cfi %fs
 	/*CFI_REL_OFFSET fs, 0;*/
 	pushl_cfi %es
 	/*CFI_REL_OFFSET es, 0;*/
 	pushl_cfi %ds
 	/*CFI_REL_OFFSET ds, 0;*/
 	pushl_cfi %eax
 	CFI_REL_OFFSET eax, 0
 	pushl_cfi %ebp
 	CFI_REL_OFFSET ebp, 0
 	pushl_cfi %edi
 	CFI_REL_OFFSET edi, 0
 	pushl_cfi %esi
 	CFI_REL_OFFSET esi, 0
 	pushl_cfi %edx
 	CFI_REL_OFFSET edx, 0
 	pushl_cfi %ecx
 	CFI_REL_OFFSET ecx, 0
 	pushl_cfi %ebx
 	CFI_REL_OFFSET ebx, 0
 	movl $(__USER_DS), %edx
 	movl %edx, %ds
 	movl %edx, %es
 	movl $(__KERNEL_PERCPU), %edx
 	movl %edx, %fs
 	SET_KERNEL_GS %edx
 .endm
 .macro RESTORE_INT_REGS
 	popl_cfi %ebx
 	CFI_RESTORE ebx
 	popl_cfi %ecx
 	CFI_RESTORE ecx
 	popl_cfi %edx
 	CFI_RESTORE edx
 	popl_cfi %esi
 	CFI_RESTORE esi
 	popl_cfi %edi
 	CFI_RESTORE edi
 	popl_cfi %ebp
 	CFI_RESTORE ebp
 	popl_cfi %eax
 	CFI_RESTORE eax
 .endm
 .macro RESTORE_REGS pop=0
 	RESTORE_INT_REGS
 1:	popl_cfi %ds
 	/*CFI_RESTORE ds;*/
 2:	popl_cfi %es
 	/*CFI_RESTORE es;*/
 3:	popl_cfi %fs
 	/*CFI_RESTORE fs;*/
 	POP_GS \pop
 .pushsection .fixup, "ax"
 4:	movl $0, (%esp)
 	jmp 1b
 5:	movl $0, (%esp)
 	jmp 2b
 6:	movl $0, (%esp)
 	jmp 3b
 .popsection
 	_ASM_EXTABLE(1b,4b)
 	_ASM_EXTABLE(2b,5b)
 	_ASM_EXTABLE(3b,6b)
 	POP_GS_EX
 .endm
 .macro RING0_INT_FRAME
 	CFI_STARTPROC simple
 	CFI_SIGNAL_FRAME
 	CFI_DEF_CFA esp, 3*4
 	/*CFI_OFFSET cs, -2*4;*/
 	CFI_OFFSET eip, -3*4
 .endm
 .macro RING0_EC_FRAME
 	CFI_STARTPROC simple
 	CFI_SIGNAL_FRAME
 	CFI_DEF_CFA esp, 4*4
 	/*CFI_OFFSET cs, -2*4;*/
 	CFI_OFFSET eip, -3*4
 .endm
 .macro RING0_PTREGS_FRAME
 	CFI_STARTPROC simple
 	CFI_SIGNAL_FRAME
 	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
 	/*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
 	CFI_OFFSET eip, PT_EIP-PT_OLDESP
 	/*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
 	/*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
 	CFI_OFFSET eax, PT_EAX-PT_OLDESP
 	CFI_OFFSET ebp, PT_EBP-PT_OLDESP
 	CFI_OFFSET edi, PT_EDI-PT_OLDESP
 	CFI_OFFSET esi, PT_ESI-PT_OLDESP
 	CFI_OFFSET edx, PT_EDX-PT_OLDESP
 	CFI_OFFSET ecx, PT_ECX-PT_OLDESP
 	CFI_OFFSET ebx, PT_EBX-PT_OLDESP
 .endm
 ENTRY(ret_from_fork)
 	CFI_STARTPROC
 	pushl_cfi %eax
 	call schedule_tail
 	GET_THREAD_INFO(%ebp)
 	popl_cfi %eax
 	pushl_cfi $0x0202		# Reset kernel eflags
 	popfl_cfi
 	jmp syscall_exit
 	CFI_ENDPROC
 END(ret_from_fork)
 ENTRY(ret_from_kernel_thread)
 	CFI_STARTPROC
 	pushl_cfi %eax
 	call schedule_tail
 	GET_THREAD_INFO(%ebp)
 	popl_cfi %eax
 	pushl_cfi $0x0202		# Reset kernel eflags
 	popfl_cfi
 	movl PT_EBP(%esp),%eax
 	call *PT_EBX(%esp)
 	movl $0,PT_EAX(%esp)
 	jmp syscall_exit
 	CFI_ENDPROC
 ENDPROC(ret_from_kernel_thread)
 /*
  * Interrupt exit functions should be protected against kprobes
  */
 	.pushsection .kprobes.text, "ax"
 /*
  * Return to user mode is not as complex as all this looks,
  * but we want the default path for a system call return to
  * go as quickly as possible which is why some of this is
  * less clear than it otherwise should be.
  */
 	# userspace resumption stub bypassing syscall exit tracing
 	ALIGN
 	RING0_PTREGS_FRAME
 ret_from_exception:
 	preempt_stop(CLBR_ANY)
 ret_from_intr:
 	GET_THREAD_INFO(%ebp)
 #ifdef CONFIG_VM86
 	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS and CS
 	movb PT_CS(%esp), %al
 	andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
 #else
 	/*
 	 * We can be coming here from child spawned by kernel_thread().
 	 */
 	movl PT_CS(%esp), %eax
 	andl $SEGMENT_RPL_MASK, %eax
 #endif
 	cmpl $USER_RPL, %eax
 	jb resume_kernel		# not returning to v8086 or userspace
 ENTRY(resume_userspace)
 	LOCKDEP_SYS_EXIT
  	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
 					# setting need_resched or sigpending
 					# between sampling and the iret
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done on
 					# int/exception return?
 	jne work_pending
 	jmp restore_all
 END(ret_from_exception)
 #ifdef CONFIG_PREEMPT
 ENTRY(resume_kernel)
 	DISABLE_INTERRUPTS(CLBR_ANY)
 	cmpl $0,TI_preempt_count(%ebp)	# non-zero preempt_count ?
 	jnz restore_all
 need_resched:
 	movl TI_flags(%ebp), %ecx	# need_resched set ?
 	testb $_TIF_NEED_RESCHED, %cl
 	jz restore_all
 	testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)	# interrupts off (exception path) ?
 	jz restore_all
 	call preempt_schedule_irq
 	jmp need_resched
 END(resume_kernel)
 #endif
 	CFI_ENDPROC
 /*
  * End of kprobes section
  */
 	.popsection
 /* SYSENTER_RETURN points to after the "sysenter" instruction in
    the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */
 	# sysenter call handler stub
 ENTRY(ia32_sysenter_target)
 	CFI_STARTPROC simple
 	CFI_SIGNAL_FRAME
 	CFI_DEF_CFA esp, 0
 	CFI_REGISTER esp, ebp
 	movl TSS_sysenter_sp0(%esp),%esp
 sysenter_past_esp:
 	/*
 	 * Interrupts are disabled here, but we can't trace it until
 	 * enough kernel state to call TRACE_IRQS_OFF can be called - but
 	 * we immediately enable interrupts at that point anyway.
 	 */
 	pushl_cfi $__USER_DS
 	/*CFI_REL_OFFSET ss, 0*/
 	pushl_cfi %ebp
 	CFI_REL_OFFSET esp, 0
 	pushfl_cfi
 	orl $X86_EFLAGS_IF, (%esp)
 	pushl_cfi $__USER_CS
 	/*CFI_REL_OFFSET cs, 0*/
 	/*
 	 * Push current_thread_info()->sysenter_return to the stack.
 	 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
 	 * pushed above; +8 corresponds to copy_thread's esp0 setting.
 	 */
 	pushl_cfi ((TI_sysenter_return)-THREAD_SIZE+8+4*4)(%esp)
 	CFI_REL_OFFSET eip, 0
 	pushl_cfi %eax
 	SAVE_ALL
 	ENABLE_INTERRUPTS(CLBR_NONE)
 /*
  * Load the potential sixth argument from user stack.
  * Careful about security.
  */
 	cmpl $__PAGE_OFFSET-3,%ebp
 	jae syscall_fault
 	ASM_STAC
 1:	movl (%ebp),%ebp
 	ASM_CLAC
 	movl %ebp,PT_EBP(%esp)
 	_ASM_EXTABLE(1b,syscall_fault)
 	GET_THREAD_INFO(%ebp)
 	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 	jnz sysenter_audit
 sysenter_do_call:
 	cmpl $(NR_syscalls), %eax
 	jae syscall_badsys
 	call *sys_call_table(,%eax,4)
 	movl %eax,PT_EAX(%esp)
 	LOCKDEP_SYS_EXIT
 	DISABLE_INTERRUPTS(CLBR_ANY)
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	testl $_TIF_ALLWORK_MASK, %ecx
 	jne sysexit_audit
 sysenter_exit:
 /* if something modifies registers it must also disable sysexit */
 	movl PT_EIP(%esp), %edx
 	movl PT_OLDESP(%esp), %ecx
 	xorl %ebp,%ebp
 	TRACE_IRQS_ON
 1:	mov  PT_FS(%esp), %fs
 	PTGS_TO_GS
 	ENABLE_INTERRUPTS_SYSEXIT
 #ifdef CONFIG_AUDITSYSCALL
 sysenter_audit:
 	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
 	jnz syscall_trace_entry
 	addl $4,%esp
 	CFI_ADJUST_CFA_OFFSET -4
 	/* %esi already in 8(%esp)	   6th arg: 4th syscall arg */
 	/* %edx already in 4(%esp)	   5th arg: 3rd syscall arg */
 	/* %ecx already in 0(%esp)	   4th arg: 2nd syscall arg */
 	movl %ebx,%ecx			/* 3rd arg: 1st syscall arg */
 	movl %eax,%edx			/* 2nd arg: syscall number */
 	movl $AUDIT_ARCH_I386,%eax	/* 1st arg: audit arch */
 	call __audit_syscall_entry
 	pushl_cfi %ebx
 	movl PT_EAX(%esp),%eax		/* reload syscall number */
 	jmp sysenter_do_call
 sysexit_audit:
 	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 	jne syscall_exit_work
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_ANY)
 	movl %eax,%edx		/* second arg, syscall return value */
 	cmpl $-MAX_ERRNO,%eax	/* is it an error ? */
 	setbe %al		/* 1 if so, 0 if not */
 	movzbl %al,%eax		/* zero-extend that */
 	call __audit_syscall_exit
 	DISABLE_INTERRUPTS(CLBR_ANY)
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
 	jne syscall_exit_work
 	movl PT_EAX(%esp),%eax	/* reload syscall return value */
 	jmp sysenter_exit
 #endif
 	CFI_ENDPROC
 .pushsection .fixup,"ax"
 2:	movl $0,PT_FS(%esp)
 	jmp 1b
 .popsection
 	_ASM_EXTABLE(1b,2b)
 	PTGS_TO_GS_EX
 ENDPROC(ia32_sysenter_target)
 /*
  * syscall stub including irq exit should be protected against kprobes
  */
 	.pushsection .kprobes.text, "ax"
 	# system call handler stub
 ENTRY(system_call)
 	RING0_INT_FRAME			# can't unwind into user space anyway
 	ASM_CLAC
 	pushl_cfi %eax			# save orig_eax
 	SAVE_ALL
 	GET_THREAD_INFO(%ebp)
 					# system call tracing in operation / emulation
 	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
 	jnz syscall_trace_entry
 	cmpl $(NR_syscalls), %eax
 	jae syscall_badsys
 syscall_call:
 	call *sys_call_table(,%eax,4)
 	movl %eax,PT_EAX(%esp)		# store the return value
 syscall_exit:
 	LOCKDEP_SYS_EXIT
 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
 					# setting need_resched or sigpending
 					# between sampling and the iret
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	testl $_TIF_ALLWORK_MASK, %ecx	# current->work
 	jne syscall_exit_work
 restore_all:
 	TRACE_IRQS_IRET
 restore_all_notrace:
 	movl PT_EFLAGS(%esp), %eax	# mix EFLAGS, SS and CS
 	# Warning: PT_OLDSS(%esp) contains the wrong/random values if we
 	# are returning to the kernel.
 	# See comments in process.c:copy_thread() for details.
 	movb PT_OLDSS(%esp), %ah
 	movb PT_CS(%esp), %al
 	andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
 	cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
 	CFI_REMEMBER_STATE
 	je ldt_ss			# returning to user-space with LDT SS
 restore_nocheck:
 	RESTORE_REGS 4			# skip orig_eax/error_code
 irq_return:
 	INTERRUPT_RETURN
 .section .fixup,"ax"
 ENTRY(iret_exc)
 	pushl $0			# no error code
 	pushl $do_iret_error
 	jmp error_code
 .previous
 	_ASM_EXTABLE(irq_return,iret_exc)
 	CFI_RESTORE_STATE
 ldt_ss:
 	larl PT_OLDSS(%esp), %eax
 	jnz restore_nocheck
 	testl $0x00400000, %eax		# returning to 32bit stack?
 	jnz restore_nocheck		# allright, normal return
 #ifdef CONFIG_PARAVIRT
 	/*
 	 * The kernel can't run on a non-flat stack if paravirt mode
 	 * is active.  Rather than try to fixup the high bits of
 	 * ESP, bypass this code entirely.  This may break DOSemu
 	 * and/or Wine support in a paravirt VM, although the option
 	 * is still available to implement the setting of the high
 	 * 16-bits in the INTERRUPT_RETURN paravirt-op.
 	 */
 	cmpl $0, pv_info+PARAVIRT_enabled
 	jne restore_nocheck
 #endif
 /*
  * Setup and switch to ESPFIX stack
  *
  * We're returning to userspace with a 16 bit stack. The CPU will not
  * restore the high word of ESP for us on executing iret... This is an
  * "official" bug of all the x86-compatible CPUs, which we can work
  * around to make dosemu and wine happy. We do this by preloading the
  * high word of ESP with the high word of the userspace ESP while
  * compensating for the offset by changing to the ESPFIX segment with
  * a base address that matches for the difference.
  */
 #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
 	mov %esp, %edx			/* load kernel esp */
 	mov PT_OLDESP(%esp), %eax	/* load userspace esp */
 	mov %dx, %ax			/* eax: new kernel esp */
 	sub %eax, %edx			/* offset (low word is 0) */
 	shr $16, %edx
 	mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
 	mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
 	pushl_cfi $__ESPFIX_SS
 	pushl_cfi %eax			/* new kernel esp */
 	/* Disable interrupts, but do not irqtrace this section: we
 	 * will soon execute iret and the tracer was already set to
 	 * the irqstate after the iret */
 	DISABLE_INTERRUPTS(CLBR_EAX)
 	lss (%esp), %esp		/* switch to espfix segment */
 	CFI_ADJUST_CFA_OFFSET -8
 	jmp restore_nocheck
 	CFI_ENDPROC
 ENDPROC(system_call)
 	# perform work that needs to be done immediately before resumption
 	ALIGN
 	RING0_PTREGS_FRAME		# can't unwind into user space anyway
 work_pending:
 	testb $_TIF_NEED_RESCHED, %cl
 	jz work_notifysig
 work_resched:
 	call schedule
 	LOCKDEP_SYS_EXIT
 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
 					# setting need_resched or sigpending
 					# between sampling and the iret
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	andl $_TIF_WORK_MASK, %ecx	# is there any work to be done other
 					# than syscall tracing?
 	jz restore_all
 	testb $_TIF_NEED_RESCHED, %cl
 	jnz work_resched
 work_notifysig:				# deal with pending signals and
 					# notify-resume requests
 #ifdef CONFIG_VM86
 	testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
 	movl %esp, %eax
 	jne work_notifysig_v86		# returning to kernel-space or
 					# vm86-space
 1:
 #else
 	movl %esp, %eax
 #endif
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_NONE)
 	movb PT_CS(%esp), %bl
 	andb $SEGMENT_RPL_MASK, %bl
 	cmpb $USER_RPL, %bl
 	jb resume_kernel
 	xorl %edx, %edx
 	call do_notify_resume
 	jmp resume_userspace
 #ifdef CONFIG_VM86
 	ALIGN
 work_notifysig_v86:
 	pushl_cfi %ecx			# save ti_flags for do_notify_resume
 	call save_v86_state		# %eax contains pt_regs pointer
 	popl_cfi %ecx
 	movl %eax, %esp
 	jmp 1b
 #endif
 END(work_pending)
 	# perform syscall exit tracing
 	ALIGN
 syscall_trace_entry:
 	movl $-ENOSYS,PT_EAX(%esp)
 	movl %esp, %eax
 	call syscall_trace_enter
 	/* What it returned is what we'll actually use.  */
 	cmpl $(NR_syscalls), %eax
 	jnae syscall_call
 	jmp syscall_exit
 END(syscall_trace_entry)
 	# perform syscall exit tracing
 	ALIGN
 syscall_exit_work:
 	testl $_TIF_WORK_SYSCALL_EXIT, %ecx
 	jz work_pending
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_ANY)	# could let syscall_trace_leave() call
 					# schedule() instead
 	movl %esp, %eax
 	call syscall_trace_leave
 	jmp resume_userspace
 END(syscall_exit_work)
 	CFI_ENDPROC
 	RING0_INT_FRAME			# can't unwind into user space anyway
 syscall_fault:
 	ASM_CLAC
 	GET_THREAD_INFO(%ebp)
 	movl $-EFAULT,PT_EAX(%esp)
 	jmp resume_userspace
 END(syscall_fault)
 syscall_badsys:
 	movl $-ENOSYS,PT_EAX(%esp)
 	jmp resume_userspace
 END(syscall_badsys)
 	CFI_ENDPROC
 /*
  * End of kprobes section
  */
 	.popsection
 .macro FIXUP_ESPFIX_STACK
 /*
  * Switch back for ESPFIX stack to the normal zerobased stack
  *
  * We can't call C functions using the ESPFIX stack. This code reads
  * the high word of the segment base from the GDT and swiches to the
  * normal stack and adjusts ESP with the matching offset.
  */
 	/* fixup the stack */
 	mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
 	mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
 	shl $16, %eax
 	addl %esp, %eax			/* the adjusted stack pointer */
 	pushl_cfi $__KERNEL_DS
 	pushl_cfi %eax
 	lss (%esp), %esp		/* switch to the normal stack segment */
 	CFI_ADJUST_CFA_OFFSET -8
 .endm
 .macro UNWIND_ESPFIX_STACK
 	movl %ss, %eax
 	/* see if on espfix stack */
 	cmpw $__ESPFIX_SS, %ax
 	jne 27f
 	movl $__KERNEL_DS, %eax
 	movl %eax, %ds
 	movl %eax, %es
 	/* switch to normal stack */
 	FIXUP_ESPFIX_STACK
 27:
 .endm
 /*
  * Build the entry stubs and pointer table with some assembler magic.
  * We pack 7 stubs into a single 32-byte chunk, which will fit in a
  * single cache line on all modern x86 implementations.
  */
 .section .init.rodata,"a"
 ENTRY(interrupt)
 .section .entry.text, "ax"
 	.p2align 5
 	.p2align CONFIG_X86_L1_CACHE_SHIFT
 ENTRY(irq_entries_start)
 	RING0_INT_FRAME
 vector=FIRST_EXTERNAL_VECTOR
 .rept (NR_VECTORS-FIRST_EXTERNAL_VECTOR+6)/7
 	.balign 32
   .rept	7
     .if vector < NR_VECTORS
       .if vector <> FIRST_EXTERNAL_VECTOR
 	CFI_ADJUST_CFA_OFFSET -4
       .endif
 1:	pushl_cfi $(~vector+0x80)	/* Note: always in signed byte range */
       .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
 	jmp 2f
       .endif
       .previous
 	.long 1b
       .section .entry.text, "ax"
 vector=vector+1
     .endif
   .endr
 2:	jmp common_interrupt
 .endr
 END(irq_entries_start)
 .previous
 END(interrupt)
 .previous
 /*
  * the CPU automatically disables interrupts when executing an IRQ vector,
  * so IRQ-flags tracing has to follow that:
  */
 	.p2align CONFIG_X86_L1_CACHE_SHIFT
 common_interrupt:
 	ASM_CLAC
 	addl $-0x80,(%esp)	/* Adjust vector into the [-256,-1] range */
 	SAVE_ALL
 	TRACE_IRQS_OFF
 	movl %esp,%eax
 	call do_IRQ
 	jmp ret_from_intr
 ENDPROC(common_interrupt)
 	CFI_ENDPROC
 /*
  *  Irq entries should be protected against kprobes
  */
 	.pushsection .kprobes.text, "ax"
 #define BUILD_INTERRUPT3(name, nr, fn)	\
 ENTRY(name)				\
 	RING0_INT_FRAME;		\
 	ASM_CLAC;			\
 	pushl_cfi $~(nr);		\
 	SAVE_ALL;			\
 	TRACE_IRQS_OFF			\
 	movl %esp,%eax;			\
 	call fn;			\
 	jmp ret_from_intr;		\
 	CFI_ENDPROC;			\
 ENDPROC(name)
 #ifdef CONFIG_TRACING
 #define TRACE_BUILD_INTERRUPT(name, nr)		\
 	BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name)
 #else
 #define TRACE_BUILD_INTERRUPT(name, nr)
 #endif
 #define BUILD_INTERRUPT(name, nr) \
 	BUILD_INTERRUPT3(name, nr, smp_##name); \
 	TRACE_BUILD_INTERRUPT(name, nr)
 /* The include is where all of the SMP etc. interrupts come from */
 #include <asm/entry_arch.h>
 ENTRY(coprocessor_error)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 	pushl_cfi $do_coprocessor_error
 	jmp error_code
 	CFI_ENDPROC
 END(coprocessor_error)
 ENTRY(simd_coprocessor_error)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 #ifdef CONFIG_X86_INVD_BUG
 	/* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
 661:	pushl_cfi $do_general_protection
 662:
 .section .altinstructions,"a"
 	altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f
 .previous
 .section .altinstr_replacement,"ax"
 663:	pushl $do_simd_coprocessor_error
 664:
 .previous
 #else
 	pushl_cfi $do_simd_coprocessor_error
 #endif
 	jmp error_code
 	CFI_ENDPROC
 END(simd_coprocessor_error)
 ENTRY(device_not_available)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $-1			# mark this as an int
 	pushl_cfi $do_device_not_available
 	jmp error_code
 	CFI_ENDPROC
 END(device_not_available)
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
 	iret
 	_ASM_EXTABLE(native_iret, iret_exc)
 END(native_iret)
 ENTRY(native_irq_enable_sysexit)
 	sti
 	sysexit
 END(native_irq_enable_sysexit)
 #endif
 ENTRY(overflow)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 	pushl_cfi $do_overflow
 	jmp error_code
 	CFI_ENDPROC
 END(overflow)
 ENTRY(bounds)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 	pushl_cfi $do_bounds
 	jmp error_code
 	CFI_ENDPROC
 END(bounds)
 ENTRY(invalid_op)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 	pushl_cfi $do_invalid_op
 	jmp error_code
 	CFI_ENDPROC
 END(invalid_op)
 ENTRY(coprocessor_segment_overrun)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 	pushl_cfi $do_coprocessor_segment_overrun
 	jmp error_code
 	CFI_ENDPROC
 END(coprocessor_segment_overrun)
 ENTRY(invalid_TSS)
 	RING0_EC_FRAME
 	ASM_CLAC
 	pushl_cfi $do_invalid_TSS
 	jmp error_code
 	CFI_ENDPROC
 END(invalid_TSS)
 ENTRY(segment_not_present)
 	RING0_EC_FRAME
 	ASM_CLAC
 	pushl_cfi $do_segment_not_present
 	jmp error_code
 	CFI_ENDPROC
 END(segment_not_present)
 ENTRY(stack_segment)
 	RING0_EC_FRAME
 	ASM_CLAC
 	pushl_cfi $do_stack_segment
 	jmp error_code
 	CFI_ENDPROC
 END(stack_segment)
 ENTRY(alignment_check)
 	RING0_EC_FRAME
 	ASM_CLAC
 	pushl_cfi $do_alignment_check
 	jmp error_code
 	CFI_ENDPROC
 END(alignment_check)
 ENTRY(divide_error)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0			# no error code
 	pushl_cfi $do_divide_error
 	jmp error_code
 	CFI_ENDPROC
 END(divide_error)
 #ifdef CONFIG_X86_MCE
 ENTRY(machine_check)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 	pushl_cfi machine_check_vector
 	jmp error_code
 	CFI_ENDPROC
 END(machine_check)
 #endif
 ENTRY(spurious_interrupt_bug)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $0
 	pushl_cfi $do_spurious_interrupt_bug
 	jmp error_code
 	CFI_ENDPROC
 END(spurious_interrupt_bug)
 /*
  * End of kprobes section
  */
 	.popsection
 #ifdef CONFIG_XEN
 /* Xen doesn't set %esp to be precisely what the normal sysenter
    entrypoint expects, so fix it up before using the normal path. */
 ENTRY(xen_sysenter_target)
 	RING0_INT_FRAME
 	addl $5*4, %esp		/* remove xen-provided frame */
 	CFI_ADJUST_CFA_OFFSET -5*4
 	jmp sysenter_past_esp
 	CFI_ENDPROC
 ENTRY(xen_hypervisor_callback)
 	CFI_STARTPROC
 	pushl_cfi $-1 /* orig_ax = -1 => not a system call */
 	SAVE_ALL
 	TRACE_IRQS_OFF
 	/* Check to see if we got the event in the critical
 	   region in xen_iret_direct, after we've reenabled
 	   events and checked for pending events.  This simulates
 	   iret instruction's behaviour where it delivers a
 	   pending interrupt when enabling interrupts. */
 	movl PT_EIP(%esp),%eax
 	cmpl $xen_iret_start_crit,%eax
 	jb   1f
 	cmpl $xen_iret_end_crit,%eax
 	jae  1f
 	jmp  xen_iret_crit_fixup
 ENTRY(xen_do_upcall)
 1:	mov %esp, %eax
 	call xen_evtchn_do_upcall
 	jmp  ret_from_intr
 	CFI_ENDPROC
 ENDPROC(xen_hypervisor_callback)
 # Hypervisor uses this for application faults while it executes.
 # We get here for two reasons:
 #  1. Fault while reloading DS, ES, FS or GS
 #  2. Fault while executing IRET
 # Category 1 we fix up by reattempting the load, and zeroing the segment
 # register if the load fails.
 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
 # normal Linux return path in this case because if we use the IRET hypercall
 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
 # We distinguish between categories by maintaining a status value in EAX.
 ENTRY(xen_failsafe_callback)
 	CFI_STARTPROC
 	pushl_cfi %eax
 	movl $1,%eax
 1:	mov 4(%esp),%ds
 2:	mov 8(%esp),%es
 3:	mov 12(%esp),%fs
 4:	mov 16(%esp),%gs
 	/* EAX == 0 => Category 1 (Bad segment)
 	   EAX != 0 => Category 2 (Bad IRET) */
 	testl %eax,%eax
 	popl_cfi %eax
 	lea 16(%esp),%esp
 	CFI_ADJUST_CFA_OFFSET -16
 	jz 5f
 	jmp iret_exc
 5:	pushl_cfi $-1 /* orig_ax = -1 => not a system call */
 	SAVE_ALL
 	jmp ret_from_exception
 	CFI_ENDPROC
 .section .fixup,"ax"
 6:	xorl %eax,%eax
 	movl %eax,4(%esp)
 	jmp 1b
 7:	xorl %eax,%eax
 	movl %eax,8(%esp)
 	jmp 2b
 8:	xorl %eax,%eax
 	movl %eax,12(%esp)
 	jmp 3b
 9:	xorl %eax,%eax
 	movl %eax,16(%esp)
 	jmp 4b
 .previous
 	_ASM_EXTABLE(1b,6b)
 	_ASM_EXTABLE(2b,7b)
 	_ASM_EXTABLE(3b,8b)
 	_ASM_EXTABLE(4b,9b)
 ENDPROC(xen_failsafe_callback)
 BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
 		xen_evtchn_do_upcall)
 #endif	/* CONFIG_XEN */
 #if IS_ENABLED(CONFIG_HYPERV)
 BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
 	hyperv_vector_handler)
 #endif /* CONFIG_HYPERV */
 #ifdef CONFIG_FUNCTION_TRACER
 #ifdef CONFIG_DYNAMIC_FTRACE
 ENTRY(mcount)
 	ret
 END(mcount)
 ENTRY(ftrace_caller)
 	cmpl $0, function_trace_stop
 	jne  ftrace_stub
 	pushl %eax
 	pushl %ecx
 	pushl %edx
 	pushl $0	/* Pass NULL as regs pointer */
 	movl 4*4(%esp), %eax
 	movl 0x4(%ebp), %edx
 	leal function_trace_op, %ecx
 	subl $MCOUNT_INSN_SIZE, %eax
 .globl ftrace_call
 ftrace_call:
 	call ftrace_stub
 	addl $4,%esp	/* skip NULL pointer */
 	popl %edx
 	popl %ecx
 	popl %eax
 ftrace_ret:
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 .globl ftrace_graph_call
 ftrace_graph_call:
 	jmp ftrace_stub
 #endif
 .globl ftrace_stub
 ftrace_stub:
 	ret
 END(ftrace_caller)
 ENTRY(ftrace_regs_caller)
 	pushf	/* push flags before compare (in cs location) */
 	cmpl $0, function_trace_stop
 	jne ftrace_restore_flags
 	/*
 	 * i386 does not save SS and ESP when coming from kernel.
 	 * Instead, to get sp, &regs->sp is used (see ptrace.h).
 	 * Unfortunately, that means eflags must be at the same location
 	 * as the current return ip is. We move the return ip into the
 	 * ip location, and move flags into the return ip location.
 	 */
 	pushl 4(%esp)	/* save return ip into ip slot */
 	pushl $0	/* Load 0 into orig_ax */
 	pushl %gs
 	pushl %fs
 	pushl %es
 	pushl %ds
 	pushl %eax
 	pushl %ebp
 	pushl %edi
 	pushl %esi
 	pushl %edx
 	pushl %ecx
 	pushl %ebx
 	movl 13*4(%esp), %eax	/* Get the saved flags */
 	movl %eax, 14*4(%esp)	/* Move saved flags into regs->flags location */
 				/* clobbering return ip */
 	movl $__KERNEL_CS,13*4(%esp)
 	movl 12*4(%esp), %eax	/* Load ip (1st parameter) */
 	subl $MCOUNT_INSN_SIZE, %eax	/* Adjust ip */
 	movl 0x4(%ebp), %edx	/* Load parent ip (2nd parameter) */
 	leal function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
 	pushl %esp		/* Save pt_regs as 4th parameter */
 GLOBAL(ftrace_regs_call)
 	call ftrace_stub
 	addl $4, %esp		/* Skip pt_regs */
 	movl 14*4(%esp), %eax	/* Move flags back into cs */
 	movl %eax, 13*4(%esp)	/* Needed to keep addl from modifying flags */
 	movl 12*4(%esp), %eax	/* Get return ip from regs->ip */
 	movl %eax, 14*4(%esp)	/* Put return ip back for ret */
 	popl %ebx
 	popl %ecx
 	popl %edx
 	popl %esi
 	popl %edi
 	popl %ebp
 	popl %eax
 	popl %ds
 	popl %es
 	popl %fs
 	popl %gs
 	addl $8, %esp		/* Skip orig_ax and ip */
 	popf			/* Pop flags at end (no addl to corrupt flags) */
 	jmp ftrace_ret
 ftrace_restore_flags:
 	popf
 	jmp  ftrace_stub
 #else /* ! CONFIG_DYNAMIC_FTRACE */
 ENTRY(mcount)
+	cmpl $__PAGE_OFFSET, %esp
+	jb ftrace_stub		/* Paging not enabled yet? */
 	cmpl $0, function_trace_stop
 	jne  ftrace_stub
 	cmpl $ftrace_stub, ftrace_trace_function
 	jnz trace
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	cmpl $ftrace_stub, ftrace_graph_return
 	jnz ftrace_graph_caller
 	cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
 	jnz ftrace_graph_caller
 #endif
 .globl ftrace_stub
 ftrace_stub:
 	ret
 	/* taken from glibc */
 trace:
 	pushl %eax
 	pushl %ecx
 	pushl %edx
 	movl 0xc(%esp), %eax
 	movl 0x4(%ebp), %edx
 	subl $MCOUNT_INSN_SIZE, %eax
 	call *ftrace_trace_function
 	popl %edx
 	popl %ecx
 	popl %eax
 	jmp ftrace_stub
 END(mcount)
 #endif /* CONFIG_DYNAMIC_FTRACE */
 #endif /* CONFIG_FUNCTION_TRACER */
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 ENTRY(ftrace_graph_caller)
 	pushl %eax
 	pushl %ecx
 	pushl %edx
 	movl 0xc(%esp), %edx
 	lea 0x4(%ebp), %eax
 	movl (%ebp), %ecx
 	subl $MCOUNT_INSN_SIZE, %edx
 	call prepare_ftrace_return
 	popl %edx
 	popl %ecx
 	popl %eax
 	ret
 END(ftrace_graph_caller)
 .globl return_to_handler
 return_to_handler:
 	pushl %eax
 	pushl %edx
 	movl %ebp, %eax
 	call ftrace_return_to_handler
 	movl %eax, %ecx
 	popl %edx
 	popl %eax
 	jmp *%ecx
 #endif
 /*
  * Some functions should be protected against kprobes
  */
 	.pushsection .kprobes.text, "ax"
 ENTRY(page_fault)
 	RING0_EC_FRAME
 	ASM_CLAC
 	pushl_cfi $do_page_fault
 	ALIGN
 error_code:
 	/* the function address is in %gs's slot on the stack */
 	pushl_cfi %fs
 	/*CFI_REL_OFFSET fs, 0*/
 	pushl_cfi %es
 	/*CFI_REL_OFFSET es, 0*/
 	pushl_cfi %ds
 	/*CFI_REL_OFFSET ds, 0*/
 	pushl_cfi %eax
 	CFI_REL_OFFSET eax, 0
 	pushl_cfi %ebp
 	CFI_REL_OFFSET ebp, 0
 	pushl_cfi %edi
 	CFI_REL_OFFSET edi, 0
 	pushl_cfi %esi
 	CFI_REL_OFFSET esi, 0
 	pushl_cfi %edx
 	CFI_REL_OFFSET edx, 0
 	pushl_cfi %ecx
 	CFI_REL_OFFSET ecx, 0
 	pushl_cfi %ebx
 	CFI_REL_OFFSET ebx, 0
 	cld
 	movl $(__KERNEL_PERCPU), %ecx
 	movl %ecx, %fs
 	UNWIND_ESPFIX_STACK
 	GS_TO_REG %ecx
 	movl PT_GS(%esp), %edi		# get the function address
 	movl PT_ORIG_EAX(%esp), %edx	# get the error code
 	movl $-1, PT_ORIG_EAX(%esp)	# no syscall to restart
 	REG_TO_PTGS %ecx
 	SET_KERNEL_GS %ecx
 	movl $(__USER_DS), %ecx
 	movl %ecx, %ds
 	movl %ecx, %es
 	TRACE_IRQS_OFF
 	movl %esp,%eax			# pt_regs pointer
 	call *%edi
 	jmp ret_from_exception
 	CFI_ENDPROC
 END(page_fault)
 /*
  * Debug traps and NMI can happen at the one SYSENTER instruction
  * that sets up the real kernel stack. Check here, since we can't
  * allow the wrong stack to be used.
  *
  * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
  * already pushed 3 words if it hits on the sysenter instruction:
  * eflags, cs and eip.
  *
  * We just load the right stack, and push the three (known) values
  * by hand onto the new stack - while updating the return eip past
  * the instruction that would have done it for sysenter.
  */
 .macro FIX_STACK offset ok label
 	cmpw $__KERNEL_CS, 4(%esp)
 	jne \ok
 \label:
 	movl TSS_sysenter_sp0 + \offset(%esp), %esp
 	CFI_DEF_CFA esp, 0
 	CFI_UNDEFINED eip
 	pushfl_cfi
 	pushl_cfi $__KERNEL_CS
 	pushl_cfi $sysenter_past_esp
 	CFI_REL_OFFSET eip, 0
 .endm
 ENTRY(debug)
 	RING0_INT_FRAME
 	ASM_CLAC
 	cmpl $ia32_sysenter_target,(%esp)
 	jne debug_stack_correct
 	FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
 debug_stack_correct:
 	pushl_cfi $-1			# mark this as an int
 	SAVE_ALL
 	TRACE_IRQS_OFF
 	xorl %edx,%edx			# error code 0
 	movl %esp,%eax			# pt_regs pointer
 	call do_debug
 	jmp ret_from_exception
 	CFI_ENDPROC
 END(debug)
 /*
  * NMI is doubly nasty. It can happen _while_ we're handling
  * a debug fault, and the debug fault hasn't yet been able to
  * clear up the stack. So we first check whether we got  an
  * NMI on the sysenter entry path, but after that we need to
  * check whether we got an NMI on the debug path where the debug
  * fault happened on the sysenter path.
  */
 ENTRY(nmi)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi %eax
 	movl %ss, %eax
 	cmpw $__ESPFIX_SS, %ax
 	popl_cfi %eax
 	je nmi_espfix_stack
 	cmpl $ia32_sysenter_target,(%esp)
 	je nmi_stack_fixup
 	pushl_cfi %eax
 	movl %esp,%eax
 	/* Do not access memory above the end of our stack page,
 	 * it might not exist.
 	 */
 	andl $(THREAD_SIZE-1),%eax
 	cmpl $(THREAD_SIZE-20),%eax
 	popl_cfi %eax
 	jae nmi_stack_correct
 	cmpl $ia32_sysenter_target,12(%esp)
 	je nmi_debug_stack_check
 nmi_stack_correct:
 	/* We have a RING0_INT_FRAME here */
 	pushl_cfi %eax
 	SAVE_ALL
 	xorl %edx,%edx		# zero error code
 	movl %esp,%eax		# pt_regs pointer
 	call do_nmi
 	jmp restore_all_notrace
 	CFI_ENDPROC
 nmi_stack_fixup:
 	RING0_INT_FRAME
 	FIX_STACK 12, nmi_stack_correct, 1
 	jmp nmi_stack_correct
 nmi_debug_stack_check:
 	/* We have a RING0_INT_FRAME here */
 	cmpw $__KERNEL_CS,16(%esp)
 	jne nmi_stack_correct
 	cmpl $debug,(%esp)
 	jb nmi_stack_correct
 	cmpl $debug_esp_fix_insn,(%esp)
 	ja nmi_stack_correct
 	FIX_STACK 24, nmi_stack_correct, 1
 	jmp nmi_stack_correct
 nmi_espfix_stack:
 	/* We have a RING0_INT_FRAME here.
 	 *
 	 * create the pointer to lss back
 	 */
 	pushl_cfi %ss
 	pushl_cfi %esp
 	addl $4, (%esp)
 	/* copy the iret frame of 12 bytes */
 	.rept 3
 	pushl_cfi 16(%esp)
 	.endr
 	pushl_cfi %eax
 	SAVE_ALL
 	FIXUP_ESPFIX_STACK		# %eax == %esp
 	xorl %edx,%edx			# zero error code
 	call do_nmi
 	RESTORE_REGS
 	lss 12+4(%esp), %esp		# back to espfix stack
 	CFI_ADJUST_CFA_OFFSET -24
 	jmp irq_return
 	CFI_ENDPROC
 END(nmi)
 ENTRY(int3)
 	RING0_INT_FRAME
 	ASM_CLAC
 	pushl_cfi $-1			# mark this as an int
 	SAVE_ALL
 	TRACE_IRQS_OFF
 	xorl %edx,%edx		# zero error code
 	movl %esp,%eax		# pt_regs pointer
 	call do_int3
 	jmp ret_from_exception
 	CFI_ENDPROC
 END(int3)
 ENTRY(general_protection)
 	RING0_EC_FRAME
 	pushl_cfi $do_general_protection
 	jmp error_code
 	CFI_ENDPROC
 END(general_protection)
 #ifdef CONFIG_KVM_GUEST
 ENTRY(async_page_fault)
 	RING0_EC_FRAME
 	ASM_CLAC
 	pushl_cfi $do_async_page_fault
 	jmp error_code
 	CFI_ENDPROC
 END(async_page_fault)
 #endif
 /*
  * End of kprobes section
  */
 	.popsection

kernel/rcupdate.c

Diff comments View file @ 7eb6952

kernel/trace/ftrace.c

Diff comments View file @ 7eb6952

 /*
  * Infrastructure for profiling code inserted by 'gcc -pg'.
  *
  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
  *
  * Originally ported from the -rt patch by:
  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
  *
  * Based on code in the latency_tracer, that is:
  *
  *  Copyright (C) 2004-2006 Ingo Molnar
  *  Copyright (C) 2004 Nadia Yvette Chambers
  */
 #include <linux/stop_machine.h>
 #include <linux/clocksource.h>
 #include <linux/kallsyms.h>
 #include <linux/seq_file.h>
 #include <linux/suspend.h>
 #include <linux/debugfs.h>
 #include <linux/hardirq.h>
 #include <linux/kthread.h>
 #include <linux/uaccess.h>
 #include <linux/bsearch.h>
 #include <linux/module.h>
 #include <linux/ftrace.h>
 #include <linux/sysctl.h>
 #include <linux/slab.h>
 #include <linux/ctype.h>
 #include <linux/sort.h>
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/rcupdate.h>
 #include <trace/events/sched.h>
 #include <asm/setup.h>
 #include "trace_output.h"
 #include "trace_stat.h"
 #define FTRACE_WARN_ON(cond)			\
 	({					\
 		int ___r = cond;		\
 		if (WARN_ON(___r))		\
 			ftrace_kill();		\
 		___r;				\
 	})
 #define FTRACE_WARN_ON_ONCE(cond)		\
 	({					\
 		int ___r = cond;		\
 		if (WARN_ON_ONCE(___r))		\
 			ftrace_kill();		\
 		___r;				\
 	})
 /* hash bits for specific function selection */
 #define FTRACE_HASH_BITS 7
 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
 #define FTRACE_HASH_DEFAULT_BITS 10
 #define FTRACE_HASH_MAX_BITS 12
 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)
 #ifdef CONFIG_DYNAMIC_FTRACE
 #define INIT_REGEX_LOCK(opsname)	\
 	.regex_lock	= __MUTEX_INITIALIZER(opsname.regex_lock),
 #else
 #define INIT_REGEX_LOCK(opsname)
 #endif
 static struct ftrace_ops ftrace_list_end __read_mostly = {
 	.func		= ftrace_stub,
 	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
 };
 /* ftrace_enabled is a method to turn ftrace on or off */
 int ftrace_enabled __read_mostly;
 static int last_ftrace_enabled;
 /* Quick disabling of function tracer. */
 int function_trace_stop __read_mostly;
 /* Current function tracing op */
 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
 /* List for set_ftrace_pid's pids. */
 LIST_HEAD(ftrace_pids);
 struct ftrace_pid {
 	struct list_head list;
 	struct pid *pid;
 };
 /*
  * ftrace_disabled is set when an anomaly is discovered.
  * ftrace_disabled is much stronger than ftrace_enabled.
  */
 static int ftrace_disabled __read_mostly;
 static DEFINE_MUTEX(ftrace_lock);
 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
 static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
 static struct ftrace_ops global_ops;
 static struct ftrace_ops control_ops;
 #if ARCH_SUPPORTS_FTRACE_OPS
 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 				 struct ftrace_ops *op, struct pt_regs *regs);
 #else
 /* See comment below, where ftrace_ops_list_func is defined */
 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
 #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
 #endif
 /*
  * Traverse the ftrace_global_list, invoking all entries.  The reason that we
  * can use rcu_dereference_raw_notrace() is that elements removed from this list
  * are simply leaked, so there is no need to interact with a grace-period
  * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
  * concurrent insertions into the ftrace_global_list.
  *
  * Silly Alpha and silly pointer-speculation compiler optimizations!
  */
 #define do_for_each_ftrace_op(op, list)			\
 	op = rcu_dereference_raw_notrace(list);			\
 	do
 /*
  * Optimized for just a single item in the list (as that is the normal case).
  */
 #define while_for_each_ftrace_op(op)				\
 	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
 	       unlikely((op) != &ftrace_list_end))
 static inline void ftrace_ops_init(struct ftrace_ops *ops)
 {
 #ifdef CONFIG_DYNAMIC_FTRACE
 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
 		mutex_init(&ops->regex_lock);
 		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
 	}
 #endif
 }
 /**
  * ftrace_nr_registered_ops - return number of ops registered
  *
  * Returns the number of ftrace_ops registered and tracing functions
  */
 int ftrace_nr_registered_ops(void)
 {
 	struct ftrace_ops *ops;
 	int cnt = 0;
 	mutex_lock(&ftrace_lock);
 	for (ops = ftrace_ops_list;
 	     ops != &ftrace_list_end; ops = ops->next)
 		cnt++;
 	mutex_unlock(&ftrace_lock);
 	return cnt;
 }
 static void
 ftrace_global_list_func(unsigned long ip, unsigned long parent_ip,
 			struct ftrace_ops *op, struct pt_regs *regs)
 {
 	int bit;
 	bit = trace_test_and_set_recursion(TRACE_GLOBAL_START, TRACE_GLOBAL_MAX);
 	if (bit < 0)
 		return;
 	do_for_each_ftrace_op(op, ftrace_global_list) {
 		op->func(ip, parent_ip, op, regs);
 	} while_for_each_ftrace_op(op);
 	trace_clear_recursion(bit);
 }
 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
 			    struct ftrace_ops *op, struct pt_regs *regs)
 {
 	if (!test_tsk_trace_trace(current))
 		return;
 	ftrace_pid_function(ip, parent_ip, op, regs);
 }
 static void set_ftrace_pid_function(ftrace_func_t func)
 {
 	/* do not set ftrace_pid_function to itself! */
 	if (func != ftrace_pid_func)
 		ftrace_pid_function = func;
 }
 /**
  * clear_ftrace_function - reset the ftrace function
  *
  * This NULLs the ftrace function and in essence stops
  * tracing.  There may be lag
  */
 void clear_ftrace_function(void)
 {
 	ftrace_trace_function = ftrace_stub;
 	ftrace_pid_function = ftrace_stub;
 }
 static void control_ops_disable_all(struct ftrace_ops *ops)
 {
 	int cpu;
 	for_each_possible_cpu(cpu)
 		*per_cpu_ptr(ops->disabled, cpu) = 1;
 }
 static int control_ops_alloc(struct ftrace_ops *ops)
 {
 	int __percpu *disabled;
 	disabled = alloc_percpu(int);
 	if (!disabled)
 		return -ENOMEM;
 	ops->disabled = disabled;
 	control_ops_disable_all(ops);
 	return 0;
 }
 static void control_ops_free(struct ftrace_ops *ops)
 {
 	free_percpu(ops->disabled);
 }
 static void update_global_ops(void)
 {
 	ftrace_func_t func;
 	/*
 	 * If there's only one function registered, then call that
 	 * function directly. Otherwise, we need to iterate over the
 	 * registered callers.
 	 */
 	if (ftrace_global_list == &ftrace_list_end ||
 	    ftrace_global_list->next == &ftrace_list_end) {
 		func = ftrace_global_list->func;
 		/*
 		 * As we are calling the function directly.
 		 * If it does not have recursion protection,
 		 * the function_trace_op needs to be updated
 		 * accordingly.
 		 */
 		if (ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE)
 			global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
 		else
 			global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE;
 	} else {
 		func = ftrace_global_list_func;
 		/* The list has its own recursion protection. */
 		global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
 	}
 	/* If we filter on pids, update to use the pid function */
 	if (!list_empty(&ftrace_pids)) {
 		set_ftrace_pid_function(func);
 		func = ftrace_pid_func;
 	}
 	global_ops.func = func;
 }
 static void update_ftrace_function(void)
 {
 	ftrace_func_t func;
 	update_global_ops();
 	/*
 	 * If we are at the end of the list and this ops is
 	 * recursion safe and not dynamic and the arch supports passing ops,
 	 * then have the mcount trampoline call the function directly.
 	 */
 	if (ftrace_ops_list == &ftrace_list_end ||
 	    (ftrace_ops_list->next == &ftrace_list_end &&
 	     !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) &&
 	     (ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) &&
 	     !FTRACE_FORCE_LIST_FUNC)) {
 		/* Set the ftrace_ops that the arch callback uses */
 		if (ftrace_ops_list == &global_ops)
 			function_trace_op = ftrace_global_list;
 		else
 			function_trace_op = ftrace_ops_list;
 		func = ftrace_ops_list->func;
 	} else {
 		/* Just use the default ftrace_ops */
 		function_trace_op = &ftrace_list_end;
 		func = ftrace_ops_list_func;
 	}
 	ftrace_trace_function = func;
 }
 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 {
 	ops->next = *list;
 	/*
 	 * We are entering ops into the list but another
 	 * CPU might be walking that list. We need to make sure
 	 * the ops->next pointer is valid before another CPU sees
 	 * the ops pointer included into the list.
 	 */
 	rcu_assign_pointer(*list, ops);
 }
 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 {
 	struct ftrace_ops **p;
 	/*
 	 * If we are removing the last function, then simply point
 	 * to the ftrace_stub.
 	 */
 	if (*list == ops && ops->next == &ftrace_list_end) {
 		*list = &ftrace_list_end;
 		return 0;
 	}
 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
 		if (*p == ops)
 			break;
 	if (*p != ops)
 		return -1;
 	*p = (*p)->next;
 	return 0;
 }
 static void add_ftrace_list_ops(struct ftrace_ops **list,
 				struct ftrace_ops *main_ops,
 				struct ftrace_ops *ops)
 {
 	int first = *list == &ftrace_list_end;
 	add_ftrace_ops(list, ops);
 	if (first)
 		add_ftrace_ops(&ftrace_ops_list, main_ops);
 }
 static int remove_ftrace_list_ops(struct ftrace_ops **list,
 				  struct ftrace_ops *main_ops,
 				  struct ftrace_ops *ops)
 {
 	int ret = remove_ftrace_ops(list, ops);
 	if (!ret && *list == &ftrace_list_end)
 		ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
 	return ret;
 }
 static int __register_ftrace_function(struct ftrace_ops *ops)
 {
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	if (FTRACE_WARN_ON(ops == &global_ops))
 		return -EINVAL;
 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
 		return -EBUSY;
 	/* We don't support both control and global flags set. */
 	if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
 		return -EINVAL;
 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 	/*
 	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
 	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
 	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
 	 */
 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
 	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
 		return -EINVAL;
 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
 		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
 #endif
 	if (!core_kernel_data((unsigned long)ops))
 		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 		add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);
 		ops->flags |= FTRACE_OPS_FL_ENABLED;
 	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
 		if (control_ops_alloc(ops))
 			return -ENOMEM;
 		add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
 	} else
 		add_ftrace_ops(&ftrace_ops_list, ops);
 	if (ftrace_enabled)
 		update_ftrace_function();
 	return 0;
 }
 static void ftrace_sync(struct work_struct *work)
 {
 	/*
 	 * This function is just a stub to implement a hard force
 	 * of synchronize_sched(). This requires synchronizing
 	 * tasks even in userspace and idle.
 	 *
 	 * Yes, function tracing is rude.
 	 */
 }
 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 {
 	int ret;
 	if (ftrace_disabled)
 		return -ENODEV;
 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
 		return -EBUSY;
 	if (FTRACE_WARN_ON(ops == &global_ops))
 		return -EINVAL;
 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 		ret = remove_ftrace_list_ops(&ftrace_global_list,
 					     &global_ops, ops);
 		if (!ret)
 			ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
 		ret = remove_ftrace_list_ops(&ftrace_control_list,
 					     &control_ops, ops);
 		if (!ret) {
 			/*
 			 * The ftrace_ops is now removed from the list,
 			 * so there'll be no new users. We must ensure
 			 * all current users are done before we free
 			 * the control data.
 			 * Note synchronize_sched() is not enough, as we
 			 * use preempt_disable() to do RCU, but the function
 			 * tracer can be called where RCU is not active
 			 * (before user_exit()).
 			 */
 			schedule_on_each_cpu(ftrace_sync);
 			control_ops_free(ops);
 		}
 	} else
 		ret = remove_ftrace_ops(&ftrace_ops_list, ops);
 	if (ret < 0)
 		return ret;
 	if (ftrace_enabled)
 		update_ftrace_function();
 	/*
 	 * Dynamic ops may be freed, we must make sure that all
 	 * callers are done before leaving this function.
 	 *
 	 * Again, normal synchronize_sched() is not good enough.
 	 * We need to do a hard force of sched synchronization.
 	 */
 	if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
 		schedule_on_each_cpu(ftrace_sync);
 	return 0;
 }
 static void ftrace_update_pid_func(void)
 {
 	/* Only do something if we are tracing something */
 	if (ftrace_trace_function == ftrace_stub)
 		return;
 	update_ftrace_function();
 }
 #ifdef CONFIG_FUNCTION_PROFILER
 struct ftrace_profile {
 	struct hlist_node		node;
 	unsigned long			ip;
 	unsigned long			counter;
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	unsigned long long		time;
 	unsigned long long		time_squared;
 #endif
 };
 struct ftrace_profile_page {
 	struct ftrace_profile_page	*next;
 	unsigned long			index;
 	struct ftrace_profile		records[];
 };
 struct ftrace_profile_stat {
 	atomic_t			disabled;
 	struct hlist_head		*hash;
 	struct ftrace_profile_page	*pages;
 	struct ftrace_profile_page	*start;
 	struct tracer_stat		stat;
 };
 #define PROFILE_RECORDS_SIZE						\
 	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 #define PROFILES_PER_PAGE					\
 	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 static int ftrace_profile_enabled __read_mostly;
 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 static DEFINE_MUTEX(ftrace_profile_lock);
 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 #define FTRACE_PROFILE_HASH_BITS 10
 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
 static void *
 function_stat_next(void *v, int idx)
 {
 	struct ftrace_profile *rec = v;
 	struct ftrace_profile_page *pg;
 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
  again:
 	if (idx != 0)
 		rec++;
 	if ((void *)rec >= (void *)&pg->records[pg->index]) {
 		pg = pg->next;
 		if (!pg)
 			return NULL;
 		rec = &pg->records[0];
 		if (!rec->counter)
 			goto again;
 	}
 	return rec;
 }
 static void *function_stat_start(struct tracer_stat *trace)
 {
 	struct ftrace_profile_stat *stat =
 		container_of(trace, struct ftrace_profile_stat, stat);
 	if (!stat || !stat->start)
 		return NULL;
 	return function_stat_next(&stat->start->records[0], 0);
 }
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 /* function graph compares on total time */
 static int function_stat_cmp(void *p1, void *p2)
 {
 	struct ftrace_profile *a = p1;
 	struct ftrace_profile *b = p2;
 	if (a->time < b->time)
 		return -1;
 	if (a->time > b->time)
 		return 1;
 	else
 		return 0;
 }
 #else
 /* not function graph compares against hits */
 static int function_stat_cmp(void *p1, void *p2)
 {
 	struct ftrace_profile *a = p1;
 	struct ftrace_profile *b = p2;
 	if (a->counter < b->counter)
 		return -1;
 	if (a->counter > b->counter)
 		return 1;
 	else
 		return 0;
 }
 #endif
 static int function_stat_headers(struct seq_file *m)
 {
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	seq_printf(m, "  Function                               "
 		   "Hit    Time            Avg             s^2\n"
 		      "  --------                               "
 		   "---    ----            ---             ---\n");
 #else
 	seq_printf(m, "  Function                               Hit\n"
 		      "  --------                               ---\n");
 #endif
 	return 0;
 }
 static int function_stat_show(struct seq_file *m, void *v)
 {
 	struct ftrace_profile *rec = v;
 	char str[KSYM_SYMBOL_LEN];
 	int ret = 0;
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	static struct trace_seq s;
 	unsigned long long avg;
 	unsigned long long stddev;
 #endif
 	mutex_lock(&ftrace_profile_lock);
 	/* we raced with function_profile_reset() */
 	if (unlikely(rec->counter == 0)) {
 		ret = -EBUSY;
 		goto out;
 	}
 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	seq_printf(m, "    ");
 	avg = rec->time;
 	do_div(avg, rec->counter);
 	/* Sample standard deviation (s^2) */
 	if (rec->counter <= 1)
 		stddev = 0;
 	else {
 		/*
 		 * Apply Welford's method:
 		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
 		 */
 		stddev = rec->counter * rec->time_squared -
 			 rec->time * rec->time;
 		/*
 		 * Divide only 1000 for ns^2 -> us^2 conversion.
 		 * trace_print_graph_duration will divide 1000 again.
 		 */
 		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
 	}
 	trace_seq_init(&s);
 	trace_print_graph_duration(rec->time, &s);
 	trace_seq_puts(&s, "    ");
 	trace_print_graph_duration(avg, &s);
 	trace_seq_puts(&s, "    ");
 	trace_print_graph_duration(stddev, &s);
 	trace_print_seq(m, &s);
 #endif
 	seq_putc(m, '\n');
 out:
 	mutex_unlock(&ftrace_profile_lock);
 	return ret;
 }
 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 {
 	struct ftrace_profile_page *pg;
 	pg = stat->pages = stat->start;
 	while (pg) {
 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 		pg->index = 0;
 		pg = pg->next;
 	}
 	memset(stat->hash, 0,
 	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 }
 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 {
 	struct ftrace_profile_page *pg;
 	int functions;
 	int pages;
 	int i;
 	/* If we already allocated, do nothing */
 	if (stat->pages)
 		return 0;
 	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 	if (!stat->pages)
 		return -ENOMEM;
 #ifdef CONFIG_DYNAMIC_FTRACE
 	functions = ftrace_update_tot_cnt;
 #else
 	/*
 	 * We do not know the number of functions that exist because
 	 * dynamic tracing is what counts them. With past experience
 	 * we have around 20K functions. That should be more than enough.
 	 * It is highly unlikely we will execute every function in
 	 * the kernel.
 	 */
 	functions = 20000;
 #endif
 	pg = stat->start = stat->pages;
 	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 	for (i = 1; i < pages; i++) {
 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 		if (!pg->next)
 			goto out_free;
 		pg = pg->next;
 	}
 	return 0;
  out_free:
 	pg = stat->start;
 	while (pg) {
 		unsigned long tmp = (unsigned long)pg;
 		pg = pg->next;
 		free_page(tmp);
 	}
 	stat->pages = NULL;
 	stat->start = NULL;
 	return -ENOMEM;
 }
 static int ftrace_profile_init_cpu(int cpu)
 {
 	struct ftrace_profile_stat *stat;
 	int size;
 	stat = &per_cpu(ftrace_profile_stats, cpu);
 	if (stat->hash) {
 		/* If the profile is already created, simply reset it */
 		ftrace_profile_reset(stat);
 		return 0;
 	}
 	/*
 	 * We are profiling all functions, but usually only a few thousand
 	 * functions are hit. We'll make a hash of 1024 items.
 	 */
 	size = FTRACE_PROFILE_HASH_SIZE;
 	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 	if (!stat->hash)
 		return -ENOMEM;
 	/* Preallocate the function profiling pages */
 	if (ftrace_profile_pages_init(stat) < 0) {
 		kfree(stat->hash);
 		stat->hash = NULL;
 		return -ENOMEM;
 	}
 	return 0;
 }
 static int ftrace_profile_init(void)
 {
 	int cpu;
 	int ret = 0;
 	for_each_online_cpu(cpu) {
 		ret = ftrace_profile_init_cpu(cpu);
 		if (ret)
 			break;
 	}
 	return ret;
 }
 /* interrupts must be disabled */
 static struct ftrace_profile *
 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 {
 	struct ftrace_profile *rec;
 	struct hlist_head *hhd;
 	unsigned long key;
 	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
 	hhd = &stat->hash[key];
 	if (hlist_empty(hhd))
 		return NULL;
 	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
 		if (rec->ip == ip)
 			return rec;
 	}
 	return NULL;
 }
 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 			       struct ftrace_profile *rec)
 {
 	unsigned long key;
 	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 }
 /*
  * The memory is already allocated, this simply finds a new record to use.
  */
 static struct ftrace_profile *
 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 {
 	struct ftrace_profile *rec = NULL;
 	/* prevent recursion (from NMIs) */
 	if (atomic_inc_return(&stat->disabled) != 1)
 		goto out;
 	/*
 	 * Try to find the function again since an NMI
 	 * could have added it
 	 */
 	rec = ftrace_find_profiled_func(stat, ip);
 	if (rec)
 		goto out;
 	if (stat->pages->index == PROFILES_PER_PAGE) {
 		if (!stat->pages->next)
 			goto out;
 		stat->pages = stat->pages->next;
 	}
 	rec = &stat->pages->records[stat->pages->index++];
 	rec->ip = ip;
 	ftrace_add_profile(stat, rec);
  out:
 	atomic_dec(&stat->disabled);
 	return rec;
 }
 static void
 function_profile_call(unsigned long ip, unsigned long parent_ip,
 		      struct ftrace_ops *ops, struct pt_regs *regs)
 {
 	struct ftrace_profile_stat *stat;
 	struct ftrace_profile *rec;
 	unsigned long flags;
 	if (!ftrace_profile_enabled)
 		return;
 	local_irq_save(flags);
 	stat = &__get_cpu_var(ftrace_profile_stats);
 	if (!stat->hash || !ftrace_profile_enabled)
 		goto out;
 	rec = ftrace_find_profiled_func(stat, ip);
 	if (!rec) {
 		rec = ftrace_profile_alloc(stat, ip);
 		if (!rec)
 			goto out;
 	}
 	rec->counter++;
  out:
 	local_irq_restore(flags);
 }
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 {
 	function_profile_call(trace->func, 0, NULL, NULL);
 	return 1;
 }
 static void profile_graph_return(struct ftrace_graph_ret *trace)
 {
 	struct ftrace_profile_stat *stat;
 	unsigned long long calltime;
 	struct ftrace_profile *rec;
 	unsigned long flags;
 	local_irq_save(flags);
 	stat = &__get_cpu_var(ftrace_profile_stats);
 	if (!stat->hash || !ftrace_profile_enabled)
 		goto out;
 	/* If the calltime was zero'd ignore it */
 	if (!trace->calltime)
 		goto out;
 	calltime = trace->rettime - trace->calltime;
 	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 		int index;
 		index = trace->depth;
 		/* Append this call time to the parent time to subtract */
 		if (index)
 			current->ret_stack[index - 1].subtime += calltime;
 		if (current->ret_stack[index].subtime < calltime)
 			calltime -= current->ret_stack[index].subtime;
 		else
 			calltime = 0;
 	}
 	rec = ftrace_find_profiled_func(stat, trace->func);
 	if (rec) {
 		rec->time += calltime;
 		rec->time_squared += calltime * calltime;
 	}
  out:
 	local_irq_restore(flags);
 }
 static int register_ftrace_profiler(void)
 {
 	return register_ftrace_graph(&profile_graph_return,
 				     &profile_graph_entry);
 }
 static void unregister_ftrace_profiler(void)
 {
 	unregister_ftrace_graph();
 }
 #else
 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
 	.func		= function_profile_call,
 	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
 	INIT_REGEX_LOCK(ftrace_profile_ops)
 };
 static int register_ftrace_profiler(void)
 {
 	return register_ftrace_function(&ftrace_profile_ops);
 }
 static void unregister_ftrace_profiler(void)
 {
 	unregister_ftrace_function(&ftrace_profile_ops);
 }
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 static ssize_t
 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 		     size_t cnt, loff_t *ppos)
 {
 	unsigned long val;
 	int ret;
 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 	if (ret)
 		return ret;
 	val = !!val;
 	mutex_lock(&ftrace_profile_lock);
 	if (ftrace_profile_enabled ^ val) {
 		if (val) {
 			ret = ftrace_profile_init();
 			if (ret < 0) {
 				cnt = ret;
 				goto out;
 			}
 			ret = register_ftrace_profiler();
 			if (ret < 0) {
 				cnt = ret;
 				goto out;
 			}
 			ftrace_profile_enabled = 1;
 		} else {
 			ftrace_profile_enabled = 0;
 			/*
 			 * unregister_ftrace_profiler calls stop_machine
 			 * so this acts like an synchronize_sched.
 			 */
 			unregister_ftrace_profiler();
 		}
 	}
  out:
 	mutex_unlock(&ftrace_profile_lock);
 	*ppos += cnt;
 	return cnt;
 }
 static ssize_t
 ftrace_profile_read(struct file *filp, char __user *ubuf,
 		     size_t cnt, loff_t *ppos)
 {
 	char buf[64];		/* big enough to hold a number */
 	int r;
 	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 }
 static const struct file_operations ftrace_profile_fops = {
 	.open		= tracing_open_generic,
 	.read		= ftrace_profile_read,
 	.write		= ftrace_profile_write,
 	.llseek		= default_llseek,
 };
 /* used to initialize the real stat files */
 static struct tracer_stat function_stats __initdata = {
 	.name		= "functions",
 	.stat_start	= function_stat_start,
 	.stat_next	= function_stat_next,
 	.stat_cmp	= function_stat_cmp,
 	.stat_headers	= function_stat_headers,
 	.stat_show	= function_stat_show
 };
 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 {
 	struct ftrace_profile_stat *stat;
 	struct dentry *entry;
 	char *name;
 	int ret;
 	int cpu;
 	for_each_possible_cpu(cpu) {
 		stat = &per_cpu(ftrace_profile_stats, cpu);
 		/* allocate enough for function name + cpu number */
 		name = kmalloc(32, GFP_KERNEL);
 		if (!name) {
 			/*
 			 * The files created are permanent, if something happens
 			 * we still do not free memory.
 			 */
 			WARN(1,
 			     "Could not allocate stat file for cpu %d\n",
 			     cpu);
 			return;
 		}
 		stat->stat = function_stats;
 		snprintf(name, 32, "function%d", cpu);
 		stat->stat.name = name;
 		ret = register_stat_tracer(&stat->stat);
 		if (ret) {
 			WARN(1,
 			     "Could not register function stat for cpu %d\n",
 			     cpu);
 			kfree(name);
 			return;
 		}
 	}
 	entry = debugfs_create_file("function_profile_enabled", 0644,
 				    d_tracer, NULL, &ftrace_profile_fops);
 	if (!entry)
 		pr_warning("Could not create debugfs "
 			   "'function_profile_enabled' entry\n");
 }
 #else /* CONFIG_FUNCTION_PROFILER */
 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 {
 }
 #endif /* CONFIG_FUNCTION_PROFILER */
 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 loff_t
 ftrace_filter_lseek(struct file *file, loff_t offset, int whence)
 {
 	loff_t ret;
 	if (file->f_mode & FMODE_READ)
 		ret = seq_lseek(file, offset, whence);
 	else
 		file->f_pos = ret = 1;
 	return ret;
 }
 #ifdef CONFIG_DYNAMIC_FTRACE
 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 # error Dynamic ftrace depends on MCOUNT_RECORD
 #endif
 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 struct ftrace_func_probe {
 	struct hlist_node	node;
 	struct ftrace_probe_ops	*ops;
 	unsigned long		flags;
 	unsigned long		ip;
 	void			*data;
 	struct list_head	free_list;
 };
 struct ftrace_func_entry {
 	struct hlist_node hlist;
 	unsigned long ip;
 };
 struct ftrace_hash {
 	unsigned long		size_bits;
 	struct hlist_head	*buckets;
 	unsigned long		count;
 	struct rcu_head		rcu;
 };
 /*
  * We make these constant because no one should touch them,
  * but they are used as the default "empty hash", to avoid allocating
  * it all the time. These are in a read only section such that if
  * anyone does try to modify it, it will cause an exception.
  */
 static const struct hlist_head empty_buckets[1];
 static const struct ftrace_hash empty_hash = {
 	.buckets = (struct hlist_head *)empty_buckets,
 };
 #define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
 static struct ftrace_ops global_ops = {
 	.func			= ftrace_stub,
 	.notrace_hash		= EMPTY_HASH,
 	.filter_hash		= EMPTY_HASH,
 	.flags			= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
 	INIT_REGEX_LOCK(global_ops)
 };
 struct ftrace_page {
 	struct ftrace_page	*next;
 	struct dyn_ftrace	*records;
 	int			index;
 	int			size;
 };
 static struct ftrace_page *ftrace_new_pgs;
 #define ENTRY_SIZE sizeof(struct dyn_ftrace)
 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
 /* estimate from running different kernels */
 #define NR_TO_INIT		10000
 static struct ftrace_page	*ftrace_pages_start;
 static struct ftrace_page	*ftrace_pages;
 static bool ftrace_hash_empty(struct ftrace_hash *hash)
 {
 	return !hash || !hash->count;
 }
 static struct ftrace_func_entry *
 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
 {
 	unsigned long key;
 	struct ftrace_func_entry *entry;
 	struct hlist_head *hhd;
 	if (ftrace_hash_empty(hash))
 		return NULL;
 	if (hash->size_bits > 0)
 		key = hash_long(ip, hash->size_bits);
 	else
 		key = 0;
 	hhd = &hash->buckets[key];
 	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
 		if (entry->ip == ip)
 			return entry;
 	}
 	return NULL;
 }
 static void __add_hash_entry(struct ftrace_hash *hash,
 			     struct ftrace_func_entry *entry)
 {
 	struct hlist_head *hhd;
 	unsigned long key;
 	if (hash->size_bits)
 		key = hash_long(entry->ip, hash->size_bits);
 	else
 		key = 0;
 	hhd = &hash->buckets[key];
 	hlist_add_head(&entry->hlist, hhd);
 	hash->count++;
 }
 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
 {
 	struct ftrace_func_entry *entry;
 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 	if (!entry)
 		return -ENOMEM;
 	entry->ip = ip;
 	__add_hash_entry(hash, entry);
 	return 0;
 }
 static void
 free_hash_entry(struct ftrace_hash *hash,
 		  struct ftrace_func_entry *entry)
 {
 	hlist_del(&entry->hlist);
 	kfree(entry);
 	hash->count--;
 }
 static void
 remove_hash_entry(struct ftrace_hash *hash,
 		  struct ftrace_func_entry *entry)
 {
 	hlist_del(&entry->hlist);
 	hash->count--;
 }
 static void ftrace_hash_clear(struct ftrace_hash *hash)
 {
 	struct hlist_head *hhd;
 	struct hlist_node *tn;
 	struct ftrace_func_entry *entry;
 	int size = 1 << hash->size_bits;
 	int i;
 	if (!hash->count)
 		return;
 	for (i = 0; i < size; i++) {
 		hhd = &hash->buckets[i];
 		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
 			free_hash_entry(hash, entry);
 	}
 	FTRACE_WARN_ON(hash->count);
 }
 static void free_ftrace_hash(struct ftrace_hash *hash)
 {
 	if (!hash || hash == EMPTY_HASH)
 		return;
 	ftrace_hash_clear(hash);
 	kfree(hash->buckets);
 	kfree(hash);
 }
 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
 {
 	struct ftrace_hash *hash;
 	hash = container_of(rcu, struct ftrace_hash, rcu);
 	free_ftrace_hash(hash);
 }
 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
 {
 	if (!hash || hash == EMPTY_HASH)
 		return;
 	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
 }
 void ftrace_free_filter(struct ftrace_ops *ops)
 {
 	ftrace_ops_init(ops);
 	free_ftrace_hash(ops->filter_hash);
 	free_ftrace_hash(ops->notrace_hash);
 }
 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
 {
 	struct ftrace_hash *hash;
 	int size;
 	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
 	if (!hash)
 		return NULL;
 	size = 1 << size_bits;
 	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
 	if (!hash->buckets) {
 		kfree(hash);
 		return NULL;
 	}
 	hash->size_bits = size_bits;
 	return hash;
 }
 static struct ftrace_hash *
 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
 {
 	struct ftrace_func_entry *entry;
 	struct ftrace_hash *new_hash;
 	int size;
 	int ret;
 	int i;
 	new_hash = alloc_ftrace_hash(size_bits);
 	if (!new_hash)
 		return NULL;
 	/* Empty hash? */
 	if (ftrace_hash_empty(hash))
 		return new_hash;
 	size = 1 << hash->size_bits;
 	for (i = 0; i < size; i++) {
 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
 			ret = add_hash_entry(new_hash, entry->ip);
 			if (ret < 0)
 				goto free_hash;
 		}
 	}
 	FTRACE_WARN_ON(new_hash->count != hash->count);
 	return new_hash;
  free_hash:
 	free_ftrace_hash(new_hash);
 	return NULL;
 }
 static void
 ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
 static void
 ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
 static int
 ftrace_hash_move(struct ftrace_ops *ops, int enable,
 		 struct ftrace_hash **dst, struct ftrace_hash *src)
 {
 	struct ftrace_func_entry *entry;
 	struct hlist_node *tn;
 	struct hlist_head *hhd;
 	struct ftrace_hash *old_hash;
 	struct ftrace_hash *new_hash;
 	int size = src->count;
 	int bits = 0;
 	int ret;
 	int i;
 	/*
 	 * Remove the current set, update the hash and add
 	 * them back.
 	 */
 	ftrace_hash_rec_disable(ops, enable);
 	/*
 	 * If the new source is empty, just free dst and assign it
 	 * the empty_hash.
 	 */
 	if (!src->count) {
 		free_ftrace_hash_rcu(*dst);
 		rcu_assign_pointer(*dst, EMPTY_HASH);
 		/* still need to update the function records */
 		ret = 0;
 		goto out;
 	}
 	/*
 	 * Make the hash size about 1/2 the # found
 	 */
 	for (size /= 2; size; size >>= 1)
 		bits++;
 	/* Don't allocate too much */
 	if (bits > FTRACE_HASH_MAX_BITS)
 		bits = FTRACE_HASH_MAX_BITS;
 	ret = -ENOMEM;
 	new_hash = alloc_ftrace_hash(bits);
 	if (!new_hash)
 		goto out;
 	size = 1 << src->size_bits;
 	for (i = 0; i < size; i++) {
 		hhd = &src->buckets[i];
 		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
 			remove_hash_entry(src, entry);
 			__add_hash_entry(new_hash, entry);
 		}
 	}
 	old_hash = *dst;
 	rcu_assign_pointer(*dst, new_hash);
 	free_ftrace_hash_rcu(old_hash);
 	ret = 0;
  out:
 	/*
 	 * Enable regardless of ret:
 	 *  On success, we enable the new hash.
 	 *  On failure, we re-enable the original hash.
 	 */
 	ftrace_hash_rec_enable(ops, enable);
 	return ret;
 }
 /*
  * Test the hashes for this ops to see if we want to call
  * the ops->func or not.
  *
  * It's a match if the ip is in the ops->filter_hash or
  * the filter_hash does not exist or is empty,
  *  AND
  * the ip is not in the ops->notrace_hash.
  *
  * This needs to be called with preemption disabled as
  * the hashes are freed with call_rcu_sched().
  */
 static int
 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
 {
 	struct ftrace_hash *filter_hash;
 	struct ftrace_hash *notrace_hash;
 	int ret;
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 	/*
 	 * There's a small race when adding ops that the ftrace handler
 	 * that wants regs, may be called without them. We can not
 	 * allow that handler to be called if regs is NULL.
 	 */
 	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
 		return 0;
 #endif
 	filter_hash = rcu_dereference_raw_notrace(ops->filter_hash);
 	notrace_hash = rcu_dereference_raw_notrace(ops->notrace_hash);
 	if ((ftrace_hash_empty(filter_hash) ||
 	     ftrace_lookup_ip(filter_hash, ip)) &&
 	    (ftrace_hash_empty(notrace_hash) ||
 	     !ftrace_lookup_ip(notrace_hash, ip)))
 		ret = 1;
 	else
 		ret = 0;
 	return ret;
 }
 /*
  * This is a double for. Do not use 'break' to break out of the loop,
  * you must use a goto.
  */
 #define do_for_each_ftrace_rec(pg, rec)					\
 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
 		int _____i;						\
 		for (_____i = 0; _____i < pg->index; _____i++) {	\
 			rec = &pg->records[_____i];
 #define while_for_each_ftrace_rec()		\
 		}				\
 	}
 static int ftrace_cmp_recs(const void *a, const void *b)
 {
 	const struct dyn_ftrace *key = a;
 	const struct dyn_ftrace *rec = b;
 	if (key->flags < rec->ip)
 		return -1;
 	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
 		return 1;
 	return 0;
 }
 static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
 {
 	struct ftrace_page *pg;
 	struct dyn_ftrace *rec;
 	struct dyn_ftrace key;
 	key.ip = start;
 	key.flags = end;	/* overload flags, as it is unsigned long */
 	for (pg = ftrace_pages_start; pg; pg = pg->next) {
 		if (end < pg->records[0].ip ||
 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
 			continue;
 		rec = bsearch(&key, pg->records, pg->index,
 			      sizeof(struct dyn_ftrace),
 			      ftrace_cmp_recs);
 		if (rec)
 			return rec->ip;
 	}
 	return 0;
 }
 /**
  * ftrace_location - return true if the ip giving is a traced location
  * @ip: the instruction pointer to check
  *
  * Returns rec->ip if @ip given is a pointer to a ftrace location.
  * That is, the instruction that is either a NOP or call to
  * the function tracer. It checks the ftrace internal tables to
  * determine if the address belongs or not.
  */
 unsigned long ftrace_location(unsigned long ip)
 {
 	return ftrace_location_range(ip, ip);
 }
 /**
  * ftrace_text_reserved - return true if range contains an ftrace location
  * @start: start of range to search
  * @end: end of range to search (inclusive). @end points to the last byte to check.
  *
  * Returns 1 if @start and @end contains a ftrace location.
  * That is, the instruction that is either a NOP or call to
  * the function tracer. It checks the ftrace internal tables to
  * determine if the address belongs or not.
  */
 int ftrace_text_reserved(void *start, void *end)
 {
 	unsigned long ret;
 	ret = ftrace_location_range((unsigned long)start,
 				    (unsigned long)end);
 	return (int)!!ret;
 }
 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
 				     int filter_hash,
 				     bool inc)
 {
 	struct ftrace_hash *hash;
 	struct ftrace_hash *other_hash;
 	struct ftrace_page *pg;
 	struct dyn_ftrace *rec;
 	int count = 0;
 	int all = 0;
 	/* Only update if the ops has been registered */
 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
 		return;
 	/*
 	 * In the filter_hash case:
 	 *   If the count is zero, we update all records.
 	 *   Otherwise we just update the items in the hash.
 	 *
 	 * In the notrace_hash case:
 	 *   We enable the update in the hash.
 	 *   As disabling notrace means enabling the tracing,
 	 *   and enabling notrace means disabling, the inc variable
 	 *   gets inversed.
 	 */
 	if (filter_hash) {
 		hash = ops->filter_hash;
 		other_hash = ops->notrace_hash;
 		if (ftrace_hash_empty(hash))
 			all = 1;
 	} else {
 		inc = !inc;
 		hash = ops->notrace_hash;
 		other_hash = ops->filter_hash;
 		/*
 		 * If the notrace hash has no items,
 		 * then there's nothing to do.
 		 */
 		if (ftrace_hash_empty(hash))
 			return;
 	}
 	do_for_each_ftrace_rec(pg, rec) {
 		int in_other_hash = 0;
 		int in_hash = 0;
 		int match = 0;
 		if (all) {
 			/*
 			 * Only the filter_hash affects all records.
 			 * Update if the record is not in the notrace hash.
 			 */
 			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
 				match = 1;
 		} else {
 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
 			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
 			/*
 			 *
 			 */
 			if (filter_hash && in_hash && !in_other_hash)
 				match = 1;
 			else if (!filter_hash && in_hash &&
 				 (in_other_hash || ftrace_hash_empty(other_hash)))
 				match = 1;
 		}
 		if (!match)
 			continue;
 		if (inc) {
 			rec->flags++;
 			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
 				return;
 			/*
 			 * If any ops wants regs saved for this function
 			 * then all ops will get saved regs.
 			 */
 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
 				rec->flags |= FTRACE_FL_REGS;
 		} else {
 			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
 				return;
 			rec->flags--;
 		}
 		count++;
 		/* Shortcut, if we handled all records, we are done. */
 		if (!all && count == hash->count)
 			return;
 	} while_for_each_ftrace_rec();
 }
 static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
 				    int filter_hash)
 {
 	__ftrace_hash_rec_update(ops, filter_hash, 0);
 }
 static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
 				   int filter_hash)
 {
 	__ftrace_hash_rec_update(ops, filter_hash, 1);
 }
 static void print_ip_ins(const char *fmt, unsigned char *p)
 {
 	int i;
 	printk(KERN_CONT "%s", fmt);
 	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 }
 /**
  * ftrace_bug - report and shutdown function tracer
  * @failed: The failed type (EFAULT, EINVAL, EPERM)
  * @ip: The address that failed
  *
  * The arch code that enables or disables the function tracing
  * can call ftrace_bug() when it has detected a problem in
  * modifying the code. @failed should be one of either:
  * EFAULT - if the problem happens on reading the @ip address
  * EINVAL - if what is read at @ip is not what was expected
  * EPERM - if the problem happens on writting to the @ip address
  */
 void ftrace_bug(int failed, unsigned long ip)
 {
 	switch (failed) {
 	case -EFAULT:
 		FTRACE_WARN_ON_ONCE(1);
 		pr_info("ftrace faulted on modifying ");
 		print_ip_sym(ip);
 		break;
 	case -EINVAL:
 		FTRACE_WARN_ON_ONCE(1);
 		pr_info("ftrace failed to modify ");
 		print_ip_sym(ip);
 		print_ip_ins(" actual: ", (unsigned char *)ip);
 		printk(KERN_CONT "\n");
 		break;
 	case -EPERM:
 		FTRACE_WARN_ON_ONCE(1);
 		pr_info("ftrace faulted on writing ");
 		print_ip_sym(ip);
 		break;
 	default:
 		FTRACE_WARN_ON_ONCE(1);
 		pr_info("ftrace faulted on unknown error ");
 		print_ip_sym(ip);
 	}
 }
 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
 {
 	unsigned long flag = 0UL;
 	/*
 	 * If we are updating calls:
 	 *
 	 *   If the record has a ref count, then we need to enable it
 	 *   because someone is using it.
 	 *
 	 *   Otherwise we make sure its disabled.
 	 *
 	 * If we are disabling calls, then disable all records that
 	 * are enabled.
 	 */
 	if (enable && (rec->flags & ~FTRACE_FL_MASK))
 		flag = FTRACE_FL_ENABLED;
 	/*
 	 * If enabling and the REGS flag does not match the REGS_EN, then
 	 * do not ignore this record. Set flags to fail the compare against
 	 * ENABLED.
 	 */
 	if (flag &&
 	    (!(rec->flags & FTRACE_FL_REGS) != !(rec->flags & FTRACE_FL_REGS_EN)))
 		flag |= FTRACE_FL_REGS;
 	/* If the state of this record hasn't changed, then do nothing */
 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
 		return FTRACE_UPDATE_IGNORE;
 	if (flag) {
 		/* Save off if rec is being enabled (for return value) */
 		flag ^= rec->flags & FTRACE_FL_ENABLED;
 		if (update) {
 			rec->flags |= FTRACE_FL_ENABLED;
 			if (flag & FTRACE_FL_REGS) {
 				if (rec->flags & FTRACE_FL_REGS)
 					rec->flags |= FTRACE_FL_REGS_EN;
 				else
 					rec->flags &= ~FTRACE_FL_REGS_EN;
 			}
 		}
 		/*
 		 * If this record is being updated from a nop, then
 		 *   return UPDATE_MAKE_CALL.
 		 * Otherwise, if the EN flag is set, then return
 		 *   UPDATE_MODIFY_CALL_REGS to tell the caller to convert
 		 *   from the non-save regs, to a save regs function.
 		 * Otherwise,
 		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
 		 *   from the save regs, to a non-save regs function.
 		 */
 		if (flag & FTRACE_FL_ENABLED)
 			return FTRACE_UPDATE_MAKE_CALL;
 		else if (rec->flags & FTRACE_FL_REGS_EN)
 			return FTRACE_UPDATE_MODIFY_CALL_REGS;
 		else
 			return FTRACE_UPDATE_MODIFY_CALL;
 	}
 	if (update) {
 		/* If there's no more users, clear all flags */
 		if (!(rec->flags & ~FTRACE_FL_MASK))
 			rec->flags = 0;
 		else
 			/* Just disable the record (keep REGS state) */
 			rec->flags &= ~FTRACE_FL_ENABLED;
 	}
 	return FTRACE_UPDATE_MAKE_NOP;
 }
 /**
  * ftrace_update_record, set a record that now is tracing or not
  * @rec: the record to update
  * @enable: set to 1 if the record is tracing, zero to force disable
  *
  * The records that represent all functions that can be traced need
  * to be updated when tracing has been enabled.
  */
 int ftrace_update_record(struct dyn_ftrace *rec, int enable)
 {
 	return ftrace_check_record(rec, enable, 1);
 }
 /**
  * ftrace_test_record, check if the record has been enabled or not
  * @rec: the record to test
  * @enable: set to 1 to check if enabled, 0 if it is disabled
  *
  * The arch code may need to test if a record is already set to
  * tracing to determine how to modify the function code that it
  * represents.
  */
 int ftrace_test_record(struct dyn_ftrace *rec, int enable)
 {
 	return ftrace_check_record(rec, enable, 0);
 }
 static int
 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 {
 	unsigned long ftrace_old_addr;
 	unsigned long ftrace_addr;
 	int ret;
 	ret = ftrace_update_record(rec, enable);
 	if (rec->flags & FTRACE_FL_REGS)
 		ftrace_addr = (unsigned long)FTRACE_REGS_ADDR;
 	else
 		ftrace_addr = (unsigned long)FTRACE_ADDR;
 	switch (ret) {
 	case FTRACE_UPDATE_IGNORE:
 		return 0;
 	case FTRACE_UPDATE_MAKE_CALL:
 		return ftrace_make_call(rec, ftrace_addr);
 	case FTRACE_UPDATE_MAKE_NOP:
 		return ftrace_make_nop(NULL, rec, ftrace_addr);
 	case FTRACE_UPDATE_MODIFY_CALL_REGS:
 	case FTRACE_UPDATE_MODIFY_CALL:
 		if (rec->flags & FTRACE_FL_REGS)
 			ftrace_old_addr = (unsigned long)FTRACE_ADDR;
 		else
 			ftrace_old_addr = (unsigned long)FTRACE_REGS_ADDR;
 		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
 	}
 	return -1; /* unknow ftrace bug */
 }
 void __weak ftrace_replace_code(int enable)
 {
 	struct dyn_ftrace *rec;
 	struct ftrace_page *pg;
 	int failed;
 	if (unlikely(ftrace_disabled))
 		return;
 	do_for_each_ftrace_rec(pg, rec) {
 		failed = __ftrace_replace_code(rec, enable);
 		if (failed) {
 			ftrace_bug(failed, rec->ip);
 			/* Stop processing */
 			return;
 		}
 	} while_for_each_ftrace_rec();
 }
 struct ftrace_rec_iter {
 	struct ftrace_page	*pg;
 	int			index;
 };
 /**
  * ftrace_rec_iter_start, start up iterating over traced functions
  *
  * Returns an iterator handle that is used to iterate over all
  * the records that represent address locations where functions
  * are traced.
  *
  * May return NULL if no records are available.
  */
 struct ftrace_rec_iter *ftrace_rec_iter_start(void)
 {
 	/*
 	 * We only use a single iterator.
 	 * Protected by the ftrace_lock mutex.
 	 */
 	static struct ftrace_rec_iter ftrace_rec_iter;
 	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
 	iter->pg = ftrace_pages_start;
 	iter->index = 0;
 	/* Could have empty pages */
 	while (iter->pg && !iter->pg->index)
 		iter->pg = iter->pg->next;
 	if (!iter->pg)
 		return NULL;
 	return iter;
 }
 /**
  * ftrace_rec_iter_next, get the next record to process.
  * @iter: The handle to the iterator.
  *
  * Returns the next iterator after the given iterator @iter.
  */
 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
 {
 	iter->index++;
 	if (iter->index >= iter->pg->index) {
 		iter->pg = iter->pg->next;
 		iter->index = 0;
 		/* Could have empty pages */
 		while (iter->pg && !iter->pg->index)
 			iter->pg = iter->pg->next;
 	}
 	if (!iter->pg)
 		return NULL;
 	return iter;
 }
 /**
  * ftrace_rec_iter_record, get the record at the iterator location
  * @iter: The current iterator location
  *
  * Returns the record that the current @iter is at.
  */
 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
 {
 	return &iter->pg->records[iter->index];
 }
 static int
 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
 {
 	unsigned long ip;
 	int ret;
 	ip = rec->ip;
 	if (unlikely(ftrace_disabled))
 		return 0;
 	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
 	if (ret) {
 		ftrace_bug(ret, ip);
 		return 0;
 	}
 	return 1;
 }
 /*
  * archs can override this function if they must do something
  * before the modifying code is performed.
  */
 int __weak ftrace_arch_code_modify_prepare(void)
 {
 	return 0;
 }
 /*
  * archs can override this function if they must do something
  * after the modifying code is performed.
  */
 int __weak ftrace_arch_code_modify_post_process(void)
 {
 	return 0;
 }
 void ftrace_modify_all_code(int command)
 {
+	int update = command & FTRACE_UPDATE_TRACE_FUNC;
+	/*
+	 * If the ftrace_caller calls a ftrace_ops func directly,
+	 * we need to make sure that it only traces functions it
+	 * expects to trace. When doing the switch of functions,
+	 * we need to update to the ftrace_ops_list_func first
+	 * before the transition between old and new calls are set,
+	 * as the ftrace_ops_list_func will check the ops hashes
+	 * to make sure the ops are having the right functions
+	 * traced.
+	 */
+	if (update)
+		ftrace_update_ftrace_func(ftrace_ops_list_func);
 	if (command & FTRACE_UPDATE_CALLS)
 		ftrace_replace_code(1);
 	else if (command & FTRACE_DISABLE_CALLS)
 		ftrace_replace_code(0);
-	if (command & FTRACE_UPDATE_TRACE_FUNC)
+	if (update && ftrace_trace_function != ftrace_ops_list_func)
 		ftrace_update_ftrace_func(ftrace_trace_function);
 	if (command & FTRACE_START_FUNC_RET)
 		ftrace_enable_ftrace_graph_caller();
 	else if (command & FTRACE_STOP_FUNC_RET)
 		ftrace_disable_ftrace_graph_caller();
 }
 static int __ftrace_modify_code(void *data)
 {
 	int *command = data;
 	ftrace_modify_all_code(*command);
 	return 0;
 }
 /**
  * ftrace_run_stop_machine, go back to the stop machine method
  * @command: The command to tell ftrace what to do
  *
  * If an arch needs to fall back to the stop machine method, the
  * it can call this function.
  */
 void ftrace_run_stop_machine(int command)
 {
 	stop_machine(__ftrace_modify_code, &command, NULL);
 }
 /**
  * arch_ftrace_update_code, modify the code to trace or not trace
  * @command: The command that needs to be done
  *
  * Archs can override this function if it does not need to
  * run stop_machine() to modify code.
  */
 void __weak arch_ftrace_update_code(int command)
 {
 	ftrace_run_stop_machine(command);
 }
 static void ftrace_run_update_code(int command)
 {
 	int ret;
 	ret = ftrace_arch_code_modify_prepare();
 	FTRACE_WARN_ON(ret);
 	if (ret)
 		return;
 	/*
 	 * Do not call function tracer while we update the code.
 	 * We are in stop machine.
 	 */
 	function_trace_stop++;
 	/*
 	 * By default we use stop_machine() to modify the code.
 	 * But archs can do what ever they want as long as it
 	 * is safe. The stop_machine() is the safest, but also
 	 * produces the most overhead.
 	 */
 	arch_ftrace_update_code(command);
 	function_trace_stop--;
 	ret = ftrace_arch_code_modify_post_process();
 	FTRACE_WARN_ON(ret);
 }
 static ftrace_func_t saved_ftrace_func;
 static int ftrace_start_up;
 static int global_start_up;
 static void ftrace_startup_enable(int command)
 {
 	if (saved_ftrace_func != ftrace_trace_function) {
 		saved_ftrace_func = ftrace_trace_function;
 		command |= FTRACE_UPDATE_TRACE_FUNC;
 	}
 	if (!command || !ftrace_enabled)
 		return;
 	ftrace_run_update_code(command);
 }
 static int ftrace_startup(struct ftrace_ops *ops, int command)
 {
 	bool hash_enable = true;
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	ftrace_start_up++;
 	command |= FTRACE_UPDATE_CALLS;
 	/* ops marked global share the filter hashes */
 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 		ops = &global_ops;
 		/* Don't update hash if global is already set */
 		if (global_start_up)
 			hash_enable = false;
 		global_start_up++;
 	}
 	ops->flags |= FTRACE_OPS_FL_ENABLED;
 	if (hash_enable)
 		ftrace_hash_rec_enable(ops, 1);
 	ftrace_startup_enable(command);
 	return 0;
 }
 static void ftrace_shutdown(struct ftrace_ops *ops, int command)
 {
 	bool hash_disable = true;
 	if (unlikely(ftrace_disabled))
 		return;
 	ftrace_start_up--;
 	/*
 	 * Just warn in case of unbalance, no need to kill ftrace, it's not
 	 * critical but the ftrace_call callers may be never nopped again after
 	 * further ftrace uses.
 	 */
 	WARN_ON_ONCE(ftrace_start_up < 0);
 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 		ops = &global_ops;
 		global_start_up--;
 		WARN_ON_ONCE(global_start_up < 0);
 		/* Don't update hash if global still has users */
 		if (global_start_up) {
 			WARN_ON_ONCE(!ftrace_start_up);
 			hash_disable = false;
 		}
 	}
 	if (hash_disable)
 		ftrace_hash_rec_disable(ops, 1);
 	if (ops != &global_ops || !global_start_up)
 		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 	command |= FTRACE_UPDATE_CALLS;
 	if (saved_ftrace_func != ftrace_trace_function) {
 		saved_ftrace_func = ftrace_trace_function;
 		command |= FTRACE_UPDATE_TRACE_FUNC;
 	}
 	if (!command || !ftrace_enabled)
 		return;
 	ftrace_run_update_code(command);
 }
 static void ftrace_startup_sysctl(void)
 {
 	if (unlikely(ftrace_disabled))
 		return;
 	/* Force update next time */
 	saved_ftrace_func = NULL;
 	/* ftrace_start_up is true if we want ftrace running */
 	if (ftrace_start_up)
 		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
 }
 static void ftrace_shutdown_sysctl(void)
 {
 	if (unlikely(ftrace_disabled))
 		return;
 	/* ftrace_start_up is true if ftrace is running */
 	if (ftrace_start_up)
 		ftrace_run_update_code(FTRACE_DISABLE_CALLS);
 }
 static cycle_t		ftrace_update_time;
 static unsigned long	ftrace_update_cnt;
 unsigned long		ftrace_update_tot_cnt;
 static inline int ops_traces_mod(struct ftrace_ops *ops)
 {
 	/*
 	 * Filter_hash being empty will default to trace module.
 	 * But notrace hash requires a test of individual module functions.
 	 */
 	return ftrace_hash_empty(ops->filter_hash) &&
 		ftrace_hash_empty(ops->notrace_hash);
 }
 /*
  * Check if the current ops references the record.
  *
  * If the ops traces all functions, then it was already accounted for.
  * If the ops does not trace the current record function, skip it.
  * If the ops ignores the function via notrace filter, skip it.
  */
 static inline bool
 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
 {
 	/* If ops isn't enabled, ignore it */
 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
 		return 0;
 	/* If ops traces all mods, we already accounted for it */
 	if (ops_traces_mod(ops))
 		return 0;
 	/* The function must be in the filter */
 	if (!ftrace_hash_empty(ops->filter_hash) &&
 	    !ftrace_lookup_ip(ops->filter_hash, rec->ip))
 		return 0;
 	/* If in notrace hash, we ignore it too */
 	if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))
 		return 0;
 	return 1;
 }
 static int referenced_filters(struct dyn_ftrace *rec)
 {
 	struct ftrace_ops *ops;
 	int cnt = 0;
 	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
 		if (ops_references_rec(ops, rec))
 		    cnt++;
 	}
 	return cnt;
 }
 static int ftrace_update_code(struct module *mod)
 {
 	struct ftrace_page *pg;
 	struct dyn_ftrace *p;
 	cycle_t start, stop;
 	unsigned long ref = 0;
 	bool test = false;
 	int i;
 	/*
 	 * When adding a module, we need to check if tracers are
 	 * currently enabled and if they are set to trace all functions.
 	 * If they are, we need to enable the module functions as well
 	 * as update the reference counts for those function records.
 	 */
 	if (mod) {
 		struct ftrace_ops *ops;
 		for (ops = ftrace_ops_list;
 		     ops != &ftrace_list_end; ops = ops->next) {
 			if (ops->flags & FTRACE_OPS_FL_ENABLED) {
 				if (ops_traces_mod(ops))
 					ref++;
 				else
 					test = true;
 			}
 		}
 	}
 	start = ftrace_now(raw_smp_processor_id());
 	ftrace_update_cnt = 0;
 	for (pg = ftrace_new_pgs; pg; pg = pg->next) {
 		for (i = 0; i < pg->index; i++) {
 			int cnt = ref;
 			/* If something went wrong, bail without enabling anything */
 			if (unlikely(ftrace_disabled))
 				return -1;
 			p = &pg->records[i];
 			if (test)
 				cnt += referenced_filters(p);
 			p->flags = cnt;
 			/*
 			 * Do the initial record conversion from mcount jump
 			 * to the NOP instructions.
 			 */
 			if (!ftrace_code_disable(mod, p))
 				break;
 			ftrace_update_cnt++;
 			/*
 			 * If the tracing is enabled, go ahead and enable the record.
 			 *
 			 * The reason not to enable the record immediatelly is the
 			 * inherent check of ftrace_make_nop/ftrace_make_call for
 			 * correct previous instructions.  Making first the NOP
 			 * conversion puts the module to the correct state, thus
 			 * passing the ftrace_make_call check.
 			 */
 			if (ftrace_start_up && cnt) {
 				int failed = __ftrace_replace_code(p, 1);
 				if (failed)
 					ftrace_bug(failed, p->ip);
 			}
 		}
 	}
 	ftrace_new_pgs = NULL;
 	stop = ftrace_now(raw_smp_processor_id());
 	ftrace_update_time = stop - start;
 	ftrace_update_tot_cnt += ftrace_update_cnt;
 	return 0;
 }
 static int ftrace_allocate_records(struct ftrace_page *pg, int count)
 {
 	int order;
 	int cnt;
 	if (WARN_ON(!count))
 		return -EINVAL;
 	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
 	/*
 	 * We want to fill as much as possible. No more than a page
 	 * may be empty.
 	 */
 	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
 		order--;
  again:
 	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
 	if (!pg->records) {
 		/* if we can't allocate this size, try something smaller */
 		if (!order)
 			return -ENOMEM;
 		order >>= 1;
 		goto again;
 	}
 	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
 	pg->size = cnt;
 	if (cnt > count)
 		cnt = count;
 	return cnt;
 }
 static struct ftrace_page *
 ftrace_allocate_pages(unsigned long num_to_init)
 {
 	struct ftrace_page *start_pg;
 	struct ftrace_page *pg;
 	int order;
 	int cnt;
 	if (!num_to_init)
 		return 0;
 	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 	if (!pg)
 		return NULL;
 	/*
 	 * Try to allocate as much as possible in one continues
 	 * location that fills in all of the space. We want to
 	 * waste as little space as possible.
 	 */
 	for (;;) {
 		cnt = ftrace_allocate_records(pg, num_to_init);
 		if (cnt < 0)
 			goto free_pages;
 		num_to_init -= cnt;
 		if (!num_to_init)
 			break;
 		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
 		if (!pg->next)
 			goto free_pages;
 		pg = pg->next;
 	}
 	return start_pg;
  free_pages:
 	while (start_pg) {
 		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
 		free_pages((unsigned long)pg->records, order);
 		start_pg = pg->next;
 		kfree(pg);
 		pg = start_pg;
 	}
 	pr_info("ftrace: FAILED to allocate memory for functions\n");
 	return NULL;
 }
 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 {
 	int cnt;
 	if (!num_to_init) {
 		pr_info("ftrace: No functions to be traced?\n");
 		return -1;
 	}
 	cnt = num_to_init / ENTRIES_PER_PAGE;
 	pr_info("ftrace: allocating %ld entries in %d pages\n",
 		num_to_init, cnt + 1);
 	return 0;
 }
 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 struct ftrace_iterator {
 	loff_t				pos;
 	loff_t				func_pos;
 	struct ftrace_page		*pg;
 	struct dyn_ftrace		*func;
 	struct ftrace_func_probe	*probe;
 	struct trace_parser		parser;
 	struct ftrace_hash		*hash;
 	struct ftrace_ops		*ops;
 	int				hidx;
 	int				idx;
 	unsigned			flags;
 };
 static void *
 t_hash_next(struct seq_file *m, loff_t *pos)
 {
 	struct ftrace_iterator *iter = m->private;
 	struct hlist_node *hnd = NULL;
 	struct hlist_head *hhd;
 	(*pos)++;
 	iter->pos = *pos;
 	if (iter->probe)
 		hnd = &iter->probe->node;
  retry:
 	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
 		return NULL;
 	hhd = &ftrace_func_hash[iter->hidx];
 	if (hlist_empty(hhd)) {
 		iter->hidx++;
 		hnd = NULL;
 		goto retry;
 	}
 	if (!hnd)
 		hnd = hhd->first;
 	else {
 		hnd = hnd->next;
 		if (!hnd) {
 			iter->hidx++;
 			goto retry;
 		}
 	}
 	if (WARN_ON_ONCE(!hnd))
 		return NULL;
 	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
 	return iter;
 }
 static void *t_hash_start(struct seq_file *m, loff_t *pos)
 {
 	struct ftrace_iterator *iter = m->private;
 	void *p = NULL;
 	loff_t l;
 	if (!(iter->flags & FTRACE_ITER_DO_HASH))
 		return NULL;
 	if (iter->func_pos > *pos)
 		return NULL;
 	iter->hidx = 0;
 	for (l = 0; l <= (*pos - iter->func_pos); ) {
 		p = t_hash_next(m, &l);
 		if (!p)
 			break;
 	}
 	if (!p)
 		return NULL;
 	/* Only set this if we have an item */
 	iter->flags |= FTRACE_ITER_HASH;
 	return iter;
 }
 static int
 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
 {
 	struct ftrace_func_probe *rec;
 	rec = iter->probe;
 	if (WARN_ON_ONCE(!rec))
 		return -EIO;
 	if (rec->ops->print)
 		return rec->ops->print(m, rec->ip, rec->ops, rec->data);
 	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
 	if (rec->data)
 		seq_printf(m, ":%p", rec->data);
 	seq_putc(m, '\n');
 	return 0;
 }
 static void *
 t_next(struct seq_file *m, void *v, loff_t *pos)
 {
 	struct ftrace_iterator *iter = m->private;
 	struct ftrace_ops *ops = iter->ops;
 	struct dyn_ftrace *rec = NULL;
 	if (unlikely(ftrace_disabled))
 		return NULL;
 	if (iter->flags & FTRACE_ITER_HASH)
 		return t_hash_next(m, pos);
 	(*pos)++;
 	iter->pos = iter->func_pos = *pos;
 	if (iter->flags & FTRACE_ITER_PRINTALL)
 		return t_hash_start(m, pos);
  retry:
 	if (iter->idx >= iter->pg->index) {
 		if (iter->pg->next) {
 			iter->pg = iter->pg->next;
 			iter->idx = 0;
 			goto retry;
 		}
 	} else {
 		rec = &iter->pg->records[iter->idx++];
 		if (((iter->flags & FTRACE_ITER_FILTER) &&
 		     !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
 		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
 		     !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
 		    ((iter->flags & FTRACE_ITER_ENABLED) &&
 		     !(rec->flags & FTRACE_FL_ENABLED))) {
 			rec = NULL;
 			goto retry;
 		}
 	}
 	if (!rec)
 		return t_hash_start(m, pos);
 	iter->func = rec;
 	return iter;
 }
 static void reset_iter_read(struct ftrace_iterator *iter)
 {
 	iter->pos = 0;
 	iter->func_pos = 0;
 	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
 }
 static void *t_start(struct seq_file *m, loff_t *pos)
 {
 	struct ftrace_iterator *iter = m->private;
 	struct ftrace_ops *ops = iter->ops;
 	void *p = NULL;
 	loff_t l;
 	mutex_lock(&ftrace_lock);
 	if (unlikely(ftrace_disabled))
 		return NULL;
 	/*
 	 * If an lseek was done, then reset and start from beginning.
 	 */
 	if (*pos < iter->pos)
 		reset_iter_read(iter);
 	/*
 	 * For set_ftrace_filter reading, if we have the filter
 	 * off, we can short cut and just print out that all
 	 * functions are enabled.
 	 */
 	if (iter->flags & FTRACE_ITER_FILTER &&
 	    ftrace_hash_empty(ops->filter_hash)) {
 		if (*pos > 0)
 			return t_hash_start(m, pos);
 		iter->flags |= FTRACE_ITER_PRINTALL;
 		/* reset in case of seek/pread */
 		iter->flags &= ~FTRACE_ITER_HASH;
 		return iter;
 	}
 	if (iter->flags & FTRACE_ITER_HASH)
 		return t_hash_start(m, pos);
 	/*
 	 * Unfortunately, we need to restart at ftrace_pages_start
 	 * every time we let go of the ftrace_mutex. This is because
 	 * those pointers can change without the lock.
 	 */
 	iter->pg = ftrace_pages_start;
 	iter->idx = 0;
 	for (l = 0; l <= *pos; ) {
 		p = t_next(m, p, &l);
 		if (!p)
 			break;
 	}
 	if (!p)
 		return t_hash_start(m, pos);
 	return iter;
 }
 static void t_stop(struct seq_file *m, void *p)
 {
 	mutex_unlock(&ftrace_lock);
 }
 static int t_show(struct seq_file *m, void *v)
 {
 	struct ftrace_iterator *iter = m->private;
 	struct dyn_ftrace *rec;
 	if (iter->flags & FTRACE_ITER_HASH)
 		return t_hash_show(m, iter);
 	if (iter->flags & FTRACE_ITER_PRINTALL) {
 		seq_printf(m, "#### all functions enabled ####\n");
 		return 0;
 	}
 	rec = iter->func;
 	if (!rec)
 		return 0;
 	seq_printf(m, "%ps", (void *)rec->ip);
 	if (iter->flags & FTRACE_ITER_ENABLED)
 		seq_printf(m, " (%ld)%s",
 			   rec->flags & ~FTRACE_FL_MASK,
 			   rec->flags & FTRACE_FL_REGS ? " R" : "");
 	seq_printf(m, "\n");
 	return 0;
 }
 static const struct seq_operations show_ftrace_seq_ops = {
 	.start = t_start,
 	.next = t_next,
 	.stop = t_stop,
 	.show = t_show,
 };
 static int
 ftrace_avail_open(struct inode *inode, struct file *file)
 {
 	struct ftrace_iterator *iter;
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
 	if (iter) {
 		iter->pg = ftrace_pages_start;
 		iter->ops = &global_ops;
 	}
 	return iter ? 0 : -ENOMEM;
 }
 static int
 ftrace_enabled_open(struct inode *inode, struct file *file)
 {
 	struct ftrace_iterator *iter;
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
 	if (iter) {
 		iter->pg = ftrace_pages_start;
 		iter->flags = FTRACE_ITER_ENABLED;
 		iter->ops = &global_ops;
 	}
 	return iter ? 0 : -ENOMEM;
 }
 static void ftrace_filter_reset(struct ftrace_hash *hash)
 {
 	mutex_lock(&ftrace_lock);
 	ftrace_hash_clear(hash);
 	mutex_unlock(&ftrace_lock);
 }
 /**
  * ftrace_regex_open - initialize function tracer filter files
  * @ops: The ftrace_ops that hold the hash filters
  * @flag: The type of filter to process
  * @inode: The inode, usually passed in to your open routine
  * @file: The file, usually passed in to your open routine
  *
  * ftrace_regex_open() initializes the filter files for the
  * @ops. Depending on @flag it may process the filter hash or
  * the notrace hash of @ops. With this called from the open
  * routine, you can use ftrace_filter_write() for the write
  * routine if @flag has FTRACE_ITER_FILTER set, or
  * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
  * ftrace_filter_lseek() should be used as the lseek routine, and
  * release must call ftrace_regex_release().
  */
 int
 ftrace_regex_open(struct ftrace_ops *ops, int flag,
 		  struct inode *inode, struct file *file)
 {
 	struct ftrace_iterator *iter;
 	struct ftrace_hash *hash;
 	int ret = 0;
 	ftrace_ops_init(ops);
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 	if (!iter)
 		return -ENOMEM;
 	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
 		kfree(iter);
 		return -ENOMEM;
 	}
 	iter->ops = ops;
 	iter->flags = flag;
 	mutex_lock(&ops->regex_lock);
 	if (flag & FTRACE_ITER_NOTRACE)
 		hash = ops->notrace_hash;
 	else
 		hash = ops->filter_hash;
 	if (file->f_mode & FMODE_WRITE) {
 		iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
 		if (!iter->hash) {
 			trace_parser_put(&iter->parser);
 			kfree(iter);
 			ret = -ENOMEM;
 			goto out_unlock;
 		}
 	}
 	if ((file->f_mode & FMODE_WRITE) &&
 	    (file->f_flags & O_TRUNC))
 		ftrace_filter_reset(iter->hash);
 	if (file->f_mode & FMODE_READ) {
 		iter->pg = ftrace_pages_start;
 		ret = seq_open(file, &show_ftrace_seq_ops);
 		if (!ret) {
 			struct seq_file *m = file->private_data;
 			m->private = iter;
 		} else {
 			/* Failed */
 			free_ftrace_hash(iter->hash);
 			trace_parser_put(&iter->parser);
 			kfree(iter);
 		}
 	} else
 		file->private_data = iter;
  out_unlock:
 	mutex_unlock(&ops->regex_lock);
 	return ret;
 }
 static int
 ftrace_filter_open(struct inode *inode, struct file *file)
 {
 	return ftrace_regex_open(&global_ops,
 			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
 			inode, file);
 }
 static int
 ftrace_notrace_open(struct inode *inode, struct file *file)
 {
 	return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
 				 inode, file);
 }
 static int ftrace_match(char *str, char *regex, int len, int type)
 {
 	int matched = 0;
 	int slen;
 	switch (type) {
 	case MATCH_FULL:
 		if (strcmp(str, regex) == 0)
 			matched = 1;
 		break;
 	case MATCH_FRONT_ONLY:
 		if (strncmp(str, regex, len) == 0)
 			matched = 1;
 		break;
 	case MATCH_MIDDLE_ONLY:
 		if (strstr(str, regex))
 			matched = 1;
 		break;
 	case MATCH_END_ONLY:
 		slen = strlen(str);
 		if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
 			matched = 1;
 		break;
 	}
 	return matched;
 }
 static int
 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
 {
 	struct ftrace_func_entry *entry;
 	int ret = 0;
 	entry = ftrace_lookup_ip(hash, rec->ip);
 	if (not) {
 		/* Do nothing if it doesn't exist */
 		if (!entry)
 			return 0;
 		free_hash_entry(hash, entry);
 	} else {
 		/* Do nothing if it exists */
 		if (entry)
 			return 0;
 		ret = add_hash_entry(hash, rec->ip);
 	}
 	return ret;
 }
 static int
 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
 		    char *regex, int len, int type)
 {
 	char str[KSYM_SYMBOL_LEN];
 	char *modname;
 	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
 	if (mod) {
 		/* module lookup requires matching the module */
 		if (!modname || strcmp(modname, mod))
 			return 0;
 		/* blank search means to match all funcs in the mod */
 		if (!len)
 			return 1;
 	}
 	return ftrace_match(str, regex, len, type);
 }
 static int
 match_records(struct ftrace_hash *hash, char *buff,
 	      int len, char *mod, int not)
 {
 	unsigned search_len = 0;
 	struct ftrace_page *pg;
 	struct dyn_ftrace *rec;
 	int type = MATCH_FULL;
 	char *search = buff;
 	int found = 0;
 	int ret;
 	if (len) {
 		type = filter_parse_regex(buff, len, &search, &not);
 		search_len = strlen(search);
 	}
 	mutex_lock(&ftrace_lock);
 	if (unlikely(ftrace_disabled))
 		goto out_unlock;
 	do_for_each_ftrace_rec(pg, rec) {
 		if (ftrace_match_record(rec, mod, search, search_len, type)) {
 			ret = enter_record(hash, rec, not);
 			if (ret < 0) {
 				found = ret;
 				goto out_unlock;
 			}
 			found = 1;
 		}
 	} while_for_each_ftrace_rec();
  out_unlock:
 	mutex_unlock(&ftrace_lock);
 	return found;
 }
 static int
 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
 {
 	return match_records(hash, buff, len, NULL, 0);
 }
 static int
 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
 {
 	int not = 0;
 	/* blank or '*' mean the same */
 	if (strcmp(buff, "*") == 0)
 		buff[0] = 0;
 	/* handle the case of 'dont filter this module' */
 	if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
 		buff[0] = 0;
 		not = 1;
 	}
 	return match_records(hash, buff, strlen(buff), mod, not);
 }
 /*
  * We register the module command as a template to show others how
  * to register the a command as well.
  */
 static int
 ftrace_mod_callback(struct ftrace_hash *hash,
 		    char *func, char *cmd, char *param, int enable)
 {
 	char *mod;
 	int ret = -EINVAL;
 	/*
 	 * cmd == 'mod' because we only registered this func
 	 * for the 'mod' ftrace_func_command.
 	 * But if you register one func with multiple commands,
 	 * you can tell which command was used by the cmd
 	 * parameter.
 	 */
 	/* we must have a module name */
 	if (!param)
 		return ret;
 	mod = strsep(&param, ":");
 	if (!strlen(mod))
 		return ret;
 	ret = ftrace_match_module_records(hash, func, mod);
 	if (!ret)
 		ret = -EINVAL;
 	if (ret < 0)
 		return ret;
 	return 0;
 }
 static struct ftrace_func_command ftrace_mod_cmd = {
 	.name			= "mod",
 	.func			= ftrace_mod_callback,
 };
 static int __init ftrace_mod_cmd_init(void)
 {
 	return register_ftrace_command(&ftrace_mod_cmd);
 }
 core_initcall(ftrace_mod_cmd_init);
 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
 				      struct ftrace_ops *op, struct pt_regs *pt_regs)
 {
 	struct ftrace_func_probe *entry;
 	struct hlist_head *hhd;
 	unsigned long key;
 	key = hash_long(ip, FTRACE_HASH_BITS);
 	hhd = &ftrace_func_hash[key];
 	if (hlist_empty(hhd))
 		return;
 	/*
 	 * Disable preemption for these calls to prevent a RCU grace
 	 * period. This syncs the hash iteration and freeing of items
 	 * on the hash. rcu_read_lock is too dangerous here.
 	 */
 	preempt_disable_notrace();
 	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
 		if (entry->ip == ip)
 			entry->ops->func(ip, parent_ip, &entry->data);
 	}
 	preempt_enable_notrace();
 }
 static struct ftrace_ops trace_probe_ops __read_mostly =
 {
 	.func		= function_trace_probe_call,
 	.flags		= FTRACE_OPS_FL_INITIALIZED,
 	INIT_REGEX_LOCK(trace_probe_ops)
 };
 static int ftrace_probe_registered;
 static void __enable_ftrace_function_probe(void)
 {
 	int ret;
 	int i;
 	if (ftrace_probe_registered) {
 		/* still need to update the function call sites */
 		if (ftrace_enabled)
 			ftrace_run_update_code(FTRACE_UPDATE_CALLS);
 		return;
 	}
 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
 		struct hlist_head *hhd = &ftrace_func_hash[i];
 		if (hhd->first)
 			break;
 	}
 	/* Nothing registered? */
 	if (i == FTRACE_FUNC_HASHSIZE)
 		return;
 	ret = __register_ftrace_function(&trace_probe_ops);
 	if (!ret)
 		ret = ftrace_startup(&trace_probe_ops, 0);
 	ftrace_probe_registered = 1;
 }
 static void __disable_ftrace_function_probe(void)
 {
 	int ret;
 	int i;
 	if (!ftrace_probe_registered)
 		return;
 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
 		struct hlist_head *hhd = &ftrace_func_hash[i];
 		if (hhd->first)
 			return;
 	}
 	/* no more funcs left */
 	ret = __unregister_ftrace_function(&trace_probe_ops);
 	if (!ret)
 		ftrace_shutdown(&trace_probe_ops, 0);
 	ftrace_probe_registered = 0;
 }
 static void ftrace_free_entry(struct ftrace_func_probe *entry)
 {
 	if (entry->ops->free)
 		entry->ops->free(entry->ops, entry->ip, &entry->data);
 	kfree(entry);
 }
 int
 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 			      void *data)
 {
 	struct ftrace_func_probe *entry;
 	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
 	struct ftrace_hash *hash;
 	struct ftrace_page *pg;
 	struct dyn_ftrace *rec;
 	int type, len, not;
 	unsigned long key;
 	int count = 0;
 	char *search;
 	int ret;
 	type = filter_parse_regex(glob, strlen(glob), &search, &not);
 	len = strlen(search);
 	/* we do not support '!' for function probes */
 	if (WARN_ON(not))
 		return -EINVAL;
 	mutex_lock(&trace_probe_ops.regex_lock);
 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
 	if (!hash) {
 		count = -ENOMEM;
 		goto out;
 	}
 	if (unlikely(ftrace_disabled)) {
 		count = -ENODEV;
 		goto out;
 	}
 	mutex_lock(&ftrace_lock);
 	do_for_each_ftrace_rec(pg, rec) {
 		if (!ftrace_match_record(rec, NULL, search, len, type))
 			continue;
 		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 		if (!entry) {
 			/* If we did not process any, then return error */
 			if (!count)
 				count = -ENOMEM;
 			goto out_unlock;
 		}
 		count++;
 		entry->data = data;
 		/*
 		 * The caller might want to do something special
 		 * for each function we find. We call the callback
 		 * to give the caller an opportunity to do so.
 		 */
 		if (ops->init) {
 			if (ops->init(ops, rec->ip, &entry->data) < 0) {
 				/* caller does not like this func */
 				kfree(entry);
 				continue;
 			}
 		}
 		ret = enter_record(hash, rec, 0);
 		if (ret < 0) {
 			kfree(entry);
 			count = ret;
 			goto out_unlock;
 		}
 		entry->ops = ops;
 		entry->ip = rec->ip;
 		key = hash_long(entry->ip, FTRACE_HASH_BITS);
 		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
 	} while_for_each_ftrace_rec();
 	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
 	if (ret < 0)
 		count = ret;
 	__enable_ftrace_function_probe();
  out_unlock:
 	mutex_unlock(&ftrace_lock);
  out:
 	mutex_unlock(&trace_probe_ops.regex_lock);
 	free_ftrace_hash(hash);
 	return count;
 }
 enum {
 	PROBE_TEST_FUNC		= 1,
 	PROBE_TEST_DATA		= 2
 };
 static void
 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 				  void *data, int flags)
 {
 	struct ftrace_func_entry *rec_entry;
 	struct ftrace_func_probe *entry;
 	struct ftrace_func_probe *p;
 	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
 	struct list_head free_list;
 	struct ftrace_hash *hash;
 	struct hlist_node *tmp;
 	char str[KSYM_SYMBOL_LEN];
 	int type = MATCH_FULL;
 	int i, len = 0;
 	char *search;
 	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
 		glob = NULL;
 	else if (glob) {
 		int not;
 		type = filter_parse_regex(glob, strlen(glob), &search, &not);
 		len = strlen(search);
 		/* we do not support '!' for function probes */
 		if (WARN_ON(not))
 			return;
 	}
 	mutex_lock(&trace_probe_ops.regex_lock);
 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
 	if (!hash)
 		/* Hmm, should report this somehow */
 		goto out_unlock;
 	INIT_LIST_HEAD(&free_list);
 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
 		struct hlist_head *hhd = &ftrace_func_hash[i];
 		hlist_for_each_entry_safe(entry, tmp, hhd, node) {
 			/* break up if statements for readability */
 			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
 				continue;
 			if ((flags & PROBE_TEST_DATA) && entry->data != data)
 				continue;
 			/* do this last, since it is the most expensive */
 			if (glob) {
 				kallsyms_lookup(entry->ip, NULL, NULL,
 						NULL, str);
 				if (!ftrace_match(str, glob, len, type))
 					continue;
 			}
 			rec_entry = ftrace_lookup_ip(hash, entry->ip);
 			/* It is possible more than one entry had this ip */
 			if (rec_entry)
 				free_hash_entry(hash, rec_entry);
 			hlist_del_rcu(&entry->node);
 			list_add(&entry->free_list, &free_list);
 		}
 	}
 	mutex_lock(&ftrace_lock);
 	__disable_ftrace_function_probe();
 	/*
 	 * Remove after the disable is called. Otherwise, if the last
 	 * probe is removed, a null hash means *all enabled*.
 	 */
 	ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
 	synchronize_sched();
 	list_for_each_entry_safe(entry, p, &free_list, free_list) {
 		list_del(&entry->free_list);
 		ftrace_free_entry(entry);
 	}
 	mutex_unlock(&ftrace_lock);
  out_unlock:
 	mutex_unlock(&trace_probe_ops.regex_lock);
 	free_ftrace_hash(hash);
 }
 void
 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 				void *data)
 {
 	__unregister_ftrace_function_probe(glob, ops, data,
 					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
 }
 void
 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
 {
 	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
 }
 void unregister_ftrace_function_probe_all(char *glob)
 {
 	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
 }
 static LIST_HEAD(ftrace_commands);
 static DEFINE_MUTEX(ftrace_cmd_mutex);
 int register_ftrace_command(struct ftrace_func_command *cmd)
 {
 	struct ftrace_func_command *p;
 	int ret = 0;
 	mutex_lock(&ftrace_cmd_mutex);
 	list_for_each_entry(p, &ftrace_commands, list) {
 		if (strcmp(cmd->name, p->name) == 0) {
 			ret = -EBUSY;
 			goto out_unlock;
 		}
 	}
 	list_add(&cmd->list, &ftrace_commands);
  out_unlock:
 	mutex_unlock(&ftrace_cmd_mutex);
 	return ret;
 }
 int unregister_ftrace_command(struct ftrace_func_command *cmd)
 {
 	struct ftrace_func_command *p, *n;
 	int ret = -ENODEV;
 	mutex_lock(&ftrace_cmd_mutex);
 	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
 		if (strcmp(cmd->name, p->name) == 0) {
 			ret = 0;
 			list_del_init(&p->list);
 			goto out_unlock;
 		}
 	}
  out_unlock:
 	mutex_unlock(&ftrace_cmd_mutex);
 	return ret;
 }
 static int ftrace_process_regex(struct ftrace_hash *hash,
 				char *buff, int len, int enable)
 {
 	char *func, *command, *next = buff;
 	struct ftrace_func_command *p;
 	int ret = -EINVAL;
 	func = strsep(&next, ":");
 	if (!next) {
 		ret = ftrace_match_records(hash, func, len);
 		if (!ret)
 			ret = -EINVAL;
 		if (ret < 0)
 			return ret;
 		return 0;
 	}
 	/* command found */
 	command = strsep(&next, ":");
 	mutex_lock(&ftrace_cmd_mutex);
 	list_for_each_entry(p, &ftrace_commands, list) {
 		if (strcmp(p->name, command) == 0) {
 			ret = p->func(hash, func, command, next, enable);
 			goto out_unlock;
 		}
 	}
  out_unlock:
 	mutex_unlock(&ftrace_cmd_mutex);
 	return ret;
 }
 static ssize_t
 ftrace_regex_write(struct file *file, const char __user *ubuf,
 		   size_t cnt, loff_t *ppos, int enable)
 {
 	struct ftrace_iterator *iter;
 	struct trace_parser *parser;
 	ssize_t ret, read;
 	if (!cnt)
 		return 0;
 	if (file->f_mode & FMODE_READ) {
 		struct seq_file *m = file->private_data;
 		iter = m->private;
 	} else
 		iter = file->private_data;
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	/* iter->hash is a local copy, so we don't need regex_lock */
 	parser = &iter->parser;
 	read = trace_get_user(parser, ubuf, cnt, ppos);
 	if (read >= 0 && trace_parser_loaded(parser) &&
 	    !trace_parser_cont(parser)) {
 		ret = ftrace_process_regex(iter->hash, parser->buffer,
 					   parser->idx, enable);
 		trace_parser_clear(parser);
 		if (ret < 0)
 			goto out;
 	}
 	ret = read;
  out:
 	return ret;
 }
 ssize_t
 ftrace_filter_write(struct file *file, const char __user *ubuf,
 		    size_t cnt, loff_t *ppos)
 {
 	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
 }
 ssize_t
 ftrace_notrace_write(struct file *file, const char __user *ubuf,
 		     size_t cnt, loff_t *ppos)
 {
 	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
 }
 static int
 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
 {
 	struct ftrace_func_entry *entry;
 	if (!ftrace_location(ip))
 		return -EINVAL;
 	if (remove) {
 		entry = ftrace_lookup_ip(hash, ip);
 		if (!entry)
 			return -ENOENT;
 		free_hash_entry(hash, entry);
 		return 0;
 	}
 	return add_hash_entry(hash, ip);
 }
 static void ftrace_ops_update_code(struct ftrace_ops *ops)
 {
 	if (ops->flags & FTRACE_OPS_FL_ENABLED && ftrace_enabled)
 		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
 }
 static int
 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
 		unsigned long ip, int remove, int reset, int enable)
 {
 	struct ftrace_hash **orig_hash;
 	struct ftrace_hash *hash;
 	int ret;
 	/* All global ops uses the global ops filters */
 	if (ops->flags & FTRACE_OPS_FL_GLOBAL)
 		ops = &global_ops;
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	mutex_lock(&ops->regex_lock);
 	if (enable)
 		orig_hash = &ops->filter_hash;
 	else
 		orig_hash = &ops->notrace_hash;
 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
 	if (!hash) {
 		ret = -ENOMEM;
 		goto out_regex_unlock;
 	}
 	if (reset)
 		ftrace_filter_reset(hash);
 	if (buf && !ftrace_match_records(hash, buf, len)) {
 		ret = -EINVAL;
 		goto out_regex_unlock;
 	}
 	if (ip) {
 		ret = ftrace_match_addr(hash, ip, remove);
 		if (ret < 0)
 			goto out_regex_unlock;
 	}
 	mutex_lock(&ftrace_lock);
 	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
 	if (!ret)
 		ftrace_ops_update_code(ops);
 	mutex_unlock(&ftrace_lock);
  out_regex_unlock:
 	mutex_unlock(&ops->regex_lock);
 	free_ftrace_hash(hash);
 	return ret;
 }
 static int
 ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
 		int reset, int enable)
 {
 	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
 }
 /**
  * ftrace_set_filter_ip - set a function to filter on in ftrace by address
  * @ops - the ops to set the filter with
  * @ip - the address to add to or remove from the filter.
  * @remove - non zero to remove the ip from the filter
  * @reset - non zero to reset all filters before applying this filter.
  *
  * Filters denote which functions should be enabled when tracing is enabled
  * If @ip is NULL, it failes to update filter.
  */
 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
 			 int remove, int reset)
 {
 	ftrace_ops_init(ops);
 	return ftrace_set_addr(ops, ip, remove, reset, 1);
 }
 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
 static int
 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
 		 int reset, int enable)
 {
 	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
 }
 /**
  * ftrace_set_filter - set a function to filter on in ftrace
  * @ops - the ops to set the filter with
  * @buf - the string that holds the function filter text.
  * @len - the length of the string.
  * @reset - non zero to reset all filters before applying this filter.
  *
  * Filters denote which functions should be enabled when tracing is enabled.
  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  */
 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
 		       int len, int reset)
 {
 	ftrace_ops_init(ops);
 	return ftrace_set_regex(ops, buf, len, reset, 1);
 }
 EXPORT_SYMBOL_GPL(ftrace_set_filter);
 /**
  * ftrace_set_notrace - set a function to not trace in ftrace
  * @ops - the ops to set the notrace filter with
  * @buf - the string that holds the function notrace text.
  * @len - the length of the string.
  * @reset - non zero to reset all filters before applying this filter.
  *
  * Notrace Filters denote which functions should not be enabled when tracing
  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  * for tracing.
  */
 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
 			int len, int reset)
 {
 	ftrace_ops_init(ops);
 	return ftrace_set_regex(ops, buf, len, reset, 0);
 }
 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
 /**
  * ftrace_set_filter - set a function to filter on in ftrace
  * @ops - the ops to set the filter with
  * @buf - the string that holds the function filter text.
  * @len - the length of the string.
  * @reset - non zero to reset all filters before applying this filter.
  *
  * Filters denote which functions should be enabled when tracing is enabled.
  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  */
 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
 {
 	ftrace_set_regex(&global_ops, buf, len, reset, 1);
 }
 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
 /**
  * ftrace_set_notrace - set a function to not trace in ftrace
  * @ops - the ops to set the notrace filter with
  * @buf - the string that holds the function notrace text.
  * @len - the length of the string.
  * @reset - non zero to reset all filters before applying this filter.
  *
  * Notrace Filters denote which functions should not be enabled when tracing
  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  * for tracing.
  */
 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
 {
 	ftrace_set_regex(&global_ops, buf, len, reset, 0);
 }
 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
 /*
  * command line interface to allow users to set filters on boot up.
  */
 #define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
 /* Used by function selftest to not test if filter is set */
 bool ftrace_filter_param __initdata;
 static int __init set_ftrace_notrace(char *str)
 {
 	ftrace_filter_param = true;
 	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
 	return 1;
 }
 __setup("ftrace_notrace=", set_ftrace_notrace);
 static int __init set_ftrace_filter(char *str)
 {
 	ftrace_filter_param = true;
 	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
 	return 1;
 }
 __setup("ftrace_filter=", set_ftrace_filter);
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
 static int __init set_graph_function(char *str)
 {
 	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
 	return 1;
 }
 __setup("ftrace_graph_filter=", set_graph_function);
 static void __init set_ftrace_early_graph(char *buf)
 {
 	int ret;
 	char *func;
 	while (buf) {
 		func = strsep(&buf, ",");
 		/* we allow only one expression at a time */
 		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
 				      func);
 		if (ret)
 			printk(KERN_DEBUG "ftrace: function %s not "
 					  "traceable\n", func);
 	}
 }
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 void __init
 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
 {
 	char *func;
 	ftrace_ops_init(ops);
 	while (buf) {
 		func = strsep(&buf, ",");
 		ftrace_set_regex(ops, func, strlen(func), 0, enable);
 	}
 }
 static void __init set_ftrace_early_filters(void)
 {
 	if (ftrace_filter_buf[0])
 		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
 	if (ftrace_notrace_buf[0])
 		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	if (ftrace_graph_buf[0])
 		set_ftrace_early_graph(ftrace_graph_buf);
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 }
 int ftrace_regex_release(struct inode *inode, struct file *file)
 {
 	struct seq_file *m = (struct seq_file *)file->private_data;
 	struct ftrace_iterator *iter;
 	struct ftrace_hash **orig_hash;
 	struct trace_parser *parser;
 	int filter_hash;
 	int ret;
 	if (file->f_mode & FMODE_READ) {
 		iter = m->private;
 		seq_release(inode, file);
 	} else
 		iter = file->private_data;
 	parser = &iter->parser;
 	if (trace_parser_loaded(parser)) {
 		parser->buffer[parser->idx] = 0;
 		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
 	}
 	trace_parser_put(parser);
 	mutex_lock(&iter->ops->regex_lock);
 	if (file->f_mode & FMODE_WRITE) {
 		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
 		if (filter_hash)
 			orig_hash = &iter->ops->filter_hash;
 		else
 			orig_hash = &iter->ops->notrace_hash;
 		mutex_lock(&ftrace_lock);
 		ret = ftrace_hash_move(iter->ops, filter_hash,
 				       orig_hash, iter->hash);
 		if (!ret)
 			ftrace_ops_update_code(iter->ops);
 		mutex_unlock(&ftrace_lock);
 	}
 	mutex_unlock(&iter->ops->regex_lock);
 	free_ftrace_hash(iter->hash);
 	kfree(iter);
 	return 0;
 }
 static const struct file_operations ftrace_avail_fops = {
 	.open = ftrace_avail_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
 	.release = seq_release_private,
 };
 static const struct file_operations ftrace_enabled_fops = {
 	.open = ftrace_enabled_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
 	.release = seq_release_private,
 };
 static const struct file_operations ftrace_filter_fops = {
 	.open = ftrace_filter_open,
 	.read = seq_read,
 	.write = ftrace_filter_write,
 	.llseek = ftrace_filter_lseek,
 	.release = ftrace_regex_release,
 };
 static const struct file_operations ftrace_notrace_fops = {
 	.open = ftrace_notrace_open,
 	.read = seq_read,
 	.write = ftrace_notrace_write,
 	.llseek = ftrace_filter_lseek,
 	.release = ftrace_regex_release,
 };
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 static DEFINE_MUTEX(graph_lock);
 int ftrace_graph_count;
 int ftrace_graph_filter_enabled;
 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
 static void *
 __g_next(struct seq_file *m, loff_t *pos)
 {
 	if (*pos >= ftrace_graph_count)
 		return NULL;
 	return &ftrace_graph_funcs[*pos];
 }
 static void *
 g_next(struct seq_file *m, void *v, loff_t *pos)
 {
 	(*pos)++;
 	return __g_next(m, pos);
 }
 static void *g_start(struct seq_file *m, loff_t *pos)
 {
 	mutex_lock(&graph_lock);
 	/* Nothing, tell g_show to print all functions are enabled */
 	if (!ftrace_graph_filter_enabled && !*pos)
 		return (void *)1;
 	return __g_next(m, pos);
 }
 static void g_stop(struct seq_file *m, void *p)
 {
 	mutex_unlock(&graph_lock);
 }
 static int g_show(struct seq_file *m, void *v)
 {
 	unsigned long *ptr = v;
 	if (!ptr)
 		return 0;
 	if (ptr == (unsigned long *)1) {
 		seq_printf(m, "#### all functions enabled ####\n");
 		return 0;
 	}
 	seq_printf(m, "%ps\n", (void *)*ptr);
 	return 0;
 }
 static const struct seq_operations ftrace_graph_seq_ops = {
 	.start = g_start,
 	.next = g_next,
 	.stop = g_stop,
 	.show = g_show,
 };
 static int
 ftrace_graph_open(struct inode *inode, struct file *file)
 {
 	int ret = 0;
 	if (unlikely(ftrace_disabled))
 		return -ENODEV;
 	mutex_lock(&graph_lock);
 	if ((file->f_mode & FMODE_WRITE) &&
 	    (file->f_flags & O_TRUNC)) {
 		ftrace_graph_filter_enabled = 0;
 		ftrace_graph_count = 0;
 		memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
 	}
 	mutex_unlock(&graph_lock);
 	if (file->f_mode & FMODE_READ)
 		ret = seq_open(file, &ftrace_graph_seq_ops);
 	return ret;
 }
 static int
 ftrace_graph_release(struct inode *inode, struct file *file)
 {
 	if (file->f_mode & FMODE_READ)
 		seq_release(inode, file);
 	return 0;
 }
 static int
 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
 {
 	struct dyn_ftrace *rec;
 	struct ftrace_page *pg;
 	int search_len;
 	int fail = 1;
 	int type, not;
 	char *search;
 	bool exists;
 	int i;
 	/* decode regex */
 	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
 	if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
 		return -EBUSY;
 	search_len = strlen(search);
 	mutex_lock(&ftrace_lock);
 	if (unlikely(ftrace_disabled)) {
 		mutex_unlock(&ftrace_lock);
 		return -ENODEV;
 	}
 	do_for_each_ftrace_rec(pg, rec) {
 		if (ftrace_match_record(rec, NULL, search, search_len, type)) {
 			/* if it is in the array */
 			exists = false;
 			for (i = 0; i < *idx; i++) {
 				if (array[i] == rec->ip) {
 					exists = true;
 					break;
 				}
 			}
 			if (!not) {
 				fail = 0;
 				if (!exists) {
 					array[(*idx)++] = rec->ip;
 					if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
 						goto out;
 				}
 			} else {
 				if (exists) {
 					array[i] = array[--(*idx)];
 					array[*idx] = 0;
 					fail = 0;
 				}
 			}
 		}
 	} while_for_each_ftrace_rec();
 out:
 	mutex_unlock(&ftrace_lock);
 	if (fail)
 		return -EINVAL;
 	ftrace_graph_filter_enabled = !!(*idx);
 	return 0;
 }
 static ssize_t
 ftrace_graph_write(struct file *file, const char __user *ubuf,
 		   size_t cnt, loff_t *ppos)
 {
 	struct trace_parser parser;
 	ssize_t read, ret;
 	if (!cnt)
 		return 0;
 	mutex_lock(&graph_lock);
 	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
 		ret = -ENOMEM;
 		goto out_unlock;
 	}
 	read = trace_get_user(&parser, ubuf, cnt, ppos);
 	if (read >= 0 && trace_parser_loaded((&parser))) {
 		parser.buffer[parser.idx] = 0;
 		/* we allow only one expression at a time */
 		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
 					parser.buffer);
 		if (ret)
 			goto out_free;
 	}
 	ret = read;
 out_free:
 	trace_parser_put(&parser);
 out_unlock:
 	mutex_unlock(&graph_lock);
 	return ret;
 }
 static const struct file_operations ftrace_graph_fops = {
 	.open		= ftrace_graph_open,
 	.read		= seq_read,
 	.write		= ftrace_graph_write,
 	.llseek		= ftrace_filter_lseek,
 	.release	= ftrace_graph_release,
 };
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
 {
 	trace_create_file("available_filter_functions", 0444,
 			d_tracer, NULL, &ftrace_avail_fops);
 	trace_create_file("enabled_functions", 0444,
 			d_tracer, NULL, &ftrace_enabled_fops);
 	trace_create_file("set_ftrace_filter", 0644, d_tracer,
 			NULL, &ftrace_filter_fops);
 	trace_create_file("set_ftrace_notrace", 0644, d_tracer,
 				    NULL, &ftrace_notrace_fops);
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	trace_create_file("set_graph_function", 0444, d_tracer,
 				    NULL,
 				    &ftrace_graph_fops);
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 	return 0;
 }
 static int ftrace_cmp_ips(const void *a, const void *b)
 {
 	const unsigned long *ipa = a;
 	const unsigned long *ipb = b;
 	if (*ipa > *ipb)
 		return 1;
 	if (*ipa < *ipb)
 		return -1;
 	return 0;
 }
 static void ftrace_swap_ips(void *a, void *b, int size)
 {
 	unsigned long *ipa = a;
 	unsigned long *ipb = b;
 	unsigned long t;
 	t = *ipa;
 	*ipa = *ipb;
 	*ipb = t;
 }
 static int ftrace_process_locs(struct module *mod,
 			       unsigned long *start,
 			       unsigned long *end)
 {
 	struct ftrace_page *start_pg;
 	struct ftrace_page *pg;
 	struct dyn_ftrace *rec;
 	unsigned long count;
 	unsigned long *p;
 	unsigned long addr;
 	unsigned long flags = 0; /* Shut up gcc */
 	int ret = -ENOMEM;
 	count = end - start;
 	if (!count)
 		return 0;
 	sort(start, count, sizeof(*start),
 	     ftrace_cmp_ips, ftrace_swap_ips);
 	start_pg = ftrace_allocate_pages(count);
 	if (!start_pg)
 		return -ENOMEM;
 	mutex_lock(&ftrace_lock);
 	/*
 	 * Core and each module needs their own pages, as
 	 * modules will free them when they are removed.
 	 * Force a new page to be allocated for modules.
 	 */
 	if (!mod) {
 		WARN_ON(ftrace_pages || ftrace_pages_start);
 		/* First initialization */
 		ftrace_pages = ftrace_pages_start = start_pg;
 	} else {
 		if (!ftrace_pages)
 			goto out;
 		if (WARN_ON(ftrace_pages->next)) {
 			/* Hmm, we have free pages? */
 			while (ftrace_pages->next)
 				ftrace_pages = ftrace_pages->next;
 		}
 		ftrace_pages->next = start_pg;
 	}
 	p = start;
 	pg = start_pg;
 	while (p < end) {
 		addr = ftrace_call_adjust(*p++);
 		/*
 		 * Some architecture linkers will pad between
 		 * the different mcount_loc sections of different
 		 * object files to satisfy alignments.
 		 * Skip any NULL pointers.
 		 */
 		if (!addr)
 			continue;
 		if (pg->index == pg->size) {
 			/* We should have allocated enough */
 			if (WARN_ON(!pg->next))
 				break;
 			pg = pg->next;
 		}
 		rec = &pg->records[pg->index++];
 		rec->ip = addr;
 	}
 	/* We should have used all pages */
 	WARN_ON(pg->next);
 	/* Assign the last page to ftrace_pages */
 	ftrace_pages = pg;
 	/* These new locations need to be initialized */
 	ftrace_new_pgs = start_pg;
 	/*
 	 * We only need to disable interrupts on start up
 	 * because we are modifying code that an interrupt
 	 * may execute, and the modification is not atomic.
 	 * But for modules, nothing runs the code we modify
 	 * until we are finished with it, and there's no
 	 * reason to cause large interrupt latencies while we do it.
 	 */
 	if (!mod)
 		local_irq_save(flags);
 	ftrace_update_code(mod);
 	if (!mod)
 		local_irq_restore(flags);
 	ret = 0;
  out:
 	mutex_unlock(&ftrace_lock);
 	return ret;
 }
 #ifdef CONFIG_MODULES
 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
 void ftrace_release_mod(struct module *mod)
 {
 	struct dyn_ftrace *rec;
 	struct ftrace_page **last_pg;
 	struct ftrace_page *pg;
 	int order;
 	mutex_lock(&ftrace_lock);
 	if (ftrace_disabled)
 		goto out_unlock;
 	/*
 	 * Each module has its own ftrace_pages, remove
 	 * them from the list.
 	 */
 	last_pg = &ftrace_pages_start;
 	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
 		rec = &pg->records[0];
 		if (within_module_core(rec->ip, mod)) {
 			/*
 			 * As core pages are first, the first
 			 * page should never be a module page.
 			 */
 			if (WARN_ON(pg == ftrace_pages_start))
 				goto out_unlock;
 			/* Check if we are deleting the last page */
 			if (pg == ftrace_pages)
 				ftrace_pages = next_to_ftrace_page(last_pg);
 			*last_pg = pg->next;
 			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
 			free_pages((unsigned long)pg->records, order);
 			kfree(pg);
 		} else
 			last_pg = &pg->next;
 	}
  out_unlock:
 	mutex_unlock(&ftrace_lock);
 }
 static void ftrace_init_module(struct module *mod,
 			       unsigned long *start, unsigned long *end)
 {
 	if (ftrace_disabled || start == end)
 		return;
 	ftrace_process_locs(mod, start, end);
 }
 static int ftrace_module_notify_enter(struct notifier_block *self,
 				      unsigned long val, void *data)
 {
 	struct module *mod = data;
 	if (val == MODULE_STATE_COMING)
 		ftrace_init_module(mod, mod->ftrace_callsites,
 				   mod->ftrace_callsites +
 				   mod->num_ftrace_callsites);
 	return 0;
 }
 static int ftrace_module_notify_exit(struct notifier_block *self,
 				     unsigned long val, void *data)
 {
 	struct module *mod = data;
 	if (val == MODULE_STATE_GOING)
 		ftrace_release_mod(mod);
 	return 0;
 }
 #else
 static int ftrace_module_notify_enter(struct notifier_block *self,
 				      unsigned long val, void *data)
 {
 	return 0;
 }
 static int ftrace_module_notify_exit(struct notifier_block *self,
 				     unsigned long val, void *data)
 {
 	return 0;
 }
 #endif /* CONFIG_MODULES */
 struct notifier_block ftrace_module_enter_nb = {
 	.notifier_call = ftrace_module_notify_enter,
 	.priority = INT_MAX,	/* Run before anything that can use kprobes */
 };
 struct notifier_block ftrace_module_exit_nb = {
 	.notifier_call = ftrace_module_notify_exit,
 	.priority = INT_MIN,	/* Run after anything that can remove kprobes */
 };
 extern unsigned long __start_mcount_loc[];
 extern unsigned long __stop_mcount_loc[];
 void __init ftrace_init(void)
 {
 	unsigned long count, addr, flags;
 	int ret;
 	/* Keep the ftrace pointer to the stub */
 	addr = (unsigned long)ftrace_stub;
 	local_irq_save(flags);
 	ftrace_dyn_arch_init(&addr);
 	local_irq_restore(flags);
 	/* ftrace_dyn_arch_init places the return code in addr */
 	if (addr)
 		goto failed;
 	count = __stop_mcount_loc - __start_mcount_loc;
 	ret = ftrace_dyn_table_alloc(count);
 	if (ret)
 		goto failed;
 	last_ftrace_enabled = ftrace_enabled = 1;
 	ret = ftrace_process_locs(NULL,
 				  __start_mcount_loc,
 				  __stop_mcount_loc);
 	ret = register_module_notifier(&ftrace_module_enter_nb);
 	if (ret)
 		pr_warning("Failed to register trace ftrace module enter notifier\n");
 	ret = register_module_notifier(&ftrace_module_exit_nb);
 	if (ret)
 		pr_warning("Failed to register trace ftrace module exit notifier\n");
 	set_ftrace_early_filters();
 	return;
  failed:
 	ftrace_disabled = 1;
 }
 #else
 static struct ftrace_ops global_ops = {
 	.func			= ftrace_stub,
 	.flags			= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
 	INIT_REGEX_LOCK(global_ops)
 };
 static int __init ftrace_nodyn_init(void)
 {
 	ftrace_enabled = 1;
 	return 0;
 }
 core_initcall(ftrace_nodyn_init);
 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
 static inline void ftrace_startup_enable(int command) { }
 /* Keep as macros so we do not need to define the commands */
 # define ftrace_startup(ops, command)			\
 	({						\
 		(ops)->flags |= FTRACE_OPS_FL_ENABLED;	\
 		0;					\
 	})
 # define ftrace_shutdown(ops, command)	do { } while (0)
 # define ftrace_startup_sysctl()	do { } while (0)
 # define ftrace_shutdown_sysctl()	do { } while (0)
 static inline int
 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
 {
 	return 1;
 }
 #endif /* CONFIG_DYNAMIC_FTRACE */
 static void
 ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
 			struct ftrace_ops *op, struct pt_regs *regs)
 {
 	if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))
 		return;
 	/*
 	 * Some of the ops may be dynamically allocated,
 	 * they must be freed after a synchronize_sched().
 	 */
 	preempt_disable_notrace();
 	trace_recursion_set(TRACE_CONTROL_BIT);
 	do_for_each_ftrace_op(op, ftrace_control_list) {
 		if (!(op->flags & FTRACE_OPS_FL_STUB) &&
 		    !ftrace_function_local_disabled(op) &&
 		    ftrace_ops_test(op, ip, regs))
 			op->func(ip, parent_ip, op, regs);
 	} while_for_each_ftrace_op(op);
 	trace_recursion_clear(TRACE_CONTROL_BIT);
 	preempt_enable_notrace();
 }
 static struct ftrace_ops control_ops = {
 	.func	= ftrace_ops_control_func,
 	.flags	= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
 	INIT_REGEX_LOCK(control_ops)
 };
 static inline void
 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 		       struct ftrace_ops *ignored, struct pt_regs *regs)
 {
 	struct ftrace_ops *op;
 	int bit;
 	if (function_trace_stop)
 		return;
 	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
 	if (bit < 0)
 		return;
 	/*
 	 * Some of the ops may be dynamically allocated,
 	 * they must be freed after a synchronize_sched().
 	 */
 	preempt_disable_notrace();
 	do_for_each_ftrace_op(op, ftrace_ops_list) {
 		if (ftrace_ops_test(op, ip, regs))
 			op->func(ip, parent_ip, op, regs);
 	} while_for_each_ftrace_op(op);
 	preempt_enable_notrace();
 	trace_clear_recursion(bit);
 }
 /*
  * Some archs only support passing ip and parent_ip. Even though
  * the list function ignores the op parameter, we do not want any
  * C side effects, where a function is called without the caller
  * sending a third parameter.
  * Archs are to support both the regs and ftrace_ops at the same time.
  * If they support ftrace_ops, it is assumed they support regs.
  * If call backs want to use regs, they must either check for regs
  * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
  * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
  * An architecture can pass partial regs with ftrace_ops and still
  * set the ARCH_SUPPORT_FTARCE_OPS.
  */
 #if ARCH_SUPPORTS_FTRACE_OPS
 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 				 struct ftrace_ops *op, struct pt_regs *regs)
 {
 	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
 }
 #else
 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
 {
 	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
 }
 #endif
 static void clear_ftrace_swapper(void)
 {
 	struct task_struct *p;
 	int cpu;
 	get_online_cpus();
 	for_each_online_cpu(cpu) {
 		p = idle_task(cpu);
 		clear_tsk_trace_trace(p);
 	}
 	put_online_cpus();
 }
 static void set_ftrace_swapper(void)
 {
 	struct task_struct *p;
 	int cpu;
 	get_online_cpus();
 	for_each_online_cpu(cpu) {
 		p = idle_task(cpu);
 		set_tsk_trace_trace(p);
 	}
 	put_online_cpus();
 }
 static void clear_ftrace_pid(struct pid *pid)
 {
 	struct task_struct *p;
 	rcu_read_lock();
 	do_each_pid_task(pid, PIDTYPE_PID, p) {
 		clear_tsk_trace_trace(p);
 	} while_each_pid_task(pid, PIDTYPE_PID, p);
 	rcu_read_unlock();
 	put_pid(pid);
 }
 static void set_ftrace_pid(struct pid *pid)
 {
 	struct task_struct *p;
 	rcu_read_lock();
 	do_each_pid_task(pid, PIDTYPE_PID, p) {
 		set_tsk_trace_trace(p);
 	} while_each_pid_task(pid, PIDTYPE_PID, p);
 	rcu_read_unlock();
 }
 static void clear_ftrace_pid_task(struct pid *pid)
 {
 	if (pid == ftrace_swapper_pid)
 		clear_ftrace_swapper();
 	else
 		clear_ftrace_pid(pid);
 }
 static void set_ftrace_pid_task(struct pid *pid)
 {
 	if (pid == ftrace_swapper_pid)
 		set_ftrace_swapper();
 	else
 		set_ftrace_pid(pid);
 }
 static int ftrace_pid_add(int p)
 {
 	struct pid *pid;
 	struct ftrace_pid *fpid;
 	int ret = -EINVAL;
 	mutex_lock(&ftrace_lock);
 	if (!p)
 		pid = ftrace_swapper_pid;
 	else
 		pid = find_get_pid(p);
 	if (!pid)
 		goto out;
 	ret = 0;
 	list_for_each_entry(fpid, &ftrace_pids, list)
 		if (fpid->pid == pid)
 			goto out_put;
 	ret = -ENOMEM;
 	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
 	if (!fpid)
 		goto out_put;
 	list_add(&fpid->list, &ftrace_pids);
 	fpid->pid = pid;
 	set_ftrace_pid_task(pid);
 	ftrace_update_pid_func();
 	ftrace_startup_enable(0);
 	mutex_unlock(&ftrace_lock);
 	return 0;
 out_put:
 	if (pid != ftrace_swapper_pid)
 		put_pid(pid);
 out:
 	mutex_unlock(&ftrace_lock);
 	return ret;
 }
 static void ftrace_pid_reset(void)
 {
 	struct ftrace_pid *fpid, *safe;
 	mutex_lock(&ftrace_lock);
 	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
 		struct pid *pid = fpid->pid;
 		clear_ftrace_pid_task(pid);
 		list_del(&fpid->list);
 		kfree(fpid);
 	}
 	ftrace_update_pid_func();
 	ftrace_startup_enable(0);
 	mutex_unlock(&ftrace_lock);
 }
 static void *fpid_start(struct seq_file *m, loff_t *pos)
 {
 	mutex_lock(&ftrace_lock);
 	if (list_empty(&ftrace_pids) && (!*pos))
 		return (void *) 1;
 	return seq_list_start(&ftrace_pids, *pos);
 }
 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
 {
 	if (v == (void *)1)
 		return NULL;
 	return seq_list_next(v, &ftrace_pids, pos);
 }
 static void fpid_stop(struct seq_file *m, void *p)
 {
 	mutex_unlock(&ftrace_lock);
 }
 static int fpid_show(struct seq_file *m, void *v)
 {
 	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
 	if (v == (void *)1) {
 		seq_printf(m, "no pid\n");
 		return 0;
 	}
 	if (fpid->pid == ftrace_swapper_pid)
 		seq_printf(m, "swapper tasks\n");
 	else
 		seq_printf(m, "%u\n", pid_vnr(fpid->pid));
 	return 0;
 }
 static const struct seq_operations ftrace_pid_sops = {
 	.start = fpid_start,
 	.next = fpid_next,
 	.stop = fpid_stop,
 	.show = fpid_show,
 };
 static int
 ftrace_pid_open(struct inode *inode, struct file *file)
 {
 	int ret = 0;
 	if ((file->f_mode & FMODE_WRITE) &&
 	    (file->f_flags & O_TRUNC))
 		ftrace_pid_reset();
 	if (file->f_mode & FMODE_READ)
 		ret = seq_open(file, &ftrace_pid_sops);
 	return ret;
 }
 static ssize_t
 ftrace_pid_write(struct file *filp, const char __user *ubuf,
 		   size_t cnt, loff_t *ppos)
 {
 	char buf[64], *tmp;
 	long val;
 	int ret;
 	if (cnt >= sizeof(buf))
 		return -EINVAL;
 	if (copy_from_user(&buf, ubuf, cnt))
 		return -EFAULT;
 	buf[cnt] = 0;
 	/*
 	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
 	 * to clean the filter quietly.
 	 */
 	tmp = strstrip(buf);
 	if (strlen(tmp) == 0)
 		return 1;
 	ret = kstrtol(tmp, 10, &val);
 	if (ret < 0)
 		return ret;
 	ret = ftrace_pid_add(val);
 	return ret ? ret : cnt;
 }
 static int
 ftrace_pid_release(struct inode *inode, struct file *file)
 {
 	if (file->f_mode & FMODE_READ)
 		seq_release(inode, file);
 	return 0;
 }
 static const struct file_operations ftrace_pid_fops = {
 	.open		= ftrace_pid_open,
 	.write		= ftrace_pid_write,
 	.read		= seq_read,
 	.llseek		= ftrace_filter_lseek,
 	.release	= ftrace_pid_release,
 };
 static __init int ftrace_init_debugfs(void)
 {
 	struct dentry *d_tracer;
 	d_tracer = tracing_init_dentry();
 	if (!d_tracer)
 		return 0;
 	ftrace_init_dyn_debugfs(d_tracer);
 	trace_create_file("set_ftrace_pid", 0644, d_tracer,
 			    NULL, &ftrace_pid_fops);
 	ftrace_profile_debugfs(d_tracer);
 	return 0;
 }
 fs_initcall(ftrace_init_debugfs);
 /**
  * ftrace_kill - kill ftrace
  *
  * This function should be used by panic code. It stops ftrace
  * but in a not so nice way. If you need to simply kill ftrace
  * from a non-atomic section, use ftrace_kill.
  */
 void ftrace_kill(void)
 {
 	ftrace_disabled = 1;
 	ftrace_enabled = 0;
 	clear_ftrace_function();
 }
 /**
  * Test if ftrace is dead or not.
  */
 int ftrace_is_dead(void)
 {
 	return ftrace_disabled;
 }
 /**
  * register_ftrace_function - register a function for profiling
  * @ops - ops structure that holds the function for profiling.
  *
  * Register a function to be called by all functions in the
  * kernel.
  *
  * Note: @ops->func and all the functions it calls must be labeled
  *       with "notrace", otherwise it will go into a
  *       recursive loop.
  */
 int register_ftrace_function(struct ftrace_ops *ops)
 {
 	int ret = -1;
 	ftrace_ops_init(ops);
 	mutex_lock(&ftrace_lock);
 	ret = __register_ftrace_function(ops);
 	if (!ret)
 		ret = ftrace_startup(ops, 0);
 	mutex_unlock(&ftrace_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(register_ftrace_function);
 /**
  * unregister_ftrace_function - unregister a function for profiling.
  * @ops - ops structure that holds the function to unregister
  *
  * Unregister a function that was added to be called by ftrace profiling.
  */
 int unregister_ftrace_function(struct ftrace_ops *ops)
 {
 	int ret;
 	mutex_lock(&ftrace_lock);
 	ret = __unregister_ftrace_function(ops);
 	if (!ret)
 		ftrace_shutdown(ops, 0);
 	mutex_unlock(&ftrace_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
 int
 ftrace_enable_sysctl(struct ctl_table *table, int write,
 		     void __user *buffer, size_t *lenp,
 		     loff_t *ppos)
 {
 	int ret = -ENODEV;
 	mutex_lock(&ftrace_lock);
 	if (unlikely(ftrace_disabled))
 		goto out;
 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
 	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
 		goto out;
 	last_ftrace_enabled = !!ftrace_enabled;
 	if (ftrace_enabled) {
 		ftrace_startup_sysctl();
 		/* we are starting ftrace again */
 		if (ftrace_ops_list != &ftrace_list_end)
 			update_ftrace_function();
 	} else {
 		/* stopping ftrace calls (just send to ftrace_stub) */
 		ftrace_trace_function = ftrace_stub;
 		ftrace_shutdown_sysctl();
 	}
  out:
 	mutex_unlock(&ftrace_lock);
 	return ret;
 }
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 static int ftrace_graph_active;
 static struct notifier_block ftrace_suspend_notifier;
 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
 {
 	return 0;
 }
 /* The callbacks that hook a function */
 trace_func_graph_ret_t ftrace_graph_return =
 			(trace_func_graph_ret_t)ftrace_stub;
 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
 {
 	int i;
 	int ret = 0;
 	unsigned long flags;
 	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
 	struct task_struct *g, *t;
 	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
 		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
 					* sizeof(struct ftrace_ret_stack),
 					GFP_KERNEL);
 		if (!ret_stack_list[i]) {
 			start = 0;
 			end = i;
 			ret = -ENOMEM;
 			goto free;
 		}
 	}
 	read_lock_irqsave(&tasklist_lock, flags);
 	do_each_thread(g, t) {
 		if (start == end) {
 			ret = -EAGAIN;
 			goto unlock;
 		}
 		if (t->ret_stack == NULL) {
 			atomic_set(&t->tracing_graph_pause, 0);
 			atomic_set(&t->trace_overrun, 0);
 			t->curr_ret_stack = -1;
 			/* Make sure the tasks see the -1 first: */
 			smp_wmb();
 			t->ret_stack = ret_stack_list[start++];
 		}
 	} while_each_thread(g, t);
 unlock:
 	read_unlock_irqrestore(&tasklist_lock, flags);
 free:
 	for (i = start; i < end; i++)
 		kfree(ret_stack_list[i]);
 	return ret;
 }
 static void
 ftrace_graph_probe_sched_switch(void *ignore,
 			struct task_struct *prev, struct task_struct *next)
 {
 	unsigned long long timestamp;
 	int index;
 	/*
 	 * Does the user want to count the time a function was asleep.
 	 * If so, do not update the time stamps.
 	 */
 	if (trace_flags & TRACE_ITER_SLEEP_TIME)
 		return;
 	timestamp = trace_clock_local();
 	prev->ftrace_timestamp = timestamp;
 	/* only process tasks that we timestamped */
 	if (!next->ftrace_timestamp)
 		return;
 	/*
 	 * Update all the counters in next to make up for the
 	 * time next was sleeping.
 	 */
 	timestamp -= next->ftrace_timestamp;
 	for (index = next->curr_ret_stack; index >= 0; index--)
 		next->ret_stack[index].calltime += timestamp;
 }
 /* Allocate a return stack for each task */
 static int start_graph_tracing(void)
 {
 	struct ftrace_ret_stack **ret_stack_list;
 	int ret, cpu;
 	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
 				sizeof(struct ftrace_ret_stack *),
 				GFP_KERNEL);
 	if (!ret_stack_list)
 		return -ENOMEM;
 	/* The cpu_boot init_task->ret_stack will never be freed */
 	for_each_online_cpu(cpu) {
 		if (!idle_task(cpu)->ret_stack)
 			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
 	}
 	do {
 		ret = alloc_retstack_tasklist(ret_stack_list);
 	} while (ret == -EAGAIN);
 	if (!ret) {
 		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
 		if (ret)
 			pr_info("ftrace_graph: Couldn't activate tracepoint"
 				" probe to kernel_sched_switch\n");
 	}
 	kfree(ret_stack_list);
 	return ret;
 }
 /*
  * Hibernation protection.
  * The state of the current task is too much unstable during
  * suspend/restore to disk. We want to protect against that.
  */
 static int
 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
 							void *unused)
 {
 	switch (state) {
 	case PM_HIBERNATION_PREPARE:
 		pause_graph_tracing();
 		break;
 	case PM_POST_HIBERNATION:
 		unpause_graph_tracing();
 		break;
 	}
 	return NOTIFY_DONE;
 }
 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
 			trace_func_graph_ent_t entryfunc)
 {
 	int ret = 0;
 	mutex_lock(&ftrace_lock);
 	/* we currently allow only one tracer registered at a time */
 	if (ftrace_graph_active) {
 		ret = -EBUSY;
 		goto out;
 	}
 	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
 	register_pm_notifier(&ftrace_suspend_notifier);
 	ftrace_graph_active++;
 	ret = start_graph_tracing();
 	if (ret) {
 		ftrace_graph_active--;
 		goto out;
 	}
 	ftrace_graph_return = retfunc;
 	ftrace_graph_entry = entryfunc;
 	ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
 out:
 	mutex_unlock(&ftrace_lock);
 	return ret;
 }
 void unregister_ftrace_graph(void)
 {
 	mutex_lock(&ftrace_lock);
 	if (unlikely(!ftrace_graph_active))
 		goto out;
 	ftrace_graph_active--;
 	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
 	ftrace_graph_entry = ftrace_graph_entry_stub;
 	ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
 	unregister_pm_notifier(&ftrace_suspend_notifier);
 	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
  out:
 	mutex_unlock(&ftrace_lock);
 }
 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
 static void
 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
 {
 	atomic_set(&t->tracing_graph_pause, 0);
 	atomic_set(&t->trace_overrun, 0);
 	t->ftrace_timestamp = 0;
 	/* make curr_ret_stack visible before we add the ret_stack */
 	smp_wmb();
 	t->ret_stack = ret_stack;
 }
 /*
  * Allocate a return stack for the idle task. May be the first
  * time through, or it may be done by CPU hotplug online.
  */
 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
 {
 	t->curr_ret_stack = -1;
 	/*
 	 * The idle task has no parent, it either has its own
 	 * stack or no stack at all.
 	 */
 	if (t->ret_stack)
 		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
 	if (ftrace_graph_active) {
 		struct ftrace_ret_stack *ret_stack;
 		ret_stack = per_cpu(idle_ret_stack, cpu);
 		if (!ret_stack) {
 			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
 					    * sizeof(struct ftrace_ret_stack),
 					    GFP_KERNEL);
 			if (!ret_stack)
 				return;
 			per_cpu(idle_ret_stack, cpu) = ret_stack;
 		}
 		graph_init_task(t, ret_stack);
 	}
 }
 /* Allocate a return stack for newly created task */
 void ftrace_graph_init_task(struct task_struct *t)
 {
 	/* Make sure we do not use the parent ret_stack */
 	t->ret_stack = NULL;
 	t->curr_ret_stack = -1;
 	if (ftrace_graph_active) {
 		struct ftrace_ret_stack *ret_stack;
 		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
 				* sizeof(struct ftrace_ret_stack),
 				GFP_KERNEL);
 		if (!ret_stack)
 			return;
 		graph_init_task(t, ret_stack);
 	}
 }
 void ftrace_graph_exit_task(struct task_struct *t)
 {
 	struct ftrace_ret_stack	*ret_stack = t->ret_stack;
 	t->ret_stack = NULL;
 	/* NULL must become visible to IRQs before we free it: */
 	barrier();
 	kfree(ret_stack);
 }
 void ftrace_graph_stop(void)
 {
 	ftrace_stop();
 }
 #endif

kernel/trace/trace.c

Diff comments View file @ 7eb6952

 /*
  * ring buffer based function tracer
  *
  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
  *
  * Originally taken from the RT patch by:
  *    Arnaldo Carvalho de Melo <acme@redhat.com>
  *
  * Based on code from the latency_tracer, that is:
  *  Copyright (C) 2004-2006 Ingo Molnar
  *  Copyright (C) 2004 Nadia Yvette Chambers
  */
 #include <linux/ring_buffer.h>
 #include <generated/utsrelease.h>
 #include <linux/stacktrace.h>
 #include <linux/writeback.h>
 #include <linux/kallsyms.h>
 #include <linux/seq_file.h>
 #include <linux/notifier.h>
 #include <linux/irqflags.h>
 #include <linux/debugfs.h>
 #include <linux/pagemap.h>
 #include <linux/hardirq.h>
 #include <linux/linkage.h>
 #include <linux/uaccess.h>
 #include <linux/kprobes.h>
 #include <linux/ftrace.h>
 #include <linux/module.h>
 #include <linux/percpu.h>
 #include <linux/splice.h>
 #include <linux/kdebug.h>
 #include <linux/string.h>
 #include <linux/rwsem.h>
 #include <linux/slab.h>
 #include <linux/ctype.h>
 #include <linux/init.h>
 #include <linux/poll.h>
 #include <linux/nmi.h>
 #include <linux/fs.h>
 #include <linux/sched/rt.h>
 #include "trace.h"
 #include "trace_output.h"
 /*
  * On boot up, the ring buffer is set to the minimum size, so that
  * we do not waste memory on systems that are not using tracing.
  */
 bool ring_buffer_expanded;
 /*
  * We need to change this state when a selftest is running.
  * A selftest will lurk into the ring-buffer to count the
  * entries inserted during the selftest although some concurrent
  * insertions into the ring-buffer such as trace_printk could occurred
  * at the same time, giving false positive or negative results.
  */
 static bool __read_mostly tracing_selftest_running;
 /*
  * If a tracer is running, we do not want to run SELFTEST.
  */
 bool __read_mostly tracing_selftest_disabled;
 /* For tracers that don't implement custom flags */
 static struct tracer_opt dummy_tracer_opt[] = {
 	{ }
 };
 static struct tracer_flags dummy_tracer_flags = {
 	.val = 0,
 	.opts = dummy_tracer_opt
 };
 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
 {
 	return 0;
 }
 /*
  * To prevent the comm cache from being overwritten when no
  * tracing is active, only save the comm when a trace event
  * occurred.
  */
 static DEFINE_PER_CPU(bool, trace_cmdline_save);
 /*
  * Kill all tracing for good (never come back).
  * It is initialized to 1 but will turn to zero if the initialization
  * of the tracer is successful. But that is the only place that sets
  * this back to zero.
  */
 static int tracing_disabled = 1;
 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
 cpumask_var_t __read_mostly	tracing_buffer_mask;
 /*
  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  *
  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  * is set, then ftrace_dump is called. This will output the contents
  * of the ftrace buffers to the console.  This is very useful for
  * capturing traces that lead to crashes and outputing it to a
  * serial console.
  *
  * It is default off, but you can enable it with either specifying
  * "ftrace_dump_on_oops" in the kernel command line, or setting
  * /proc/sys/kernel/ftrace_dump_on_oops
  * Set 1 if you want to dump buffers of all CPUs
  * Set 2 if you want to dump the buffer of the CPU that triggered oops
  */
 enum ftrace_dump_mode ftrace_dump_on_oops;
 /* When set, tracing will stop when a WARN*() is hit */
 int __disable_trace_on_warning;
 static int tracing_set_tracer(const char *buf);
 #define MAX_TRACER_SIZE		100
 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
 static char *default_bootup_tracer;
 static bool allocate_snapshot;
 static int __init set_cmdline_ftrace(char *str)
 {
 	strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
 	default_bootup_tracer = bootup_tracer_buf;
 	/* We are using ftrace early, expand it */
 	ring_buffer_expanded = true;
 	return 1;
 }
 __setup("ftrace=", set_cmdline_ftrace);
 static int __init set_ftrace_dump_on_oops(char *str)
 {
 	if (*str++ != '=' || !*str) {
 		ftrace_dump_on_oops = DUMP_ALL;
 		return 1;
 	}
 	if (!strcmp("orig_cpu", str)) {
 		ftrace_dump_on_oops = DUMP_ORIG;
                 return 1;
         }
         return 0;
 }
 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
 static int __init stop_trace_on_warning(char *str)
 {
 	__disable_trace_on_warning = 1;
 	return 1;
 }
 __setup("traceoff_on_warning=", stop_trace_on_warning);
 static int __init boot_alloc_snapshot(char *str)
 {
 	allocate_snapshot = true;
 	/* We also need the main ring buffer expanded */
 	ring_buffer_expanded = true;
 	return 1;
 }
 __setup("alloc_snapshot", boot_alloc_snapshot);
 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
 static char *trace_boot_options __initdata;
 static int __init set_trace_boot_options(char *str)
 {
 	strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
 	trace_boot_options = trace_boot_options_buf;
 	return 0;
 }
 __setup("trace_options=", set_trace_boot_options);
 unsigned long long ns2usecs(cycle_t nsec)
 {
 	nsec += 500;
 	do_div(nsec, 1000);
 	return nsec;
 }
 /*
  * The global_trace is the descriptor that holds the tracing
  * buffers for the live tracing. For each CPU, it contains
  * a link list of pages that will store trace entries. The
  * page descriptor of the pages in the memory is used to hold
  * the link list by linking the lru item in the page descriptor
  * to each of the pages in the buffer per CPU.
  *
  * For each active CPU there is a data field that holds the
  * pages for the buffer for that CPU. Each CPU has the same number
  * of pages allocated for its buffer.
  */
 static struct trace_array	global_trace;
 LIST_HEAD(ftrace_trace_arrays);
 int trace_array_get(struct trace_array *this_tr)
 {
 	struct trace_array *tr;
 	int ret = -ENODEV;
 	mutex_lock(&trace_types_lock);
 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
 		if (tr == this_tr) {
 			tr->ref++;
 			ret = 0;
 			break;
 		}
 	}
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static void __trace_array_put(struct trace_array *this_tr)
 {
 	WARN_ON(!this_tr->ref);
 	this_tr->ref--;
 }
 void trace_array_put(struct trace_array *this_tr)
 {
 	mutex_lock(&trace_types_lock);
 	__trace_array_put(this_tr);
 	mutex_unlock(&trace_types_lock);
 }
 int filter_current_check_discard(struct ring_buffer *buffer,
 				 struct ftrace_event_call *call, void *rec,
 				 struct ring_buffer_event *event)
 {
 	return filter_check_discard(call, rec, buffer, event);
 }
 EXPORT_SYMBOL_GPL(filter_current_check_discard);
 cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
 {
 	u64 ts;
 	/* Early boot up does not have a buffer yet */
 	if (!buf->buffer)
 		return trace_clock_local();
 	ts = ring_buffer_time_stamp(buf->buffer, cpu);
 	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
 	return ts;
 }
 cycle_t ftrace_now(int cpu)
 {
 	return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
 }
 /**
  * tracing_is_enabled - Show if global_trace has been disabled
  *
  * Shows if the global trace has been enabled or not. It uses the
  * mirror flag "buffer_disabled" to be used in fast paths such as for
  * the irqsoff tracer. But it may be inaccurate due to races. If you
  * need to know the accurate state, use tracing_is_on() which is a little
  * slower, but accurate.
  */
 int tracing_is_enabled(void)
 {
 	/*
 	 * For quick access (irqsoff uses this in fast path), just
 	 * return the mirror variable of the state of the ring buffer.
 	 * It's a little racy, but we don't really care.
 	 */
 	smp_rmb();
 	return !global_trace.buffer_disabled;
 }
 /*
  * trace_buf_size is the size in bytes that is allocated
  * for a buffer. Note, the number of bytes is always rounded
  * to page size.
  *
  * This number is purposely set to a low number of 16384.
  * If the dump on oops happens, it will be much appreciated
  * to not have to wait for all that output. Anyway this can be
  * boot time and run time configurable.
  */
 #define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */
 static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 /* trace_types holds a link list of available tracers. */
 static struct tracer		*trace_types __read_mostly;
 /*
  * trace_types_lock is used to protect the trace_types list.
  */
 DEFINE_MUTEX(trace_types_lock);
 /*
  * serialize the access of the ring buffer
  *
  * ring buffer serializes readers, but it is low level protection.
  * The validity of the events (which returns by ring_buffer_peek() ..etc)
  * are not protected by ring buffer.
  *
  * The content of events may become garbage if we allow other process consumes
  * these events concurrently:
  *   A) the page of the consumed events may become a normal page
  *      (not reader page) in ring buffer, and this page will be rewrited
  *      by events producer.
  *   B) The page of the consumed events may become a page for splice_read,
  *      and this page will be returned to system.
  *
  * These primitives allow multi process access to different cpu ring buffer
  * concurrently.
  *
  * These primitives don't distinguish read-only and read-consume access.
  * Multi read-only access are also serialized.
  */
 #ifdef CONFIG_SMP
 static DECLARE_RWSEM(all_cpu_access_lock);
 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
 static inline void trace_access_lock(int cpu)
 {
 	if (cpu == RING_BUFFER_ALL_CPUS) {
 		/* gain it for accessing the whole ring buffer. */
 		down_write(&all_cpu_access_lock);
 	} else {
 		/* gain it for accessing a cpu ring buffer. */
 		/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
 		down_read(&all_cpu_access_lock);
 		/* Secondly block other access to this @cpu ring buffer. */
 		mutex_lock(&per_cpu(cpu_access_lock, cpu));
 	}
 }
 static inline void trace_access_unlock(int cpu)
 {
 	if (cpu == RING_BUFFER_ALL_CPUS) {
 		up_write(&all_cpu_access_lock);
 	} else {
 		mutex_unlock(&per_cpu(cpu_access_lock, cpu));
 		up_read(&all_cpu_access_lock);
 	}
 }
 static inline void trace_access_lock_init(void)
 {
 	int cpu;
 	for_each_possible_cpu(cpu)
 		mutex_init(&per_cpu(cpu_access_lock, cpu));
 }
 #else
 static DEFINE_MUTEX(access_lock);
 static inline void trace_access_lock(int cpu)
 {
 	(void)cpu;
 	mutex_lock(&access_lock);
 }
 static inline void trace_access_unlock(int cpu)
 {
 	(void)cpu;
 	mutex_unlock(&access_lock);
 }
 static inline void trace_access_lock_init(void)
 {
 }
 #endif
 /* trace_flags holds trace_options default values */
 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
 	TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
 	TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
 	TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
 static void tracer_tracing_on(struct trace_array *tr)
 {
 	if (tr->trace_buffer.buffer)
 		ring_buffer_record_on(tr->trace_buffer.buffer);
 	/*
 	 * This flag is looked at when buffers haven't been allocated
 	 * yet, or by some tracers (like irqsoff), that just want to
 	 * know if the ring buffer has been disabled, but it can handle
 	 * races of where it gets disabled but we still do a record.
 	 * As the check is in the fast path of the tracers, it is more
 	 * important to be fast than accurate.
 	 */
 	tr->buffer_disabled = 0;
 	/* Make the flag seen by readers */
 	smp_wmb();
 }
 /**
  * tracing_on - enable tracing buffers
  *
  * This function enables tracing buffers that may have been
  * disabled with tracing_off.
  */
 void tracing_on(void)
 {
 	tracer_tracing_on(&global_trace);
 }
 EXPORT_SYMBOL_GPL(tracing_on);
 /**
  * __trace_puts - write a constant string into the trace buffer.
  * @ip:	   The address of the caller
  * @str:   The constant string to write
  * @size:  The size of the string.
  */
 int __trace_puts(unsigned long ip, const char *str, int size)
 {
 	struct ring_buffer_event *event;
 	struct ring_buffer *buffer;
 	struct print_entry *entry;
 	unsigned long irq_flags;
 	int alloc;
 	alloc = sizeof(*entry) + size + 2; /* possible \n added */
 	local_save_flags(irq_flags);
 	buffer = global_trace.trace_buffer.buffer;
 	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
 					  irq_flags, preempt_count());
 	if (!event)
 		return 0;
 	entry = ring_buffer_event_data(event);
 	entry->ip = ip;
 	memcpy(&entry->buf, str, size);
 	/* Add a newline if necessary */
 	if (entry->buf[size - 1] != '\n') {
 		entry->buf[size] = '\n';
 		entry->buf[size + 1] = '\0';
 	} else
 		entry->buf[size] = '\0';
 	__buffer_unlock_commit(buffer, event);
 	return size;
 }
 EXPORT_SYMBOL_GPL(__trace_puts);
 /**
  * __trace_bputs - write the pointer to a constant string into trace buffer
  * @ip:	   The address of the caller
  * @str:   The constant string to write to the buffer to
  */
 int __trace_bputs(unsigned long ip, const char *str)
 {
 	struct ring_buffer_event *event;
 	struct ring_buffer *buffer;
 	struct bputs_entry *entry;
 	unsigned long irq_flags;
 	int size = sizeof(struct bputs_entry);
 	local_save_flags(irq_flags);
 	buffer = global_trace.trace_buffer.buffer;
 	event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
 					  irq_flags, preempt_count());
 	if (!event)
 		return 0;
 	entry = ring_buffer_event_data(event);
 	entry->ip			= ip;
 	entry->str			= str;
 	__buffer_unlock_commit(buffer, event);
 	return 1;
 }
 EXPORT_SYMBOL_GPL(__trace_bputs);
 #ifdef CONFIG_TRACER_SNAPSHOT
 /**
  * trace_snapshot - take a snapshot of the current buffer.
  *
  * This causes a swap between the snapshot buffer and the current live
  * tracing buffer. You can use this to take snapshots of the live
  * trace when some condition is triggered, but continue to trace.
  *
  * Note, make sure to allocate the snapshot with either
  * a tracing_snapshot_alloc(), or by doing it manually
  * with: echo 1 > /sys/kernel/debug/tracing/snapshot
  *
  * If the snapshot buffer is not allocated, it will stop tracing.
  * Basically making a permanent snapshot.
  */
 void tracing_snapshot(void)
 {
 	struct trace_array *tr = &global_trace;
 	struct tracer *tracer = tr->current_trace;
 	unsigned long flags;
 	if (in_nmi()) {
 		internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
 		internal_trace_puts("*** snapshot is being ignored        ***\n");
 		return;
 	}
 	if (!tr->allocated_snapshot) {
 		internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
 		internal_trace_puts("*** stopping trace here!   ***\n");
 		tracing_off();
 		return;
 	}
 	/* Note, snapshot can not be used when the tracer uses it */
 	if (tracer->use_max_tr) {
 		internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
 		internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
 		return;
 	}
 	local_irq_save(flags);
 	update_max_tr(tr, current, smp_processor_id());
 	local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(tracing_snapshot);
 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
 					struct trace_buffer *size_buf, int cpu_id);
 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
 static int alloc_snapshot(struct trace_array *tr)
 {
 	int ret;
 	if (!tr->allocated_snapshot) {
 		/* allocate spare buffer */
 		ret = resize_buffer_duplicate_size(&tr->max_buffer,
 				   &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
 		if (ret < 0)
 			return ret;
 		tr->allocated_snapshot = true;
 	}
 	return 0;
 }
 void free_snapshot(struct trace_array *tr)
 {
 	/*
 	 * We don't free the ring buffer. instead, resize it because
 	 * The max_tr ring buffer has some state (e.g. ring->clock) and
 	 * we want preserve it.
 	 */
 	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
 	set_buffer_entries(&tr->max_buffer, 1);
 	tracing_reset_online_cpus(&tr->max_buffer);
 	tr->allocated_snapshot = false;
 }
 /**
  * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
  *
  * This is similar to trace_snapshot(), but it will allocate the
  * snapshot buffer if it isn't already allocated. Use this only
  * where it is safe to sleep, as the allocation may sleep.
  *
  * This causes a swap between the snapshot buffer and the current live
  * tracing buffer. You can use this to take snapshots of the live
  * trace when some condition is triggered, but continue to trace.
  */
 void tracing_snapshot_alloc(void)
 {
 	struct trace_array *tr = &global_trace;
 	int ret;
 	ret = alloc_snapshot(tr);
 	if (WARN_ON(ret < 0))
 		return;
 	tracing_snapshot();
 }
 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
 #else
 void tracing_snapshot(void)
 {
 	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
 }
 EXPORT_SYMBOL_GPL(tracing_snapshot);
 void tracing_snapshot_alloc(void)
 {
 	/* Give warning */
 	tracing_snapshot();
 }
 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
 #endif /* CONFIG_TRACER_SNAPSHOT */
 static void tracer_tracing_off(struct trace_array *tr)
 {
 	if (tr->trace_buffer.buffer)
 		ring_buffer_record_off(tr->trace_buffer.buffer);
 	/*
 	 * This flag is looked at when buffers haven't been allocated
 	 * yet, or by some tracers (like irqsoff), that just want to
 	 * know if the ring buffer has been disabled, but it can handle
 	 * races of where it gets disabled but we still do a record.
 	 * As the check is in the fast path of the tracers, it is more
 	 * important to be fast than accurate.
 	 */
 	tr->buffer_disabled = 1;
 	/* Make the flag seen by readers */
 	smp_wmb();
 }
 /**
  * tracing_off - turn off tracing buffers
  *
  * This function stops the tracing buffers from recording data.
  * It does not disable any overhead the tracers themselves may
  * be causing. This function simply causes all recording to
  * the ring buffers to fail.
  */
 void tracing_off(void)
 {
 	tracer_tracing_off(&global_trace);
 }
 EXPORT_SYMBOL_GPL(tracing_off);
 void disable_trace_on_warning(void)
 {
 	if (__disable_trace_on_warning)
 		tracing_off();
 }
 /**
  * tracer_tracing_is_on - show real state of ring buffer enabled
  * @tr : the trace array to know if ring buffer is enabled
  *
  * Shows real state of the ring buffer if it is enabled or not.
  */
 static int tracer_tracing_is_on(struct trace_array *tr)
 {
 	if (tr->trace_buffer.buffer)
 		return ring_buffer_record_is_on(tr->trace_buffer.buffer);
 	return !tr->buffer_disabled;
 }
 /**
  * tracing_is_on - show state of ring buffers enabled
  */
 int tracing_is_on(void)
 {
 	return tracer_tracing_is_on(&global_trace);
 }
 EXPORT_SYMBOL_GPL(tracing_is_on);
 static int __init set_buf_size(char *str)
 {
 	unsigned long buf_size;
 	if (!str)
 		return 0;
 	buf_size = memparse(str, &str);
 	/* nr_entries can not be zero */
 	if (buf_size == 0)
 		return 0;
 	trace_buf_size = buf_size;
 	return 1;
 }
 __setup("trace_buf_size=", set_buf_size);
 static int __init set_tracing_thresh(char *str)
 {
 	unsigned long threshold;
 	int ret;
 	if (!str)
 		return 0;
 	ret = kstrtoul(str, 0, &threshold);
 	if (ret < 0)
 		return 0;
 	tracing_thresh = threshold * 1000;
 	return 1;
 }
 __setup("tracing_thresh=", set_tracing_thresh);
 unsigned long nsecs_to_usecs(unsigned long nsecs)
 {
 	return nsecs / 1000;
 }
 /* These must match the bit postions in trace_iterator_flags */
 static const char *trace_options[] = {
 	"print-parent",
 	"sym-offset",
 	"sym-addr",
 	"verbose",
 	"raw",
 	"hex",
 	"bin",
 	"block",
 	"stacktrace",
 	"trace_printk",
 	"ftrace_preempt",
 	"branch",
 	"annotate",
 	"userstacktrace",
 	"sym-userobj",
 	"printk-msg-only",
 	"context-info",
 	"latency-format",
 	"sleep-time",
 	"graph-time",
 	"record-cmd",
 	"overwrite",
 	"disable_on_free",
 	"irq-info",
 	"markers",
 	"function-trace",
 	NULL
 };
 static struct {
 	u64 (*func)(void);
 	const char *name;
 	int in_ns;		/* is this clock in nanoseconds? */
 } trace_clocks[] = {
 	{ trace_clock_local,	"local",	1 },
 	{ trace_clock_global,	"global",	1 },
 	{ trace_clock_counter,	"counter",	0 },
 	{ trace_clock_jiffies,	"uptime",	1 },
 	{ trace_clock,		"perf",		1 },
 	ARCH_TRACE_CLOCKS
 };
 /*
  * trace_parser_get_init - gets the buffer for trace parser
  */
 int trace_parser_get_init(struct trace_parser *parser, int size)
 {
 	memset(parser, 0, sizeof(*parser));
 	parser->buffer = kmalloc(size, GFP_KERNEL);
 	if (!parser->buffer)
 		return 1;
 	parser->size = size;
 	return 0;
 }
 /*
  * trace_parser_put - frees the buffer for trace parser
  */
 void trace_parser_put(struct trace_parser *parser)
 {
 	kfree(parser->buffer);
 }
 /*
  * trace_get_user - reads the user input string separated by  space
  * (matched by isspace(ch))
  *
  * For each string found the 'struct trace_parser' is updated,
  * and the function returns.
  *
  * Returns number of bytes read.
  *
  * See kernel/trace/trace.h for 'struct trace_parser' details.
  */
 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
 	size_t cnt, loff_t *ppos)
 {
 	char ch;
 	size_t read = 0;
 	ssize_t ret;
 	if (!*ppos)
 		trace_parser_clear(parser);
 	ret = get_user(ch, ubuf++);
 	if (ret)
 		goto out;
 	read++;
 	cnt--;
 	/*
 	 * The parser is not finished with the last write,
 	 * continue reading the user input without skipping spaces.
 	 */
 	if (!parser->cont) {
 		/* skip white space */
 		while (cnt && isspace(ch)) {
 			ret = get_user(ch, ubuf++);
 			if (ret)
 				goto out;
 			read++;
 			cnt--;
 		}
 		/* only spaces were written */
 		if (isspace(ch)) {
 			*ppos += read;
 			ret = read;
 			goto out;
 		}
 		parser->idx = 0;
 	}
 	/* read the non-space input */
 	while (cnt && !isspace(ch)) {
 		if (parser->idx < parser->size - 1)
 			parser->buffer[parser->idx++] = ch;
 		else {
 			ret = -EINVAL;
 			goto out;
 		}
 		ret = get_user(ch, ubuf++);
 		if (ret)
 			goto out;
 		read++;
 		cnt--;
 	}
 	/* We either got finished input or we have to wait for another call. */
 	if (isspace(ch)) {
 		parser->buffer[parser->idx] = 0;
 		parser->cont = false;
 	} else {
 		parser->cont = true;
 		parser->buffer[parser->idx++] = ch;
 	}
 	*ppos += read;
 	ret = read;
 out:
 	return ret;
 }
 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
 {
 	int len;
 	int ret;
 	if (!cnt)
 		return 0;
 	if (s->len <= s->readpos)
 		return -EBUSY;
 	len = s->len - s->readpos;
 	if (cnt > len)
 		cnt = len;
 	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
 	if (ret == cnt)
 		return -EFAULT;
 	cnt -= ret;
 	s->readpos += cnt;
 	return cnt;
 }
 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
 {
 	int len;
 	if (s->len <= s->readpos)
 		return -EBUSY;
 	len = s->len - s->readpos;
 	if (cnt > len)
 		cnt = len;
 	memcpy(buf, s->buffer + s->readpos, cnt);
 	s->readpos += cnt;
 	return cnt;
 }
 /*
  * ftrace_max_lock is used to protect the swapping of buffers
  * when taking a max snapshot. The buffers themselves are
  * protected by per_cpu spinlocks. But the action of the swap
  * needs its own lock.
  *
  * This is defined as a arch_spinlock_t in order to help
  * with performance when lockdep debugging is enabled.
  *
  * It is also used in other places outside the update_max_tr
  * so it needs to be defined outside of the
  * CONFIG_TRACER_MAX_TRACE.
  */
 static arch_spinlock_t ftrace_max_lock =
 	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 unsigned long __read_mostly	tracing_thresh;
 #ifdef CONFIG_TRACER_MAX_TRACE
 unsigned long __read_mostly	tracing_max_latency;
 /*
  * Copy the new maximum trace into the separate maximum-trace
  * structure. (this way the maximum trace is permanently saved,
  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
  */
 static void
 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 {
 	struct trace_buffer *trace_buf = &tr->trace_buffer;
 	struct trace_buffer *max_buf = &tr->max_buffer;
 	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
 	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
 	max_buf->cpu = cpu;
 	max_buf->time_start = data->preempt_timestamp;
 	max_data->saved_latency = tracing_max_latency;
 	max_data->critical_start = data->critical_start;
 	max_data->critical_end = data->critical_end;
 	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
 	max_data->pid = tsk->pid;
 	/*
 	 * If tsk == current, then use current_uid(), as that does not use
 	 * RCU. The irq tracer can be called out of RCU scope.
 	 */
 	if (tsk == current)
 		max_data->uid = current_uid();
 	else
 		max_data->uid = task_uid(tsk);
 	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 	max_data->policy = tsk->policy;
 	max_data->rt_priority = tsk->rt_priority;
 	/* record this tasks comm */
 	tracing_record_cmdline(tsk);
 }
 /**
  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
  * @tr: tracer
  * @tsk: the task with the latency
  * @cpu: The cpu that initiated the trace.
  *
  * Flip the buffers between the @tr and the max_tr and record information
  * about which task was the cause of this latency.
  */
 void
 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 {
 	struct ring_buffer *buf;
 	if (tr->stop_count)
 		return;
 	WARN_ON_ONCE(!irqs_disabled());
 	if (!tr->allocated_snapshot) {
 		/* Only the nop tracer should hit this when disabling */
 		WARN_ON_ONCE(tr->current_trace != &nop_trace);
 		return;
 	}
 	arch_spin_lock(&ftrace_max_lock);
 	buf = tr->trace_buffer.buffer;
 	tr->trace_buffer.buffer = tr->max_buffer.buffer;
 	tr->max_buffer.buffer = buf;
 	__update_max_tr(tr, tsk, cpu);
 	arch_spin_unlock(&ftrace_max_lock);
 }
 /**
  * update_max_tr_single - only copy one trace over, and reset the rest
  * @tr - tracer
  * @tsk - task with the latency
  * @cpu - the cpu of the buffer to copy.
  *
  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
  */
 void
 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 {
 	int ret;
 	if (tr->stop_count)
 		return;
 	WARN_ON_ONCE(!irqs_disabled());
 	if (!tr->allocated_snapshot) {
 		/* Only the nop tracer should hit this when disabling */
 		WARN_ON_ONCE(tr->current_trace != &nop_trace);
 		return;
 	}
 	arch_spin_lock(&ftrace_max_lock);
 	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
 	if (ret == -EBUSY) {
 		/*
 		 * We failed to swap the buffer due to a commit taking
 		 * place on this CPU. We fail to record, but we reset
 		 * the max trace buffer (no one writes directly to it)
 		 * and flag that it failed.
 		 */
 		trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
 			"Failed to swap buffers due to commit in progress\n");
 	}
 	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
 	__update_max_tr(tr, tsk, cpu);
 	arch_spin_unlock(&ftrace_max_lock);
 }
 #endif /* CONFIG_TRACER_MAX_TRACE */
 static void default_wait_pipe(struct trace_iterator *iter)
 {
 	/* Iterators are static, they should be filled or empty */
 	if (trace_buffer_iter(iter, iter->cpu_file))
 		return;
 	ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
 }
 #ifdef CONFIG_FTRACE_STARTUP_TEST
 static int run_tracer_selftest(struct tracer *type)
 {
 	struct trace_array *tr = &global_trace;
 	struct tracer *saved_tracer = tr->current_trace;
 	int ret;
 	if (!type->selftest || tracing_selftest_disabled)
 		return 0;
 	/*
 	 * Run a selftest on this tracer.
 	 * Here we reset the trace buffer, and set the current
 	 * tracer to be this tracer. The tracer can then run some
 	 * internal tracing to verify that everything is in order.
 	 * If we fail, we do not register this tracer.
 	 */
 	tracing_reset_online_cpus(&tr->trace_buffer);
 	tr->current_trace = type;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (type->use_max_tr) {
 		/* If we expanded the buffers, make sure the max is expanded too */
 		if (ring_buffer_expanded)
 			ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
 					   RING_BUFFER_ALL_CPUS);
 		tr->allocated_snapshot = true;
 	}
 #endif
 	/* the test is responsible for initializing and enabling */
 	pr_info("Testing tracer %s: ", type->name);
 	ret = type->selftest(type, tr);
 	/* the test is responsible for resetting too */
 	tr->current_trace = saved_tracer;
 	if (ret) {
 		printk(KERN_CONT "FAILED!\n");
 		/* Add the warning after printing 'FAILED' */
 		WARN_ON(1);
 		return -1;
 	}
 	/* Only reset on passing, to avoid touching corrupted buffers */
 	tracing_reset_online_cpus(&tr->trace_buffer);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (type->use_max_tr) {
 		tr->allocated_snapshot = false;
 		/* Shrink the max buffer again */
 		if (ring_buffer_expanded)
 			ring_buffer_resize(tr->max_buffer.buffer, 1,
 					   RING_BUFFER_ALL_CPUS);
 	}
 #endif
 	printk(KERN_CONT "PASSED\n");
 	return 0;
 }
 #else
 static inline int run_tracer_selftest(struct tracer *type)
 {
 	return 0;
 }
 #endif /* CONFIG_FTRACE_STARTUP_TEST */
 /**
  * register_tracer - register a tracer with the ftrace system.
  * @type - the plugin for the tracer
  *
  * Register a new plugin tracer.
  */
 int register_tracer(struct tracer *type)
 {
 	struct tracer *t;
 	int ret = 0;
 	if (!type->name) {
 		pr_info("Tracer must have a name\n");
 		return -1;
 	}
 	if (strlen(type->name) >= MAX_TRACER_SIZE) {
 		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
 		return -1;
 	}
 	mutex_lock(&trace_types_lock);
 	tracing_selftest_running = true;
 	for (t = trace_types; t; t = t->next) {
 		if (strcmp(type->name, t->name) == 0) {
 			/* already found */
 			pr_info("Tracer %s already registered\n",
 				type->name);
 			ret = -1;
 			goto out;
 		}
 	}
 	if (!type->set_flag)
 		type->set_flag = &dummy_set_flag;
 	if (!type->flags)
 		type->flags = &dummy_tracer_flags;
 	else
 		if (!type->flags->opts)
 			type->flags->opts = dummy_tracer_opt;
 	if (!type->wait_pipe)
 		type->wait_pipe = default_wait_pipe;
 	ret = run_tracer_selftest(type);
 	if (ret < 0)
 		goto out;
 	type->next = trace_types;
 	trace_types = type;
  out:
 	tracing_selftest_running = false;
 	mutex_unlock(&trace_types_lock);
 	if (ret || !default_bootup_tracer)
 		goto out_unlock;
 	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
 		goto out_unlock;
 	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
 	/* Do we want this tracer to start on bootup? */
 	tracing_set_tracer(type->name);
 	default_bootup_tracer = NULL;
 	/* disable other selftests, since this will break it. */
 	tracing_selftest_disabled = true;
 #ifdef CONFIG_FTRACE_STARTUP_TEST
 	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
 	       type->name);
 #endif
  out_unlock:
 	return ret;
 }
 void tracing_reset(struct trace_buffer *buf, int cpu)
 {
 	struct ring_buffer *buffer = buf->buffer;
 	if (!buffer)
 		return;
 	ring_buffer_record_disable(buffer);
 	/* Make sure all commits have finished */
 	synchronize_sched();
 	ring_buffer_reset_cpu(buffer, cpu);
 	ring_buffer_record_enable(buffer);
 }
 void tracing_reset_online_cpus(struct trace_buffer *buf)
 {
 	struct ring_buffer *buffer = buf->buffer;
 	int cpu;
 	if (!buffer)
 		return;
 	ring_buffer_record_disable(buffer);
 	/* Make sure all commits have finished */
 	synchronize_sched();
 	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
 	for_each_online_cpu(cpu)
 		ring_buffer_reset_cpu(buffer, cpu);
 	ring_buffer_record_enable(buffer);
 }
 /* Must have trace_types_lock held */
 void tracing_reset_all_online_cpus(void)
 {
 	struct trace_array *tr;
 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
 		tracing_reset_online_cpus(&tr->trace_buffer);
 #ifdef CONFIG_TRACER_MAX_TRACE
 		tracing_reset_online_cpus(&tr->max_buffer);
 #endif
 	}
 }
 #define SAVED_CMDLINES 128
 #define NO_CMDLINE_MAP UINT_MAX
 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 static int cmdline_idx;
 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
 /* temporary disable recording */
 static atomic_t trace_record_cmdline_disabled __read_mostly;
 static void trace_init_cmdlines(void)
 {
 	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
 	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
 	cmdline_idx = 0;
 }
 int is_tracing_stopped(void)
 {
 	return global_trace.stop_count;
 }
 /**
  * ftrace_off_permanent - disable all ftrace code permanently
  *
  * This should only be called when a serious anomally has
  * been detected.  This will turn off the function tracing,
  * ring buffers, and other tracing utilites. It takes no
  * locks and can be called from any context.
  */
 void ftrace_off_permanent(void)
 {
 	tracing_disabled = 1;
 	ftrace_stop();
 	tracing_off_permanent();
 }
 /**
  * tracing_start - quick start of the tracer
  *
  * If tracing is enabled but was stopped by tracing_stop,
  * this will start the tracer back up.
  */
 void tracing_start(void)
 {
 	struct ring_buffer *buffer;
 	unsigned long flags;
 	if (tracing_disabled)
 		return;
 	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
 	if (--global_trace.stop_count) {
 		if (global_trace.stop_count < 0) {
 			/* Someone screwed up their debugging */
 			WARN_ON_ONCE(1);
 			global_trace.stop_count = 0;
 		}
 		goto out;
 	}
 	/* Prevent the buffers from switching */
 	arch_spin_lock(&ftrace_max_lock);
 	buffer = global_trace.trace_buffer.buffer;
 	if (buffer)
 		ring_buffer_record_enable(buffer);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	buffer = global_trace.max_buffer.buffer;
 	if (buffer)
 		ring_buffer_record_enable(buffer);
 #endif
 	arch_spin_unlock(&ftrace_max_lock);
 	ftrace_start();
  out:
 	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
 }
 static void tracing_start_tr(struct trace_array *tr)
 {
 	struct ring_buffer *buffer;
 	unsigned long flags;
 	if (tracing_disabled)
 		return;
 	/* If global, we need to also start the max tracer */
 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
 		return tracing_start();
 	raw_spin_lock_irqsave(&tr->start_lock, flags);
 	if (--tr->stop_count) {
 		if (tr->stop_count < 0) {
 			/* Someone screwed up their debugging */
 			WARN_ON_ONCE(1);
 			tr->stop_count = 0;
 		}
 		goto out;
 	}
 	buffer = tr->trace_buffer.buffer;
 	if (buffer)
 		ring_buffer_record_enable(buffer);
  out:
 	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
 }
 /**
  * tracing_stop - quick stop of the tracer
  *
  * Light weight way to stop tracing. Use in conjunction with
  * tracing_start.
  */
 void tracing_stop(void)
 {
 	struct ring_buffer *buffer;
 	unsigned long flags;
 	ftrace_stop();
 	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
 	if (global_trace.stop_count++)
 		goto out;
 	/* Prevent the buffers from switching */
 	arch_spin_lock(&ftrace_max_lock);
 	buffer = global_trace.trace_buffer.buffer;
 	if (buffer)
 		ring_buffer_record_disable(buffer);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	buffer = global_trace.max_buffer.buffer;
 	if (buffer)
 		ring_buffer_record_disable(buffer);
 #endif
 	arch_spin_unlock(&ftrace_max_lock);
  out:
 	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
 }
 static void tracing_stop_tr(struct trace_array *tr)
 {
 	struct ring_buffer *buffer;
 	unsigned long flags;
 	/* If global, we need to also stop the max tracer */
 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
 		return tracing_stop();
 	raw_spin_lock_irqsave(&tr->start_lock, flags);
 	if (tr->stop_count++)
 		goto out;
 	buffer = tr->trace_buffer.buffer;
 	if (buffer)
 		ring_buffer_record_disable(buffer);
  out:
 	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
 }
 void trace_stop_cmdline_recording(void);
 static void trace_save_cmdline(struct task_struct *tsk)
 {
 	unsigned pid, idx;
 	if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
 		return;
 	/*
 	 * It's not the end of the world if we don't get
 	 * the lock, but we also don't want to spin
 	 * nor do we want to disable interrupts,
 	 * so if we miss here, then better luck next time.
 	 */
 	if (!arch_spin_trylock(&trace_cmdline_lock))
 		return;
 	idx = map_pid_to_cmdline[tsk->pid];
 	if (idx == NO_CMDLINE_MAP) {
 		idx = (cmdline_idx + 1) % SAVED_CMDLINES;
 		/*
 		 * Check whether the cmdline buffer at idx has a pid
 		 * mapped. We are going to overwrite that entry so we
 		 * need to clear the map_pid_to_cmdline. Otherwise we
 		 * would read the new comm for the old pid.
 		 */
 		pid = map_cmdline_to_pid[idx];
 		if (pid != NO_CMDLINE_MAP)
 			map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
 		map_cmdline_to_pid[idx] = tsk->pid;
 		map_pid_to_cmdline[tsk->pid] = idx;
 		cmdline_idx = idx;
 	}
 	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
 	arch_spin_unlock(&trace_cmdline_lock);
 }
 void trace_find_cmdline(int pid, char comm[])
 {
 	unsigned map;
 	if (!pid) {
 		strcpy(comm, "<idle>");
 		return;
 	}
 	if (WARN_ON_ONCE(pid < 0)) {
 		strcpy(comm, "<XXX>");
 		return;
 	}
 	if (pid > PID_MAX_DEFAULT) {
 		strcpy(comm, "<...>");
 		return;
 	}
 	preempt_disable();
 	arch_spin_lock(&trace_cmdline_lock);
 	map = map_pid_to_cmdline[pid];
 	if (map != NO_CMDLINE_MAP)
 		strcpy(comm, saved_cmdlines[map]);
 	else
 		strcpy(comm, "<...>");
 	arch_spin_unlock(&trace_cmdline_lock);
 	preempt_enable();
 }
 void tracing_record_cmdline(struct task_struct *tsk)
 {
 	if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
 		return;
 	if (!__this_cpu_read(trace_cmdline_save))
 		return;
 	__this_cpu_write(trace_cmdline_save, false);
 	trace_save_cmdline(tsk);
 }
 void
 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
 			     int pc)
 {
 	struct task_struct *tsk = current;
 	entry->preempt_count		= pc & 0xff;
 	entry->pid			= (tsk) ? tsk->pid : 0;
 	entry->flags =
 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
 		(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
 #else
 		TRACE_FLAG_IRQS_NOSUPPORT |
 #endif
 		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
 		((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
 		(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
 }
 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
 struct ring_buffer_event *
 trace_buffer_lock_reserve(struct ring_buffer *buffer,
 			  int type,
 			  unsigned long len,
 			  unsigned long flags, int pc)
 {
 	struct ring_buffer_event *event;
 	event = ring_buffer_lock_reserve(buffer, len);
 	if (event != NULL) {
 		struct trace_entry *ent = ring_buffer_event_data(event);
 		tracing_generic_entry_update(ent, flags, pc);
 		ent->type = type;
 	}
 	return event;
 }
 void
 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
 {
 	__this_cpu_write(trace_cmdline_save, true);
 	ring_buffer_unlock_commit(buffer, event);
 }
 static inline void
 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
 			     struct ring_buffer_event *event,
 			     unsigned long flags, int pc)
 {
 	__buffer_unlock_commit(buffer, event);
 	ftrace_trace_stack(buffer, flags, 6, pc);
 	ftrace_trace_userstack(buffer, flags, pc);
 }
 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
 				struct ring_buffer_event *event,
 				unsigned long flags, int pc)
 {
 	__trace_buffer_unlock_commit(buffer, event, flags, pc);
 }
 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
 struct ring_buffer_event *
 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
 			  struct ftrace_event_file *ftrace_file,
 			  int type, unsigned long len,
 			  unsigned long flags, int pc)
 {
 	*current_rb = ftrace_file->tr->trace_buffer.buffer;
 	return trace_buffer_lock_reserve(*current_rb,
 					 type, len, flags, pc);
 }
 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
 struct ring_buffer_event *
 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
 				  int type, unsigned long len,
 				  unsigned long flags, int pc)
 {
 	*current_rb = global_trace.trace_buffer.buffer;
 	return trace_buffer_lock_reserve(*current_rb,
 					 type, len, flags, pc);
 }
 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
 					struct ring_buffer_event *event,
 					unsigned long flags, int pc)
 {
 	__trace_buffer_unlock_commit(buffer, event, flags, pc);
 }
 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
 				     struct ring_buffer_event *event,
 				     unsigned long flags, int pc,
 				     struct pt_regs *regs)
 {
 	__buffer_unlock_commit(buffer, event);
 	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
 	ftrace_trace_userstack(buffer, flags, pc);
 }
 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
 					 struct ring_buffer_event *event)
 {
 	ring_buffer_discard_commit(buffer, event);
 }
 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
 void
 trace_function(struct trace_array *tr,
 	       unsigned long ip, unsigned long parent_ip, unsigned long flags,
 	       int pc)
 {
 	struct ftrace_event_call *call = &event_function;
 	struct ring_buffer *buffer = tr->trace_buffer.buffer;
 	struct ring_buffer_event *event;
 	struct ftrace_entry *entry;
 	/* If we are reading the ring buffer, don't trace */
 	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
 		return;
 	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
 					  flags, pc);
 	if (!event)
 		return;
 	entry	= ring_buffer_event_data(event);
 	entry->ip			= ip;
 	entry->parent_ip		= parent_ip;
 	if (!filter_check_discard(call, entry, buffer, event))
 		__buffer_unlock_commit(buffer, event);
 }
 #ifdef CONFIG_STACKTRACE
 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
 struct ftrace_stack {
 	unsigned long		calls[FTRACE_STACK_MAX_ENTRIES];
 };
 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
 static void __ftrace_trace_stack(struct ring_buffer *buffer,
 				 unsigned long flags,
 				 int skip, int pc, struct pt_regs *regs)
 {
 	struct ftrace_event_call *call = &event_kernel_stack;
 	struct ring_buffer_event *event;
 	struct stack_entry *entry;
 	struct stack_trace trace;
 	int use_stack;
 	int size = FTRACE_STACK_ENTRIES;
 	trace.nr_entries	= 0;
 	trace.skip		= skip;
 	/*
 	 * Since events can happen in NMIs there's no safe way to
 	 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
 	 * or NMI comes in, it will just have to use the default
 	 * FTRACE_STACK_SIZE.
 	 */
 	preempt_disable_notrace();
 	use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
 	/*
 	 * We don't need any atomic variables, just a barrier.
 	 * If an interrupt comes in, we don't care, because it would
 	 * have exited and put the counter back to what we want.
 	 * We just need a barrier to keep gcc from moving things
 	 * around.
 	 */
 	barrier();
 	if (use_stack == 1) {
 		trace.entries		= &__get_cpu_var(ftrace_stack).calls[0];
 		trace.max_entries	= FTRACE_STACK_MAX_ENTRIES;
 		if (regs)
 			save_stack_trace_regs(regs, &trace);
 		else
 			save_stack_trace(&trace);
 		if (trace.nr_entries > size)
 			size = trace.nr_entries;
 	} else
 		/* From now on, use_stack is a boolean */
 		use_stack = 0;
 	size *= sizeof(unsigned long);
 	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
 					  sizeof(*entry) + size, flags, pc);
 	if (!event)
 		goto out;
 	entry = ring_buffer_event_data(event);
 	memset(&entry->caller, 0, size);
 	if (use_stack)
 		memcpy(&entry->caller, trace.entries,
 		       trace.nr_entries * sizeof(unsigned long));
 	else {
 		trace.max_entries	= FTRACE_STACK_ENTRIES;
 		trace.entries		= entry->caller;
 		if (regs)
 			save_stack_trace_regs(regs, &trace);
 		else
 			save_stack_trace(&trace);
 	}
 	entry->size = trace.nr_entries;
 	if (!filter_check_discard(call, entry, buffer, event))
 		__buffer_unlock_commit(buffer, event);
  out:
 	/* Again, don't let gcc optimize things here */
 	barrier();
 	__this_cpu_dec(ftrace_stack_reserve);
 	preempt_enable_notrace();
 }
 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
 			     int skip, int pc, struct pt_regs *regs)
 {
 	if (!(trace_flags & TRACE_ITER_STACKTRACE))
 		return;
 	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
 }
 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
 			int skip, int pc)
 {
 	if (!(trace_flags & TRACE_ITER_STACKTRACE))
 		return;
 	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
 }
 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
 		   int pc)
 {
 	__ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
 }
 /**
  * trace_dump_stack - record a stack back trace in the trace buffer
  * @skip: Number of functions to skip (helper handlers)
  */
 void trace_dump_stack(int skip)
 {
 	unsigned long flags;
 	if (tracing_disabled || tracing_selftest_running)
 		return;
 	local_save_flags(flags);
 	/*
 	 * Skip 3 more, seems to get us at the caller of
 	 * this function.
 	 */
 	skip += 3;
 	__ftrace_trace_stack(global_trace.trace_buffer.buffer,
 			     flags, skip, preempt_count(), NULL);
 }
 static DEFINE_PER_CPU(int, user_stack_count);
 void
 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
 {
 	struct ftrace_event_call *call = &event_user_stack;
 	struct ring_buffer_event *event;
 	struct userstack_entry *entry;
 	struct stack_trace trace;
 	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
 		return;
 	/*
 	 * NMIs can not handle page faults, even with fix ups.
 	 * The save user stack can (and often does) fault.
 	 */
 	if (unlikely(in_nmi()))
 		return;
 	/*
 	 * prevent recursion, since the user stack tracing may
 	 * trigger other kernel events.
 	 */
 	preempt_disable();
 	if (__this_cpu_read(user_stack_count))
 		goto out;
 	__this_cpu_inc(user_stack_count);
 	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
 					  sizeof(*entry), flags, pc);
 	if (!event)
 		goto out_drop_count;
 	entry	= ring_buffer_event_data(event);
 	entry->tgid		= current->tgid;
 	memset(&entry->caller, 0, sizeof(entry->caller));
 	trace.nr_entries	= 0;
 	trace.max_entries	= FTRACE_STACK_ENTRIES;
 	trace.skip		= 0;
 	trace.entries		= entry->caller;
 	save_stack_trace_user(&trace);
 	if (!filter_check_discard(call, entry, buffer, event))
 		__buffer_unlock_commit(buffer, event);
  out_drop_count:
 	__this_cpu_dec(user_stack_count);
  out:
 	preempt_enable();
 }
 #ifdef UNUSED
 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
 {
 	ftrace_trace_userstack(tr, flags, preempt_count());
 }
 #endif /* UNUSED */
 #endif /* CONFIG_STACKTRACE */
 /* created for use with alloc_percpu */
 struct trace_buffer_struct {
 	char buffer[TRACE_BUF_SIZE];
 };
 static struct trace_buffer_struct *trace_percpu_buffer;
 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
 static struct trace_buffer_struct *trace_percpu_irq_buffer;
 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
 /*
  * The buffer used is dependent on the context. There is a per cpu
  * buffer for normal context, softirq contex, hard irq context and
  * for NMI context. Thise allows for lockless recording.
  *
  * Note, if the buffers failed to be allocated, then this returns NULL
  */
 static char *get_trace_buf(void)
 {
 	struct trace_buffer_struct *percpu_buffer;
 	/*
 	 * If we have allocated per cpu buffers, then we do not
 	 * need to do any locking.
 	 */
 	if (in_nmi())
 		percpu_buffer = trace_percpu_nmi_buffer;
 	else if (in_irq())
 		percpu_buffer = trace_percpu_irq_buffer;
 	else if (in_softirq())
 		percpu_buffer = trace_percpu_sirq_buffer;
 	else
 		percpu_buffer = trace_percpu_buffer;
 	if (!percpu_buffer)
 		return NULL;
 	return this_cpu_ptr(&percpu_buffer->buffer[0]);
 }
 static int alloc_percpu_trace_buffer(void)
 {
 	struct trace_buffer_struct *buffers;
 	struct trace_buffer_struct *sirq_buffers;
 	struct trace_buffer_struct *irq_buffers;
 	struct trace_buffer_struct *nmi_buffers;
 	buffers = alloc_percpu(struct trace_buffer_struct);
 	if (!buffers)
 		goto err_warn;
 	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
 	if (!sirq_buffers)
 		goto err_sirq;
 	irq_buffers = alloc_percpu(struct trace_buffer_struct);
 	if (!irq_buffers)
 		goto err_irq;
 	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
 	if (!nmi_buffers)
 		goto err_nmi;
 	trace_percpu_buffer = buffers;
 	trace_percpu_sirq_buffer = sirq_buffers;
 	trace_percpu_irq_buffer = irq_buffers;
 	trace_percpu_nmi_buffer = nmi_buffers;
 	return 0;
  err_nmi:
 	free_percpu(irq_buffers);
  err_irq:
 	free_percpu(sirq_buffers);
  err_sirq:
 	free_percpu(buffers);
  err_warn:
 	WARN(1, "Could not allocate percpu trace_printk buffer");
 	return -ENOMEM;
 }
 static int buffers_allocated;
 void trace_printk_init_buffers(void)
 {
 	if (buffers_allocated)
 		return;
 	if (alloc_percpu_trace_buffer())
 		return;
 	pr_info("ftrace: Allocated trace_printk buffers\n");
 	/* Expand the buffers to set size */
 	tracing_update_buffers();
 	buffers_allocated = 1;
 	/*
 	 * trace_printk_init_buffers() can be called by modules.
 	 * If that happens, then we need to start cmdline recording
 	 * directly here. If the global_trace.buffer is already
 	 * allocated here, then this was called by module code.
 	 */
 	if (global_trace.trace_buffer.buffer)
 		tracing_start_cmdline_record();
 }
 void trace_printk_start_comm(void)
 {
 	/* Start tracing comms if trace printk is set */
 	if (!buffers_allocated)
 		return;
 	tracing_start_cmdline_record();
 }
 static void trace_printk_start_stop_comm(int enabled)
 {
 	if (!buffers_allocated)
 		return;
 	if (enabled)
 		tracing_start_cmdline_record();
 	else
 		tracing_stop_cmdline_record();
 }
 /**
  * trace_vbprintk - write binary msg to tracing buffer
  *
  */
 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
 {
 	struct ftrace_event_call *call = &event_bprint;
 	struct ring_buffer_event *event;
 	struct ring_buffer *buffer;
 	struct trace_array *tr = &global_trace;
 	struct bprint_entry *entry;
 	unsigned long flags;
 	char *tbuffer;
 	int len = 0, size, pc;
 	if (unlikely(tracing_selftest_running || tracing_disabled))
 		return 0;
 	/* Don't pollute graph traces with trace_vprintk internals */
 	pause_graph_tracing();
 	pc = preempt_count();
 	preempt_disable_notrace();
 	tbuffer = get_trace_buf();
 	if (!tbuffer) {
 		len = 0;
 		goto out;
 	}
 	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
 	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
 		goto out;
 	local_save_flags(flags);
 	size = sizeof(*entry) + sizeof(u32) * len;
 	buffer = tr->trace_buffer.buffer;
 	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
 					  flags, pc);
 	if (!event)
 		goto out;
 	entry = ring_buffer_event_data(event);
 	entry->ip			= ip;
 	entry->fmt			= fmt;
 	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
 	if (!filter_check_discard(call, entry, buffer, event)) {
 		__buffer_unlock_commit(buffer, event);
 		ftrace_trace_stack(buffer, flags, 6, pc);
 	}
 out:
 	preempt_enable_notrace();
 	unpause_graph_tracing();
 	return len;
 }
 EXPORT_SYMBOL_GPL(trace_vbprintk);
 static int
 __trace_array_vprintk(struct ring_buffer *buffer,
 		      unsigned long ip, const char *fmt, va_list args)
 {
 	struct ftrace_event_call *call = &event_print;
 	struct ring_buffer_event *event;
 	int len = 0, size, pc;
 	struct print_entry *entry;
 	unsigned long flags;
 	char *tbuffer;
 	if (tracing_disabled || tracing_selftest_running)
 		return 0;
 	/* Don't pollute graph traces with trace_vprintk internals */
 	pause_graph_tracing();
 	pc = preempt_count();
 	preempt_disable_notrace();
 	tbuffer = get_trace_buf();
 	if (!tbuffer) {
 		len = 0;
 		goto out;
 	}
 	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
 	if (len > TRACE_BUF_SIZE)
 		goto out;
 	local_save_flags(flags);
 	size = sizeof(*entry) + len + 1;
 	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
 					  flags, pc);
 	if (!event)
 		goto out;
 	entry = ring_buffer_event_data(event);
 	entry->ip = ip;
 	memcpy(&entry->buf, tbuffer, len);
 	entry->buf[len] = '\0';
 	if (!filter_check_discard(call, entry, buffer, event)) {
 		__buffer_unlock_commit(buffer, event);
 		ftrace_trace_stack(buffer, flags, 6, pc);
 	}
  out:
 	preempt_enable_notrace();
 	unpause_graph_tracing();
 	return len;
 }
 int trace_array_vprintk(struct trace_array *tr,
 			unsigned long ip, const char *fmt, va_list args)
 {
 	return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
 }
 int trace_array_printk(struct trace_array *tr,
 		       unsigned long ip, const char *fmt, ...)
 {
 	int ret;
 	va_list ap;
 	if (!(trace_flags & TRACE_ITER_PRINTK))
 		return 0;
 	va_start(ap, fmt);
 	ret = trace_array_vprintk(tr, ip, fmt, ap);
 	va_end(ap);
 	return ret;
 }
 int trace_array_printk_buf(struct ring_buffer *buffer,
 			   unsigned long ip, const char *fmt, ...)
 {
 	int ret;
 	va_list ap;
 	if (!(trace_flags & TRACE_ITER_PRINTK))
 		return 0;
 	va_start(ap, fmt);
 	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
 	va_end(ap);
 	return ret;
 }
 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
 {
 	return trace_array_vprintk(&global_trace, ip, fmt, args);
 }
 EXPORT_SYMBOL_GPL(trace_vprintk);
 static void trace_iterator_increment(struct trace_iterator *iter)
 {
 	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
 	iter->idx++;
 	if (buf_iter)
 		ring_buffer_read(buf_iter, NULL);
 }
 static struct trace_entry *
 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
 		unsigned long *lost_events)
 {
 	struct ring_buffer_event *event;
 	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
 	if (buf_iter)
 		event = ring_buffer_iter_peek(buf_iter, ts);
 	else
 		event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
 					 lost_events);
 	if (event) {
 		iter->ent_size = ring_buffer_event_length(event);
 		return ring_buffer_event_data(event);
 	}
 	iter->ent_size = 0;
 	return NULL;
 }
 static struct trace_entry *
 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
 		  unsigned long *missing_events, u64 *ent_ts)
 {
 	struct ring_buffer *buffer = iter->trace_buffer->buffer;
 	struct trace_entry *ent, *next = NULL;
 	unsigned long lost_events = 0, next_lost = 0;
 	int cpu_file = iter->cpu_file;
 	u64 next_ts = 0, ts;
 	int next_cpu = -1;
 	int next_size = 0;
 	int cpu;
 	/*
 	 * If we are in a per_cpu trace file, don't bother by iterating over
 	 * all cpu and peek directly.
 	 */
 	if (cpu_file > RING_BUFFER_ALL_CPUS) {
 		if (ring_buffer_empty_cpu(buffer, cpu_file))
 			return NULL;
 		ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
 		if (ent_cpu)
 			*ent_cpu = cpu_file;
 		return ent;
 	}
 	for_each_tracing_cpu(cpu) {
 		if (ring_buffer_empty_cpu(buffer, cpu))
 			continue;
 		ent = peek_next_entry(iter, cpu, &ts, &lost_events);
 		/*
 		 * Pick the entry with the smallest timestamp:
 		 */
 		if (ent && (!next || ts < next_ts)) {
 			next = ent;
 			next_cpu = cpu;
 			next_ts = ts;
 			next_lost = lost_events;
 			next_size = iter->ent_size;
 		}
 	}
 	iter->ent_size = next_size;
 	if (ent_cpu)
 		*ent_cpu = next_cpu;
 	if (ent_ts)
 		*ent_ts = next_ts;
 	if (missing_events)
 		*missing_events = next_lost;
 	return next;
 }
 /* Find the next real entry, without updating the iterator itself */
 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
 					  int *ent_cpu, u64 *ent_ts)
 {
 	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
 }
 /* Find the next real entry, and increment the iterator to the next entry */
 void *trace_find_next_entry_inc(struct trace_iterator *iter)
 {
 	iter->ent = __find_next_entry(iter, &iter->cpu,
 				      &iter->lost_events, &iter->ts);
 	if (iter->ent)
 		trace_iterator_increment(iter);
 	return iter->ent ? iter : NULL;
 }
 static void trace_consume(struct trace_iterator *iter)
 {
 	ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
 			    &iter->lost_events);
 }
 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
 {
 	struct trace_iterator *iter = m->private;
 	int i = (int)*pos;
 	void *ent;
 	WARN_ON_ONCE(iter->leftover);
 	(*pos)++;
 	/* can't go backwards */
 	if (iter->idx > i)
 		return NULL;
 	if (iter->idx < 0)
 		ent = trace_find_next_entry_inc(iter);
 	else
 		ent = iter;
 	while (ent && iter->idx < i)
 		ent = trace_find_next_entry_inc(iter);
 	iter->pos = *pos;
 	return ent;
 }
 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
 {
 	struct ring_buffer_event *event;
 	struct ring_buffer_iter *buf_iter;
 	unsigned long entries = 0;
 	u64 ts;
 	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
 	buf_iter = trace_buffer_iter(iter, cpu);
 	if (!buf_iter)
 		return;
 	ring_buffer_iter_reset(buf_iter);
 	/*
 	 * We could have the case with the max latency tracers
 	 * that a reset never took place on a cpu. This is evident
 	 * by the timestamp being before the start of the buffer.
 	 */
 	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
 		if (ts >= iter->trace_buffer->time_start)
 			break;
 		entries++;
 		ring_buffer_read(buf_iter, NULL);
 	}
 	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
 }
 /*
  * The current tracer is copied to avoid a global locking
  * all around.
  */
 static void *s_start(struct seq_file *m, loff_t *pos)
 {
 	struct trace_iterator *iter = m->private;
 	struct trace_array *tr = iter->tr;
 	int cpu_file = iter->cpu_file;
 	void *p = NULL;
 	loff_t l = 0;
 	int cpu;
 	/*
 	 * copy the tracer to avoid using a global lock all around.
 	 * iter->trace is a copy of current_trace, the pointer to the
 	 * name may be used instead of a strcmp(), as iter->trace->name
 	 * will point to the same string as current_trace->name.
 	 */
 	mutex_lock(&trace_types_lock);
 	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
 		*iter->trace = *tr->current_trace;
 	mutex_unlock(&trace_types_lock);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (iter->snapshot && iter->trace->use_max_tr)
 		return ERR_PTR(-EBUSY);
 #endif
 	if (!iter->snapshot)
 		atomic_inc(&trace_record_cmdline_disabled);
 	if (*pos != iter->pos) {
 		iter->ent = NULL;
 		iter->cpu = 0;
 		iter->idx = -1;
 		if (cpu_file == RING_BUFFER_ALL_CPUS) {
 			for_each_tracing_cpu(cpu)
 				tracing_iter_reset(iter, cpu);
 		} else
 			tracing_iter_reset(iter, cpu_file);
 		iter->leftover = 0;
 		for (p = iter; p && l < *pos; p = s_next(m, p, &l))
 			;
 	} else {
 		/*
 		 * If we overflowed the seq_file before, then we want
 		 * to just reuse the trace_seq buffer again.
 		 */
 		if (iter->leftover)
 			p = iter;
 		else {
 			l = *pos - 1;
 			p = s_next(m, p, &l);
 		}
 	}
 	trace_event_read_lock();
 	trace_access_lock(cpu_file);
 	return p;
 }
 static void s_stop(struct seq_file *m, void *p)
 {
 	struct trace_iterator *iter = m->private;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (iter->snapshot && iter->trace->use_max_tr)
 		return;
 #endif
 	if (!iter->snapshot)
 		atomic_dec(&trace_record_cmdline_disabled);
 	trace_access_unlock(iter->cpu_file);
 	trace_event_read_unlock();
 }
 static void
 get_total_entries(struct trace_buffer *buf,
 		  unsigned long *total, unsigned long *entries)
 {
 	unsigned long count;
 	int cpu;
 	*total = 0;
 	*entries = 0;
 	for_each_tracing_cpu(cpu) {
 		count = ring_buffer_entries_cpu(buf->buffer, cpu);
 		/*
 		 * If this buffer has skipped entries, then we hold all
 		 * entries for the trace and we need to ignore the
 		 * ones before the time stamp.
 		 */
 		if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
 			count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
 			/* total is the same as the entries */
 			*total += count;
 		} else
 			*total += count +
 				ring_buffer_overrun_cpu(buf->buffer, cpu);
 		*entries += count;
 	}
 }
 static void print_lat_help_header(struct seq_file *m)
 {
 	seq_puts(m, "#                  _------=> CPU#            \n");
 	seq_puts(m, "#                 / _-----=> irqs-off        \n");
 	seq_puts(m, "#                | / _----=> need-resched    \n");
 	seq_puts(m, "#                || / _---=> hardirq/softirq \n");
 	seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
 	seq_puts(m, "#                |||| /     delay             \n");
 	seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
 	seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
 }
 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
 {
 	unsigned long total;
 	unsigned long entries;
 	get_total_entries(buf, &total, &entries);
 	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
 		   entries, total, num_online_cpus());
 	seq_puts(m, "#\n");
 }
 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
 {
 	print_event_info(buf, m);
 	seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
 	seq_puts(m, "#              | |       |          |         |\n");
 }
 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
 {
 	print_event_info(buf, m);
 	seq_puts(m, "#                              _-----=> irqs-off\n");
 	seq_puts(m, "#                             / _----=> need-resched\n");
 	seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
 	seq_puts(m, "#                            || / _--=> preempt-depth\n");
 	seq_puts(m, "#                            ||| /     delay\n");
 	seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
 	seq_puts(m, "#              | |       |   ||||       |         |\n");
 }
 void
 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
 {
 	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
 	struct trace_buffer *buf = iter->trace_buffer;
 	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
 	struct tracer *type = iter->trace;
 	unsigned long entries;
 	unsigned long total;
 	const char *name = "preemption";
 	name = type->name;
 	get_total_entries(buf, &total, &entries);
 	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
 		   name, UTS_RELEASE);
 	seq_puts(m, "# -----------------------------------"
 		 "---------------------------------\n");
 	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
 		   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
 		   nsecs_to_usecs(data->saved_latency),
 		   entries,
 		   total,
 		   buf->cpu,
 #if defined(CONFIG_PREEMPT_NONE)
 		   "server",
 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
 		   "desktop",
 #elif defined(CONFIG_PREEMPT)
 		   "preempt",
 #else
 		   "unknown",
 #endif
 		   /* These are reserved for later use */
 		   0, 0, 0, 0);
 #ifdef CONFIG_SMP
 	seq_printf(m, " #P:%d)\n", num_online_cpus());
 #else
 	seq_puts(m, ")\n");
 #endif
 	seq_puts(m, "#    -----------------\n");
 	seq_printf(m, "#    | task: %.16s-%d "
 		   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
 		   data->comm, data->pid,
 		   from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
 		   data->policy, data->rt_priority);
 	seq_puts(m, "#    -----------------\n");
 	if (data->critical_start) {
 		seq_puts(m, "#  => started at: ");
 		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
 		trace_print_seq(m, &iter->seq);
 		seq_puts(m, "\n#  => ended at:   ");
 		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
 		trace_print_seq(m, &iter->seq);
 		seq_puts(m, "\n#\n");
 	}
 	seq_puts(m, "#\n");
 }
 static void test_cpu_buff_start(struct trace_iterator *iter)
 {
 	struct trace_seq *s = &iter->seq;
 	if (!(trace_flags & TRACE_ITER_ANNOTATE))
 		return;
 	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
 		return;
 	if (cpumask_test_cpu(iter->cpu, iter->started))
 		return;
 	if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
 		return;
 	cpumask_set_cpu(iter->cpu, iter->started);
 	/* Don't print started cpu buffer for the first entry of the trace */
 	if (iter->idx > 1)
 		trace_seq_printf(s, "##### CPU %u buffer started ####\n",
 				iter->cpu);
 }
 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
 {
 	struct trace_seq *s = &iter->seq;
 	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
 	struct trace_entry *entry;
 	struct trace_event *event;
 	entry = iter->ent;
 	test_cpu_buff_start(iter);
 	event = ftrace_find_event(entry->type);
 	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
 		if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
 			if (!trace_print_lat_context(iter))
 				goto partial;
 		} else {
 			if (!trace_print_context(iter))
 				goto partial;
 		}
 	}
 	if (event)
 		return event->funcs->trace(iter, sym_flags, event);
 	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
 		goto partial;
 	return TRACE_TYPE_HANDLED;
 partial:
 	return TRACE_TYPE_PARTIAL_LINE;
 }
 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
 {
 	struct trace_seq *s = &iter->seq;
 	struct trace_entry *entry;
 	struct trace_event *event;
 	entry = iter->ent;
 	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
 		if (!trace_seq_printf(s, "%d %d %llu ",
 				      entry->pid, iter->cpu, iter->ts))
 			goto partial;
 	}
 	event = ftrace_find_event(entry->type);
 	if (event)
 		return event->funcs->raw(iter, 0, event);
 	if (!trace_seq_printf(s, "%d ?\n", entry->type))
 		goto partial;
 	return TRACE_TYPE_HANDLED;
 partial:
 	return TRACE_TYPE_PARTIAL_LINE;
 }
 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
 {
 	struct trace_seq *s = &iter->seq;
 	unsigned char newline = '\n';
 	struct trace_entry *entry;
 	struct trace_event *event;
 	entry = iter->ent;
 	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
 		SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
 		SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
 		SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
 	}
 	event = ftrace_find_event(entry->type);
 	if (event) {
 		enum print_line_t ret = event->funcs->hex(iter, 0, event);
 		if (ret != TRACE_TYPE_HANDLED)
 			return ret;
 	}
 	SEQ_PUT_FIELD_RET(s, newline);
 	return TRACE_TYPE_HANDLED;
 }
 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
 {
 	struct trace_seq *s = &iter->seq;
 	struct trace_entry *entry;
 	struct trace_event *event;
 	entry = iter->ent;
 	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
 		SEQ_PUT_FIELD_RET(s, entry->pid);
 		SEQ_PUT_FIELD_RET(s, iter->cpu);
 		SEQ_PUT_FIELD_RET(s, iter->ts);
 	}
 	event = ftrace_find_event(entry->type);
 	return event ? event->funcs->binary(iter, 0, event) :
 		TRACE_TYPE_HANDLED;
 }
 int trace_empty(struct trace_iterator *iter)
 {
 	struct ring_buffer_iter *buf_iter;
 	int cpu;
 	/* If we are looking at one CPU buffer, only check that one */
 	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
 		cpu = iter->cpu_file;
 		buf_iter = trace_buffer_iter(iter, cpu);
 		if (buf_iter) {
 			if (!ring_buffer_iter_empty(buf_iter))
 				return 0;
 		} else {
 			if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
 				return 0;
 		}
 		return 1;
 	}
 	for_each_tracing_cpu(cpu) {
 		buf_iter = trace_buffer_iter(iter, cpu);
 		if (buf_iter) {
 			if (!ring_buffer_iter_empty(buf_iter))
 				return 0;
 		} else {
 			if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
 				return 0;
 		}
 	}
 	return 1;
 }
 /*  Called with trace_event_read_lock() held. */
 enum print_line_t print_trace_line(struct trace_iterator *iter)
 {
 	enum print_line_t ret;
 	if (iter->lost_events &&
 	    !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
 				 iter->cpu, iter->lost_events))
 		return TRACE_TYPE_PARTIAL_LINE;
 	if (iter->trace && iter->trace->print_line) {
 		ret = iter->trace->print_line(iter);
 		if (ret != TRACE_TYPE_UNHANDLED)
 			return ret;
 	}
 	if (iter->ent->type == TRACE_BPUTS &&
 			trace_flags & TRACE_ITER_PRINTK &&
 			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
 		return trace_print_bputs_msg_only(iter);
 	if (iter->ent->type == TRACE_BPRINT &&
 			trace_flags & TRACE_ITER_PRINTK &&
 			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
 		return trace_print_bprintk_msg_only(iter);
 	if (iter->ent->type == TRACE_PRINT &&
 			trace_flags & TRACE_ITER_PRINTK &&
 			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
 		return trace_print_printk_msg_only(iter);
 	if (trace_flags & TRACE_ITER_BIN)
 		return print_bin_fmt(iter);
 	if (trace_flags & TRACE_ITER_HEX)
 		return print_hex_fmt(iter);
 	if (trace_flags & TRACE_ITER_RAW)
 		return print_raw_fmt(iter);
 	return print_trace_fmt(iter);
 }
 void trace_latency_header(struct seq_file *m)
 {
 	struct trace_iterator *iter = m->private;
 	/* print nothing if the buffers are empty */
 	if (trace_empty(iter))
 		return;
 	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
 		print_trace_header(m, iter);
 	if (!(trace_flags & TRACE_ITER_VERBOSE))
 		print_lat_help_header(m);
 }
 void trace_default_header(struct seq_file *m)
 {
 	struct trace_iterator *iter = m->private;
 	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
 		return;
 	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
 		/* print nothing if the buffers are empty */
 		if (trace_empty(iter))
 			return;
 		print_trace_header(m, iter);
 		if (!(trace_flags & TRACE_ITER_VERBOSE))
 			print_lat_help_header(m);
 	} else {
 		if (!(trace_flags & TRACE_ITER_VERBOSE)) {
 			if (trace_flags & TRACE_ITER_IRQ_INFO)
 				print_func_help_header_irq(iter->trace_buffer, m);
 			else
 				print_func_help_header(iter->trace_buffer, m);
 		}
 	}
 }
 static void test_ftrace_alive(struct seq_file *m)
 {
 	if (!ftrace_is_dead())
 		return;
 	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
 	seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
 }
 #ifdef CONFIG_TRACER_MAX_TRACE
 static void show_snapshot_main_help(struct seq_file *m)
 {
 	seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
 	seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
 	seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
 	seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
 	seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
 	seq_printf(m, "#                       is not a '0' or '1')\n");
 }
 static void show_snapshot_percpu_help(struct seq_file *m)
 {
 	seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
 	seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
 	seq_printf(m, "#                      Takes a snapshot of the main buffer for this cpu.\n");
 #else
 	seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
 	seq_printf(m, "#                     Must use main snapshot file to allocate.\n");
 #endif
 	seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
 	seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
 	seq_printf(m, "#                       is not a '0' or '1')\n");
 }
 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
 {
 	if (iter->tr->allocated_snapshot)
 		seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
 	else
 		seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
 	seq_printf(m, "# Snapshot commands:\n");
 	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
 		show_snapshot_main_help(m);
 	else
 		show_snapshot_percpu_help(m);
 }
 #else
 /* Should never be called */
 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
 #endif
 static int s_show(struct seq_file *m, void *v)
 {
 	struct trace_iterator *iter = v;
 	int ret;
 	if (iter->ent == NULL) {
 		if (iter->tr) {
 			seq_printf(m, "# tracer: %s\n", iter->trace->name);
 			seq_puts(m, "#\n");
 			test_ftrace_alive(m);
 		}
 		if (iter->snapshot && trace_empty(iter))
 			print_snapshot_help(m, iter);
 		else if (iter->trace && iter->trace->print_header)
 			iter->trace->print_header(m);
 		else
 			trace_default_header(m);
 	} else if (iter->leftover) {
 		/*
 		 * If we filled the seq_file buffer earlier, we
 		 * want to just show it now.
 		 */
 		ret = trace_print_seq(m, &iter->seq);
 		/* ret should this time be zero, but you never know */
 		iter->leftover = ret;
 	} else {
 		print_trace_line(iter);
 		ret = trace_print_seq(m, &iter->seq);
 		/*
 		 * If we overflow the seq_file buffer, then it will
 		 * ask us for this data again at start up.
 		 * Use that instead.
 		 *  ret is 0 if seq_file write succeeded.
 		 *        -1 otherwise.
 		 */
 		iter->leftover = ret;
 	}
 	return 0;
 }
 /*
  * Should be used after trace_array_get(), trace_types_lock
  * ensures that i_cdev was already initialized.
  */
 static inline int tracing_get_cpu(struct inode *inode)
 {
 	if (inode->i_cdev) /* See trace_create_cpu_file() */
 		return (long)inode->i_cdev - 1;
 	return RING_BUFFER_ALL_CPUS;
 }
 static const struct seq_operations tracer_seq_ops = {
 	.start		= s_start,
 	.next		= s_next,
 	.stop		= s_stop,
 	.show		= s_show,
 };
 static struct trace_iterator *
 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
 {
 	struct trace_array *tr = inode->i_private;
 	struct trace_iterator *iter;
 	int cpu;
 	if (tracing_disabled)
 		return ERR_PTR(-ENODEV);
 	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
 	if (!iter)
 		return ERR_PTR(-ENOMEM);
 	iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
 				    GFP_KERNEL);
 	if (!iter->buffer_iter)
 		goto release;
 	/*
 	 * We make a copy of the current tracer to avoid concurrent
 	 * changes on it while we are reading.
 	 */
 	mutex_lock(&trace_types_lock);
 	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
 	if (!iter->trace)
 		goto fail;
 	*iter->trace = *tr->current_trace;
 	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
 		goto fail;
 	iter->tr = tr;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	/* Currently only the top directory has a snapshot */
 	if (tr->current_trace->print_max || snapshot)
 		iter->trace_buffer = &tr->max_buffer;
 	else
 #endif
 		iter->trace_buffer = &tr->trace_buffer;
 	iter->snapshot = snapshot;
 	iter->pos = -1;
 	iter->cpu_file = tracing_get_cpu(inode);
 	mutex_init(&iter->mutex);
 	/* Notify the tracer early; before we stop tracing. */
 	if (iter->trace && iter->trace->open)
 		iter->trace->open(iter);
 	/* Annotate start of buffers if we had overruns */
 	if (ring_buffer_overruns(iter->trace_buffer->buffer))
 		iter->iter_flags |= TRACE_FILE_ANNOTATE;
 	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
 	if (trace_clocks[tr->clock_id].in_ns)
 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
 	/* stop the trace while dumping if we are not opening "snapshot" */
 	if (!iter->snapshot)
 		tracing_stop_tr(tr);
 	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
 		for_each_tracing_cpu(cpu) {
 			iter->buffer_iter[cpu] =
 				ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
 		}
 		ring_buffer_read_prepare_sync();
 		for_each_tracing_cpu(cpu) {
 			ring_buffer_read_start(iter->buffer_iter[cpu]);
 			tracing_iter_reset(iter, cpu);
 		}
 	} else {
 		cpu = iter->cpu_file;
 		iter->buffer_iter[cpu] =
 			ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
 		ring_buffer_read_prepare_sync();
 		ring_buffer_read_start(iter->buffer_iter[cpu]);
 		tracing_iter_reset(iter, cpu);
 	}
 	mutex_unlock(&trace_types_lock);
 	return iter;
  fail:
 	mutex_unlock(&trace_types_lock);
 	kfree(iter->trace);
 	kfree(iter->buffer_iter);
 release:
 	seq_release_private(inode, file);
 	return ERR_PTR(-ENOMEM);
 }
 int tracing_open_generic(struct inode *inode, struct file *filp)
 {
 	if (tracing_disabled)
 		return -ENODEV;
 	filp->private_data = inode->i_private;
 	return 0;
 }
 /*
  * Open and update trace_array ref count.
  * Must have the current trace_array passed to it.
  */
 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
 {
 	struct trace_array *tr = inode->i_private;
 	if (tracing_disabled)
 		return -ENODEV;
 	if (trace_array_get(tr) < 0)
 		return -ENODEV;
 	filp->private_data = inode->i_private;
 	return 0;
 }
 static int tracing_release(struct inode *inode, struct file *file)
 {
 	struct trace_array *tr = inode->i_private;
 	struct seq_file *m = file->private_data;
 	struct trace_iterator *iter;
 	int cpu;
 	if (!(file->f_mode & FMODE_READ)) {
 		trace_array_put(tr);
 		return 0;
 	}
 	/* Writes do not use seq_file */
 	iter = m->private;
 	mutex_lock(&trace_types_lock);
 	for_each_tracing_cpu(cpu) {
 		if (iter->buffer_iter[cpu])
 			ring_buffer_read_finish(iter->buffer_iter[cpu]);
 	}
 	if (iter->trace && iter->trace->close)
 		iter->trace->close(iter);
 	if (!iter->snapshot)
 		/* reenable tracing if it was previously enabled */
 		tracing_start_tr(tr);
 	__trace_array_put(tr);
 	mutex_unlock(&trace_types_lock);
 	mutex_destroy(&iter->mutex);
 	free_cpumask_var(iter->started);
 	kfree(iter->trace);
 	kfree(iter->buffer_iter);
 	seq_release_private(inode, file);
 	return 0;
 }
 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
 {
 	struct trace_array *tr = inode->i_private;
 	trace_array_put(tr);
 	return 0;
 }
 static int tracing_single_release_tr(struct inode *inode, struct file *file)
 {
 	struct trace_array *tr = inode->i_private;
 	trace_array_put(tr);
 	return single_release(inode, file);
 }
 static int tracing_open(struct inode *inode, struct file *file)
 {
 	struct trace_array *tr = inode->i_private;
 	struct trace_iterator *iter;
 	int ret = 0;
 	if (trace_array_get(tr) < 0)
 		return -ENODEV;
 	/* If this file was open for write, then erase contents */
 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
 		int cpu = tracing_get_cpu(inode);
 		if (cpu == RING_BUFFER_ALL_CPUS)
 			tracing_reset_online_cpus(&tr->trace_buffer);
 		else
 			tracing_reset(&tr->trace_buffer, cpu);
 	}
 	if (file->f_mode & FMODE_READ) {
 		iter = __tracing_open(inode, file, false);
 		if (IS_ERR(iter))
 			ret = PTR_ERR(iter);
 		else if (trace_flags & TRACE_ITER_LATENCY_FMT)
 			iter->iter_flags |= TRACE_FILE_LAT_FMT;
 	}
 	if (ret < 0)
 		trace_array_put(tr);
 	return ret;
 }
 static void *
 t_next(struct seq_file *m, void *v, loff_t *pos)
 {
 	struct tracer *t = v;
 	(*pos)++;
 	if (t)
 		t = t->next;
 	return t;
 }
 static void *t_start(struct seq_file *m, loff_t *pos)
 {
 	struct tracer *t;
 	loff_t l = 0;
 	mutex_lock(&trace_types_lock);
 	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
 		;
 	return t;
 }
 static void t_stop(struct seq_file *m, void *p)
 {
 	mutex_unlock(&trace_types_lock);
 }
 static int t_show(struct seq_file *m, void *v)
 {
 	struct tracer *t = v;
 	if (!t)
 		return 0;
 	seq_printf(m, "%s", t->name);
 	if (t->next)
 		seq_putc(m, ' ');
 	else
 		seq_putc(m, '\n');
 	return 0;
 }
 static const struct seq_operations show_traces_seq_ops = {
 	.start		= t_start,
 	.next		= t_next,
 	.stop		= t_stop,
 	.show		= t_show,
 };
 static int show_traces_open(struct inode *inode, struct file *file)
 {
 	if (tracing_disabled)
 		return -ENODEV;
 	return seq_open(file, &show_traces_seq_ops);
 }
 static ssize_t
 tracing_write_stub(struct file *filp, const char __user *ubuf,
 		   size_t count, loff_t *ppos)
 {
 	return count;
 }
 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
 {
 	if (file->f_mode & FMODE_READ)
 		return seq_lseek(file, offset, origin);
 	else
 		return 0;
 }
 static const struct file_operations tracing_fops = {
 	.open		= tracing_open,
 	.read		= seq_read,
 	.write		= tracing_write_stub,
 	.llseek		= tracing_seek,
 	.release	= tracing_release,
 };
 static const struct file_operations show_traces_fops = {
 	.open		= show_traces_open,
 	.read		= seq_read,
 	.release	= seq_release,
 	.llseek		= seq_lseek,
 };
 /*
- * Only trace on a CPU if the bitmask is set:
- */
-static cpumask_var_t tracing_cpumask;
-/*
  * The tracer itself will not take this lock, but still we want
  * to provide a consistent cpumask to user-space:
  */
 static DEFINE_MUTEX(tracing_cpumask_update_lock);
 /*
  * Temporary storage for the character representation of the
  * CPU bitmask (and one more byte for the newline):
  */
 static char mask_str[NR_CPUS + 1];
 static ssize_t
 tracing_cpumask_read(struct file *filp, char __user *ubuf,
 		     size_t count, loff_t *ppos)
 {
+	struct trace_array *tr = file_inode(filp)->i_private;
 	int len;
 	mutex_lock(&tracing_cpumask_update_lock);
-	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
+	len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
 	if (count - len < 2) {
 		count = -EINVAL;
 		goto out_err;
 	}
 	len += sprintf(mask_str + len, "\n");
 	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
 out_err:
 	mutex_unlock(&tracing_cpumask_update_lock);
 	return count;
 }
 static ssize_t
 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
 		      size_t count, loff_t *ppos)
 {
-	struct trace_array *tr = filp->private_data;
+	struct trace_array *tr = file_inode(filp)->i_private;
 	cpumask_var_t tracing_cpumask_new;
 	int err, cpu;
 	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
 		return -ENOMEM;
 	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
 	if (err)
 		goto err_unlock;
 	mutex_lock(&tracing_cpumask_update_lock);
 	local_irq_disable();
 	arch_spin_lock(&ftrace_max_lock);
 	for_each_tracing_cpu(cpu) {
 		/*
 		 * Increase/decrease the disabled counter if we are
 		 * about to flip a bit in the cpumask:
 		 */
-		if (cpumask_test_cpu(cpu, tracing_cpumask) &&
+		if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
 				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
 			atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
 			ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
 		}
-		if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
+		if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
 				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
 			atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
 			ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
 		}
 	}
 	arch_spin_unlock(&ftrace_max_lock);
 	local_irq_enable();
-	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
+	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
 	mutex_unlock(&tracing_cpumask_update_lock);
 	free_cpumask_var(tracing_cpumask_new);
 	return count;
 err_unlock:
 	free_cpumask_var(tracing_cpumask_new);
 	return err;
 }
 static const struct file_operations tracing_cpumask_fops = {
-	.open		= tracing_open_generic,
+	.open		= tracing_open_generic_tr,
 	.read		= tracing_cpumask_read,
 	.write		= tracing_cpumask_write,
+	.release	= tracing_release_generic_tr,
 	.llseek		= generic_file_llseek,
 };
 static int tracing_trace_options_show(struct seq_file *m, void *v)
 {
 	struct tracer_opt *trace_opts;
 	struct trace_array *tr = m->private;
 	u32 tracer_flags;
 	int i;
 	mutex_lock(&trace_types_lock);
 	tracer_flags = tr->current_trace->flags->val;
 	trace_opts = tr->current_trace->flags->opts;
 	for (i = 0; trace_options[i]; i++) {
 		if (trace_flags & (1 << i))
 			seq_printf(m, "%s\n", trace_options[i]);
 		else
 			seq_printf(m, "no%s\n", trace_options[i]);
 	}
 	for (i = 0; trace_opts[i].name; i++) {
 		if (tracer_flags & trace_opts[i].bit)
 			seq_printf(m, "%s\n", trace_opts[i].name);
 		else
 			seq_printf(m, "no%s\n", trace_opts[i].name);
 	}
 	mutex_unlock(&trace_types_lock);
 	return 0;
 }
 static int __set_tracer_option(struct tracer *trace,
 			       struct tracer_flags *tracer_flags,
 			       struct tracer_opt *opts, int neg)
 {
 	int ret;
 	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
 	if (ret)
 		return ret;
 	if (neg)
 		tracer_flags->val &= ~opts->bit;
 	else
 		tracer_flags->val |= opts->bit;
 	return 0;
 }
 /* Try to assign a tracer specific option */
 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
 {
 	struct tracer_flags *tracer_flags = trace->flags;
 	struct tracer_opt *opts = NULL;
 	int i;
 	for (i = 0; tracer_flags->opts[i].name; i++) {
 		opts = &tracer_flags->opts[i];
 		if (strcmp(cmp, opts->name) == 0)
 			return __set_tracer_option(trace, trace->flags,
 						   opts, neg);
 	}
 	return -EINVAL;
 }
 /* Some tracers require overwrite to stay enabled */
 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
 {
 	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
 		return -1;
 	return 0;
 }
 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
 {
 	/* do nothing if flag is already set */
 	if (!!(trace_flags & mask) == !!enabled)
 		return 0;
 	/* Give the tracer a chance to approve the change */
 	if (tr->current_trace->flag_changed)
 		if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
 			return -EINVAL;
 	if (enabled)
 		trace_flags |= mask;
 	else
 		trace_flags &= ~mask;
 	if (mask == TRACE_ITER_RECORD_CMD)
 		trace_event_enable_cmd_record(enabled);
 	if (mask == TRACE_ITER_OVERWRITE) {
 		ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
 #ifdef CONFIG_TRACER_MAX_TRACE
 		ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
 #endif
 	}
 	if (mask == TRACE_ITER_PRINTK)
 		trace_printk_start_stop_comm(enabled);
 	return 0;
 }
 static int trace_set_options(struct trace_array *tr, char *option)
 {
 	char *cmp;
 	int neg = 0;
 	int ret = -ENODEV;
 	int i;
 	cmp = strstrip(option);
 	if (strncmp(cmp, "no", 2) == 0) {
 		neg = 1;
 		cmp += 2;
 	}
 	mutex_lock(&trace_types_lock);
 	for (i = 0; trace_options[i]; i++) {
 		if (strcmp(cmp, trace_options[i]) == 0) {
 			ret = set_tracer_flag(tr, 1 << i, !neg);
 			break;
 		}
 	}
 	/* If no option could be set, test the specific tracer options */
 	if (!trace_options[i])
 		ret = set_tracer_option(tr->current_trace, cmp, neg);
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static ssize_t
 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
 			size_t cnt, loff_t *ppos)
 {
 	struct seq_file *m = filp->private_data;
 	struct trace_array *tr = m->private;
 	char buf[64];
 	int ret;
 	if (cnt >= sizeof(buf))
 		return -EINVAL;
 	if (copy_from_user(&buf, ubuf, cnt))
 		return -EFAULT;
 	buf[cnt] = 0;
 	ret = trace_set_options(tr, buf);
 	if (ret < 0)
 		return ret;
 	*ppos += cnt;
 	return cnt;
 }
 static int tracing_trace_options_open(struct inode *inode, struct file *file)
 {
 	struct trace_array *tr = inode->i_private;
 	int ret;
 	if (tracing_disabled)
 		return -ENODEV;
 	if (trace_array_get(tr) < 0)
 		return -ENODEV;
 	ret = single_open(file, tracing_trace_options_show, inode->i_private);
 	if (ret < 0)
 		trace_array_put(tr);
 	return ret;
 }
 static const struct file_operations tracing_iter_fops = {
 	.open		= tracing_trace_options_open,
 	.read		= seq_read,
 	.llseek		= seq_lseek,
 	.release	= tracing_single_release_tr,
 	.write		= tracing_trace_options_write,
 };
 static const char readme_msg[] =
 	"tracing mini-HOWTO:\n\n"
 	"# echo 0 > tracing_on : quick way to disable tracing\n"
 	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
 	" Important files:\n"
 	"  trace\t\t\t- The static contents of the buffer\n"
 	"\t\t\t  To clear the buffer write into this file: echo > trace\n"
 	"  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
 	"  current_tracer\t- function and latency tracers\n"
 	"  available_tracers\t- list of configured tracers for current_tracer\n"
 	"  buffer_size_kb\t- view and modify size of per cpu buffer\n"
 	"  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
 	"  trace_clock\t\t-change the clock used to order events\n"
 	"       local:   Per cpu clock but may not be synced across CPUs\n"
 	"      global:   Synced across CPUs but slows tracing down.\n"
 	"     counter:   Not a clock, but just an increment\n"
 	"      uptime:   Jiffy counter from time of boot\n"
 	"        perf:   Same clock that perf events use\n"
 #ifdef CONFIG_X86_64
 	"     x86-tsc:   TSC cycle counter\n"
 #endif
 	"\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
 	"  tracing_cpumask\t- Limit which CPUs to trace\n"
 	"  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
 	"\t\t\t  Remove sub-buffer with rmdir\n"
 	"  trace_options\t\t- Set format or modify how tracing happens\n"
 	"\t\t\t  Disable an option by adding a suffix 'no' to the option name\n"
 #ifdef CONFIG_DYNAMIC_FTRACE
 	"\n  available_filter_functions - list of functions that can be filtered on\n"
 	"  set_ftrace_filter\t- echo function name in here to only trace these functions\n"
 	"            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
 	"            modules: Can select a group via module\n"
 	"             Format: :mod:<module-name>\n"
 	"             example: echo :mod:ext3 > set_ftrace_filter\n"
 	"            triggers: a command to perform when function is hit\n"
 	"              Format: <function>:<trigger>[:count]\n"
 	"             trigger: traceon, traceoff\n"
 	"                      enable_event:<system>:<event>\n"
 	"                      disable_event:<system>:<event>\n"
 #ifdef CONFIG_STACKTRACE
 	"                      stacktrace\n"
 #endif
 #ifdef CONFIG_TRACER_SNAPSHOT
 	"                      snapshot\n"
 #endif
 	"             example: echo do_fault:traceoff > set_ftrace_filter\n"
 	"                      echo do_trap:traceoff:3 > set_ftrace_filter\n"
 	"             The first one will disable tracing every time do_fault is hit\n"
 	"             The second will disable tracing at most 3 times when do_trap is hit\n"
 	"               The first time do trap is hit and it disables tracing, the counter\n"
 	"               will decrement to 2. If tracing is already disabled, the counter\n"
 	"               will not decrement. It only decrements when the trigger did work\n"
 	"             To remove trigger without count:\n"
 	"               echo '!<function>:<trigger> > set_ftrace_filter\n"
 	"             To remove trigger with a count:\n"
 	"               echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
 	"  set_ftrace_notrace\t- echo function name in here to never trace.\n"
 	"            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
 	"            modules: Can select a group via module command :mod:\n"
 	"            Does not accept triggers\n"
 #endif /* CONFIG_DYNAMIC_FTRACE */
 #ifdef CONFIG_FUNCTION_TRACER
 	"  set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
 #endif
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	"  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
 	"  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
 #endif
 #ifdef CONFIG_TRACER_SNAPSHOT
 	"\n  snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
 	"\t\t\t  Read the contents for more information\n"
 #endif
 #ifdef CONFIG_STACK_TRACER
 	"  stack_trace\t\t- Shows the max stack trace when active\n"
 	"  stack_max_size\t- Shows current max stack size that was traced\n"
 	"\t\t\t  Write into this file to reset the max size (trigger a new trace)\n"
 #ifdef CONFIG_DYNAMIC_FTRACE
 	"  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
 #endif
 #endif /* CONFIG_STACK_TRACER */
 ;
 static ssize_t
 tracing_readme_read(struct file *filp, char __user *ubuf,
 		       size_t cnt, loff_t *ppos)
 {
 	return simple_read_from_buffer(ubuf, cnt, ppos,
 					readme_msg, strlen(readme_msg));
 }
 static const struct file_operations tracing_readme_fops = {
 	.open		= tracing_open_generic,
 	.read		= tracing_readme_read,
 	.llseek		= generic_file_llseek,
 };
 static ssize_t
 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
 				size_t cnt, loff_t *ppos)
 {
 	char *buf_comm;
 	char *file_buf;
 	char *buf;
 	int len = 0;
 	int pid;
 	int i;
 	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
 	if (!file_buf)
 		return -ENOMEM;
 	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
 	if (!buf_comm) {
 		kfree(file_buf);
 		return -ENOMEM;
 	}
 	buf = file_buf;
 	for (i = 0; i < SAVED_CMDLINES; i++) {
 		int r;
 		pid = map_cmdline_to_pid[i];
 		if (pid == -1 || pid == NO_CMDLINE_MAP)
 			continue;
 		trace_find_cmdline(pid, buf_comm);
 		r = sprintf(buf, "%d %s\n", pid, buf_comm);
 		buf += r;
 		len += r;
 	}
 	len = simple_read_from_buffer(ubuf, cnt, ppos,
 				      file_buf, len);
 	kfree(file_buf);
 	kfree(buf_comm);
 	return len;
 }
 static const struct file_operations tracing_saved_cmdlines_fops = {
     .open       = tracing_open_generic,
     .read       = tracing_saved_cmdlines_read,
     .llseek	= generic_file_llseek,
 };
 static ssize_t
 tracing_set_trace_read(struct file *filp, char __user *ubuf,
 		       size_t cnt, loff_t *ppos)
 {
 	struct trace_array *tr = filp->private_data;
 	char buf[MAX_TRACER_SIZE+2];
 	int r;
 	mutex_lock(&trace_types_lock);
 	r = sprintf(buf, "%s\n", tr->current_trace->name);
 	mutex_unlock(&trace_types_lock);
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 }
 int tracer_init(struct tracer *t, struct trace_array *tr)
 {
 	tracing_reset_online_cpus(&tr->trace_buffer);
 	return t->init(tr);
 }
 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
 {
 	int cpu;
 	for_each_tracing_cpu(cpu)
 		per_cpu_ptr(buf->data, cpu)->entries = val;
 }
 #ifdef CONFIG_TRACER_MAX_TRACE
 /* resize @tr's buffer to the size of @size_tr's entries */
 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
 					struct trace_buffer *size_buf, int cpu_id)
 {
 	int cpu, ret = 0;
 	if (cpu_id == RING_BUFFER_ALL_CPUS) {
 		for_each_tracing_cpu(cpu) {
 			ret = ring_buffer_resize(trace_buf->buffer,
 				 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
 			if (ret < 0)
 				break;
 			per_cpu_ptr(trace_buf->data, cpu)->entries =
 				per_cpu_ptr(size_buf->data, cpu)->entries;
 		}
 	} else {
 		ret = ring_buffer_resize(trace_buf->buffer,
 				 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
 		if (ret == 0)
 			per_cpu_ptr(trace_buf->data, cpu_id)->entries =
 				per_cpu_ptr(size_buf->data, cpu_id)->entries;
 	}
 	return ret;
 }
 #endif /* CONFIG_TRACER_MAX_TRACE */
 static int __tracing_resize_ring_buffer(struct trace_array *tr,
 					unsigned long size, int cpu)
 {
 	int ret;
 	/*
 	 * If kernel or user changes the size of the ring buffer
 	 * we use the size that was given, and we can forget about
 	 * expanding it later.
 	 */
 	ring_buffer_expanded = true;
 	/* May be called before buffers are initialized */
 	if (!tr->trace_buffer.buffer)
 		return 0;
 	ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
 	if (ret < 0)
 		return ret;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
 	    !tr->current_trace->use_max_tr)
 		goto out;
 	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
 	if (ret < 0) {
 		int r = resize_buffer_duplicate_size(&tr->trace_buffer,
 						     &tr->trace_buffer, cpu);
 		if (r < 0) {
 			/*
 			 * AARGH! We are left with different
 			 * size max buffer!!!!
 			 * The max buffer is our "snapshot" buffer.
 			 * When a tracer needs a snapshot (one of the
 			 * latency tracers), it swaps the max buffer
 			 * with the saved snap shot. We succeeded to
 			 * update the size of the main buffer, but failed to
 			 * update the size of the max buffer. But when we tried
 			 * to reset the main buffer to the original size, we
 			 * failed there too. This is very unlikely to
 			 * happen, but if it does, warn and kill all
 			 * tracing.
 			 */
 			WARN_ON(1);
 			tracing_disabled = 1;
 		}
 		return ret;
 	}
 	if (cpu == RING_BUFFER_ALL_CPUS)
 		set_buffer_entries(&tr->max_buffer, size);
 	else
 		per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
  out:
 #endif /* CONFIG_TRACER_MAX_TRACE */
 	if (cpu == RING_BUFFER_ALL_CPUS)
 		set_buffer_entries(&tr->trace_buffer, size);
 	else
 		per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
 	return ret;
 }
 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
 					  unsigned long size, int cpu_id)
 {
 	int ret = size;
 	mutex_lock(&trace_types_lock);
 	if (cpu_id != RING_BUFFER_ALL_CPUS) {
 		/* make sure, this cpu is enabled in the mask */
 		if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
 			ret = -EINVAL;
 			goto out;
 		}
 	}
 	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
 	if (ret < 0)
 		ret = -ENOMEM;
 out:
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 /**
  * tracing_update_buffers - used by tracing facility to expand ring buffers
  *
  * To save on memory when the tracing is never used on a system with it
  * configured in. The ring buffers are set to a minimum size. But once
  * a user starts to use the tracing facility, then they need to grow
  * to their default size.
  *
  * This function is to be called when a tracer is about to be used.
  */
 int tracing_update_buffers(void)
 {
 	int ret = 0;
 	mutex_lock(&trace_types_lock);
 	if (!ring_buffer_expanded)
 		ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
 						RING_BUFFER_ALL_CPUS);
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 struct trace_option_dentry;
 static struct trace_option_dentry *
 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
 static void
 destroy_trace_option_files(struct trace_option_dentry *topts);
 static int tracing_set_tracer(const char *buf)
 {
 	static struct trace_option_dentry *topts;
 	struct trace_array *tr = &global_trace;
 	struct tracer *t;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	bool had_max_tr;
 #endif
 	int ret = 0;
 	mutex_lock(&trace_types_lock);
 	if (!ring_buffer_expanded) {
 		ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
 						RING_BUFFER_ALL_CPUS);
 		if (ret < 0)
 			goto out;
 		ret = 0;
 	}
 	for (t = trace_types; t; t = t->next) {
 		if (strcmp(t->name, buf) == 0)
 			break;
 	}
 	if (!t) {
 		ret = -EINVAL;
 		goto out;
 	}
 	if (t == tr->current_trace)
 		goto out;
 	trace_branch_disable();
 	tr->current_trace->enabled = false;
 	if (tr->current_trace->reset)
 		tr->current_trace->reset(tr);
 	/* Current trace needs to be nop_trace before synchronize_sched */
 	tr->current_trace = &nop_trace;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	had_max_tr = tr->allocated_snapshot;
 	if (had_max_tr && !t->use_max_tr) {
 		/*
 		 * We need to make sure that the update_max_tr sees that
 		 * current_trace changed to nop_trace to keep it from
 		 * swapping the buffers after we resize it.
 		 * The update_max_tr is called from interrupts disabled
 		 * so a synchronized_sched() is sufficient.
 		 */
 		synchronize_sched();
 		free_snapshot(tr);
 	}
 #endif
 	destroy_trace_option_files(topts);
 	topts = create_trace_option_files(tr, t);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (t->use_max_tr && !had_max_tr) {
 		ret = alloc_snapshot(tr);
 		if (ret < 0)
 			goto out;
 	}
 #endif
 	if (t->init) {
 		ret = tracer_init(t, tr);
 		if (ret)
 			goto out;
 	}
 	tr->current_trace = t;
 	tr->current_trace->enabled = true;
 	trace_branch_enable(tr);
  out:
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static ssize_t
 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
 			size_t cnt, loff_t *ppos)
 {
 	char buf[MAX_TRACER_SIZE+1];
 	int i;
 	size_t ret;
 	int err;
 	ret = cnt;
 	if (cnt > MAX_TRACER_SIZE)
 		cnt = MAX_TRACER_SIZE;
 	if (copy_from_user(&buf, ubuf, cnt))
 		return -EFAULT;
 	buf[cnt] = 0;
 	/* strip ending whitespace. */
 	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
 		buf[i] = 0;
 	err = tracing_set_tracer(buf);
 	if (err)
 		return err;
 	*ppos += ret;
 	return ret;
 }
 static ssize_t
 tracing_max_lat_read(struct file *filp, char __user *ubuf,
 		     size_t cnt, loff_t *ppos)
 {
 	unsigned long *ptr = filp->private_data;
 	char buf[64];
 	int r;
 	r = snprintf(buf, sizeof(buf), "%ld\n",
 		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
 	if (r > sizeof(buf))
 		r = sizeof(buf);
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 }
 static ssize_t
 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
 		      size_t cnt, loff_t *ppos)
 {
 	unsigned long *ptr = filp->private_data;
 	unsigned long val;
 	int ret;
 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 	if (ret)
 		return ret;
 	*ptr = val * 1000;
 	return cnt;
 }
 static int tracing_open_pipe(struct inode *inode, struct file *filp)
 {
 	struct trace_array *tr = inode->i_private;
 	struct trace_iterator *iter;
 	int ret = 0;
 	if (tracing_disabled)
 		return -ENODEV;
 	if (trace_array_get(tr) < 0)
 		return -ENODEV;
 	mutex_lock(&trace_types_lock);
 	/* create a buffer to store the information to pass to userspace */
 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 	if (!iter) {
 		ret = -ENOMEM;
 		__trace_array_put(tr);
 		goto out;
 	}
 	/*
 	 * We make a copy of the current tracer to avoid concurrent
 	 * changes on it while we are reading.
 	 */
 	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
 	if (!iter->trace) {
 		ret = -ENOMEM;
 		goto fail;
 	}
 	*iter->trace = *tr->current_trace;
 	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
 		ret = -ENOMEM;
 		goto fail;
 	}
 	/* trace pipe does not show start of buffer */
 	cpumask_setall(iter->started);
 	if (trace_flags & TRACE_ITER_LATENCY_FMT)
 		iter->iter_flags |= TRACE_FILE_LAT_FMT;
 	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
 	if (trace_clocks[tr->clock_id].in_ns)
 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
 	iter->tr = tr;
 	iter->trace_buffer = &tr->trace_buffer;
 	iter->cpu_file = tracing_get_cpu(inode);
 	mutex_init(&iter->mutex);
 	filp->private_data = iter;
 	if (iter->trace->pipe_open)
 		iter->trace->pipe_open(iter);
 	nonseekable_open(inode, filp);
 out:
 	mutex_unlock(&trace_types_lock);
 	return ret;
 fail:
 	kfree(iter->trace);
 	kfree(iter);
 	__trace_array_put(tr);
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static int tracing_release_pipe(struct inode *inode, struct file *file)
 {
 	struct trace_iterator *iter = file->private_data;
 	struct trace_array *tr = inode->i_private;
 	mutex_lock(&trace_types_lock);
 	if (iter->trace->pipe_close)
 		iter->trace->pipe_close(iter);
 	mutex_unlock(&trace_types_lock);
 	free_cpumask_var(iter->started);
 	mutex_destroy(&iter->mutex);
 	kfree(iter->trace);
 	kfree(iter);
 	trace_array_put(tr);
 	return 0;
 }
 static unsigned int
 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
 {
 	/* Iterators are static, they should be filled or empty */
 	if (trace_buffer_iter(iter, iter->cpu_file))
 		return POLLIN | POLLRDNORM;
 	if (trace_flags & TRACE_ITER_BLOCK)
 		/*
 		 * Always select as readable when in blocking mode
 		 */
 		return POLLIN | POLLRDNORM;
 	else
 		return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
 					     filp, poll_table);
 }
 static unsigned int
 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
 {
 	struct trace_iterator *iter = filp->private_data;
 	return trace_poll(iter, filp, poll_table);
 }
 /*
  * This is a make-shift waitqueue.
  * A tracer might use this callback on some rare cases:
  *
  *  1) the current tracer might hold the runqueue lock when it wakes up
  *     a reader, hence a deadlock (sched, function, and function graph tracers)
  *  2) the function tracers, trace all functions, we don't want
  *     the overhead of calling wake_up and friends
  *     (and tracing them too)
  *
  *     Anyway, this is really very primitive wakeup.
  */
 void poll_wait_pipe(struct trace_iterator *iter)
 {
 	set_current_state(TASK_INTERRUPTIBLE);
 	/* sleep for 100 msecs, and try again. */
 	schedule_timeout(HZ / 10);
 }
 /* Must be called with trace_types_lock mutex held. */
 static int tracing_wait_pipe(struct file *filp)
 {
 	struct trace_iterator *iter = filp->private_data;
 	while (trace_empty(iter)) {
 		if ((filp->f_flags & O_NONBLOCK)) {
 			return -EAGAIN;
 		}
 		mutex_unlock(&iter->mutex);
 		iter->trace->wait_pipe(iter);
 		mutex_lock(&iter->mutex);
 		if (signal_pending(current))
 			return -EINTR;
 		/*
 		 * We block until we read something and tracing is disabled.
 		 * We still block if tracing is disabled, but we have never
 		 * read anything. This allows a user to cat this file, and
 		 * then enable tracing. But after we have read something,
 		 * we give an EOF when tracing is again disabled.
 		 *
 		 * iter->pos will be 0 if we haven't read anything.
 		 */
 		if (!tracing_is_on() && iter->pos)
 			break;
 	}
 	return 1;
 }
 /*
  * Consumer reader.
  */
 static ssize_t
 tracing_read_pipe(struct file *filp, char __user *ubuf,
 		  size_t cnt, loff_t *ppos)
 {
 	struct trace_iterator *iter = filp->private_data;
 	struct trace_array *tr = iter->tr;
 	ssize_t sret;
 	/* return any leftover data */
 	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
 	if (sret != -EBUSY)
 		return sret;
 	trace_seq_init(&iter->seq);
 	/* copy the tracer to avoid using a global lock all around */
 	mutex_lock(&trace_types_lock);
 	if (unlikely(iter->trace->name != tr->current_trace->name))
 		*iter->trace = *tr->current_trace;
 	mutex_unlock(&trace_types_lock);
 	/*
 	 * Avoid more than one consumer on a single file descriptor
 	 * This is just a matter of traces coherency, the ring buffer itself
 	 * is protected.
 	 */
 	mutex_lock(&iter->mutex);
 	if (iter->trace->read) {
 		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
 		if (sret)
 			goto out;
 	}
 waitagain:
 	sret = tracing_wait_pipe(filp);
 	if (sret <= 0)
 		goto out;
 	/* stop when tracing is finished */
 	if (trace_empty(iter)) {
 		sret = 0;
 		goto out;
 	}
 	if (cnt >= PAGE_SIZE)
 		cnt = PAGE_SIZE - 1;
 	/* reset all but tr, trace, and overruns */
 	memset(&iter->seq, 0,
 	       sizeof(struct trace_iterator) -
 	       offsetof(struct trace_iterator, seq));
 	cpumask_clear(iter->started);
 	iter->pos = -1;
 	trace_event_read_lock();
 	trace_access_lock(iter->cpu_file);
 	while (trace_find_next_entry_inc(iter) != NULL) {
 		enum print_line_t ret;
 		int len = iter->seq.len;
 		ret = print_trace_line(iter);
 		if (ret == TRACE_TYPE_PARTIAL_LINE) {
 			/* don't print partial lines */
 			iter->seq.len = len;
 			break;
 		}
 		if (ret != TRACE_TYPE_NO_CONSUME)
 			trace_consume(iter);
 		if (iter->seq.len >= cnt)
 			break;
 		/*
 		 * Setting the full flag means we reached the trace_seq buffer
 		 * size and we should leave by partial output condition above.
 		 * One of the trace_seq_* functions is not used properly.
 		 */
 		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
 			  iter->ent->type);
 	}
 	trace_access_unlock(iter->cpu_file);
 	trace_event_read_unlock();
 	/* Now copy what we have to the user */
 	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
 	if (iter->seq.readpos >= iter->seq.len)
 		trace_seq_init(&iter->seq);
 	/*
 	 * If there was nothing to send to user, in spite of consuming trace
 	 * entries, go back to wait for more entries.
 	 */
 	if (sret == -EBUSY)
 		goto waitagain;
 out:
 	mutex_unlock(&iter->mutex);
 	return sret;
 }
 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
 				     struct pipe_buffer *buf)
 {
 	__free_page(buf->page);
 }
 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
 				     unsigned int idx)
 {
 	__free_page(spd->pages[idx]);
 }
 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
 	.can_merge		= 0,
 	.map			= generic_pipe_buf_map,
 	.unmap			= generic_pipe_buf_unmap,
 	.confirm		= generic_pipe_buf_confirm,
 	.release		= tracing_pipe_buf_release,
 	.steal			= generic_pipe_buf_steal,
 	.get			= generic_pipe_buf_get,
 };
 static size_t
 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
 {
 	size_t count;
 	int ret;
 	/* Seq buffer is page-sized, exactly what we need. */
 	for (;;) {
 		count = iter->seq.len;
 		ret = print_trace_line(iter);
 		count = iter->seq.len - count;
 		if (rem < count) {
 			rem = 0;
 			iter->seq.len -= count;
 			break;
 		}
 		if (ret == TRACE_TYPE_PARTIAL_LINE) {
 			iter->seq.len -= count;
 			break;
 		}
 		if (ret != TRACE_TYPE_NO_CONSUME)
 			trace_consume(iter);
 		rem -= count;
 		if (!trace_find_next_entry_inc(iter))	{
 			rem = 0;
 			iter->ent = NULL;
 			break;
 		}
 	}
 	return rem;
 }
 static ssize_t tracing_splice_read_pipe(struct file *filp,
 					loff_t *ppos,
 					struct pipe_inode_info *pipe,
 					size_t len,
 					unsigned int flags)
 {
 	struct page *pages_def[PIPE_DEF_BUFFERS];
 	struct partial_page partial_def[PIPE_DEF_BUFFERS];
 	struct trace_iterator *iter = filp->private_data;
 	struct splice_pipe_desc spd = {
 		.pages		= pages_def,
 		.partial	= partial_def,
 		.nr_pages	= 0, /* This gets updated below. */
 		.nr_pages_max	= PIPE_DEF_BUFFERS,
 		.flags		= flags,
 		.ops		= &tracing_pipe_buf_ops,
 		.spd_release	= tracing_spd_release_pipe,
 	};
 	struct trace_array *tr = iter->tr;
 	ssize_t ret;
 	size_t rem;
 	unsigned int i;
 	if (splice_grow_spd(pipe, &spd))
 		return -ENOMEM;
 	/* copy the tracer to avoid using a global lock all around */
 	mutex_lock(&trace_types_lock);
 	if (unlikely(iter->trace->name != tr->current_trace->name))
 		*iter->trace = *tr->current_trace;
 	mutex_unlock(&trace_types_lock);
 	mutex_lock(&iter->mutex);
 	if (iter->trace->splice_read) {
 		ret = iter->trace->splice_read(iter, filp,
 					       ppos, pipe, len, flags);
 		if (ret)
 			goto out_err;
 	}
 	ret = tracing_wait_pipe(filp);
 	if (ret <= 0)
 		goto out_err;
 	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
 		ret = -EFAULT;
 		goto out_err;
 	}
 	trace_event_read_lock();
 	trace_access_lock(iter->cpu_file);
 	/* Fill as many pages as possible. */
 	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
 		spd.pages[i] = alloc_page(GFP_KERNEL);
 		if (!spd.pages[i])
 			break;
 		rem = tracing_fill_pipe_page(rem, iter);
 		/* Copy the data into the page, so we can start over. */
 		ret = trace_seq_to_buffer(&iter->seq,
 					  page_address(spd.pages[i]),
 					  iter->seq.len);
 		if (ret < 0) {
 			__free_page(spd.pages[i]);
 			break;
 		}
 		spd.partial[i].offset = 0;
 		spd.partial[i].len = iter->seq.len;
 		trace_seq_init(&iter->seq);
 	}
 	trace_access_unlock(iter->cpu_file);
 	trace_event_read_unlock();
 	mutex_unlock(&iter->mutex);
 	spd.nr_pages = i;
 	ret = splice_to_pipe(pipe, &spd);
 out:
 	splice_shrink_spd(&spd);
 	return ret;
 out_err:
 	mutex_unlock(&iter->mutex);
 	goto out;
 }
 static ssize_t
 tracing_entries_read(struct file *filp, char __user *ubuf,
 		     size_t cnt, loff_t *ppos)
 {
 	struct inode *inode = file_inode(filp);
 	struct trace_array *tr = inode->i_private;
 	int cpu = tracing_get_cpu(inode);
 	char buf[64];
 	int r = 0;
 	ssize_t ret;
 	mutex_lock(&trace_types_lock);
 	if (cpu == RING_BUFFER_ALL_CPUS) {
 		int cpu, buf_size_same;
 		unsigned long size;
 		size = 0;
 		buf_size_same = 1;
 		/* check if all cpu sizes are same */
 		for_each_tracing_cpu(cpu) {
 			/* fill in the size from first enabled cpu */
 			if (size == 0)
 				size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
 			if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
 				buf_size_same = 0;
 				break;
 			}
 		}
 		if (buf_size_same) {
 			if (!ring_buffer_expanded)
 				r = sprintf(buf, "%lu (expanded: %lu)\n",
 					    size >> 10,
 					    trace_buf_size >> 10);
 			else
 				r = sprintf(buf, "%lu\n", size >> 10);
 		} else
 			r = sprintf(buf, "X\n");
 	} else
 		r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
 	mutex_unlock(&trace_types_lock);
 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 	return ret;
 }
 static ssize_t
 tracing_entries_write(struct file *filp, const char __user *ubuf,
 		      size_t cnt, loff_t *ppos)
 {
 	struct inode *inode = file_inode(filp);
 	struct trace_array *tr = inode->i_private;
 	unsigned long val;
 	int ret;
 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 	if (ret)
 		return ret;
 	/* must have at least 1 entry */
 	if (!val)
 		return -EINVAL;
 	/* value is in KB */
 	val <<= 10;
 	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
 	if (ret < 0)
 		return ret;
 	*ppos += cnt;
 	return cnt;
 }
 static ssize_t
 tracing_total_entries_read(struct file *filp, char __user *ubuf,
 				size_t cnt, loff_t *ppos)
 {
 	struct trace_array *tr = filp->private_data;
 	char buf[64];
 	int r, cpu;
 	unsigned long size = 0, expanded_size = 0;
 	mutex_lock(&trace_types_lock);
 	for_each_tracing_cpu(cpu) {
 		size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
 		if (!ring_buffer_expanded)
 			expanded_size += trace_buf_size >> 10;
 	}
 	if (ring_buffer_expanded)
 		r = sprintf(buf, "%lu\n", size);
 	else
 		r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
 	mutex_unlock(&trace_types_lock);
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 }
 static ssize_t
 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
 			  size_t cnt, loff_t *ppos)
 {
 	/*
 	 * There is no need to read what the user has written, this function
 	 * is just to make sure that there is no error when "echo" is used
 	 */
 	*ppos += cnt;
 	return cnt;
 }
 static int
 tracing_free_buffer_release(struct inode *inode, struct file *filp)
 {
 	struct trace_array *tr = inode->i_private;
 	/* disable tracing ? */
 	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
 		tracer_tracing_off(tr);
 	/* resize the ring buffer to 0 */
 	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
 	trace_array_put(tr);
 	return 0;
 }
 static ssize_t
 tracing_mark_write(struct file *filp, const char __user *ubuf,
 					size_t cnt, loff_t *fpos)
 {
 	unsigned long addr = (unsigned long)ubuf;
 	struct trace_array *tr = filp->private_data;
 	struct ring_buffer_event *event;
 	struct ring_buffer *buffer;
 	struct print_entry *entry;
 	unsigned long irq_flags;
 	struct page *pages[2];
 	void *map_page[2];
 	int nr_pages = 1;
 	ssize_t written;
 	int offset;
 	int size;
 	int len;
 	int ret;
 	int i;
 	if (tracing_disabled)
 		return -EINVAL;
 	if (!(trace_flags & TRACE_ITER_MARKERS))
 		return -EINVAL;
 	if (cnt > TRACE_BUF_SIZE)
 		cnt = TRACE_BUF_SIZE;
 	/*
 	 * Userspace is injecting traces into the kernel trace buffer.
 	 * We want to be as non intrusive as possible.
 	 * To do so, we do not want to allocate any special buffers
 	 * or take any locks, but instead write the userspace data
 	 * straight into the ring buffer.
 	 *
 	 * First we need to pin the userspace buffer into memory,
 	 * which, most likely it is, because it just referenced it.
 	 * But there's no guarantee that it is. By using get_user_pages_fast()
 	 * and kmap_atomic/kunmap_atomic() we can get access to the
 	 * pages directly. We then write the data directly into the
 	 * ring buffer.
 	 */
 	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
 	/* check if we cross pages */
 	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
 		nr_pages = 2;
 	offset = addr & (PAGE_SIZE - 1);
 	addr &= PAGE_MASK;
 	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
 	if (ret < nr_pages) {
 		while (--ret >= 0)
 			put_page(pages[ret]);
 		written = -EFAULT;
 		goto out;
 	}
 	for (i = 0; i < nr_pages; i++)
 		map_page[i] = kmap_atomic(pages[i]);
 	local_save_flags(irq_flags);
 	size = sizeof(*entry) + cnt + 2; /* possible \n added */
 	buffer = tr->trace_buffer.buffer;
 	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
 					  irq_flags, preempt_count());
 	if (!event) {
 		/* Ring buffer disabled, return as if not open for write */
 		written = -EBADF;
 		goto out_unlock;
 	}
 	entry = ring_buffer_event_data(event);
 	entry->ip = _THIS_IP_;
 	if (nr_pages == 2) {
 		len = PAGE_SIZE - offset;
 		memcpy(&entry->buf, map_page[0] + offset, len);
 		memcpy(&entry->buf[len], map_page[1], cnt - len);
 	} else
 		memcpy(&entry->buf, map_page[0] + offset, cnt);
 	if (entry->buf[cnt - 1] != '\n') {
 		entry->buf[cnt] = '\n';
 		entry->buf[cnt + 1] = '\0';
 	} else
 		entry->buf[cnt] = '\0';
 	__buffer_unlock_commit(buffer, event);
 	written = cnt;
 	*fpos += written;
  out_unlock:
 	for (i = 0; i < nr_pages; i++){
 		kunmap_atomic(map_page[i]);
 		put_page(pages[i]);
 	}
  out:
 	return written;
 }
 static int tracing_clock_show(struct seq_file *m, void *v)
 {
 	struct trace_array *tr = m->private;
 	int i;
 	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
 		seq_printf(m,
 			"%s%s%s%s", i ? " " : "",
 			i == tr->clock_id ? "[" : "", trace_clocks[i].name,
 			i == tr->clock_id ? "]" : "");
 	seq_putc(m, '\n');
 	return 0;
 }
 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
 				   size_t cnt, loff_t *fpos)
 {
 	struct seq_file *m = filp->private_data;
 	struct trace_array *tr = m->private;
 	char buf[64];
 	const char *clockstr;
 	int i;
 	if (cnt >= sizeof(buf))
 		return -EINVAL;
 	if (copy_from_user(&buf, ubuf, cnt))
 		return -EFAULT;
 	buf[cnt] = 0;
 	clockstr = strstrip(buf);
 	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
 		if (strcmp(trace_clocks[i].name, clockstr) == 0)
 			break;
 	}
 	if (i == ARRAY_SIZE(trace_clocks))
 		return -EINVAL;
 	mutex_lock(&trace_types_lock);
 	tr->clock_id = i;
 	ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
 	/*
 	 * New clock may not be consistent with the previous clock.
 	 * Reset the buffer so that it doesn't have incomparable timestamps.
 	 */
 	tracing_reset_online_cpus(&tr->trace_buffer);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
 		ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
 	tracing_reset_online_cpus(&tr->max_buffer);
 #endif
 	mutex_unlock(&trace_types_lock);
 	*fpos += cnt;
 	return cnt;
 }
 static int tracing_clock_open(struct inode *inode, struct file *file)
 {
 	struct trace_array *tr = inode->i_private;
 	int ret;
 	if (tracing_disabled)
 		return -ENODEV;
 	if (trace_array_get(tr))
 		return -ENODEV;
 	ret = single_open(file, tracing_clock_show, inode->i_private);
 	if (ret < 0)
 		trace_array_put(tr);
 	return ret;
 }
 struct ftrace_buffer_info {
 	struct trace_iterator	iter;
 	void			*spare;
 	unsigned int		read;
 };
 #ifdef CONFIG_TRACER_SNAPSHOT
 static int tracing_snapshot_open(struct inode *inode, struct file *file)
 {
 	struct trace_array *tr = inode->i_private;
 	struct trace_iterator *iter;
 	struct seq_file *m;
 	int ret = 0;
 	if (trace_array_get(tr) < 0)
 		return -ENODEV;
 	if (file->f_mode & FMODE_READ) {
 		iter = __tracing_open(inode, file, true);
 		if (IS_ERR(iter))
 			ret = PTR_ERR(iter);
 	} else {
 		/* Writes still need the seq_file to hold the private data */
 		ret = -ENOMEM;
 		m = kzalloc(sizeof(*m), GFP_KERNEL);
 		if (!m)
 			goto out;
 		iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 		if (!iter) {
 			kfree(m);
 			goto out;
 		}
 		ret = 0;
 		iter->tr = tr;
 		iter->trace_buffer = &tr->max_buffer;
 		iter->cpu_file = tracing_get_cpu(inode);
 		m->private = iter;
 		file->private_data = m;
 	}
 out:
 	if (ret < 0)
 		trace_array_put(tr);
 	return ret;
 }
 static ssize_t
 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
 		       loff_t *ppos)
 {
 	struct seq_file *m = filp->private_data;
 	struct trace_iterator *iter = m->private;
 	struct trace_array *tr = iter->tr;
 	unsigned long val;
 	int ret;
 	ret = tracing_update_buffers();
 	if (ret < 0)
 		return ret;
 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 	if (ret)
 		return ret;
 	mutex_lock(&trace_types_lock);
 	if (tr->current_trace->use_max_tr) {
 		ret = -EBUSY;
 		goto out;
 	}
 	switch (val) {
 	case 0:
 		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
 			ret = -EINVAL;
 			break;
 		}
 		if (tr->allocated_snapshot)
 			free_snapshot(tr);
 		break;
 	case 1:
 /* Only allow per-cpu swap if the ring buffer supports it */
 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
 		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
 			ret = -EINVAL;
 			break;
 		}
 #endif
 		if (!tr->allocated_snapshot) {
 			ret = alloc_snapshot(tr);
 			if (ret < 0)
 				break;
 		}
 		local_irq_disable();
 		/* Now, we're going to swap */
 		if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
 			update_max_tr(tr, current, smp_processor_id());
 		else
 			update_max_tr_single(tr, current, iter->cpu_file);
 		local_irq_enable();
 		break;
 	default:
 		if (tr->allocated_snapshot) {
 			if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
 				tracing_reset_online_cpus(&tr->max_buffer);
 			else
 				tracing_reset(&tr->max_buffer, iter->cpu_file);
 		}
 		break;
 	}
 	if (ret >= 0) {
 		*ppos += cnt;
 		ret = cnt;
 	}
 out:
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static int tracing_snapshot_release(struct inode *inode, struct file *file)
 {
 	struct seq_file *m = file->private_data;
 	int ret;
 	ret = tracing_release(inode, file);
 	if (file->f_mode & FMODE_READ)
 		return ret;
 	/* If write only, the seq_file is just a stub */
 	if (m)
 		kfree(m->private);
 	kfree(m);
 	return 0;
 }
 static int tracing_buffers_open(struct inode *inode, struct file *filp);
 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
 				    size_t count, loff_t *ppos);
 static int tracing_buffers_release(struct inode *inode, struct file *file);
 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
 		   struct pipe_inode_info *pipe, size_t len, unsigned int flags);
 static int snapshot_raw_open(struct inode *inode, struct file *filp)
 {
 	struct ftrace_buffer_info *info;
 	int ret;
 	ret = tracing_buffers_open(inode, filp);
 	if (ret < 0)
 		return ret;
 	info = filp->private_data;
 	if (info->iter.trace->use_max_tr) {
 		tracing_buffers_release(inode, filp);
 		return -EBUSY;
 	}
 	info->iter.snapshot = true;
 	info->iter.trace_buffer = &info->iter.tr->max_buffer;
 	return ret;
 }
 #endif /* CONFIG_TRACER_SNAPSHOT */
 static const struct file_operations tracing_max_lat_fops = {
 	.open		= tracing_open_generic,
 	.read		= tracing_max_lat_read,
 	.write		= tracing_max_lat_write,
 	.llseek		= generic_file_llseek,
 };
 static const struct file_operations set_tracer_fops = {
 	.open		= tracing_open_generic,
 	.read		= tracing_set_trace_read,
 	.write		= tracing_set_trace_write,
 	.llseek		= generic_file_llseek,
 };
 static const struct file_operations tracing_pipe_fops = {
 	.open		= tracing_open_pipe,
 	.poll		= tracing_poll_pipe,
 	.read		= tracing_read_pipe,
 	.splice_read	= tracing_splice_read_pipe,
 	.release	= tracing_release_pipe,
 	.llseek		= no_llseek,
 };
 static const struct file_operations tracing_entries_fops = {
 	.open		= tracing_open_generic_tr,
 	.read		= tracing_entries_read,
 	.write		= tracing_entries_write,
 	.llseek		= generic_file_llseek,
 	.release	= tracing_release_generic_tr,
 };
 static const struct file_operations tracing_total_entries_fops = {
 	.open		= tracing_open_generic_tr,
 	.read		= tracing_total_entries_read,
 	.llseek		= generic_file_llseek,
 	.release	= tracing_release_generic_tr,
 };
 static const struct file_operations tracing_free_buffer_fops = {
 	.open		= tracing_open_generic_tr,
 	.write		= tracing_free_buffer_write,
 	.release	= tracing_free_buffer_release,
 };
 static const struct file_operations tracing_mark_fops = {
 	.open		= tracing_open_generic_tr,
 	.write		= tracing_mark_write,
 	.llseek		= generic_file_llseek,
 	.release	= tracing_release_generic_tr,
 };
 static const struct file_operations trace_clock_fops = {
 	.open		= tracing_clock_open,
 	.read		= seq_read,
 	.llseek		= seq_lseek,
 	.release	= tracing_single_release_tr,
 	.write		= tracing_clock_write,
 };
 #ifdef CONFIG_TRACER_SNAPSHOT
 static const struct file_operations snapshot_fops = {
 	.open		= tracing_snapshot_open,
 	.read		= seq_read,
 	.write		= tracing_snapshot_write,
 	.llseek		= tracing_seek,
 	.release	= tracing_snapshot_release,
 };
 static const struct file_operations snapshot_raw_fops = {
 	.open		= snapshot_raw_open,
 	.read		= tracing_buffers_read,
 	.release	= tracing_buffers_release,
 	.splice_read	= tracing_buffers_splice_read,
 	.llseek		= no_llseek,
 };
 #endif /* CONFIG_TRACER_SNAPSHOT */
 static int tracing_buffers_open(struct inode *inode, struct file *filp)
 {
 	struct trace_array *tr = inode->i_private;
 	struct ftrace_buffer_info *info;
 	int ret;
 	if (tracing_disabled)
 		return -ENODEV;
 	if (trace_array_get(tr) < 0)
 		return -ENODEV;
 	info = kzalloc(sizeof(*info), GFP_KERNEL);
 	if (!info) {
 		trace_array_put(tr);
 		return -ENOMEM;
 	}
 	mutex_lock(&trace_types_lock);
 	info->iter.tr		= tr;
 	info->iter.cpu_file	= tracing_get_cpu(inode);
 	info->iter.trace	= tr->current_trace;
 	info->iter.trace_buffer = &tr->trace_buffer;
 	info->spare		= NULL;
 	/* Force reading ring buffer for first read */
 	info->read		= (unsigned int)-1;
 	filp->private_data = info;
 	mutex_unlock(&trace_types_lock);
 	ret = nonseekable_open(inode, filp);
 	if (ret < 0)
 		trace_array_put(tr);
 	return ret;
 }
 static unsigned int
 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
 {
 	struct ftrace_buffer_info *info = filp->private_data;
 	struct trace_iterator *iter = &info->iter;
 	return trace_poll(iter, filp, poll_table);
 }
 static ssize_t
 tracing_buffers_read(struct file *filp, char __user *ubuf,
 		     size_t count, loff_t *ppos)
 {
 	struct ftrace_buffer_info *info = filp->private_data;
 	struct trace_iterator *iter = &info->iter;
 	ssize_t ret;
 	ssize_t size;
 	if (!count)
 		return 0;
 	mutex_lock(&trace_types_lock);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
 		size = -EBUSY;
 		goto out_unlock;
 	}
 #endif
 	if (!info->spare)
 		info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
 							  iter->cpu_file);
 	size = -ENOMEM;
 	if (!info->spare)
 		goto out_unlock;
 	/* Do we have previous read data to read? */
 	if (info->read < PAGE_SIZE)
 		goto read;
  again:
 	trace_access_lock(iter->cpu_file);
 	ret = ring_buffer_read_page(iter->trace_buffer->buffer,
 				    &info->spare,
 				    count,
 				    iter->cpu_file, 0);
 	trace_access_unlock(iter->cpu_file);
 	if (ret < 0) {
 		if (trace_empty(iter)) {
 			if ((filp->f_flags & O_NONBLOCK)) {
 				size = -EAGAIN;
 				goto out_unlock;
 			}
 			mutex_unlock(&trace_types_lock);
 			iter->trace->wait_pipe(iter);
 			mutex_lock(&trace_types_lock);
 			if (signal_pending(current)) {
 				size = -EINTR;
 				goto out_unlock;
 			}
 			goto again;
 		}
 		size = 0;
 		goto out_unlock;
 	}
 	info->read = 0;
  read:
 	size = PAGE_SIZE - info->read;
 	if (size > count)
 		size = count;
 	ret = copy_to_user(ubuf, info->spare + info->read, size);
 	if (ret == size) {
 		size = -EFAULT;
 		goto out_unlock;
 	}
 	size -= ret;
 	*ppos += size;
 	info->read += size;
  out_unlock:
 	mutex_unlock(&trace_types_lock);
 	return size;
 }
 static int tracing_buffers_release(struct inode *inode, struct file *file)
 {
 	struct ftrace_buffer_info *info = file->private_data;
 	struct trace_iterator *iter = &info->iter;
 	mutex_lock(&trace_types_lock);
 	__trace_array_put(iter->tr);
 	if (info->spare)
 		ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
 	kfree(info);
 	mutex_unlock(&trace_types_lock);
 	return 0;
 }
 struct buffer_ref {
 	struct ring_buffer	*buffer;
 	void			*page;
 	int			ref;
 };
 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
 				    struct pipe_buffer *buf)
 {
 	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
 	if (--ref->ref)
 		return;
 	ring_buffer_free_read_page(ref->buffer, ref->page);
 	kfree(ref);
 	buf->private = 0;
 }
 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
 				struct pipe_buffer *buf)
 {
 	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
 	ref->ref++;
 }
 /* Pipe buffer operations for a buffer. */
 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
 	.can_merge		= 0,
 	.map			= generic_pipe_buf_map,
 	.unmap			= generic_pipe_buf_unmap,
 	.confirm		= generic_pipe_buf_confirm,
 	.release		= buffer_pipe_buf_release,
 	.steal			= generic_pipe_buf_steal,
 	.get			= buffer_pipe_buf_get,
 };
 /*
  * Callback from splice_to_pipe(), if we need to release some pages
  * at the end of the spd in case we error'ed out in filling the pipe.
  */
 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
 {
 	struct buffer_ref *ref =
 		(struct buffer_ref *)spd->partial[i].private;
 	if (--ref->ref)
 		return;
 	ring_buffer_free_read_page(ref->buffer, ref->page);
 	kfree(ref);
 	spd->partial[i].private = 0;
 }
 static ssize_t
 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
 			    struct pipe_inode_info *pipe, size_t len,
 			    unsigned int flags)
 {
 	struct ftrace_buffer_info *info = file->private_data;
 	struct trace_iterator *iter = &info->iter;
 	struct partial_page partial_def[PIPE_DEF_BUFFERS];
 	struct page *pages_def[PIPE_DEF_BUFFERS];
 	struct splice_pipe_desc spd = {
 		.pages		= pages_def,
 		.partial	= partial_def,
 		.nr_pages_max	= PIPE_DEF_BUFFERS,
 		.flags		= flags,
 		.ops		= &buffer_pipe_buf_ops,
 		.spd_release	= buffer_spd_release,
 	};
 	struct buffer_ref *ref;
 	int entries, size, i;
 	ssize_t ret;
 	mutex_lock(&trace_types_lock);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
 		ret = -EBUSY;
 		goto out;
 	}
 #endif
 	if (splice_grow_spd(pipe, &spd)) {
 		ret = -ENOMEM;
 		goto out;
 	}
 	if (*ppos & (PAGE_SIZE - 1)) {
 		ret = -EINVAL;
 		goto out;
 	}
 	if (len & (PAGE_SIZE - 1)) {
 		if (len < PAGE_SIZE) {
 			ret = -EINVAL;
 			goto out;
 		}
 		len &= PAGE_MASK;
 	}
  again:
 	trace_access_lock(iter->cpu_file);
 	entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
 	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
 		struct page *page;
 		int r;
 		ref = kzalloc(sizeof(*ref), GFP_KERNEL);
 		if (!ref)
 			break;
 		ref->ref = 1;
 		ref->buffer = iter->trace_buffer->buffer;
 		ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
 		if (!ref->page) {
 			kfree(ref);
 			break;
 		}
 		r = ring_buffer_read_page(ref->buffer, &ref->page,
 					  len, iter->cpu_file, 1);
 		if (r < 0) {
 			ring_buffer_free_read_page(ref->buffer, ref->page);
 			kfree(ref);
 			break;
 		}
 		/*
 		 * zero out any left over data, this is going to
 		 * user land.
 		 */
 		size = ring_buffer_page_len(ref->page);
 		if (size < PAGE_SIZE)
 			memset(ref->page + size, 0, PAGE_SIZE - size);
 		page = virt_to_page(ref->page);
 		spd.pages[i] = page;
 		spd.partial[i].len = PAGE_SIZE;
 		spd.partial[i].offset = 0;
 		spd.partial[i].private = (unsigned long)ref;
 		spd.nr_pages++;
 		*ppos += PAGE_SIZE;
 		entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
 	}
 	trace_access_unlock(iter->cpu_file);
 	spd.nr_pages = i;
 	/* did we read anything? */
 	if (!spd.nr_pages) {
 		if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
 			ret = -EAGAIN;
 			goto out;
 		}
 		mutex_unlock(&trace_types_lock);
 		iter->trace->wait_pipe(iter);
 		mutex_lock(&trace_types_lock);
 		if (signal_pending(current)) {
 			ret = -EINTR;
 			goto out;
 		}
 		goto again;
 	}
 	ret = splice_to_pipe(pipe, &spd);
 	splice_shrink_spd(&spd);
 out:
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static const struct file_operations tracing_buffers_fops = {
 	.open		= tracing_buffers_open,
 	.read		= tracing_buffers_read,
 	.poll		= tracing_buffers_poll,
 	.release	= tracing_buffers_release,
 	.splice_read	= tracing_buffers_splice_read,
 	.llseek		= no_llseek,
 };
 static ssize_t
 tracing_stats_read(struct file *filp, char __user *ubuf,
 		   size_t count, loff_t *ppos)
 {
 	struct inode *inode = file_inode(filp);
 	struct trace_array *tr = inode->i_private;
 	struct trace_buffer *trace_buf = &tr->trace_buffer;
 	int cpu = tracing_get_cpu(inode);
 	struct trace_seq *s;
 	unsigned long cnt;
 	unsigned long long t;
 	unsigned long usec_rem;
 	s = kmalloc(sizeof(*s), GFP_KERNEL);
 	if (!s)
 		return -ENOMEM;
 	trace_seq_init(s);
 	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
 	trace_seq_printf(s, "entries: %ld\n", cnt);
 	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
 	trace_seq_printf(s, "overrun: %ld\n", cnt);
 	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
 	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
 	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
 	trace_seq_printf(s, "bytes: %ld\n", cnt);
 	if (trace_clocks[tr->clock_id].in_ns) {
 		/* local or global for trace_clock */
 		t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
 		usec_rem = do_div(t, USEC_PER_SEC);
 		trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
 								t, usec_rem);
 		t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
 		usec_rem = do_div(t, USEC_PER_SEC);
 		trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
 	} else {
 		/* counter or tsc mode for trace_clock */
 		trace_seq_printf(s, "oldest event ts: %llu\n",
 				ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
 		trace_seq_printf(s, "now ts: %llu\n",
 				ring_buffer_time_stamp(trace_buf->buffer, cpu));
 	}
 	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
 	trace_seq_printf(s, "dropped events: %ld\n", cnt);
 	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
 	trace_seq_printf(s, "read events: %ld\n", cnt);
 	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
 	kfree(s);
 	return count;
 }
 static const struct file_operations tracing_stats_fops = {
 	.open		= tracing_open_generic_tr,
 	.read		= tracing_stats_read,
 	.llseek		= generic_file_llseek,
 	.release	= tracing_release_generic_tr,
 };
 #ifdef CONFIG_DYNAMIC_FTRACE
 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
 {
 	return 0;
 }
 static ssize_t
 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
 		  size_t cnt, loff_t *ppos)
 {
 	static char ftrace_dyn_info_buffer[1024];
 	static DEFINE_MUTEX(dyn_info_mutex);
 	unsigned long *p = filp->private_data;
 	char *buf = ftrace_dyn_info_buffer;
 	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
 	int r;
 	mutex_lock(&dyn_info_mutex);
 	r = sprintf(buf, "%ld ", *p);
 	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
 	buf[r++] = '\n';
 	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 	mutex_unlock(&dyn_info_mutex);
 	return r;
 }
 static const struct file_operations tracing_dyn_info_fops = {
 	.open		= tracing_open_generic,
 	.read		= tracing_read_dyn_info,
 	.llseek		= generic_file_llseek,
 };
 #endif /* CONFIG_DYNAMIC_FTRACE */
 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
 static void
 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
 {
 	tracing_snapshot();
 }
 static void
 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
 {
 	unsigned long *count = (long *)data;
 	if (!*count)
 		return;
 	if (*count != -1)
 		(*count)--;
 	tracing_snapshot();
 }
 static int
 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
 		      struct ftrace_probe_ops *ops, void *data)
 {
 	long count = (long)data;
 	seq_printf(m, "%ps:", (void *)ip);
 	seq_printf(m, "snapshot");
 	if (count == -1)
 		seq_printf(m, ":unlimited\n");
 	else
 		seq_printf(m, ":count=%ld\n", count);
 	return 0;
 }
 static struct ftrace_probe_ops snapshot_probe_ops = {
 	.func			= ftrace_snapshot,
 	.print			= ftrace_snapshot_print,
 };
 static struct ftrace_probe_ops snapshot_count_probe_ops = {
 	.func			= ftrace_count_snapshot,
 	.print			= ftrace_snapshot_print,
 };
 static int
 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
 			       char *glob, char *cmd, char *param, int enable)
 {
 	struct ftrace_probe_ops *ops;
 	void *count = (void *)-1;
 	char *number;
 	int ret;
 	/* hash funcs only work with set_ftrace_filter */
 	if (!enable)
 		return -EINVAL;
 	ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
 	if (glob[0] == '!') {
 		unregister_ftrace_function_probe_func(glob+1, ops);
 		return 0;
 	}
 	if (!param)
 		goto out_reg;
 	number = strsep(&param, ":");
 	if (!strlen(number))
 		goto out_reg;
 	/*
 	 * We use the callback data field (which is a pointer)
 	 * as our counter.
 	 */
 	ret = kstrtoul(number, 0, (unsigned long *)&count);
 	if (ret)
 		return ret;
  out_reg:
 	ret = register_ftrace_function_probe(glob, ops, count);
 	if (ret >= 0)
 		alloc_snapshot(&global_trace);
 	return ret < 0 ? ret : 0;
 }
 static struct ftrace_func_command ftrace_snapshot_cmd = {
 	.name			= "snapshot",
 	.func			= ftrace_trace_snapshot_callback,
 };
 static int register_snapshot_cmd(void)
 {
 	return register_ftrace_command(&ftrace_snapshot_cmd);
 }
 #else
 static inline int register_snapshot_cmd(void) { return 0; }
 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
 {
 	if (tr->dir)
 		return tr->dir;
 	if (!debugfs_initialized())
 		return NULL;
 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
 		tr->dir = debugfs_create_dir("tracing", NULL);
 	if (!tr->dir)
 		pr_warn_once("Could not create debugfs directory 'tracing'\n");
 	return tr->dir;
 }
 struct dentry *tracing_init_dentry(void)
 {
 	return tracing_init_dentry_tr(&global_trace);
 }
 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
 {
 	struct dentry *d_tracer;
 	if (tr->percpu_dir)
 		return tr->percpu_dir;
 	d_tracer = tracing_init_dentry_tr(tr);
 	if (!d_tracer)
 		return NULL;
 	tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
 	WARN_ONCE(!tr->percpu_dir,
 		  "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
 	return tr->percpu_dir;
 }
 static struct dentry *
 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
 		      void *data, long cpu, const struct file_operations *fops)
 {
 	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
 	if (ret) /* See tracing_get_cpu() */
 		ret->d_inode->i_cdev = (void *)(cpu + 1);
 	return ret;
 }
 static void
 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
 {
 	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
 	struct dentry *d_cpu;
 	char cpu_dir[30]; /* 30 characters should be more than enough */
 	if (!d_percpu)
 		return;
 	snprintf(cpu_dir, 30, "cpu%ld", cpu);
 	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
 	if (!d_cpu) {
 		pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
 		return;
 	}
 	/* per cpu trace_pipe */
 	trace_create_cpu_file("trace_pipe", 0444, d_cpu,
 				tr, cpu, &tracing_pipe_fops);
 	/* per cpu trace */
 	trace_create_cpu_file("trace", 0644, d_cpu,
 				tr, cpu, &tracing_fops);
 	trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
 				tr, cpu, &tracing_buffers_fops);
 	trace_create_cpu_file("stats", 0444, d_cpu,
 				tr, cpu, &tracing_stats_fops);
 	trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
 				tr, cpu, &tracing_entries_fops);
 #ifdef CONFIG_TRACER_SNAPSHOT
 	trace_create_cpu_file("snapshot", 0644, d_cpu,
 				tr, cpu, &snapshot_fops);
 	trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
 				tr, cpu, &snapshot_raw_fops);
 #endif
 }
 #ifdef CONFIG_FTRACE_SELFTEST
 /* Let selftest have access to static functions in this file */
 #include "trace_selftest.c"
 #endif
 struct trace_option_dentry {
 	struct tracer_opt		*opt;
 	struct tracer_flags		*flags;
 	struct trace_array		*tr;
 	struct dentry			*entry;
 };
 static ssize_t
 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
 			loff_t *ppos)
 {
 	struct trace_option_dentry *topt = filp->private_data;
 	char *buf;
 	if (topt->flags->val & topt->opt->bit)
 		buf = "1\n";
 	else
 		buf = "0\n";
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
 }
 static ssize_t
 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
 			 loff_t *ppos)
 {
 	struct trace_option_dentry *topt = filp->private_data;
 	unsigned long val;
 	int ret;
 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 	if (ret)
 		return ret;
 	if (val != 0 && val != 1)
 		return -EINVAL;
 	if (!!(topt->flags->val & topt->opt->bit) != val) {
 		mutex_lock(&trace_types_lock);
 		ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
 					  topt->opt, !val);
 		mutex_unlock(&trace_types_lock);
 		if (ret)
 			return ret;
 	}
 	*ppos += cnt;
 	return cnt;
 }
 static const struct file_operations trace_options_fops = {
 	.open = tracing_open_generic,
 	.read = trace_options_read,
 	.write = trace_options_write,
 	.llseek	= generic_file_llseek,
 };
 static ssize_t
 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
 			loff_t *ppos)
 {
 	long index = (long)filp->private_data;
 	char *buf;
 	if (trace_flags & (1 << index))
 		buf = "1\n";
 	else
 		buf = "0\n";
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
 }
 static ssize_t
 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
 			 loff_t *ppos)
 {
 	struct trace_array *tr = &global_trace;
 	long index = (long)filp->private_data;
 	unsigned long val;
 	int ret;
 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 	if (ret)
 		return ret;
 	if (val != 0 && val != 1)
 		return -EINVAL;
 	mutex_lock(&trace_types_lock);
 	ret = set_tracer_flag(tr, 1 << index, val);
 	mutex_unlock(&trace_types_lock);
 	if (ret < 0)
 		return ret;
 	*ppos += cnt;
 	return cnt;
 }
 static const struct file_operations trace_options_core_fops = {
 	.open = tracing_open_generic,
 	.read = trace_options_core_read,
 	.write = trace_options_core_write,
 	.llseek = generic_file_llseek,
 };
 struct dentry *trace_create_file(const char *name,
 				 umode_t mode,
 				 struct dentry *parent,
 				 void *data,
 				 const struct file_operations *fops)
 {
 	struct dentry *ret;
 	ret = debugfs_create_file(name, mode, parent, data, fops);
 	if (!ret)
 		pr_warning("Could not create debugfs '%s' entry\n", name);
 	return ret;
 }
 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
 {
 	struct dentry *d_tracer;
 	if (tr->options)
 		return tr->options;
 	d_tracer = tracing_init_dentry_tr(tr);
 	if (!d_tracer)
 		return NULL;
 	tr->options = debugfs_create_dir("options", d_tracer);
 	if (!tr->options) {
 		pr_warning("Could not create debugfs directory 'options'\n");
 		return NULL;
 	}
 	return tr->options;
 }
 static void
 create_trace_option_file(struct trace_array *tr,
 			 struct trace_option_dentry *topt,
 			 struct tracer_flags *flags,
 			 struct tracer_opt *opt)
 {
 	struct dentry *t_options;
 	t_options = trace_options_init_dentry(tr);
 	if (!t_options)
 		return;
 	topt->flags = flags;
 	topt->opt = opt;
 	topt->tr = tr;
 	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
 				    &trace_options_fops);
 }
 static struct trace_option_dentry *
 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
 {
 	struct trace_option_dentry *topts;
 	struct tracer_flags *flags;
 	struct tracer_opt *opts;
 	int cnt;
 	if (!tracer)
 		return NULL;
 	flags = tracer->flags;
 	if (!flags || !flags->opts)
 		return NULL;
 	opts = flags->opts;
 	for (cnt = 0; opts[cnt].name; cnt++)
 		;
 	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
 	if (!topts)
 		return NULL;
 	for (cnt = 0; opts[cnt].name; cnt++)
 		create_trace_option_file(tr, &topts[cnt], flags,
 					 &opts[cnt]);
 	return topts;
 }
 static void
 destroy_trace_option_files(struct trace_option_dentry *topts)
 {
 	int cnt;
 	if (!topts)
 		return;
 	for (cnt = 0; topts[cnt].opt; cnt++) {
 		if (topts[cnt].entry)
 			debugfs_remove(topts[cnt].entry);
 	}
 	kfree(topts);
 }
 static struct dentry *
 create_trace_option_core_file(struct trace_array *tr,
 			      const char *option, long index)
 {
 	struct dentry *t_options;
 	t_options = trace_options_init_dentry(tr);
 	if (!t_options)
 		return NULL;
 	return trace_create_file(option, 0644, t_options, (void *)index,
 				    &trace_options_core_fops);
 }
 static __init void create_trace_options_dir(struct trace_array *tr)
 {
 	struct dentry *t_options;
 	int i;
 	t_options = trace_options_init_dentry(tr);
 	if (!t_options)
 		return;
 	for (i = 0; trace_options[i]; i++)
 		create_trace_option_core_file(tr, trace_options[i], i);
 }
 static ssize_t
 rb_simple_read(struct file *filp, char __user *ubuf,
 	       size_t cnt, loff_t *ppos)
 {
 	struct trace_array *tr = filp->private_data;
 	char buf[64];
 	int r;
 	r = tracer_tracing_is_on(tr);
 	r = sprintf(buf, "%d\n", r);
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 }
 static ssize_t
 rb_simple_write(struct file *filp, const char __user *ubuf,
 		size_t cnt, loff_t *ppos)
 {
 	struct trace_array *tr = filp->private_data;
 	struct ring_buffer *buffer = tr->trace_buffer.buffer;
 	unsigned long val;
 	int ret;
 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 	if (ret)
 		return ret;
 	if (buffer) {
 		mutex_lock(&trace_types_lock);
 		if (val) {
 			tracer_tracing_on(tr);
 			if (tr->current_trace->start)
 				tr->current_trace->start(tr);
 		} else {
 			tracer_tracing_off(tr);
 			if (tr->current_trace->stop)
 				tr->current_trace->stop(tr);
 		}
 		mutex_unlock(&trace_types_lock);
 	}
 	(*ppos)++;
 	return cnt;
 }
 static const struct file_operations rb_simple_fops = {
 	.open		= tracing_open_generic_tr,
 	.read		= rb_simple_read,
 	.write		= rb_simple_write,
 	.release	= tracing_release_generic_tr,
 	.llseek		= default_llseek,
 };
 struct dentry *trace_instance_dir;
 static void
 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
 static int
 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
 {
 	enum ring_buffer_flags rb_flags;
 	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
 	buf->buffer = ring_buffer_alloc(size, rb_flags);
 	if (!buf->buffer)
 		return -ENOMEM;
 	buf->data = alloc_percpu(struct trace_array_cpu);
 	if (!buf->data) {
 		ring_buffer_free(buf->buffer);
 		return -ENOMEM;
 	}
 	/* Allocate the first page for all buffers */
 	set_buffer_entries(&tr->trace_buffer,
 			   ring_buffer_size(tr->trace_buffer.buffer, 0));
 	return 0;
 }
 static int allocate_trace_buffers(struct trace_array *tr, int size)
 {
 	int ret;
 	ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
 	if (ret)
 		return ret;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	ret = allocate_trace_buffer(tr, &tr->max_buffer,
 				    allocate_snapshot ? size : 1);
 	if (WARN_ON(ret)) {
 		ring_buffer_free(tr->trace_buffer.buffer);
 		free_percpu(tr->trace_buffer.data);
 		return -ENOMEM;
 	}
 	tr->allocated_snapshot = allocate_snapshot;
 	/*
 	 * Only the top level trace array gets its snapshot allocated
 	 * from the kernel command line.
 	 */
 	allocate_snapshot = false;
 #endif
 	return 0;
 }
 static int new_instance_create(const char *name)
 {
 	struct trace_array *tr;
 	int ret;
 	mutex_lock(&trace_types_lock);
 	ret = -EEXIST;
 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
 		if (tr->name && strcmp(tr->name, name) == 0)
 			goto out_unlock;
 	}
 	ret = -ENOMEM;
 	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 	if (!tr)
 		goto out_unlock;
 	tr->name = kstrdup(name, GFP_KERNEL);
 	if (!tr->name)
 		goto out_free_tr;
+	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
+		goto out_free_tr;
+	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
 	raw_spin_lock_init(&tr->start_lock);
 	tr->current_trace = &nop_trace;
 	INIT_LIST_HEAD(&tr->systems);
 	INIT_LIST_HEAD(&tr->events);
 	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
 		goto out_free_tr;
 	tr->dir = debugfs_create_dir(name, trace_instance_dir);
 	if (!tr->dir)
 		goto out_free_tr;
 	ret = event_trace_add_tracer(tr->dir, tr);
 	if (ret) {
 		debugfs_remove_recursive(tr->dir);
 		goto out_free_tr;
 	}
 	init_tracer_debugfs(tr, tr->dir);
 	list_add(&tr->list, &ftrace_trace_arrays);
 	mutex_unlock(&trace_types_lock);
 	return 0;
  out_free_tr:
 	if (tr->trace_buffer.buffer)
 		ring_buffer_free(tr->trace_buffer.buffer);
+	free_cpumask_var(tr->tracing_cpumask);
 	kfree(tr->name);
 	kfree(tr);
  out_unlock:
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static int instance_delete(const char *name)
 {
 	struct trace_array *tr;
 	int found = 0;
 	int ret;
 	mutex_lock(&trace_types_lock);
 	ret = -ENODEV;
 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
 		if (tr->name && strcmp(tr->name, name) == 0) {
 			found = 1;
 			break;
 		}
 	}
 	if (!found)
 		goto out_unlock;
 	ret = -EBUSY;
 	if (tr->ref)
 		goto out_unlock;
 	list_del(&tr->list);
 	event_trace_del_tracer(tr);
 	debugfs_remove_recursive(tr->dir);
 	free_percpu(tr->trace_buffer.data);
 	ring_buffer_free(tr->trace_buffer.buffer);
 	kfree(tr->name);
 	kfree(tr);
 	ret = 0;
  out_unlock:
 	mutex_unlock(&trace_types_lock);
 	return ret;
 }
 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
 {
 	struct dentry *parent;
 	int ret;
 	/* Paranoid: Make sure the parent is the "instances" directory */
 	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
 	if (WARN_ON_ONCE(parent != trace_instance_dir))
 		return -ENOENT;
 	/*
 	 * The inode mutex is locked, but debugfs_create_dir() will also
 	 * take the mutex. As the instances directory can not be destroyed
 	 * or changed in any other way, it is safe to unlock it, and
 	 * let the dentry try. If two users try to make the same dir at
 	 * the same time, then the new_instance_create() will determine the
 	 * winner.
 	 */
 	mutex_unlock(&inode->i_mutex);
 	ret = new_instance_create(dentry->d_iname);
 	mutex_lock(&inode->i_mutex);
 	return ret;
 }
 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
 {
 	struct dentry *parent;
 	int ret;
 	/* Paranoid: Make sure the parent is the "instances" directory */
 	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
 	if (WARN_ON_ONCE(parent != trace_instance_dir))
 		return -ENOENT;
 	/* The caller did a dget() on dentry */
 	mutex_unlock(&dentry->d_inode->i_mutex);
 	/*
 	 * The inode mutex is locked, but debugfs_create_dir() will also
 	 * take the mutex. As the instances directory can not be destroyed
 	 * or changed in any other way, it is safe to unlock it, and
 	 * let the dentry try. If two users try to make the same dir at
 	 * the same time, then the instance_delete() will determine the
 	 * winner.
 	 */
 	mutex_unlock(&inode->i_mutex);
 	ret = instance_delete(dentry->d_iname);
 	mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
 	mutex_lock(&dentry->d_inode->i_mutex);
 	return ret;
 }
 static const struct inode_operations instance_dir_inode_operations = {
 	.lookup		= simple_lookup,
 	.mkdir		= instance_mkdir,
 	.rmdir		= instance_rmdir,
 };
 static __init void create_trace_instances(struct dentry *d_tracer)
 {
 	trace_instance_dir = debugfs_create_dir("instances", d_tracer);
 	if (WARN_ON(!trace_instance_dir))
 		return;
 	/* Hijack the dir inode operations, to allow mkdir */
 	trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
 }
 static void
 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
 {
 	int cpu;
+	trace_create_file("tracing_cpumask", 0644, d_tracer,
+			  tr, &tracing_cpumask_fops);
 	trace_create_file("trace_options", 0644, d_tracer,
 			  tr, &tracing_iter_fops);
 	trace_create_file("trace", 0644, d_tracer,
 			  tr, &tracing_fops);
 	trace_create_file("trace_pipe", 0444, d_tracer,
 			  tr, &tracing_pipe_fops);
 	trace_create_file("buffer_size_kb", 0644, d_tracer,
 			  tr, &tracing_entries_fops);
 	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
 			  tr, &tracing_total_entries_fops);
 	trace_create_file("free_buffer", 0200, d_tracer,
 			  tr, &tracing_free_buffer_fops);
 	trace_create_file("trace_marker", 0220, d_tracer,
 			  tr, &tracing_mark_fops);
 	trace_create_file("trace_clock", 0644, d_tracer, tr,
 			  &trace_clock_fops);
 	trace_create_file("tracing_on", 0644, d_tracer,
 			  tr, &rb_simple_fops);
 #ifdef CONFIG_TRACER_SNAPSHOT
 	trace_create_file("snapshot", 0644, d_tracer,
 			  tr, &snapshot_fops);
 #endif
 	for_each_tracing_cpu(cpu)
 		tracing_init_debugfs_percpu(tr, cpu);
 }
 static __init int tracer_init_debugfs(void)
 {
 	struct dentry *d_tracer;
 	trace_access_lock_init();
 	d_tracer = tracing_init_dentry();
 	if (!d_tracer)
 		return 0;
 	init_tracer_debugfs(&global_trace, d_tracer);
-	trace_create_file("tracing_cpumask", 0644, d_tracer,
-			&global_trace, &tracing_cpumask_fops);
 	trace_create_file("available_tracers", 0444, d_tracer,
 			&global_trace, &show_traces_fops);
 	trace_create_file("current_tracer", 0644, d_tracer,
 			&global_trace, &set_tracer_fops);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	trace_create_file("tracing_max_latency", 0644, d_tracer,
 			&tracing_max_latency, &tracing_max_lat_fops);
 #endif
 	trace_create_file("tracing_thresh", 0644, d_tracer,
 			&tracing_thresh, &tracing_max_lat_fops);
 	trace_create_file("README", 0444, d_tracer,
 			NULL, &tracing_readme_fops);
 	trace_create_file("saved_cmdlines", 0444, d_tracer,
 			NULL, &tracing_saved_cmdlines_fops);
 #ifdef CONFIG_DYNAMIC_FTRACE
 	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
 			&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
 #endif
 	create_trace_instances(d_tracer);
 	create_trace_options_dir(&global_trace);
 	return 0;
 }
 static int trace_panic_handler(struct notifier_block *this,
 			       unsigned long event, void *unused)
 {
 	if (ftrace_dump_on_oops)
 		ftrace_dump(ftrace_dump_on_oops);
 	return NOTIFY_OK;
 }
 static struct notifier_block trace_panic_notifier = {
 	.notifier_call  = trace_panic_handler,
 	.next           = NULL,
 	.priority       = 150   /* priority: INT_MAX >= x >= 0 */
 };
 static int trace_die_handler(struct notifier_block *self,
 			     unsigned long val,
 			     void *data)
 {
 	switch (val) {
 	case DIE_OOPS:
 		if (ftrace_dump_on_oops)
 			ftrace_dump(ftrace_dump_on_oops);
 		break;
 	default:
 		break;
 	}
 	return NOTIFY_OK;
 }
 static struct notifier_block trace_die_notifier = {
 	.notifier_call = trace_die_handler,
 	.priority = 200
 };
 /*
  * printk is set to max of 1024, we really don't need it that big.
  * Nothing should be printing 1000 characters anyway.
  */
 #define TRACE_MAX_PRINT		1000
 /*
  * Define here KERN_TRACE so that we have one place to modify
  * it if we decide to change what log level the ftrace dump
  * should be at.
  */
 #define KERN_TRACE		KERN_EMERG
 void
 trace_printk_seq(struct trace_seq *s)
 {
 	/* Probably should print a warning here. */
 	if (s->len >= TRACE_MAX_PRINT)
 		s->len = TRACE_MAX_PRINT;
 	/* should be zero ended, but we are paranoid. */
 	s->buffer[s->len] = 0;
 	printk(KERN_TRACE "%s", s->buffer);
 	trace_seq_init(s);
 }
 void trace_init_global_iter(struct trace_iterator *iter)
 {
 	iter->tr = &global_trace;
 	iter->trace = iter->tr->current_trace;
 	iter->cpu_file = RING_BUFFER_ALL_CPUS;
 	iter->trace_buffer = &global_trace.trace_buffer;
 }
 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
 {
 	/* use static because iter can be a bit big for the stack */
 	static struct trace_iterator iter;
 	static atomic_t dump_running;
 	unsigned int old_userobj;
 	unsigned long flags;
 	int cnt = 0, cpu;
 	/* Only allow one dump user at a time. */
 	if (atomic_inc_return(&dump_running) != 1) {
 		atomic_dec(&dump_running);
 		return;
 	}
 	/*
 	 * Always turn off tracing when we dump.
 	 * We don't need to show trace output of what happens
 	 * between multiple crashes.
 	 *
 	 * If the user does a sysrq-z, then they can re-enable
 	 * tracing with echo 1 > tracing_on.
 	 */
 	tracing_off();
 	local_irq_save(flags);
 	/* Simulate the iterator */
 	trace_init_global_iter(&iter);
 	for_each_tracing_cpu(cpu) {
 		atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
 	}
 	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
 	/* don't look at user memory in panic mode */
 	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
 	switch (oops_dump_mode) {
 	case DUMP_ALL:
 		iter.cpu_file = RING_BUFFER_ALL_CPUS;
 		break;
 	case DUMP_ORIG:
 		iter.cpu_file = raw_smp_processor_id();
 		break;
 	case DUMP_NONE:
 		goto out_enable;
 	default:
 		printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
 		iter.cpu_file = RING_BUFFER_ALL_CPUS;
 	}
 	printk(KERN_TRACE "Dumping ftrace buffer:\n");
 	/* Did function tracer already get disabled? */
 	if (ftrace_is_dead()) {
 		printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
 		printk("#          MAY BE MISSING FUNCTION EVENTS\n");
 	}
 	/*
 	 * We need to stop all tracing on all CPUS to read the
 	 * the next buffer. This is a bit expensive, but is
 	 * not done often. We fill all what we can read,
 	 * and then release the locks again.
 	 */
 	while (!trace_empty(&iter)) {
 		if (!cnt)
 			printk(KERN_TRACE "---------------------------------\n");
 		cnt++;
 		/* reset all but tr, trace, and overruns */
 		memset(&iter.seq, 0,
 		       sizeof(struct trace_iterator) -
 		       offsetof(struct trace_iterator, seq));
 		iter.iter_flags |= TRACE_FILE_LAT_FMT;
 		iter.pos = -1;
 		if (trace_find_next_entry_inc(&iter) != NULL) {
 			int ret;
 			ret = print_trace_line(&iter);
 			if (ret != TRACE_TYPE_NO_CONSUME)
 				trace_consume(&iter);
 		}
 		touch_nmi_watchdog();
 		trace_printk_seq(&iter.seq);
 	}
 	if (!cnt)
 		printk(KERN_TRACE "   (ftrace buffer empty)\n");
 	else
 		printk(KERN_TRACE "---------------------------------\n");
  out_enable:
 	trace_flags |= old_userobj;
 	for_each_tracing_cpu(cpu) {
 		atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
 	}
  	atomic_dec(&dump_running);
 	local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(ftrace_dump);
 __init static int tracer_alloc_buffers(void)
 {
 	int ring_buf_size;
 	int ret = -ENOMEM;
 	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
 		goto out;
-	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
+	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
 		goto out_free_buffer_mask;
 	/* Only allocate trace_printk buffers if a trace_printk exists */
 	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
 		/* Must be called before global_trace.buffer is allocated */
 		trace_printk_init_buffers();
 	/* To save memory, keep the ring buffer size to its minimum */
 	if (ring_buffer_expanded)
 		ring_buf_size = trace_buf_size;
 	else
 		ring_buf_size = 1;
 	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
-	cpumask_copy(tracing_cpumask, cpu_all_mask);
+	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
 	raw_spin_lock_init(&global_trace.start_lock);
 	/* TODO: make the number of buffers hot pluggable with CPUS */
 	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
 		printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
 		WARN_ON(1);
 		goto out_free_cpumask;
 	}
 	if (global_trace.buffer_disabled)
 		tracing_off();
 	trace_init_cmdlines();
 	/*
 	 * register_tracer() might reference current_trace, so it
 	 * needs to be set before we register anything. This is
 	 * just a bootstrap of current_trace anyway.
 	 */
 	global_trace.current_trace = &nop_trace;
 	register_tracer(&nop_trace);
 	/* All seems OK, enable tracing */
 	tracing_disabled = 0;
 	atomic_notifier_chain_register(&panic_notifier_list,
 				       &trace_panic_notifier);
 	register_die_notifier(&trace_die_notifier);
 	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
 	INIT_LIST_HEAD(&global_trace.systems);
 	INIT_LIST_HEAD(&global_trace.events);
 	list_add(&global_trace.list, &ftrace_trace_arrays);
 	while (trace_boot_options) {
 		char *option;
 		option = strsep(&trace_boot_options, ",");
 		trace_set_options(&global_trace, option);
 	}
 	register_snapshot_cmd();
 	return 0;
 out_free_cpumask:
 	free_percpu(global_trace.trace_buffer.data);
 #ifdef CONFIG_TRACER_MAX_TRACE
 	free_percpu(global_trace.max_buffer.data);
 #endif
-	free_cpumask_var(tracing_cpumask);
+	free_cpumask_var(global_trace.tracing_cpumask);
 out_free_buffer_mask:
 	free_cpumask_var(tracing_buffer_mask);
 out:
 	return ret;
 }
 __init static int clear_boot_tracer(void)
 {
 	/*
 	 * The default tracer at boot buffer is an init section.
 	 * This function is called in lateinit. If we did not
 	 * find the boot tracer, then clear it out, to prevent
 	 * later registration from accessing the buffer that is
 	 * about to be freed.
 	 */
 	if (!default_bootup_tracer)
 		return 0;
 	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
 	       default_bootup_tracer);
 	default_bootup_tracer = NULL;

kernel/trace/trace.h

Diff comments View file @ 7eb6952

 #ifndef _LINUX_KERNEL_TRACE_H
 #define _LINUX_KERNEL_TRACE_H
 #include <linux/fs.h>
 #include <linux/atomic.h>
 #include <linux/sched.h>
 #include <linux/clocksource.h>
 #include <linux/ring_buffer.h>
 #include <linux/mmiotrace.h>
 #include <linux/tracepoint.h>
 #include <linux/ftrace.h>
 #include <linux/hw_breakpoint.h>
 #include <linux/trace_seq.h>
 #include <linux/ftrace_event.h>
 #ifdef CONFIG_FTRACE_SYSCALLS
 #include <asm/unistd.h>		/* For NR_SYSCALLS	     */
 #include <asm/syscall.h>	/* some archs define it here */
 #endif
 enum trace_type {
 	__TRACE_FIRST_TYPE = 0,
 	TRACE_FN,
 	TRACE_CTX,
 	TRACE_WAKE,
 	TRACE_STACK,
 	TRACE_PRINT,
 	TRACE_BPRINT,
 	TRACE_MMIO_RW,
 	TRACE_MMIO_MAP,
 	TRACE_BRANCH,
 	TRACE_GRAPH_RET,
 	TRACE_GRAPH_ENT,
 	TRACE_USER_STACK,
 	TRACE_BLK,
 	TRACE_BPUTS,
 	__TRACE_LAST_TYPE,
 };
 #undef __field
 #define __field(type, item)		type	item;
 #undef __field_struct
 #define __field_struct(type, item)	__field(type, item)
 #undef __field_desc
 #define __field_desc(type, container, item)
 #undef __array
 #define __array(type, item, size)	type	item[size];
 #undef __array_desc
 #define __array_desc(type, container, item, size)
 #undef __dynamic_array
 #define __dynamic_array(type, item)	type	item[];
 #undef F_STRUCT
 #define F_STRUCT(args...)		args
 #undef FTRACE_ENTRY
 #define FTRACE_ENTRY(name, struct_name, id, tstruct, print, filter)	\
 	struct struct_name {						\
 		struct trace_entry	ent;				\
 		tstruct							\
 	}
 #undef TP_ARGS
 #define TP_ARGS(args...)	args
 #undef FTRACE_ENTRY_DUP
 #define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk, filter)
 #undef FTRACE_ENTRY_REG
 #define FTRACE_ENTRY_REG(name, struct_name, id, tstruct, print,	\
 			 filter, regfn) \
 	FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print), \
 		     filter)
 #include "trace_entries.h"
 /*
  * syscalls are special, and need special handling, this is why
  * they are not included in trace_entries.h
  */
 struct syscall_trace_enter {
 	struct trace_entry	ent;
 	int			nr;
 	unsigned long		args[];
 };
 struct syscall_trace_exit {
 	struct trace_entry	ent;
 	int			nr;
 	long			ret;
 };
 struct kprobe_trace_entry_head {
 	struct trace_entry	ent;
 	unsigned long		ip;
 };
 struct kretprobe_trace_entry_head {
 	struct trace_entry	ent;
 	unsigned long		func;
 	unsigned long		ret_ip;
 };
 /*
  * trace_flag_type is an enumeration that holds different
  * states when a trace occurs. These are:
  *  IRQS_OFF		- interrupts were disabled
  *  IRQS_NOSUPPORT	- arch does not support irqs_disabled_flags
  *  NEED_RESCHED	- reschedule is requested
  *  HARDIRQ		- inside an interrupt handler
  *  SOFTIRQ		- inside a softirq handler
  */
 enum trace_flag_type {
 	TRACE_FLAG_IRQS_OFF		= 0x01,
 	TRACE_FLAG_IRQS_NOSUPPORT	= 0x02,
 	TRACE_FLAG_NEED_RESCHED		= 0x04,
 	TRACE_FLAG_HARDIRQ		= 0x08,
 	TRACE_FLAG_SOFTIRQ		= 0x10,
 };
 #define TRACE_BUF_SIZE		1024
 struct trace_array;
 /*
  * The CPU trace array - it consists of thousands of trace entries
  * plus some other descriptor data: (for example which task started
  * the trace, etc.)
  */
 struct trace_array_cpu {
 	atomic_t		disabled;
 	void			*buffer_page;	/* ring buffer spare */
 	unsigned long		entries;
 	unsigned long		saved_latency;
 	unsigned long		critical_start;
 	unsigned long		critical_end;
 	unsigned long		critical_sequence;
 	unsigned long		nice;
 	unsigned long		policy;
 	unsigned long		rt_priority;
 	unsigned long		skipped_entries;
 	cycle_t			preempt_timestamp;
 	pid_t			pid;
 	kuid_t			uid;
 	char			comm[TASK_COMM_LEN];
 };
 struct tracer;
 struct trace_buffer {
 	struct trace_array		*tr;
 	struct ring_buffer		*buffer;
 	struct trace_array_cpu __percpu	*data;
 	cycle_t				time_start;
 	int				cpu;
 };
 /*
  * The trace array - an array of per-CPU trace arrays. This is the
  * highest level data structure that individual tracers deal with.
  * They have on/off state as well:
  */
 struct trace_array {
 	struct list_head	list;
 	char			*name;
 	struct trace_buffer	trace_buffer;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	/*
 	 * The max_buffer is used to snapshot the trace when a maximum
 	 * latency is reached, or when the user initiates a snapshot.
 	 * Some tracers will use this to store a maximum trace while
 	 * it continues examining live traces.
 	 *
 	 * The buffers for the max_buffer are set up the same as the trace_buffer
 	 * When a snapshot is taken, the buffer of the max_buffer is swapped
 	 * with the buffer of the trace_buffer and the buffers are reset for
 	 * the trace_buffer so the tracing can continue.
 	 */
 	struct trace_buffer	max_buffer;
 	bool			allocated_snapshot;
 #endif
 	int			buffer_disabled;
 #ifdef CONFIG_FTRACE_SYSCALLS
 	int			sys_refcount_enter;
 	int			sys_refcount_exit;
 	DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
 	DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
 #endif
 	int			stop_count;
 	int			clock_id;
 	struct tracer		*current_trace;
 	unsigned int		flags;
 	raw_spinlock_t		start_lock;
 	struct dentry		*dir;
 	struct dentry		*options;
 	struct dentry		*percpu_dir;
 	struct dentry		*event_dir;
 	struct list_head	systems;
 	struct list_head	events;
+	cpumask_var_t		tracing_cpumask; /* only trace on set CPUs */
 	int			ref;
 };
 enum {
 	TRACE_ARRAY_FL_GLOBAL	= (1 << 0)
 };
 extern struct list_head ftrace_trace_arrays;
 extern struct mutex trace_types_lock;
 extern int trace_array_get(struct trace_array *tr);
 extern void trace_array_put(struct trace_array *tr);
 /*
  * The global tracer (top) should be the first trace array added,
  * but we check the flag anyway.
  */
 static inline struct trace_array *top_trace_array(void)
 {
 	struct trace_array *tr;
 	tr = list_entry(ftrace_trace_arrays.prev,
 			typeof(*tr), list);
 	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
 	return tr;
 }
 #define FTRACE_CMP_TYPE(var, type) \
 	__builtin_types_compatible_p(typeof(var), type *)
 #undef IF_ASSIGN
 #define IF_ASSIGN(var, entry, etype, id)		\
 	if (FTRACE_CMP_TYPE(var, etype)) {		\
 		var = (typeof(var))(entry);		\
 		WARN_ON(id && (entry)->type != id);	\
 		break;					\
 	}
 /* Will cause compile errors if type is not found. */
 extern void __ftrace_bad_type(void);
 /*
  * The trace_assign_type is a verifier that the entry type is
  * the same as the type being assigned. To add new types simply
  * add a line with the following format:
  *
  * IF_ASSIGN(var, ent, type, id);
  *
  *  Where "type" is the trace type that includes the trace_entry
  *  as the "ent" item. And "id" is the trace identifier that is
  *  used in the trace_type enum.
  *
  *  If the type can have more than one id, then use zero.
  */
 #define trace_assign_type(var, ent)					\
 	do {								\
 		IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN);	\
 		IF_ASSIGN(var, ent, struct ctx_switch_entry, 0);	\
 		IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK);	\
 		IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
 		IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT);	\
 		IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT);	\
 		IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS);	\
 		IF_ASSIGN(var, ent, struct trace_mmiotrace_rw,		\
 			  TRACE_MMIO_RW);				\
 		IF_ASSIGN(var, ent, struct trace_mmiotrace_map,		\
 			  TRACE_MMIO_MAP);				\
 		IF_ASSIGN(var, ent, struct trace_branch, TRACE_BRANCH); \
 		IF_ASSIGN(var, ent, struct ftrace_graph_ent_entry,	\
 			  TRACE_GRAPH_ENT);		\
 		IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry,	\
 			  TRACE_GRAPH_RET);		\
 		__ftrace_bad_type();					\
 	} while (0)
 /*
  * An option specific to a tracer. This is a boolean value.
  * The bit is the bit index that sets its value on the
  * flags value in struct tracer_flags.
  */
 struct tracer_opt {
 	const char	*name; /* Will appear on the trace_options file */
 	u32		bit; /* Mask assigned in val field in tracer_flags */
 };
 /*
  * The set of specific options for a tracer. Your tracer
  * have to set the initial value of the flags val.
  */
 struct tracer_flags {
 	u32			val;
 	struct tracer_opt	*opts;
 };
 /* Makes more easy to define a tracer opt */
 #define TRACER_OPT(s, b)	.name = #s, .bit = b
 /**
  * struct tracer - a specific tracer and its callbacks to interact with debugfs
  * @name: the name chosen to select it on the available_tracers file
  * @init: called when one switches to this tracer (echo name > current_tracer)
  * @reset: called when one switches to another tracer
  * @start: called when tracing is unpaused (echo 1 > tracing_enabled)
  * @stop: called when tracing is paused (echo 0 > tracing_enabled)
  * @open: called when the trace file is opened
  * @pipe_open: called when the trace_pipe file is opened
  * @wait_pipe: override how the user waits for traces on trace_pipe
  * @close: called when the trace file is released
  * @pipe_close: called when the trace_pipe file is released
  * @read: override the default read callback on trace_pipe
  * @splice_read: override the default splice_read callback on trace_pipe
  * @selftest: selftest to run on boot (see trace_selftest.c)
  * @print_headers: override the first lines that describe your columns
  * @print_line: callback that prints a trace
  * @set_flag: signals one of your private flags changed (trace_options file)
  * @flags: your private flags
  */
 struct tracer {
 	const char		*name;
 	int			(*init)(struct trace_array *tr);
 	void			(*reset)(struct trace_array *tr);
 	void			(*start)(struct trace_array *tr);
 	void			(*stop)(struct trace_array *tr);
 	void			(*open)(struct trace_iterator *iter);
 	void			(*pipe_open)(struct trace_iterator *iter);
 	void			(*wait_pipe)(struct trace_iterator *iter);
 	void			(*close)(struct trace_iterator *iter);
 	void			(*pipe_close)(struct trace_iterator *iter);
 	ssize_t			(*read)(struct trace_iterator *iter,
 					struct file *filp, char __user *ubuf,
 					size_t cnt, loff_t *ppos);
 	ssize_t			(*splice_read)(struct trace_iterator *iter,
 					       struct file *filp,
 					       loff_t *ppos,
 					       struct pipe_inode_info *pipe,
 					       size_t len,
 					       unsigned int flags);
 #ifdef CONFIG_FTRACE_STARTUP_TEST
 	int			(*selftest)(struct tracer *trace,
 					    struct trace_array *tr);
 #endif
 	void			(*print_header)(struct seq_file *m);
 	enum print_line_t	(*print_line)(struct trace_iterator *iter);
 	/* If you handled the flag setting, return 0 */
 	int			(*set_flag)(u32 old_flags, u32 bit, int set);
 	/* Return 0 if OK with change, else return non-zero */
 	int			(*flag_changed)(struct tracer *tracer,
 						u32 mask, int set);
 	struct tracer		*next;
 	struct tracer_flags	*flags;
 	bool			print_max;
 	bool			enabled;
 #ifdef CONFIG_TRACER_MAX_TRACE
 	bool			use_max_tr;
 #endif
 };
 /* Only current can touch trace_recursion */
 /*
  * For function tracing recursion:
  *  The order of these bits are important.
  *
  *  When function tracing occurs, the following steps are made:
  *   If arch does not support a ftrace feature:
  *    call internal function (uses INTERNAL bits) which calls...
  *   If callback is registered to the "global" list, the list
  *    function is called and recursion checks the GLOBAL bits.
  *    then this function calls...
  *   The function callback, which can use the FTRACE bits to
  *    check for recursion.
  *
  * Now if the arch does not suppport a feature, and it calls
  * the global list function which calls the ftrace callback
  * all three of these steps will do a recursion protection.
  * There's no reason to do one if the previous caller already
  * did. The recursion that we are protecting against will
  * go through the same steps again.
  *
  * To prevent the multiple recursion checks, if a recursion
  * bit is set that is higher than the MAX bit of the current
  * check, then we know that the check was made by the previous
  * caller, and we can skip the current check.
  */
 enum {
 	TRACE_BUFFER_BIT,
 	TRACE_BUFFER_NMI_BIT,
 	TRACE_BUFFER_IRQ_BIT,
 	TRACE_BUFFER_SIRQ_BIT,
 	/* Start of function recursion bits */
 	TRACE_FTRACE_BIT,
 	TRACE_FTRACE_NMI_BIT,
 	TRACE_FTRACE_IRQ_BIT,
 	TRACE_FTRACE_SIRQ_BIT,
 	/* GLOBAL_BITs must be greater than FTRACE_BITs */
 	TRACE_GLOBAL_BIT,
 	TRACE_GLOBAL_NMI_BIT,
 	TRACE_GLOBAL_IRQ_BIT,
 	TRACE_GLOBAL_SIRQ_BIT,
 	/* INTERNAL_BITs must be greater than GLOBAL_BITs */
 	TRACE_INTERNAL_BIT,
 	TRACE_INTERNAL_NMI_BIT,
 	TRACE_INTERNAL_IRQ_BIT,
 	TRACE_INTERNAL_SIRQ_BIT,
 	TRACE_CONTROL_BIT,
 /*
  * Abuse of the trace_recursion.
  * As we need a way to maintain state if we are tracing the function
  * graph in irq because we want to trace a particular function that
  * was called in irq context but we have irq tracing off. Since this
  * can only be modified by current, we can reuse trace_recursion.
  */
 	TRACE_IRQ_BIT,
 };
 #define trace_recursion_set(bit)	do { (current)->trace_recursion |= (1<<(bit)); } while (0)
 #define trace_recursion_clear(bit)	do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
 #define trace_recursion_test(bit)	((current)->trace_recursion & (1<<(bit)))
 #define TRACE_CONTEXT_BITS	4
 #define TRACE_FTRACE_START	TRACE_FTRACE_BIT
 #define TRACE_FTRACE_MAX	((1 << (TRACE_FTRACE_START + TRACE_CONTEXT_BITS)) - 1)
 #define TRACE_GLOBAL_START	TRACE_GLOBAL_BIT
 #define TRACE_GLOBAL_MAX	((1 << (TRACE_GLOBAL_START + TRACE_CONTEXT_BITS)) - 1)
 #define TRACE_LIST_START	TRACE_INTERNAL_BIT
 #define TRACE_LIST_MAX		((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
 #define TRACE_CONTEXT_MASK	TRACE_LIST_MAX
 static __always_inline int trace_get_context_bit(void)
 {
 	int bit;
 	if (in_interrupt()) {
 		if (in_nmi())
 			bit = 0;
 		else if (in_irq())
 			bit = 1;
 		else
 			bit = 2;
 	} else
 		bit = 3;
 	return bit;
 }
 static __always_inline int trace_test_and_set_recursion(int start, int max)
 {
 	unsigned int val = current->trace_recursion;
 	int bit;
 	/* A previous recursion check was made */
 	if ((val & TRACE_CONTEXT_MASK) > max)
 		return 0;
 	bit = trace_get_context_bit() + start;
 	if (unlikely(val & (1 << bit)))
 		return -1;
 	val |= 1 << bit;
 	current->trace_recursion = val;
 	barrier();
 	return bit;
 }
 static __always_inline void trace_clear_recursion(int bit)
 {
 	unsigned int val = current->trace_recursion;
 	if (!bit)
 		return;
 	bit = 1 << bit;
 	val &= ~bit;
 	barrier();
 	current->trace_recursion = val;
 }
 static inline struct ring_buffer_iter *
 trace_buffer_iter(struct trace_iterator *iter, int cpu)
 {
 	if (iter->buffer_iter && iter->buffer_iter[cpu])
 		return iter->buffer_iter[cpu];
 	return NULL;
 }
 int tracer_init(struct tracer *t, struct trace_array *tr);
 int tracing_is_enabled(void);
 void tracing_reset(struct trace_buffer *buf, int cpu);
 void tracing_reset_online_cpus(struct trace_buffer *buf);
 void tracing_reset_current(int cpu);
 void tracing_reset_all_online_cpus(void);
 int tracing_open_generic(struct inode *inode, struct file *filp);
 struct dentry *trace_create_file(const char *name,
 				 umode_t mode,
 				 struct dentry *parent,
 				 void *data,
 				 const struct file_operations *fops);
 struct dentry *tracing_init_dentry_tr(struct trace_array *tr);
 struct dentry *tracing_init_dentry(void);
 struct ring_buffer_event;
 struct ring_buffer_event *
 trace_buffer_lock_reserve(struct ring_buffer *buffer,
 			  int type,
 			  unsigned long len,
 			  unsigned long flags,
 			  int pc);
 struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
 						struct trace_array_cpu *data);
 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
 					  int *ent_cpu, u64 *ent_ts);
 void __buffer_unlock_commit(struct ring_buffer *buffer,
 			    struct ring_buffer_event *event);
 int trace_empty(struct trace_iterator *iter);
 void *trace_find_next_entry_inc(struct trace_iterator *iter);
 void trace_init_global_iter(struct trace_iterator *iter);
 void tracing_iter_reset(struct trace_iterator *iter, int cpu);
 void poll_wait_pipe(struct trace_iterator *iter);
 void tracing_sched_switch_trace(struct trace_array *tr,
 				struct task_struct *prev,
 				struct task_struct *next,
 				unsigned long flags, int pc);
 void tracing_sched_wakeup_trace(struct trace_array *tr,
 				struct task_struct *wakee,
 				struct task_struct *cur,
 				unsigned long flags, int pc);
 void trace_function(struct trace_array *tr,
 		    unsigned long ip,
 		    unsigned long parent_ip,
 		    unsigned long flags, int pc);
 void trace_graph_function(struct trace_array *tr,
 		    unsigned long ip,
 		    unsigned long parent_ip,
 		    unsigned long flags, int pc);
 void trace_latency_header(struct seq_file *m);
 void trace_default_header(struct seq_file *m);
 void print_trace_header(struct seq_file *m, struct trace_iterator *iter);
 int trace_empty(struct trace_iterator *iter);
 void trace_graph_return(struct ftrace_graph_ret *trace);
 int trace_graph_entry(struct ftrace_graph_ent *trace);
 void set_graph_array(struct trace_array *tr);
 void tracing_start_cmdline_record(void);
 void tracing_stop_cmdline_record(void);
 void tracing_sched_switch_assign_trace(struct trace_array *tr);
 void tracing_stop_sched_switch_record(void);
 void tracing_start_sched_switch_record(void);
 int register_tracer(struct tracer *type);
 int is_tracing_stopped(void);
 extern cpumask_var_t __read_mostly tracing_buffer_mask;
 #define for_each_tracing_cpu(cpu)	\
 	for_each_cpu(cpu, tracing_buffer_mask)
 extern unsigned long nsecs_to_usecs(unsigned long nsecs);
 extern unsigned long tracing_thresh;
 #ifdef CONFIG_TRACER_MAX_TRACE
 extern unsigned long tracing_max_latency;
 void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
 void update_max_tr_single(struct trace_array *tr,
 			  struct task_struct *tsk, int cpu);
 #endif /* CONFIG_TRACER_MAX_TRACE */
 #ifdef CONFIG_STACKTRACE
 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
 			int skip, int pc);
 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
 			     int skip, int pc, struct pt_regs *regs);
 void ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags,
 			    int pc);
 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
 		   int pc);
 #else
 static inline void ftrace_trace_stack(struct ring_buffer *buffer,
 				      unsigned long flags, int skip, int pc)
 {
 }
 static inline void ftrace_trace_stack_regs(struct ring_buffer *buffer,
 					   unsigned long flags, int skip,
 					   int pc, struct pt_regs *regs)
 {
 }
 static inline void ftrace_trace_userstack(struct ring_buffer *buffer,
 					  unsigned long flags, int pc)
 {
 }
 static inline void __trace_stack(struct trace_array *tr, unsigned long flags,
 				 int skip, int pc)
 {
 }
 #endif /* CONFIG_STACKTRACE */
 extern cycle_t ftrace_now(int cpu);
 extern void trace_find_cmdline(int pid, char comm[]);
 #ifdef CONFIG_DYNAMIC_FTRACE
 extern unsigned long ftrace_update_tot_cnt;
 #endif
 #define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
 extern int DYN_FTRACE_TEST_NAME(void);
 #define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
 extern int DYN_FTRACE_TEST_NAME2(void);
 extern bool ring_buffer_expanded;
 extern bool tracing_selftest_disabled;
 DECLARE_PER_CPU(int, ftrace_cpu_disabled);
 #ifdef CONFIG_FTRACE_STARTUP_TEST
 extern int trace_selftest_startup_function(struct tracer *trace,
 					   struct trace_array *tr);
 extern int trace_selftest_startup_function_graph(struct tracer *trace,
 						 struct trace_array *tr);
 extern int trace_selftest_startup_irqsoff(struct tracer *trace,
 					  struct trace_array *tr);
 extern int trace_selftest_startup_preemptoff(struct tracer *trace,
 					     struct trace_array *tr);
 extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
 						 struct trace_array *tr);
 extern int trace_selftest_startup_wakeup(struct tracer *trace,
 					 struct trace_array *tr);
 extern int trace_selftest_startup_nop(struct tracer *trace,
 					 struct trace_array *tr);
 extern int trace_selftest_startup_sched_switch(struct tracer *trace,
 					       struct trace_array *tr);
 extern int trace_selftest_startup_branch(struct tracer *trace,
 					 struct trace_array *tr);
 /*
  * Tracer data references selftest functions that only occur
  * on boot up. These can be __init functions. Thus, when selftests
  * are enabled, then the tracers need to reference __init functions.
  */
 #define __tracer_data		__refdata
 #else
 /* Tracers are seldom changed. Optimize when selftests are disabled. */
 #define __tracer_data		__read_mostly
 #endif /* CONFIG_FTRACE_STARTUP_TEST */
 extern void *head_page(struct trace_array_cpu *data);
 extern unsigned long long ns2usecs(cycle_t nsec);
 extern int
 trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
 extern int
 trace_vprintk(unsigned long ip, const char *fmt, va_list args);
 extern int
 trace_array_vprintk(struct trace_array *tr,
 		    unsigned long ip, const char *fmt, va_list args);
 int trace_array_printk(struct trace_array *tr,
 		       unsigned long ip, const char *fmt, ...);
 int trace_array_printk_buf(struct ring_buffer *buffer,
 			   unsigned long ip, const char *fmt, ...);
 void trace_printk_seq(struct trace_seq *s);
 enum print_line_t print_trace_line(struct trace_iterator *iter);
 extern unsigned long trace_flags;
 /* Standard output formatting function used for function return traces */
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 /* Flag options */
 #define TRACE_GRAPH_PRINT_OVERRUN       0x1
 #define TRACE_GRAPH_PRINT_CPU           0x2
 #define TRACE_GRAPH_PRINT_OVERHEAD      0x4
 #define TRACE_GRAPH_PRINT_PROC          0x8
 #define TRACE_GRAPH_PRINT_DURATION      0x10
 #define TRACE_GRAPH_PRINT_ABS_TIME      0x20
 extern enum print_line_t
 print_graph_function_flags(struct trace_iterator *iter, u32 flags);
 extern void print_graph_headers_flags(struct seq_file *s, u32 flags);
 extern enum print_line_t
 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s);
 extern void graph_trace_open(struct trace_iterator *iter);
 extern void graph_trace_close(struct trace_iterator *iter);
 extern int __trace_graph_entry(struct trace_array *tr,
 			       struct ftrace_graph_ent *trace,
 			       unsigned long flags, int pc);
 extern void __trace_graph_return(struct trace_array *tr,
 				 struct ftrace_graph_ret *trace,
 				 unsigned long flags, int pc);
 #ifdef CONFIG_DYNAMIC_FTRACE
 /* TODO: make this variable */
 #define FTRACE_GRAPH_MAX_FUNCS		32
 extern int ftrace_graph_filter_enabled;
 extern int ftrace_graph_count;
 extern unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS];
 static inline int ftrace_graph_addr(unsigned long addr)
 {
 	int i;
 	if (!ftrace_graph_filter_enabled)
 		return 1;
 	for (i = 0; i < ftrace_graph_count; i++) {
 		if (addr == ftrace_graph_funcs[i]) {
 			/*
 			 * If no irqs are to be traced, but a set_graph_function
 			 * is set, and called by an interrupt handler, we still
 			 * want to trace it.
 			 */
 			if (in_irq())
 				trace_recursion_set(TRACE_IRQ_BIT);
 			else
 				trace_recursion_clear(TRACE_IRQ_BIT);
 			return 1;
 		}
 	}
 	return 0;
 }
 #else
 static inline int ftrace_graph_addr(unsigned long addr)
 {
 	return 1;
 }
 #endif /* CONFIG_DYNAMIC_FTRACE */
 #else /* CONFIG_FUNCTION_GRAPH_TRACER */
 static inline enum print_line_t
 print_graph_function_flags(struct trace_iterator *iter, u32 flags)
 {
 	return TRACE_TYPE_UNHANDLED;
 }
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 extern struct list_head ftrace_pids;
 #ifdef CONFIG_FUNCTION_TRACER
 extern bool ftrace_filter_param __initdata;
 static inline int ftrace_trace_task(struct task_struct *task)
 {
 	if (list_empty(&ftrace_pids))
 		return 1;
 	return test_tsk_trace_trace(task);
 }
 extern int ftrace_is_dead(void);
 #else
 static inline int ftrace_trace_task(struct task_struct *task)
 {
 	return 1;
 }
 static inline int ftrace_is_dead(void) { return 0; }
 #endif
 int ftrace_event_is_function(struct ftrace_event_call *call);
 /*
  * struct trace_parser - servers for reading the user input separated by spaces
  * @cont: set if the input is not complete - no final space char was found
  * @buffer: holds the parsed user input
  * @idx: user input length
  * @size: buffer size
  */
 struct trace_parser {
 	bool		cont;
 	char		*buffer;
 	unsigned	idx;
 	unsigned	size;
 };
 static inline bool trace_parser_loaded(struct trace_parser *parser)
 {
 	return (parser->idx != 0);
 }
 static inline bool trace_parser_cont(struct trace_parser *parser)
 {
 	return parser->cont;
 }
 static inline void trace_parser_clear(struct trace_parser *parser)
 {
 	parser->cont = false;
 	parser->idx = 0;
 }
 extern int trace_parser_get_init(struct trace_parser *parser, int size);
 extern void trace_parser_put(struct trace_parser *parser);
 extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
 	size_t cnt, loff_t *ppos);
 /*
  * trace_iterator_flags is an enumeration that defines bit
  * positions into trace_flags that controls the output.
  *
  * NOTE: These bits must match the trace_options array in
  *       trace.c.
  */
 enum trace_iterator_flags {
 	TRACE_ITER_PRINT_PARENT		= 0x01,
 	TRACE_ITER_SYM_OFFSET		= 0x02,
 	TRACE_ITER_SYM_ADDR		= 0x04,
 	TRACE_ITER_VERBOSE		= 0x08,
 	TRACE_ITER_RAW			= 0x10,
 	TRACE_ITER_HEX			= 0x20,
 	TRACE_ITER_BIN			= 0x40,
 	TRACE_ITER_BLOCK		= 0x80,
 	TRACE_ITER_STACKTRACE		= 0x100,
 	TRACE_ITER_PRINTK		= 0x200,
 	TRACE_ITER_PREEMPTONLY		= 0x400,
 	TRACE_ITER_BRANCH		= 0x800,
 	TRACE_ITER_ANNOTATE		= 0x1000,
 	TRACE_ITER_USERSTACKTRACE       = 0x2000,
 	TRACE_ITER_SYM_USEROBJ          = 0x4000,
 	TRACE_ITER_PRINTK_MSGONLY	= 0x8000,
 	TRACE_ITER_CONTEXT_INFO		= 0x10000, /* Print pid/cpu/time */
 	TRACE_ITER_LATENCY_FMT		= 0x20000,
 	TRACE_ITER_SLEEP_TIME		= 0x40000,
 	TRACE_ITER_GRAPH_TIME		= 0x80000,
 	TRACE_ITER_RECORD_CMD		= 0x100000,
 	TRACE_ITER_OVERWRITE		= 0x200000,
 	TRACE_ITER_STOP_ON_FREE		= 0x400000,
 	TRACE_ITER_IRQ_INFO		= 0x800000,
 	TRACE_ITER_MARKERS		= 0x1000000,
 	TRACE_ITER_FUNCTION		= 0x2000000,
 };
 /*
  * TRACE_ITER_SYM_MASK masks the options in trace_flags that
  * control the output of kernel symbols.
  */
 #define TRACE_ITER_SYM_MASK \
 	(TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
 extern struct tracer nop_trace;
 #ifdef CONFIG_BRANCH_TRACER
 extern int enable_branch_tracing(struct trace_array *tr);
 extern void disable_branch_tracing(void);
 static inline int trace_branch_enable(struct trace_array *tr)
 {
 	if (trace_flags & TRACE_ITER_BRANCH)
 		return enable_branch_tracing(tr);
 	return 0;
 }
 static inline void trace_branch_disable(void)
 {
 	/* due to races, always disable */
 	disable_branch_tracing();
 }
 #else
 static inline int trace_branch_enable(struct trace_array *tr)
 {
 	return 0;
 }
 static inline void trace_branch_disable(void)
 {
 }
 #endif /* CONFIG_BRANCH_TRACER */
 /* set ring buffers to default size if not already done so */
 int tracing_update_buffers(void);
 struct ftrace_event_field {
 	struct list_head	link;
 	const char		*name;
 	const char		*type;
 	int			filter_type;
 	int			offset;
 	int			size;
 	int			is_signed;
 };
 struct event_filter {
 	int			n_preds;	/* Number assigned */
 	int			a_preds;	/* allocated */
 	struct filter_pred	*preds;
 	struct filter_pred	*root;
 	char			*filter_string;
 };
 struct event_subsystem {
 	struct list_head	list;
 	const char		*name;
 	struct event_filter	*filter;
 	int			ref_count;
 };
 struct ftrace_subsystem_dir {
 	struct list_head		list;
 	struct event_subsystem		*subsystem;
 	struct trace_array		*tr;
 	struct dentry			*entry;
 	int				ref_count;
 	int				nr_events;
 };
 #define FILTER_PRED_INVALID	((unsigned short)-1)
 #define FILTER_PRED_IS_RIGHT	(1 << 15)
 #define FILTER_PRED_FOLD	(1 << 15)
 /*
  * The max preds is the size of unsigned short with
  * two flags at the MSBs. One bit is used for both the IS_RIGHT
  * and FOLD flags. The other is reserved.
  *
  * 2^14 preds is way more than enough.
  */
 #define MAX_FILTER_PRED		16384
 struct filter_pred;
 struct regex;
 typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event);
 typedef int (*regex_match_func)(char *str, struct regex *r, int len);
 enum regex_type {
 	MATCH_FULL = 0,
 	MATCH_FRONT_ONLY,
 	MATCH_MIDDLE_ONLY,
 	MATCH_END_ONLY,
 };
 struct regex {
 	char			pattern[MAX_FILTER_STR_VAL];
 	int			len;
 	int			field_len;
 	regex_match_func	match;
 };
 struct filter_pred {
 	filter_pred_fn_t 	fn;
 	u64 			val;
 	struct regex		regex;
 	unsigned short		*ops;
 	struct ftrace_event_field *field;
 	int 			offset;
 	int 			not;
 	int 			op;
 	unsigned short		index;
 	unsigned short		parent;
 	unsigned short		left;
 	unsigned short		right;
 };
 extern enum regex_type
 filter_parse_regex(char *buff, int len, char **search, int *not);
 extern void print_event_filter(struct ftrace_event_call *call,
 			       struct trace_seq *s);
 extern int apply_event_filter(struct ftrace_event_call *call,
 			      char *filter_string);
 extern int apply_subsystem_event_filter(struct ftrace_subsystem_dir *dir,
 					char *filter_string);
 extern void print_subsystem_event_filter(struct event_subsystem *system,
 					 struct trace_seq *s);
 extern int filter_assign_type(const char *type);
 struct ftrace_event_field *
 trace_find_event_field(struct ftrace_event_call *call, char *name);
 static inline int
 filter_check_discard(struct ftrace_event_call *call, void *rec,
 		     struct ring_buffer *buffer,
 		     struct ring_buffer_event *event)
 {
 	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
 	    !filter_match_preds(call->filter, rec)) {
 		ring_buffer_discard_commit(buffer, event);
 		return 1;
 	}
 	return 0;
 }
 extern void trace_event_enable_cmd_record(bool enable);
 extern int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr);
 extern int event_trace_del_tracer(struct trace_array *tr);
 extern struct mutex event_mutex;
 extern struct list_head ftrace_events;
 extern const char *__start___trace_bprintk_fmt[];
 extern const char *__stop___trace_bprintk_fmt[];
 extern const char *__start___tracepoint_str[];
 extern const char *__stop___tracepoint_str[];
 void trace_printk_init_buffers(void);
 void trace_printk_start_comm(void);
 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
 /*
  * Normal trace_printk() and friends allocates special buffers
  * to do the manipulation, as well as saves the print formats
  * into sections to display. But the trace infrastructure wants
  * to use these without the added overhead at the price of being
  * a bit slower (used mainly for warnings, where we don't care
  * about performance). The internal_trace_puts() is for such
  * a purpose.
  */
 #define internal_trace_puts(str) __trace_puts(_THIS_IP_, str, strlen(str))
 #undef FTRACE_ENTRY
 #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter)	\
 	extern struct ftrace_event_call					\
 	__attribute__((__aligned__(4))) event_##call;
 #undef FTRACE_ENTRY_DUP
 #define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print, filter)	\
 	FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print), \
 		     filter)
 #include "trace_entries.h"
 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_FUNCTION_TRACER)
 int perf_ftrace_event_register(struct ftrace_event_call *call,
 			       enum trace_reg type, void *data);
 #else
 #define perf_ftrace_event_register NULL
 #endif
 #endif /* _LINUX_KERNEL_TRACE_H */

kernel/trace/trace_events.c

Diff comments View file @ 7eb6952

1	/*	1	/*
2	* event tracer	2	* event tracer
3	*	3	*
4	* Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>	4	* Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
5	*	5	*
6	* - Added format output of fields of the trace point.	6	* - Added format output of fields of the trace point.
7	* This was based off of work by Tom Zanussi <tzanussi@gmail.com>.	7	* This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
8	*	8	*
9	*/	9	*/
10		10
11	#include <linux/workqueue.h>	11	#include <linux/workqueue.h>
12	#include <linux/spinlock.h>	12	#include <linux/spinlock.h>
13	#include <linux/kthread.h>	13	#include <linux/kthread.h>
14	#include <linux/debugfs.h>	14	#include <linux/debugfs.h>
15	#include <linux/uaccess.h>	15	#include <linux/uaccess.h>
16	#include <linux/module.h>	16	#include <linux/module.h>
17	#include <linux/ctype.h>	17	#include <linux/ctype.h>
18	#include <linux/slab.h>	18	#include <linux/slab.h>
19	#include <linux/delay.h>	19	#include <linux/delay.h>
20		20
21	#include <asm/setup.h>	21	#include <asm/setup.h>
22		22
23	#include "trace_output.h"	23	#include "trace_output.h"
24		24
25	#undef TRACE_SYSTEM	25	#undef TRACE_SYSTEM
26	#define TRACE_SYSTEM "TRACE_SYSTEM"	26	#define TRACE_SYSTEM "TRACE_SYSTEM"
27		27
28	DEFINE_MUTEX(event_mutex);	28	DEFINE_MUTEX(event_mutex);
29		29
30	DEFINE_MUTEX(event_storage_mutex);	30	DEFINE_MUTEX(event_storage_mutex);
31	EXPORT_SYMBOL_GPL(event_storage_mutex);	31	EXPORT_SYMBOL_GPL(event_storage_mutex);
32		32
33	char event_storage[EVENT_STORAGE_SIZE];	33	char event_storage[EVENT_STORAGE_SIZE];
34	EXPORT_SYMBOL_GPL(event_storage);	34	EXPORT_SYMBOL_GPL(event_storage);
35		35
36	LIST_HEAD(ftrace_events);	36	LIST_HEAD(ftrace_events);
37	static LIST_HEAD(ftrace_common_fields);	37	static LIST_HEAD(ftrace_common_fields);
38		38
39	#define GFP_TRACE (GFP_KERNEL \| __GFP_ZERO)	39	#define GFP_TRACE (GFP_KERNEL \| __GFP_ZERO)
40		40
41	static struct kmem_cache *field_cachep;	41	static struct kmem_cache *field_cachep;
42	static struct kmem_cache *file_cachep;	42	static struct kmem_cache *file_cachep;
43		43
44	#define SYSTEM_FL_FREE_NAME (1 << 31)	44	#define SYSTEM_FL_FREE_NAME (1 << 31)
45		45
46	static inline int system_refcount(struct event_subsystem *system)	46	static inline int system_refcount(struct event_subsystem *system)
47	{	47	{
48	return system->ref_count & ~SYSTEM_FL_FREE_NAME;	48	return system->ref_count & ~SYSTEM_FL_FREE_NAME;
49	}	49	}
50		50
51	static int system_refcount_inc(struct event_subsystem *system)	51	static int system_refcount_inc(struct event_subsystem *system)
52	{	52	{
53	return (system->ref_count++) & ~SYSTEM_FL_FREE_NAME;	53	return (system->ref_count++) & ~SYSTEM_FL_FREE_NAME;
54	}	54	}
55		55
56	static int system_refcount_dec(struct event_subsystem *system)	56	static int system_refcount_dec(struct event_subsystem *system)
57	{	57	{
58	return (--system->ref_count) & ~SYSTEM_FL_FREE_NAME;	58	return (--system->ref_count) & ~SYSTEM_FL_FREE_NAME;
59	}	59	}
60		60
61	/* Double loops, do not use break, only goto's work */	61	/* Double loops, do not use break, only goto's work */
62	#define do_for_each_event_file(tr, file) \	62	#define do_for_each_event_file(tr, file) \
63	list_for_each_entry(tr, &ftrace_trace_arrays, list) { \	63	list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
64	list_for_each_entry(file, &tr->events, list)	64	list_for_each_entry(file, &tr->events, list)
65		65
66	#define do_for_each_event_file_safe(tr, file) \	66	#define do_for_each_event_file_safe(tr, file) \
67	list_for_each_entry(tr, &ftrace_trace_arrays, list) { \	67	list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
68	struct ftrace_event_file *___n; \	68	struct ftrace_event_file *___n; \
69	list_for_each_entry_safe(file, ___n, &tr->events, list)	69	list_for_each_entry_safe(file, ___n, &tr->events, list)
70		70
71	#define while_for_each_event_file() \	71	#define while_for_each_event_file() \
72	}	72	}
73		73
74	static struct list_head *	74	static struct list_head *
75	trace_get_fields(struct ftrace_event_call *event_call)	75	trace_get_fields(struct ftrace_event_call *event_call)
76	{	76	{
77	if (!event_call->class->get_fields)	77	if (!event_call->class->get_fields)
78	return &event_call->class->fields;	78	return &event_call->class->fields;
79	return event_call->class->get_fields(event_call);	79	return event_call->class->get_fields(event_call);
80	}	80	}
81		81
82	static struct ftrace_event_field *	82	static struct ftrace_event_field *
83	__find_event_field(struct list_head head, char name)	83	__find_event_field(struct list_head head, char name)
84	{	84	{
85	struct ftrace_event_field *field;	85	struct ftrace_event_field *field;
86		86
87	list_for_each_entry(field, head, link) {	87	list_for_each_entry(field, head, link) {
88	if (!strcmp(field->name, name))	88	if (!strcmp(field->name, name))
89	return field;	89	return field;
90	}	90	}
91		91
92	return NULL;	92	return NULL;
93	}	93	}
94		94
95	struct ftrace_event_field *	95	struct ftrace_event_field *
96	trace_find_event_field(struct ftrace_event_call call, char name)	96	trace_find_event_field(struct ftrace_event_call call, char name)
97	{	97	{
98	struct ftrace_event_field *field;	98	struct ftrace_event_field *field;
99	struct list_head *head;	99	struct list_head *head;
100		100
101	field = __find_event_field(&ftrace_common_fields, name);	101	field = __find_event_field(&ftrace_common_fields, name);
102	if (field)	102	if (field)
103	return field;	103	return field;
104		104
105	head = trace_get_fields(call);	105	head = trace_get_fields(call);
106	return __find_event_field(head, name);	106	return __find_event_field(head, name);
107	}	107	}
108		108
109	static int __trace_define_field(struct list_head head, const char type,	109	static int __trace_define_field(struct list_head head, const char type,
110	const char *name, int offset, int size,	110	const char *name, int offset, int size,
111	int is_signed, int filter_type)	111	int is_signed, int filter_type)
112	{	112	{
113	struct ftrace_event_field *field;	113	struct ftrace_event_field *field;
114		114
115	field = kmem_cache_alloc(field_cachep, GFP_TRACE);	115	field = kmem_cache_alloc(field_cachep, GFP_TRACE);
116	if (!field)	116	if (!field)
117	return -ENOMEM;	117	return -ENOMEM;
118		118
119	field->name = name;	119	field->name = name;
120	field->type = type;	120	field->type = type;
121		121
122	if (filter_type == FILTER_OTHER)	122	if (filter_type == FILTER_OTHER)
123	field->filter_type = filter_assign_type(type);	123	field->filter_type = filter_assign_type(type);
124	else	124	else
125	field->filter_type = filter_type;	125	field->filter_type = filter_type;
126		126
127	field->offset = offset;	127	field->offset = offset;
128	field->size = size;	128	field->size = size;
129	field->is_signed = is_signed;	129	field->is_signed = is_signed;
130		130
131	list_add(&field->link, head);	131	list_add(&field->link, head);
132		132
133	return 0;	133	return 0;
134	}	134	}
135		135
136	int trace_define_field(struct ftrace_event_call call, const char type,	136	int trace_define_field(struct ftrace_event_call call, const char type,
137	const char *name, int offset, int size, int is_signed,	137	const char *name, int offset, int size, int is_signed,
138	int filter_type)	138	int filter_type)
139	{	139	{
140	struct list_head *head;	140	struct list_head *head;
141		141
142	if (WARN_ON(!call->class))	142	if (WARN_ON(!call->class))
143	return 0;	143	return 0;
144		144
145	head = trace_get_fields(call);	145	head = trace_get_fields(call);
146	return __trace_define_field(head, type, name, offset, size,	146	return __trace_define_field(head, type, name, offset, size,
147	is_signed, filter_type);	147	is_signed, filter_type);
148	}	148	}
149	EXPORT_SYMBOL_GPL(trace_define_field);	149	EXPORT_SYMBOL_GPL(trace_define_field);
150		150
151	#define __common_field(type, item) \	151	#define __common_field(type, item) \
152	ret = __trace_define_field(&ftrace_common_fields, #type, \	152	ret = __trace_define_field(&ftrace_common_fields, #type, \
153	"common_" #item, \	153	"common_" #item, \
154	offsetof(typeof(ent), item), \	154	offsetof(typeof(ent), item), \
155	sizeof(ent.item), \	155	sizeof(ent.item), \
156	is_signed_type(type), FILTER_OTHER); \	156	is_signed_type(type), FILTER_OTHER); \
157	if (ret) \	157	if (ret) \
158	return ret;	158	return ret;
159		159
160	static int trace_define_common_fields(void)	160	static int trace_define_common_fields(void)
161	{	161	{
162	int ret;	162	int ret;
163	struct trace_entry ent;	163	struct trace_entry ent;
164		164
165	__common_field(unsigned short, type);	165	__common_field(unsigned short, type);
166	__common_field(unsigned char, flags);	166	__common_field(unsigned char, flags);
167	__common_field(unsigned char, preempt_count);	167	__common_field(unsigned char, preempt_count);
168	__common_field(int, pid);	168	__common_field(int, pid);
169		169
170	return ret;	170	return ret;
171	}	171	}
172		172
173	static void trace_destroy_fields(struct ftrace_event_call *call)	173	static void trace_destroy_fields(struct ftrace_event_call *call)
174	{	174	{
175	struct ftrace_event_field field, next;	175	struct ftrace_event_field field, next;
176	struct list_head *head;	176	struct list_head *head;
177		177
178	head = trace_get_fields(call);	178	head = trace_get_fields(call);
179	list_for_each_entry_safe(field, next, head, link) {	179	list_for_each_entry_safe(field, next, head, link) {
180	list_del(&field->link);	180	list_del(&field->link);
181	kmem_cache_free(field_cachep, field);	181	kmem_cache_free(field_cachep, field);
182	}	182	}
183	}	183	}
184		184
185	int trace_event_raw_init(struct ftrace_event_call *call)	185	int trace_event_raw_init(struct ftrace_event_call *call)
186	{	186	{
187	int id;	187	int id;
188		188
189	id = register_ftrace_event(&call->event);	189	id = register_ftrace_event(&call->event);
190	if (!id)	190	if (!id)
191	return -ENODEV;	191	return -ENODEV;
192		192
193	return 0;	193	return 0;
194	}	194	}
195	EXPORT_SYMBOL_GPL(trace_event_raw_init);	195	EXPORT_SYMBOL_GPL(trace_event_raw_init);
196		196
197	int ftrace_event_reg(struct ftrace_event_call *call,	197	int ftrace_event_reg(struct ftrace_event_call *call,
198	enum trace_reg type, void *data)	198	enum trace_reg type, void *data)
199	{	199	{
200	struct ftrace_event_file *file = data;	200	struct ftrace_event_file *file = data;
201		201
202	switch (type) {	202	switch (type) {
203	case TRACE_REG_REGISTER:	203	case TRACE_REG_REGISTER:
204	return tracepoint_probe_register(call->name,	204	return tracepoint_probe_register(call->name,
205	call->class->probe,	205	call->class->probe,
206	file);	206	file);
207	case TRACE_REG_UNREGISTER:	207	case TRACE_REG_UNREGISTER:
208	tracepoint_probe_unregister(call->name,	208	tracepoint_probe_unregister(call->name,
209	call->class->probe,	209	call->class->probe,
210	file);	210	file);
211	return 0;	211	return 0;
212		212
213	#ifdef CONFIG_PERF_EVENTS	213	#ifdef CONFIG_PERF_EVENTS
214	case TRACE_REG_PERF_REGISTER:	214	case TRACE_REG_PERF_REGISTER:
215	return tracepoint_probe_register(call->name,	215	return tracepoint_probe_register(call->name,
216	call->class->perf_probe,	216	call->class->perf_probe,
217	call);	217	call);
218	case TRACE_REG_PERF_UNREGISTER:	218	case TRACE_REG_PERF_UNREGISTER:
219	tracepoint_probe_unregister(call->name,	219	tracepoint_probe_unregister(call->name,
220	call->class->perf_probe,	220	call->class->perf_probe,
221	call);	221	call);
222	return 0;	222	return 0;
223	case TRACE_REG_PERF_OPEN:	223	case TRACE_REG_PERF_OPEN:
224	case TRACE_REG_PERF_CLOSE:	224	case TRACE_REG_PERF_CLOSE:
225	case TRACE_REG_PERF_ADD:	225	case TRACE_REG_PERF_ADD:
226	case TRACE_REG_PERF_DEL:	226	case TRACE_REG_PERF_DEL:
227	return 0;	227	return 0;
228	#endif	228	#endif
229	}	229	}
230	return 0;	230	return 0;
231	}	231	}
232	EXPORT_SYMBOL_GPL(ftrace_event_reg);	232	EXPORT_SYMBOL_GPL(ftrace_event_reg);
233		233
234	void trace_event_enable_cmd_record(bool enable)	234	void trace_event_enable_cmd_record(bool enable)
235	{	235	{
236	struct ftrace_event_file *file;	236	struct ftrace_event_file *file;
237	struct trace_array *tr;	237	struct trace_array *tr;
238		238
239	mutex_lock(&event_mutex);	239	mutex_lock(&event_mutex);
240	do_for_each_event_file(tr, file) {	240	do_for_each_event_file(tr, file) {
241		241
242	if (!(file->flags & FTRACE_EVENT_FL_ENABLED))	242	if (!(file->flags & FTRACE_EVENT_FL_ENABLED))
243	continue;	243	continue;
244		244
245	if (enable) {	245	if (enable) {
246	tracing_start_cmdline_record();	246	tracing_start_cmdline_record();
247	set_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);	247	set_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);
248	} else {	248	} else {
249	tracing_stop_cmdline_record();	249	tracing_stop_cmdline_record();
250	clear_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);	250	clear_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);
251	}	251	}
252	} while_for_each_event_file();	252	} while_for_each_event_file();
253	mutex_unlock(&event_mutex);	253	mutex_unlock(&event_mutex);
254	}	254	}
255		255
256	static int __ftrace_event_enable_disable(struct ftrace_event_file *file,	256	static int __ftrace_event_enable_disable(struct ftrace_event_file *file,
257	int enable, int soft_disable)	257	int enable, int soft_disable)
258	{	258	{
259	struct ftrace_event_call *call = file->event_call;	259	struct ftrace_event_call *call = file->event_call;
260	int ret = 0;	260	int ret = 0;
261	int disable;	261	int disable;
262		262
263	switch (enable) {	263	switch (enable) {
264	case 0:	264	case 0:
265	/*	265	/*
266	* When soft_disable is set and enable is cleared, the sm_ref	266	* When soft_disable is set and enable is cleared, the sm_ref
267	* reference counter is decremented. If it reaches 0, we want	267	* reference counter is decremented. If it reaches 0, we want
268	* to clear the SOFT_DISABLED flag but leave the event in the	268	* to clear the SOFT_DISABLED flag but leave the event in the
269	* state that it was. That is, if the event was enabled and	269	* state that it was. That is, if the event was enabled and
270	* SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED	270	* SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
271	* is set we do not want the event to be enabled before we	271	* is set we do not want the event to be enabled before we
272	* clear the bit.	272	* clear the bit.
273	*	273	*
274	* When soft_disable is not set but the SOFT_MODE flag is,	274	* When soft_disable is not set but the SOFT_MODE flag is,
275	* we do nothing. Do not disable the tracepoint, otherwise	275	* we do nothing. Do not disable the tracepoint, otherwise
276	* "soft enable"s (clearing the SOFT_DISABLED bit) wont work.	276	* "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
277	*/	277	*/
278	if (soft_disable) {	278	if (soft_disable) {
279	if (atomic_dec_return(&file->sm_ref) > 0)	279	if (atomic_dec_return(&file->sm_ref) > 0)
280	break;	280	break;
281	disable = file->flags & FTRACE_EVENT_FL_SOFT_DISABLED;	281	disable = file->flags & FTRACE_EVENT_FL_SOFT_DISABLED;
282	clear_bit(FTRACE_EVENT_FL_SOFT_MODE_BIT, &file->flags);	282	clear_bit(FTRACE_EVENT_FL_SOFT_MODE_BIT, &file->flags);
283	} else	283	} else
284	disable = !(file->flags & FTRACE_EVENT_FL_SOFT_MODE);	284	disable = !(file->flags & FTRACE_EVENT_FL_SOFT_MODE);
285		285
286	if (disable && (file->flags & FTRACE_EVENT_FL_ENABLED)) {	286	if (disable && (file->flags & FTRACE_EVENT_FL_ENABLED)) {
287	clear_bit(FTRACE_EVENT_FL_ENABLED_BIT, &file->flags);	287	clear_bit(FTRACE_EVENT_FL_ENABLED_BIT, &file->flags);
288	if (file->flags & FTRACE_EVENT_FL_RECORDED_CMD) {	288	if (file->flags & FTRACE_EVENT_FL_RECORDED_CMD) {
289	tracing_stop_cmdline_record();	289	tracing_stop_cmdline_record();
290	clear_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);	290	clear_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);
291	}	291	}
292	call->class->reg(call, TRACE_REG_UNREGISTER, file);	292	call->class->reg(call, TRACE_REG_UNREGISTER, file);
293	}	293	}
294	/* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */	294	/* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
295	if (file->flags & FTRACE_EVENT_FL_SOFT_MODE)	295	if (file->flags & FTRACE_EVENT_FL_SOFT_MODE)
296	set_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);	296	set_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);
297	else	297	else
298	clear_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);	298	clear_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);
299	break;	299	break;
300	case 1:	300	case 1:
301	/*	301	/*
302	* When soft_disable is set and enable is set, we want to	302	* When soft_disable is set and enable is set, we want to
303	* register the tracepoint for the event, but leave the event	303	* register the tracepoint for the event, but leave the event
304	* as is. That means, if the event was already enabled, we do	304	* as is. That means, if the event was already enabled, we do
305	* nothing (but set SOFT_MODE). If the event is disabled, we	305	* nothing (but set SOFT_MODE). If the event is disabled, we
306	* set SOFT_DISABLED before enabling the event tracepoint, so	306	* set SOFT_DISABLED before enabling the event tracepoint, so
307	* it still seems to be disabled.	307	* it still seems to be disabled.
308	*/	308	*/
309	if (!soft_disable)	309	if (!soft_disable)
310	clear_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);	310	clear_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);
311	else {	311	else {
312	if (atomic_inc_return(&file->sm_ref) > 1)	312	if (atomic_inc_return(&file->sm_ref) > 1)
313	break;	313	break;
314	set_bit(FTRACE_EVENT_FL_SOFT_MODE_BIT, &file->flags);	314	set_bit(FTRACE_EVENT_FL_SOFT_MODE_BIT, &file->flags);
315	}	315	}
316		316
317	if (!(file->flags & FTRACE_EVENT_FL_ENABLED)) {	317	if (!(file->flags & FTRACE_EVENT_FL_ENABLED)) {
318		318
319	/* Keep the event disabled, when going to SOFT_MODE. */	319	/* Keep the event disabled, when going to SOFT_MODE. */
320	if (soft_disable)	320	if (soft_disable)
321	set_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);	321	set_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags);
322		322
323	if (trace_flags & TRACE_ITER_RECORD_CMD) {	323	if (trace_flags & TRACE_ITER_RECORD_CMD) {
324	tracing_start_cmdline_record();	324	tracing_start_cmdline_record();
325	set_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);	325	set_bit(FTRACE_EVENT_FL_RECORDED_CMD_BIT, &file->flags);
326	}	326	}
327	ret = call->class->reg(call, TRACE_REG_REGISTER, file);	327	ret = call->class->reg(call, TRACE_REG_REGISTER, file);
328	if (ret) {	328	if (ret) {
329	tracing_stop_cmdline_record();	329	tracing_stop_cmdline_record();
330	pr_info("event trace: Could not enable event "	330	pr_info("event trace: Could not enable event "
331	"%s\n", call->name);	331	"%s\n", call->name);
332	break;	332	break;
333	}	333	}
334	set_bit(FTRACE_EVENT_FL_ENABLED_BIT, &file->flags);	334	set_bit(FTRACE_EVENT_FL_ENABLED_BIT, &file->flags);
335		335
336	/* WAS_ENABLED gets set but never cleared. */	336	/* WAS_ENABLED gets set but never cleared. */
337	call->flags \|= TRACE_EVENT_FL_WAS_ENABLED;	337	call->flags \|= TRACE_EVENT_FL_WAS_ENABLED;
338	}	338	}
339	break;	339	break;
340	}	340	}
341		341
342	return ret;	342	return ret;
343	}	343	}
344		344
345	static int ftrace_event_enable_disable(struct ftrace_event_file *file,	345	static int ftrace_event_enable_disable(struct ftrace_event_file *file,
346	int enable)	346	int enable)
347	{	347	{
348	return __ftrace_event_enable_disable(file, enable, 0);	348	return __ftrace_event_enable_disable(file, enable, 0);
349	}	349	}
350		350
351	static void ftrace_clear_events(struct trace_array *tr)	351	static void ftrace_clear_events(struct trace_array *tr)
352	{	352	{
353	struct ftrace_event_file *file;	353	struct ftrace_event_file *file;
354		354
355	mutex_lock(&event_mutex);	355	mutex_lock(&event_mutex);
356	list_for_each_entry(file, &tr->events, list) {	356	list_for_each_entry(file, &tr->events, list) {
357	ftrace_event_enable_disable(file, 0);	357	ftrace_event_enable_disable(file, 0);
358	}	358	}
359	mutex_unlock(&event_mutex);	359	mutex_unlock(&event_mutex);
360	}	360	}
361		361
362	static void __put_system(struct event_subsystem *system)	362	static void __put_system(struct event_subsystem *system)
363	{	363	{
364	struct event_filter *filter = system->filter;	364	struct event_filter *filter = system->filter;
365		365
366	WARN_ON_ONCE(system_refcount(system) == 0);	366	WARN_ON_ONCE(system_refcount(system) == 0);
367	if (system_refcount_dec(system))	367	if (system_refcount_dec(system))
368	return;	368	return;
369		369
370	list_del(&system->list);	370	list_del(&system->list);
371		371
372	if (filter) {	372	if (filter) {
373	kfree(filter->filter_string);	373	kfree(filter->filter_string);
374	kfree(filter);	374	kfree(filter);
375	}	375	}
376	if (system->ref_count & SYSTEM_FL_FREE_NAME)	376	if (system->ref_count & SYSTEM_FL_FREE_NAME)
377	kfree(system->name);	377	kfree(system->name);
378	kfree(system);	378	kfree(system);
379	}	379	}
380		380
381	static void __get_system(struct event_subsystem *system)	381	static void __get_system(struct event_subsystem *system)
382	{	382	{
383	WARN_ON_ONCE(system_refcount(system) == 0);	383	WARN_ON_ONCE(system_refcount(system) == 0);
384	system_refcount_inc(system);	384	system_refcount_inc(system);
385	}	385	}
386		386
387	static void __get_system_dir(struct ftrace_subsystem_dir *dir)	387	static void __get_system_dir(struct ftrace_subsystem_dir *dir)
388	{	388	{
389	WARN_ON_ONCE(dir->ref_count == 0);	389	WARN_ON_ONCE(dir->ref_count == 0);
390	dir->ref_count++;	390	dir->ref_count++;
391	__get_system(dir->subsystem);	391	__get_system(dir->subsystem);
392	}	392	}
393		393
394	static void __put_system_dir(struct ftrace_subsystem_dir *dir)	394	static void __put_system_dir(struct ftrace_subsystem_dir *dir)
395	{	395	{
396	WARN_ON_ONCE(dir->ref_count == 0);	396	WARN_ON_ONCE(dir->ref_count == 0);
397	/* If the subsystem is about to be freed, the dir must be too */	397	/* If the subsystem is about to be freed, the dir must be too */
398	WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);	398	WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
399		399
400	__put_system(dir->subsystem);	400	__put_system(dir->subsystem);
401	if (!--dir->ref_count)	401	if (!--dir->ref_count)
402	kfree(dir);	402	kfree(dir);
403	}	403	}
404		404
405	static void put_system(struct ftrace_subsystem_dir *dir)	405	static void put_system(struct ftrace_subsystem_dir *dir)
406	{	406	{
407	mutex_lock(&event_mutex);	407	mutex_lock(&event_mutex);
408	__put_system_dir(dir);	408	__put_system_dir(dir);
409	mutex_unlock(&event_mutex);	409	mutex_unlock(&event_mutex);
410	}	410	}
411		411
412	static void remove_subsystem(struct ftrace_subsystem_dir *dir)	412	static void remove_subsystem(struct ftrace_subsystem_dir *dir)
413	{	413	{
414	if (!dir)	414	if (!dir)
415	return;	415	return;
416		416
417	if (!--dir->nr_events) {	417	if (!--dir->nr_events) {
418	debugfs_remove_recursive(dir->entry);	418	debugfs_remove_recursive(dir->entry);
419	list_del(&dir->list);	419	list_del(&dir->list);
420	__put_system_dir(dir);	420	__put_system_dir(dir);
421	}	421	}
422	}	422	}
423		423
424	static void event_file_data(struct file filp)	424	static void event_file_data(struct file filp)
425	{	425	{
426	return ACCESS_ONCE(file_inode(filp)->i_private);	426	return ACCESS_ONCE(file_inode(filp)->i_private);
427	}	427	}
428		428
429	static void remove_event_file_dir(struct ftrace_event_file *file)	429	static void remove_event_file_dir(struct ftrace_event_file *file)
430	{	430	{
431	struct dentry *dir = file->dir;	431	struct dentry *dir = file->dir;
432	struct dentry *child;	432	struct dentry *child;
433		433
434	if (dir) {	434	if (dir) {
435	spin_lock(&dir->d_lock); /* probably unneeded */	435	spin_lock(&dir->d_lock); /* probably unneeded */
436	list_for_each_entry(child, &dir->d_subdirs, d_u.d_child) {	436	list_for_each_entry(child, &dir->d_subdirs, d_u.d_child) {
437	if (child->d_inode) /* probably unneeded */	437	if (child->d_inode) /* probably unneeded */
438	child->d_inode->i_private = NULL;	438	child->d_inode->i_private = NULL;
439	}	439	}
440	spin_unlock(&dir->d_lock);	440	spin_unlock(&dir->d_lock);
441		441
442	debugfs_remove_recursive(dir);	442	debugfs_remove_recursive(dir);
443	}	443	}
444		444
445	list_del(&file->list);	445	list_del(&file->list);
446	remove_subsystem(file->system);	446	remove_subsystem(file->system);
447	kmem_cache_free(file_cachep, file);	447	kmem_cache_free(file_cachep, file);
448	}	448	}
449		449
450	/*	450	/*
451	* __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.	451	* __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
452	*/	452	*/
453	static int	453	static int
454	__ftrace_set_clr_event_nolock(struct trace_array tr, const char match,	454	__ftrace_set_clr_event_nolock(struct trace_array tr, const char match,
455	const char sub, const char event, int set)	455	const char sub, const char event, int set)
456	{	456	{
457	struct ftrace_event_file *file;	457	struct ftrace_event_file *file;
458	struct ftrace_event_call *call;	458	struct ftrace_event_call *call;
459	int ret = -EINVAL;	459	int ret = -EINVAL;
460		460
461	list_for_each_entry(file, &tr->events, list) {	461	list_for_each_entry(file, &tr->events, list) {
462		462
463	call = file->event_call;	463	call = file->event_call;
464		464
465	if (!call->name \|\| !call->class \|\| !call->class->reg)	465	if (!call->name \|\| !call->class \|\| !call->class->reg)
466	continue;	466	continue;
467		467
468	if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)	468	if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
469	continue;	469	continue;
470		470
471	if (match &&	471	if (match &&
472	strcmp(match, call->name) != 0 &&	472	strcmp(match, call->name) != 0 &&
473	strcmp(match, call->class->system) != 0)	473	strcmp(match, call->class->system) != 0)
474	continue;	474	continue;
475		475
476	if (sub && strcmp(sub, call->class->system) != 0)	476	if (sub && strcmp(sub, call->class->system) != 0)
477	continue;	477	continue;
478		478
479	if (event && strcmp(event, call->name) != 0)	479	if (event && strcmp(event, call->name) != 0)
480	continue;	480	continue;
481		481
482	ftrace_event_enable_disable(file, set);	482	ftrace_event_enable_disable(file, set);
483		483
484	ret = 0;	484	ret = 0;
485	}	485	}
486		486
487	return ret;	487	return ret;
488	}	488	}
489		489
490	static int __ftrace_set_clr_event(struct trace_array tr, const char match,	490	static int __ftrace_set_clr_event(struct trace_array tr, const char match,
491	const char sub, const char event, int set)	491	const char sub, const char event, int set)
492	{	492	{
493	int ret;	493	int ret;
494		494
495	mutex_lock(&event_mutex);	495	mutex_lock(&event_mutex);
496	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);	496	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);
497	mutex_unlock(&event_mutex);	497	mutex_unlock(&event_mutex);
498		498
499	return ret;	499	return ret;
500	}	500	}
501		501
502	static int ftrace_set_clr_event(struct trace_array tr, char buf, int set)	502	static int ftrace_set_clr_event(struct trace_array tr, char buf, int set)
503	{	503	{
504	char event = NULL, sub = NULL, *match;	504	char event = NULL, sub = NULL, *match;
505		505
506	/*	506	/*
507	* The buf format can be <subsystem>:<event-name>	507	* The buf format can be <subsystem>:<event-name>
508	* *:<event-name> means any event by that name.	508	* *:<event-name> means any event by that name.
509	* :<event-name> is the same.	509	* :<event-name> is the same.
510	*	510	*
511	* <subsystem>:* means all events in that subsystem	511	* <subsystem>:* means all events in that subsystem
512	* <subsystem>: means the same.	512	* <subsystem>: means the same.
513	*	513	*
514	* <name> (no ':') means all events in a subsystem with	514	* <name> (no ':') means all events in a subsystem with
515	* the name <name> or any event that matches <name>	515	* the name <name> or any event that matches <name>
516	*/	516	*/
517		517
518	match = strsep(&buf, ":");	518	match = strsep(&buf, ":");
519	if (buf) {	519	if (buf) {
520	sub = match;	520	sub = match;
521	event = buf;	521	event = buf;
522	match = NULL;	522	match = NULL;
523		523
524	if (!strlen(sub) \|\| strcmp(sub, "*") == 0)	524	if (!strlen(sub) \|\| strcmp(sub, "*") == 0)
525	sub = NULL;	525	sub = NULL;
526	if (!strlen(event) \|\| strcmp(event, "*") == 0)	526	if (!strlen(event) \|\| strcmp(event, "*") == 0)
527	event = NULL;	527	event = NULL;
528	}	528	}
529		529
530	return __ftrace_set_clr_event(tr, match, sub, event, set);	530	return __ftrace_set_clr_event(tr, match, sub, event, set);
531	}	531	}
532		532
533	/**	533	/**
534	* trace_set_clr_event - enable or disable an event	534	* trace_set_clr_event - enable or disable an event
535	* @system: system name to match (NULL for any system)	535	* @system: system name to match (NULL for any system)
536	* @event: event name to match (NULL for all events, within system)	536	* @event: event name to match (NULL for all events, within system)
537	* @set: 1 to enable, 0 to disable	537	* @set: 1 to enable, 0 to disable
538	*	538	*
539	* This is a way for other parts of the kernel to enable or disable	539	* This is a way for other parts of the kernel to enable or disable
540	* event recording.	540	* event recording.
541	*	541	*
542	* Returns 0 on success, -EINVAL if the parameters do not match any	542	* Returns 0 on success, -EINVAL if the parameters do not match any
543	* registered events.	543	* registered events.
544	*/	544	*/
545	int trace_set_clr_event(const char system, const char event, int set)	545	int trace_set_clr_event(const char system, const char event, int set)
546	{	546	{
547	struct trace_array *tr = top_trace_array();	547	struct trace_array *tr = top_trace_array();
548		548
549	return __ftrace_set_clr_event(tr, NULL, system, event, set);	549	return __ftrace_set_clr_event(tr, NULL, system, event, set);
550	}	550	}
551	EXPORT_SYMBOL_GPL(trace_set_clr_event);	551	EXPORT_SYMBOL_GPL(trace_set_clr_event);
552		552
553	/* 128 should be much more than enough */	553	/* 128 should be much more than enough */
554	#define EVENT_BUF_SIZE 127	554	#define EVENT_BUF_SIZE 127
555		555
556	static ssize_t	556	static ssize_t
557	ftrace_event_write(struct file file, const char __user ubuf,	557	ftrace_event_write(struct file file, const char __user ubuf,
558	size_t cnt, loff_t *ppos)	558	size_t cnt, loff_t *ppos)
559	{	559	{
560	struct trace_parser parser;	560	struct trace_parser parser;
561	struct seq_file *m = file->private_data;	561	struct seq_file *m = file->private_data;
562	struct trace_array *tr = m->private;	562	struct trace_array *tr = m->private;
563	ssize_t read, ret;	563	ssize_t read, ret;
564		564
565	if (!cnt)	565	if (!cnt)
566	return 0;	566	return 0;
567		567
568	ret = tracing_update_buffers();	568	ret = tracing_update_buffers();
569	if (ret < 0)	569	if (ret < 0)
570	return ret;	570	return ret;
571		571
572	if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))	572	if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
573	return -ENOMEM;	573	return -ENOMEM;
574		574
575	read = trace_get_user(&parser, ubuf, cnt, ppos);	575	read = trace_get_user(&parser, ubuf, cnt, ppos);
576		576
577	if (read >= 0 && trace_parser_loaded((&parser))) {	577	if (read >= 0 && trace_parser_loaded((&parser))) {
578	int set = 1;	578	int set = 1;
579		579
580	if (*parser.buffer == '!')	580	if (*parser.buffer == '!')
581	set = 0;	581	set = 0;
582		582
583	parser.buffer[parser.idx] = 0;	583	parser.buffer[parser.idx] = 0;
584		584
585	ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);	585	ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
586	if (ret)	586	if (ret)
587	goto out_put;	587	goto out_put;
588	}	588	}
589		589
590	ret = read;	590	ret = read;
591		591
592	out_put:	592	out_put:
593	trace_parser_put(&parser);	593	trace_parser_put(&parser);
594		594
595	return ret;	595	return ret;
596	}	596	}
597		597
598	static void *	598	static void *
599	t_next(struct seq_file m, void v, loff_t *pos)	599	t_next(struct seq_file m, void v, loff_t *pos)
600	{	600	{
601	struct ftrace_event_file *file = v;	601	struct ftrace_event_file *file = v;
602	struct ftrace_event_call *call;	602	struct ftrace_event_call *call;
603	struct trace_array *tr = m->private;	603	struct trace_array *tr = m->private;
604		604
605	(*pos)++;	605	(*pos)++;
606		606
607	list_for_each_entry_continue(file, &tr->events, list) {	607	list_for_each_entry_continue(file, &tr->events, list) {
608	call = file->event_call;	608	call = file->event_call;
609	/*	609	/*
610	* The ftrace subsystem is for showing formats only.	610	* The ftrace subsystem is for showing formats only.
611	* They can not be enabled or disabled via the event files.	611	* They can not be enabled or disabled via the event files.
612	*/	612	*/
613	if (call->class && call->class->reg)	613	if (call->class && call->class->reg)
614	return file;	614	return file;
615	}	615	}
616		616
617	return NULL;	617	return NULL;
618	}	618	}
619		619
620	static void t_start(struct seq_file m, loff_t *pos)	620	static void t_start(struct seq_file m, loff_t *pos)
621	{	621	{
622	struct ftrace_event_file *file;	622	struct ftrace_event_file *file;
623	struct trace_array *tr = m->private;	623	struct trace_array *tr = m->private;
624	loff_t l;	624	loff_t l;
625		625
626	mutex_lock(&event_mutex);	626	mutex_lock(&event_mutex);
627		627
628	file = list_entry(&tr->events, struct ftrace_event_file, list);	628	file = list_entry(&tr->events, struct ftrace_event_file, list);
629	for (l = 0; l <= *pos; ) {	629	for (l = 0; l <= *pos; ) {
630	file = t_next(m, file, &l);	630	file = t_next(m, file, &l);
631	if (!file)	631	if (!file)
632	break;	632	break;
633	}	633	}
634	return file;	634	return file;
635	}	635	}
636		636
637	static void *	637	static void *
638	s_next(struct seq_file m, void v, loff_t *pos)	638	s_next(struct seq_file m, void v, loff_t *pos)
639	{	639	{
640	struct ftrace_event_file *file = v;	640	struct ftrace_event_file *file = v;
641	struct trace_array *tr = m->private;	641	struct trace_array *tr = m->private;
642		642
643	(*pos)++;	643	(*pos)++;
644		644
645	list_for_each_entry_continue(file, &tr->events, list) {	645	list_for_each_entry_continue(file, &tr->events, list) {
646	if (file->flags & FTRACE_EVENT_FL_ENABLED)	646	if (file->flags & FTRACE_EVENT_FL_ENABLED)
647	return file;	647	return file;
648	}	648	}
649		649
650	return NULL;	650	return NULL;
651	}	651	}
652		652
653	static void s_start(struct seq_file m, loff_t *pos)	653	static void s_start(struct seq_file m, loff_t *pos)
654	{	654	{
655	struct ftrace_event_file *file;	655	struct ftrace_event_file *file;
656	struct trace_array *tr = m->private;	656	struct trace_array *tr = m->private;
657	loff_t l;	657	loff_t l;
658		658
659	mutex_lock(&event_mutex);	659	mutex_lock(&event_mutex);
660		660
661	file = list_entry(&tr->events, struct ftrace_event_file, list);	661	file = list_entry(&tr->events, struct ftrace_event_file, list);
662	for (l = 0; l <= *pos; ) {	662	for (l = 0; l <= *pos; ) {
663	file = s_next(m, file, &l);	663	file = s_next(m, file, &l);
664	if (!file)	664	if (!file)
665	break;	665	break;
666	}	666	}
667	return file;	667	return file;
668	}	668	}
669		669
670	static int t_show(struct seq_file m, void v)	670	static int t_show(struct seq_file m, void v)
671	{	671	{
672	struct ftrace_event_file *file = v;	672	struct ftrace_event_file *file = v;
673	struct ftrace_event_call *call = file->event_call;	673	struct ftrace_event_call *call = file->event_call;
674		674
675	if (strcmp(call->class->system, TRACE_SYSTEM) != 0)	675	if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
676	seq_printf(m, "%s:", call->class->system);	676	seq_printf(m, "%s:", call->class->system);
677	seq_printf(m, "%s\n", call->name);	677	seq_printf(m, "%s\n", call->name);
678		678
679	return 0;	679	return 0;
680	}	680	}
681		681
682	static void t_stop(struct seq_file m, void p)	682	static void t_stop(struct seq_file m, void p)
683	{	683	{
684	mutex_unlock(&event_mutex);	684	mutex_unlock(&event_mutex);
685	}	685	}
686		686
687	static ssize_t	687	static ssize_t
688	event_enable_read(struct file filp, char __user ubuf, size_t cnt,	688	event_enable_read(struct file filp, char __user ubuf, size_t cnt,
689	loff_t *ppos)	689	loff_t *ppos)
690	{	690	{
691	struct ftrace_event_file *file;	691	struct ftrace_event_file *file;
692	unsigned long flags;	692	unsigned long flags;
693	char buf[4] = "0";	693	char buf[4] = "0";
694		694
695	mutex_lock(&event_mutex);	695	mutex_lock(&event_mutex);
696	file = event_file_data(filp);	696	file = event_file_data(filp);
697	if (likely(file))	697	if (likely(file))
698	flags = file->flags;	698	flags = file->flags;
699	mutex_unlock(&event_mutex);	699	mutex_unlock(&event_mutex);
700		700
701	if (!file)	701	if (!file)
702	return -ENODEV;	702	return -ENODEV;
703		703
704	if (flags & FTRACE_EVENT_FL_ENABLED &&	704	if (flags & FTRACE_EVENT_FL_ENABLED &&
705	!(flags & FTRACE_EVENT_FL_SOFT_DISABLED))	705	!(flags & FTRACE_EVENT_FL_SOFT_DISABLED))
706	strcpy(buf, "1");	706	strcpy(buf, "1");
707		707
708	if (flags & FTRACE_EVENT_FL_SOFT_DISABLED \|\|	708	if (flags & FTRACE_EVENT_FL_SOFT_DISABLED \|\|
709	flags & FTRACE_EVENT_FL_SOFT_MODE)	709	flags & FTRACE_EVENT_FL_SOFT_MODE)
710	strcat(buf, "*");	710	strcat(buf, "*");
711		711
712	strcat(buf, "\n");	712	strcat(buf, "\n");
713		713
714	return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));	714	return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
715	}	715	}
716		716
717	static ssize_t	717	static ssize_t
718	event_enable_write(struct file filp, const char __user ubuf, size_t cnt,	718	event_enable_write(struct file filp, const char __user ubuf, size_t cnt,
719	loff_t *ppos)	719	loff_t *ppos)
720	{	720	{
721	struct ftrace_event_file *file;	721	struct ftrace_event_file *file;
722	unsigned long val;	722	unsigned long val;
723	int ret;	723	int ret;
724		724
725	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);	725	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
726	if (ret)	726	if (ret)
727	return ret;	727	return ret;
728		728
729	ret = tracing_update_buffers();	729	ret = tracing_update_buffers();
730	if (ret < 0)	730	if (ret < 0)
731	return ret;	731	return ret;
732		732
733	switch (val) {	733	switch (val) {
734	case 0:	734	case 0:
735	case 1:	735	case 1:
736	ret = -ENODEV;	736	ret = -ENODEV;
737	mutex_lock(&event_mutex);	737	mutex_lock(&event_mutex);
738	file = event_file_data(filp);	738	file = event_file_data(filp);
739	if (likely(file))	739	if (likely(file))
740	ret = ftrace_event_enable_disable(file, val);	740	ret = ftrace_event_enable_disable(file, val);
741	mutex_unlock(&event_mutex);	741	mutex_unlock(&event_mutex);
742	break;	742	break;
743		743
744	default:	744	default:
745	return -EINVAL;	745	return -EINVAL;
746	}	746	}
747		747
748	*ppos += cnt;	748	*ppos += cnt;
749		749
750	return ret ? ret : cnt;	750	return ret ? ret : cnt;
751	}	751	}
752		752
753	static ssize_t	753	static ssize_t
754	system_enable_read(struct file filp, char __user ubuf, size_t cnt,	754	system_enable_read(struct file filp, char __user ubuf, size_t cnt,
755	loff_t *ppos)	755	loff_t *ppos)
756	{	756	{
757	const char set_to_char[4] = { '?', '0', '1', 'X' };	757	const char set_to_char[4] = { '?', '0', '1', 'X' };
758	struct ftrace_subsystem_dir *dir = filp->private_data;	758	struct ftrace_subsystem_dir *dir = filp->private_data;
759	struct event_subsystem *system = dir->subsystem;	759	struct event_subsystem *system = dir->subsystem;
760	struct ftrace_event_call *call;	760	struct ftrace_event_call *call;
761	struct ftrace_event_file *file;	761	struct ftrace_event_file *file;
762	struct trace_array *tr = dir->tr;	762	struct trace_array *tr = dir->tr;
763	char buf[2];	763	char buf[2];
764	int set = 0;	764	int set = 0;
765	int ret;	765	int ret;
766		766
767	mutex_lock(&event_mutex);	767	mutex_lock(&event_mutex);
768	list_for_each_entry(file, &tr->events, list) {	768	list_for_each_entry(file, &tr->events, list) {
769	call = file->event_call;	769	call = file->event_call;
770	if (!call->name \|\| !call->class \|\| !call->class->reg)	770	if (!call->name \|\| !call->class \|\| !call->class->reg)
771	continue;	771	continue;
772		772
773	if (system && strcmp(call->class->system, system->name) != 0)	773	if (system && strcmp(call->class->system, system->name) != 0)
774	continue;	774	continue;
775		775
776	/*	776	/*
777	* We need to find out if all the events are set	777	* We need to find out if all the events are set
778	* or if all events or cleared, or if we have	778	* or if all events or cleared, or if we have
779	* a mixture.	779	* a mixture.
780	*/	780	*/
781	set \|= (1 << !!(file->flags & FTRACE_EVENT_FL_ENABLED));	781	set \|= (1 << !!(file->flags & FTRACE_EVENT_FL_ENABLED));
782		782
783	/*	783	/*
784	* If we have a mixture, no need to look further.	784	* If we have a mixture, no need to look further.
785	*/	785	*/
786	if (set == 3)	786	if (set == 3)
787	break;	787	break;
788	}	788	}
789	mutex_unlock(&event_mutex);	789	mutex_unlock(&event_mutex);
790		790
791	buf[0] = set_to_char[set];	791	buf[0] = set_to_char[set];
792	buf[1] = '\n';	792	buf[1] = '\n';
793		793
794	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);	794	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
795		795
796	return ret;	796	return ret;
797	}	797	}
798		798
799	static ssize_t	799	static ssize_t
800	system_enable_write(struct file filp, const char __user ubuf, size_t cnt,	800	system_enable_write(struct file filp, const char __user ubuf, size_t cnt,
801	loff_t *ppos)	801	loff_t *ppos)
802	{	802	{
803	struct ftrace_subsystem_dir *dir = filp->private_data;	803	struct ftrace_subsystem_dir *dir = filp->private_data;
804	struct event_subsystem *system = dir->subsystem;	804	struct event_subsystem *system = dir->subsystem;
805	const char *name = NULL;	805	const char *name = NULL;
806	unsigned long val;	806	unsigned long val;
807	ssize_t ret;	807	ssize_t ret;
808		808
809	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);	809	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
810	if (ret)	810	if (ret)
811	return ret;	811	return ret;
812		812
813	ret = tracing_update_buffers();	813	ret = tracing_update_buffers();
814	if (ret < 0)	814	if (ret < 0)
815	return ret;	815	return ret;
816		816
817	if (val != 0 && val != 1)	817	if (val != 0 && val != 1)
818	return -EINVAL;	818	return -EINVAL;
819		819
820	/*	820	/*
821	* Opening of "enable" adds a ref count to system,	821	* Opening of "enable" adds a ref count to system,
822	* so the name is safe to use.	822	* so the name is safe to use.
823	*/	823	*/
824	if (system)	824	if (system)
825	name = system->name;	825	name = system->name;
826		826
827	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);	827	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);
828	if (ret)	828	if (ret)
829	goto out;	829	goto out;
830		830
831	ret = cnt;	831	ret = cnt;
832		832
833	out:	833	out:
834	*ppos += cnt;	834	*ppos += cnt;
835		835
836	return ret;	836	return ret;
837	}	837	}
838		838
839	enum {	839	enum {
840	FORMAT_HEADER = 1,	840	FORMAT_HEADER = 1,
841	FORMAT_FIELD_SEPERATOR = 2,	841	FORMAT_FIELD_SEPERATOR = 2,
842	FORMAT_PRINTFMT = 3,	842	FORMAT_PRINTFMT = 3,
843	};	843	};
844		844
845	static void f_next(struct seq_file m, void v, loff_t pos)	845	static void f_next(struct seq_file m, void v, loff_t pos)
846	{	846	{
847	struct ftrace_event_call *call = event_file_data(m->private);	847	struct ftrace_event_call *call = event_file_data(m->private);
848	struct list_head *common_head = &ftrace_common_fields;	848	struct list_head *common_head = &ftrace_common_fields;
849	struct list_head *head = trace_get_fields(call);	849	struct list_head *head = trace_get_fields(call);
850	struct list_head *node = v;	850	struct list_head *node = v;
851		851
852	(*pos)++;	852	(*pos)++;
853		853
854	switch ((unsigned long)v) {	854	switch ((unsigned long)v) {
855	case FORMAT_HEADER:	855	case FORMAT_HEADER:
856	node = common_head;	856	node = common_head;
857	break;	857	break;
858		858
859	case FORMAT_FIELD_SEPERATOR:	859	case FORMAT_FIELD_SEPERATOR:
860	node = head;	860	node = head;
861	break;	861	break;
862		862
863	case FORMAT_PRINTFMT:	863	case FORMAT_PRINTFMT:
864	/* all done */	864	/* all done */
865	return NULL;	865	return NULL;
866	}	866	}
867		867
868	node = node->prev;	868	node = node->prev;
869	if (node == common_head)	869	if (node == common_head)
870	return (void *)FORMAT_FIELD_SEPERATOR;	870	return (void *)FORMAT_FIELD_SEPERATOR;
871	else if (node == head)	871	else if (node == head)
872	return (void *)FORMAT_PRINTFMT;	872	return (void *)FORMAT_PRINTFMT;
873	else	873	else
874	return node;	874	return node;
875	}	875	}
876		876
877	static int f_show(struct seq_file m, void v)	877	static int f_show(struct seq_file m, void v)
878	{	878	{
879	struct ftrace_event_call *call = event_file_data(m->private);	879	struct ftrace_event_call *call = event_file_data(m->private);
880	struct ftrace_event_field *field;	880	struct ftrace_event_field *field;
881	const char *array_descriptor;	881	const char *array_descriptor;
882		882
883	switch ((unsigned long)v) {	883	switch ((unsigned long)v) {
884	case FORMAT_HEADER:	884	case FORMAT_HEADER:
885	seq_printf(m, "name: %s\n", call->name);	885	seq_printf(m, "name: %s\n", call->name);
886	seq_printf(m, "ID: %d\n", call->event.type);	886	seq_printf(m, "ID: %d\n", call->event.type);
887	seq_printf(m, "format:\n");	887	seq_printf(m, "format:\n");
888	return 0;	888	return 0;
889		889
890	case FORMAT_FIELD_SEPERATOR:	890	case FORMAT_FIELD_SEPERATOR:
891	seq_putc(m, '\n');	891	seq_putc(m, '\n');
892	return 0;	892	return 0;
893		893
894	case FORMAT_PRINTFMT:	894	case FORMAT_PRINTFMT:
895	seq_printf(m, "\nprint fmt: %s\n",	895	seq_printf(m, "\nprint fmt: %s\n",
896	call->print_fmt);	896	call->print_fmt);
897	return 0;	897	return 0;
898	}	898	}
899		899
900	field = list_entry(v, struct ftrace_event_field, link);	900	field = list_entry(v, struct ftrace_event_field, link);
901	/*	901	/*
902	* Smartly shows the array type(except dynamic array).	902	* Smartly shows the array type(except dynamic array).
903	* Normal:	903	* Normal:
904	* field:TYPE VAR	904	* field:TYPE VAR
905	* If TYPE := TYPE[LEN], it is shown:	905	* If TYPE := TYPE[LEN], it is shown:
906	* field:TYPE VAR[LEN]	906	* field:TYPE VAR[LEN]
907	*/	907	*/
908	array_descriptor = strchr(field->type, '[');	908	array_descriptor = strchr(field->type, '[');
909		909
910	if (!strncmp(field->type, "__data_loc", 10))	910	if (!strncmp(field->type, "__data_loc", 10))
911	array_descriptor = NULL;	911	array_descriptor = NULL;
912		912
913	if (!array_descriptor)	913	if (!array_descriptor)
914	seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",	914	seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
915	field->type, field->name, field->offset,	915	field->type, field->name, field->offset,
916	field->size, !!field->is_signed);	916	field->size, !!field->is_signed);
917	else	917	else
918	seq_printf(m, "\tfield:%.*s %s%s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",	918	seq_printf(m, "\tfield:%.*s %s%s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
919	(int)(array_descriptor - field->type),	919	(int)(array_descriptor - field->type),
920	field->type, field->name,	920	field->type, field->name,
921	array_descriptor, field->offset,	921	array_descriptor, field->offset,
922	field->size, !!field->is_signed);	922	field->size, !!field->is_signed);
923		923
924	return 0;	924	return 0;
925	}	925	}
926		926
927	static void f_start(struct seq_file m, loff_t *pos)	927	static void f_start(struct seq_file m, loff_t *pos)
928	{	928	{
929	void p = (void )FORMAT_HEADER;	929	void p = (void )FORMAT_HEADER;
930	loff_t l = 0;	930	loff_t l = 0;
931		931
932	/* ->stop() is called even if ->start() fails */	932	/* ->stop() is called even if ->start() fails */
933	mutex_lock(&event_mutex);	933	mutex_lock(&event_mutex);
934	if (!event_file_data(m->private))	934	if (!event_file_data(m->private))
935	return ERR_PTR(-ENODEV);	935	return ERR_PTR(-ENODEV);
936		936
937	while (l < *pos && p)	937	while (l < *pos && p)
938	p = f_next(m, p, &l);	938	p = f_next(m, p, &l);
939		939
940	return p;	940	return p;
941	}	941	}
942		942
943	static void f_stop(struct seq_file m, void p)	943	static void f_stop(struct seq_file m, void p)
944	{	944	{
945	mutex_unlock(&event_mutex);	945	mutex_unlock(&event_mutex);
946	}	946	}
947		947
948	static const struct seq_operations trace_format_seq_ops = {	948	static const struct seq_operations trace_format_seq_ops = {
949	.start = f_start,	949	.start = f_start,
950	.next = f_next,	950	.next = f_next,
951	.stop = f_stop,	951	.stop = f_stop,
952	.show = f_show,	952	.show = f_show,
953	};	953	};
954		954
955	static int trace_format_open(struct inode inode, struct file file)	955	static int trace_format_open(struct inode inode, struct file file)
956	{	956	{
957	struct seq_file *m;	957	struct seq_file *m;
958	int ret;	958	int ret;
959		959
960	ret = seq_open(file, &trace_format_seq_ops);	960	ret = seq_open(file, &trace_format_seq_ops);
961	if (ret < 0)	961	if (ret < 0)
962	return ret;	962	return ret;
963		963
964	m = file->private_data;	964	m = file->private_data;
965	m->private = file;	965	m->private = file;
966		966
967	return 0;	967	return 0;
968	}	968	}
969		969
970	static ssize_t	970	static ssize_t
971	event_id_read(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)	971	event_id_read(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)
972	{	972	{
973	int id = (long)event_file_data(filp);	973	int id = (long)event_file_data(filp);
974	char buf[32];	974	char buf[32];
975	int len;	975	int len;
976		976
977	if (*ppos)	977	if (*ppos)
978	return 0;	978	return 0;
979		979
980	if (unlikely(!id))	980	if (unlikely(!id))
981	return -ENODEV;	981	return -ENODEV;
982		982
983	len = sprintf(buf, "%d\n", id);	983	len = sprintf(buf, "%d\n", id);
984		984
985	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);	985	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
986	}	986	}
987		987
988	static ssize_t	988	static ssize_t
989	event_filter_read(struct file filp, char __user ubuf, size_t cnt,	989	event_filter_read(struct file filp, char __user ubuf, size_t cnt,
990	loff_t *ppos)	990	loff_t *ppos)
991	{	991	{
992	struct ftrace_event_call *call;	992	struct ftrace_event_call *call;
993	struct trace_seq *s;	993	struct trace_seq *s;
994	int r = -ENODEV;	994	int r = -ENODEV;
995		995
996	if (*ppos)	996	if (*ppos)
997	return 0;	997	return 0;
998		998
999	s = kmalloc(sizeof(*s), GFP_KERNEL);	999	s = kmalloc(sizeof(*s), GFP_KERNEL);
1000		1000
1001	if (!s)	1001	if (!s)
1002	return -ENOMEM;	1002	return -ENOMEM;
1003		1003
1004	trace_seq_init(s);	1004	trace_seq_init(s);
1005		1005
1006	mutex_lock(&event_mutex);	1006	mutex_lock(&event_mutex);
1007	call = event_file_data(filp);	1007	call = event_file_data(filp);
1008	if (call)	1008	if (call)
1009	print_event_filter(call, s);	1009	print_event_filter(call, s);
1010	mutex_unlock(&event_mutex);	1010	mutex_unlock(&event_mutex);
1011		1011
1012	if (call)	1012	if (call)
1013	r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);	1013	r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);
1014		1014
1015	kfree(s);	1015	kfree(s);
1016		1016
1017	return r;	1017	return r;
1018	}	1018	}
1019		1019
1020	static ssize_t	1020	static ssize_t
1021	event_filter_write(struct file filp, const char __user ubuf, size_t cnt,	1021	event_filter_write(struct file filp, const char __user ubuf, size_t cnt,
1022	loff_t *ppos)	1022	loff_t *ppos)
1023	{	1023	{
1024	struct ftrace_event_call *call;	1024	struct ftrace_event_call *call;
1025	char *buf;	1025	char *buf;
1026	int err = -ENODEV;	1026	int err = -ENODEV;
1027		1027
1028	if (cnt >= PAGE_SIZE)	1028	if (cnt >= PAGE_SIZE)
1029	return -EINVAL;	1029	return -EINVAL;
1030		1030
1031	buf = (char *)__get_free_page(GFP_TEMPORARY);	1031	buf = (char *)__get_free_page(GFP_TEMPORARY);
1032	if (!buf)	1032	if (!buf)
1033	return -ENOMEM;	1033	return -ENOMEM;
1034		1034
1035	if (copy_from_user(buf, ubuf, cnt)) {	1035	if (copy_from_user(buf, ubuf, cnt)) {
1036	free_page((unsigned long) buf);	1036	free_page((unsigned long) buf);
1037	return -EFAULT;	1037	return -EFAULT;
1038	}	1038	}
1039	buf[cnt] = '\0';	1039	buf[cnt] = '\0';
1040		1040
1041	mutex_lock(&event_mutex);	1041	mutex_lock(&event_mutex);
1042	call = event_file_data(filp);	1042	call = event_file_data(filp);
1043	if (call)	1043	if (call)
1044	err = apply_event_filter(call, buf);	1044	err = apply_event_filter(call, buf);
1045	mutex_unlock(&event_mutex);	1045	mutex_unlock(&event_mutex);
1046		1046
1047	free_page((unsigned long) buf);	1047	free_page((unsigned long) buf);
1048	if (err < 0)	1048	if (err < 0)
1049	return err;	1049	return err;
1050		1050
1051	*ppos += cnt;	1051	*ppos += cnt;
1052		1052
1053	return cnt;	1053	return cnt;
1054	}	1054	}
1055		1055
1056	static LIST_HEAD(event_subsystems);	1056	static LIST_HEAD(event_subsystems);
1057		1057
1058	static int subsystem_open(struct inode inode, struct file filp)	1058	static int subsystem_open(struct inode inode, struct file filp)
1059	{	1059	{
1060	struct event_subsystem *system = NULL;	1060	struct event_subsystem *system = NULL;
1061	struct ftrace_subsystem_dir dir = NULL; / Initialize for gcc */	1061	struct ftrace_subsystem_dir dir = NULL; / Initialize for gcc */
1062	struct trace_array *tr;	1062	struct trace_array *tr;
1063	int ret;	1063	int ret;
1064		1064
1065	/* Make sure the system still exists */	1065	/* Make sure the system still exists */
1066	mutex_lock(&trace_types_lock);	1066	mutex_lock(&trace_types_lock);
1067	mutex_lock(&event_mutex);	1067	mutex_lock(&event_mutex);
1068	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	1068	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1069	list_for_each_entry(dir, &tr->systems, list) {	1069	list_for_each_entry(dir, &tr->systems, list) {
1070	if (dir == inode->i_private) {	1070	if (dir == inode->i_private) {
1071	/* Don't open systems with no events */	1071	/* Don't open systems with no events */
1072	if (dir->nr_events) {	1072	if (dir->nr_events) {
1073	__get_system_dir(dir);	1073	__get_system_dir(dir);
1074	system = dir->subsystem;	1074	system = dir->subsystem;
1075	}	1075	}
1076	goto exit_loop;	1076	goto exit_loop;
1077	}	1077	}
1078	}	1078	}
1079	}	1079	}
1080	exit_loop:	1080	exit_loop:
1081	mutex_unlock(&event_mutex);	1081	mutex_unlock(&event_mutex);
1082	mutex_unlock(&trace_types_lock);	1082	mutex_unlock(&trace_types_lock);
1083		1083
1084	if (!system)	1084	if (!system)
1085	return -ENODEV;	1085	return -ENODEV;
1086		1086
1087	/* Some versions of gcc think dir can be uninitialized here */	1087	/* Some versions of gcc think dir can be uninitialized here */
1088	WARN_ON(!dir);	1088	WARN_ON(!dir);
1089		1089
1090	/* Still need to increment the ref count of the system */	1090	/* Still need to increment the ref count of the system */
1091	if (trace_array_get(tr) < 0) {	1091	if (trace_array_get(tr) < 0) {
1092	put_system(dir);	1092	put_system(dir);
1093	return -ENODEV;	1093	return -ENODEV;
1094	}	1094	}
1095		1095
1096	ret = tracing_open_generic(inode, filp);	1096	ret = tracing_open_generic(inode, filp);
1097	if (ret < 0) {	1097	if (ret < 0) {
1098	trace_array_put(tr);	1098	trace_array_put(tr);
1099	put_system(dir);	1099	put_system(dir);
1100	}	1100	}
1101		1101
1102	return ret;	1102	return ret;
1103	}	1103	}
1104		1104
1105	static int system_tr_open(struct inode inode, struct file filp)	1105	static int system_tr_open(struct inode inode, struct file filp)
1106	{	1106	{
1107	struct ftrace_subsystem_dir *dir;	1107	struct ftrace_subsystem_dir *dir;
1108	struct trace_array *tr = inode->i_private;	1108	struct trace_array *tr = inode->i_private;
1109	int ret;	1109	int ret;
1110		1110
1111	if (trace_array_get(tr) < 0)	1111	if (trace_array_get(tr) < 0)
1112	return -ENODEV;	1112	return -ENODEV;
1113		1113
1114	/* Make a temporary dir that has no system but points to tr */	1114	/* Make a temporary dir that has no system but points to tr */
1115	dir = kzalloc(sizeof(*dir), GFP_KERNEL);	1115	dir = kzalloc(sizeof(*dir), GFP_KERNEL);
1116	if (!dir) {	1116	if (!dir) {
1117	trace_array_put(tr);	1117	trace_array_put(tr);
1118	return -ENOMEM;	1118	return -ENOMEM;
1119	}	1119	}
1120		1120
1121	dir->tr = tr;	1121	dir->tr = tr;
1122		1122
1123	ret = tracing_open_generic(inode, filp);	1123	ret = tracing_open_generic(inode, filp);
1124	if (ret < 0) {	1124	if (ret < 0) {
1125	trace_array_put(tr);	1125	trace_array_put(tr);
1126	kfree(dir);	1126	kfree(dir);
1127	}	1127	}
1128		1128
1129	filp->private_data = dir;	1129	filp->private_data = dir;
1130		1130
1131	return ret;	1131	return ret;
1132	}	1132	}
1133		1133
1134	static int subsystem_release(struct inode inode, struct file file)	1134	static int subsystem_release(struct inode inode, struct file file)
1135	{	1135	{
1136	struct ftrace_subsystem_dir *dir = file->private_data;	1136	struct ftrace_subsystem_dir *dir = file->private_data;
1137		1137
1138	trace_array_put(dir->tr);	1138	trace_array_put(dir->tr);
1139		1139
1140	/*	1140	/*
1141	* If dir->subsystem is NULL, then this is a temporary	1141	* If dir->subsystem is NULL, then this is a temporary
1142	* descriptor that was made for a trace_array to enable	1142	* descriptor that was made for a trace_array to enable
1143	* all subsystems.	1143	* all subsystems.
1144	*/	1144	*/
1145	if (dir->subsystem)	1145	if (dir->subsystem)
1146	put_system(dir);	1146	put_system(dir);
1147	else	1147	else
1148	kfree(dir);	1148	kfree(dir);
1149		1149
1150	return 0;	1150	return 0;
1151	}	1151	}
1152		1152
1153	static ssize_t	1153	static ssize_t
1154	subsystem_filter_read(struct file filp, char __user ubuf, size_t cnt,	1154	subsystem_filter_read(struct file filp, char __user ubuf, size_t cnt,
1155	loff_t *ppos)	1155	loff_t *ppos)
1156	{	1156	{
1157	struct ftrace_subsystem_dir *dir = filp->private_data;	1157	struct ftrace_subsystem_dir *dir = filp->private_data;
1158	struct event_subsystem *system = dir->subsystem;	1158	struct event_subsystem *system = dir->subsystem;
1159	struct trace_seq *s;	1159	struct trace_seq *s;
1160	int r;	1160	int r;
1161		1161
1162	if (*ppos)	1162	if (*ppos)
1163	return 0;	1163	return 0;
1164		1164
1165	s = kmalloc(sizeof(*s), GFP_KERNEL);	1165	s = kmalloc(sizeof(*s), GFP_KERNEL);
1166	if (!s)	1166	if (!s)
1167	return -ENOMEM;	1167	return -ENOMEM;
1168		1168
1169	trace_seq_init(s);	1169	trace_seq_init(s);
1170		1170
1171	print_subsystem_event_filter(system, s);	1171	print_subsystem_event_filter(system, s);
1172	r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);	1172	r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);
1173		1173
1174	kfree(s);	1174	kfree(s);
1175		1175
1176	return r;	1176	return r;
1177	}	1177	}
1178		1178
1179	static ssize_t	1179	static ssize_t
1180	subsystem_filter_write(struct file filp, const char __user ubuf, size_t cnt,	1180	subsystem_filter_write(struct file filp, const char __user ubuf, size_t cnt,
1181	loff_t *ppos)	1181	loff_t *ppos)
1182	{	1182	{
1183	struct ftrace_subsystem_dir *dir = filp->private_data;	1183	struct ftrace_subsystem_dir *dir = filp->private_data;
1184	char *buf;	1184	char *buf;
1185	int err;	1185	int err;
1186		1186
1187	if (cnt >= PAGE_SIZE)	1187	if (cnt >= PAGE_SIZE)
1188	return -EINVAL;	1188	return -EINVAL;
1189		1189
1190	buf = (char *)__get_free_page(GFP_TEMPORARY);	1190	buf = (char *)__get_free_page(GFP_TEMPORARY);
1191	if (!buf)	1191	if (!buf)
1192	return -ENOMEM;	1192	return -ENOMEM;
1193		1193
1194	if (copy_from_user(buf, ubuf, cnt)) {	1194	if (copy_from_user(buf, ubuf, cnt)) {
1195	free_page((unsigned long) buf);	1195	free_page((unsigned long) buf);
1196	return -EFAULT;	1196	return -EFAULT;
1197	}	1197	}
1198	buf[cnt] = '\0';	1198	buf[cnt] = '\0';
1199		1199
1200	err = apply_subsystem_event_filter(dir, buf);	1200	err = apply_subsystem_event_filter(dir, buf);
1201	free_page((unsigned long) buf);	1201	free_page((unsigned long) buf);
1202	if (err < 0)	1202	if (err < 0)
1203	return err;	1203	return err;
1204		1204
1205	*ppos += cnt;	1205	*ppos += cnt;
1206		1206
1207	return cnt;	1207	return cnt;
1208	}	1208	}
1209		1209
1210	static ssize_t	1210	static ssize_t
1211	show_header(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)	1211	show_header(struct file filp, char __user ubuf, size_t cnt, loff_t *ppos)
1212	{	1212	{
1213	int (func)(struct trace_seq s) = filp->private_data;	1213	int (func)(struct trace_seq s) = filp->private_data;
1214	struct trace_seq *s;	1214	struct trace_seq *s;
1215	int r;	1215	int r;
1216		1216
1217	if (*ppos)	1217	if (*ppos)
1218	return 0;	1218	return 0;
1219		1219
1220	s = kmalloc(sizeof(*s), GFP_KERNEL);	1220	s = kmalloc(sizeof(*s), GFP_KERNEL);
1221	if (!s)	1221	if (!s)
1222	return -ENOMEM;	1222	return -ENOMEM;
1223		1223
1224	trace_seq_init(s);	1224	trace_seq_init(s);
1225		1225
1226	func(s);	1226	func(s);
1227	r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);	1227	r = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, s->len);
1228		1228
1229	kfree(s);	1229	kfree(s);
1230		1230
1231	return r;	1231	return r;
1232	}	1232	}
1233		1233
1234	static int ftrace_event_avail_open(struct inode inode, struct file file);	1234	static int ftrace_event_avail_open(struct inode inode, struct file file);
1235	static int ftrace_event_set_open(struct inode inode, struct file file);	1235	static int ftrace_event_set_open(struct inode inode, struct file file);
1236	static int ftrace_event_release(struct inode inode, struct file file);	1236	static int ftrace_event_release(struct inode inode, struct file file);
1237		1237
1238	static const struct seq_operations show_event_seq_ops = {	1238	static const struct seq_operations show_event_seq_ops = {
1239	.start = t_start,	1239	.start = t_start,
1240	.next = t_next,	1240	.next = t_next,
1241	.show = t_show,	1241	.show = t_show,
1242	.stop = t_stop,	1242	.stop = t_stop,
1243	};	1243	};
1244		1244
1245	static const struct seq_operations show_set_event_seq_ops = {	1245	static const struct seq_operations show_set_event_seq_ops = {
1246	.start = s_start,	1246	.start = s_start,
1247	.next = s_next,	1247	.next = s_next,
1248	.show = t_show,	1248	.show = t_show,
1249	.stop = t_stop,	1249	.stop = t_stop,
1250	};	1250	};
1251		1251
1252	static const struct file_operations ftrace_avail_fops = {	1252	static const struct file_operations ftrace_avail_fops = {
1253	.open = ftrace_event_avail_open,	1253	.open = ftrace_event_avail_open,
1254	.read = seq_read,	1254	.read = seq_read,
1255	.llseek = seq_lseek,	1255	.llseek = seq_lseek,
1256	.release = seq_release,	1256	.release = seq_release,
1257	};	1257	};
1258		1258
1259	static const struct file_operations ftrace_set_event_fops = {	1259	static const struct file_operations ftrace_set_event_fops = {
1260	.open = ftrace_event_set_open,	1260	.open = ftrace_event_set_open,
1261	.read = seq_read,	1261	.read = seq_read,
1262	.write = ftrace_event_write,	1262	.write = ftrace_event_write,
1263	.llseek = seq_lseek,	1263	.llseek = seq_lseek,
1264	.release = ftrace_event_release,	1264	.release = ftrace_event_release,
1265	};	1265	};
1266		1266
1267	static const struct file_operations ftrace_enable_fops = {	1267	static const struct file_operations ftrace_enable_fops = {
1268	.open = tracing_open_generic,	1268	.open = tracing_open_generic,
1269	.read = event_enable_read,	1269	.read = event_enable_read,
1270	.write = event_enable_write,	1270	.write = event_enable_write,
1271	.llseek = default_llseek,	1271	.llseek = default_llseek,
1272	};	1272	};
1273		1273
1274	static const struct file_operations ftrace_event_format_fops = {	1274	static const struct file_operations ftrace_event_format_fops = {
1275	.open = trace_format_open,	1275	.open = trace_format_open,
1276	.read = seq_read,	1276	.read = seq_read,
1277	.llseek = seq_lseek,	1277	.llseek = seq_lseek,
1278	.release = seq_release,	1278	.release = seq_release,
1279	};	1279	};
1280		1280
1281	static const struct file_operations ftrace_event_id_fops = {	1281	static const struct file_operations ftrace_event_id_fops = {
1282	.read = event_id_read,	1282	.read = event_id_read,
1283	.llseek = default_llseek,	1283	.llseek = default_llseek,
1284	};	1284	};
1285		1285
1286	static const struct file_operations ftrace_event_filter_fops = {	1286	static const struct file_operations ftrace_event_filter_fops = {
1287	.open = tracing_open_generic,	1287	.open = tracing_open_generic,
1288	.read = event_filter_read,	1288	.read = event_filter_read,
1289	.write = event_filter_write,	1289	.write = event_filter_write,
1290	.llseek = default_llseek,	1290	.llseek = default_llseek,
1291	};	1291	};
1292		1292
1293	static const struct file_operations ftrace_subsystem_filter_fops = {	1293	static const struct file_operations ftrace_subsystem_filter_fops = {
1294	.open = subsystem_open,	1294	.open = subsystem_open,
1295	.read = subsystem_filter_read,	1295	.read = subsystem_filter_read,
1296	.write = subsystem_filter_write,	1296	.write = subsystem_filter_write,
1297	.llseek = default_llseek,	1297	.llseek = default_llseek,
1298	.release = subsystem_release,	1298	.release = subsystem_release,
1299	};	1299	};
1300		1300
1301	static const struct file_operations ftrace_system_enable_fops = {	1301	static const struct file_operations ftrace_system_enable_fops = {
1302	.open = subsystem_open,	1302	.open = subsystem_open,
1303	.read = system_enable_read,	1303	.read = system_enable_read,
1304	.write = system_enable_write,	1304	.write = system_enable_write,
1305	.llseek = default_llseek,	1305	.llseek = default_llseek,
1306	.release = subsystem_release,	1306	.release = subsystem_release,
1307	};	1307	};
1308		1308
1309	static const struct file_operations ftrace_tr_enable_fops = {	1309	static const struct file_operations ftrace_tr_enable_fops = {
1310	.open = system_tr_open,	1310	.open = system_tr_open,
1311	.read = system_enable_read,	1311	.read = system_enable_read,
1312	.write = system_enable_write,	1312	.write = system_enable_write,
1313	.llseek = default_llseek,	1313	.llseek = default_llseek,
1314	.release = subsystem_release,	1314	.release = subsystem_release,
1315	};	1315	};
1316		1316
1317	static const struct file_operations ftrace_show_header_fops = {	1317	static const struct file_operations ftrace_show_header_fops = {
1318	.open = tracing_open_generic,	1318	.open = tracing_open_generic,
1319	.read = show_header,	1319	.read = show_header,
1320	.llseek = default_llseek,	1320	.llseek = default_llseek,
1321	};	1321	};
1322		1322
1323	static int	1323	static int
1324	ftrace_event_open(struct inode inode, struct file file,	1324	ftrace_event_open(struct inode inode, struct file file,
1325	const struct seq_operations *seq_ops)	1325	const struct seq_operations *seq_ops)
1326	{	1326	{
1327	struct seq_file *m;	1327	struct seq_file *m;
1328	int ret;	1328	int ret;
1329		1329
1330	ret = seq_open(file, seq_ops);	1330	ret = seq_open(file, seq_ops);
1331	if (ret < 0)	1331	if (ret < 0)
1332	return ret;	1332	return ret;
1333	m = file->private_data;	1333	m = file->private_data;
1334	/* copy tr over to seq ops */	1334	/* copy tr over to seq ops */
1335	m->private = inode->i_private;	1335	m->private = inode->i_private;
1336		1336
1337	return ret;	1337	return ret;
1338	}	1338	}
1339		1339
1340	static int ftrace_event_release(struct inode inode, struct file file)	1340	static int ftrace_event_release(struct inode inode, struct file file)
1341	{	1341	{
1342	struct trace_array *tr = inode->i_private;	1342	struct trace_array *tr = inode->i_private;
1343		1343
1344	trace_array_put(tr);	1344	trace_array_put(tr);
1345		1345
1346	return seq_release(inode, file);	1346	return seq_release(inode, file);
1347	}	1347	}
1348		1348
1349	static int	1349	static int
1350	ftrace_event_avail_open(struct inode inode, struct file file)	1350	ftrace_event_avail_open(struct inode inode, struct file file)
1351	{	1351	{
1352	const struct seq_operations *seq_ops = &show_event_seq_ops;	1352	const struct seq_operations *seq_ops = &show_event_seq_ops;
1353		1353
1354	return ftrace_event_open(inode, file, seq_ops);	1354	return ftrace_event_open(inode, file, seq_ops);
1355	}	1355	}
1356		1356
1357	static int	1357	static int
1358	ftrace_event_set_open(struct inode inode, struct file file)	1358	ftrace_event_set_open(struct inode inode, struct file file)
1359	{	1359	{
1360	const struct seq_operations *seq_ops = &show_set_event_seq_ops;	1360	const struct seq_operations *seq_ops = &show_set_event_seq_ops;
1361	struct trace_array *tr = inode->i_private;	1361	struct trace_array *tr = inode->i_private;
1362	int ret;	1362	int ret;
1363		1363
1364	if (trace_array_get(tr) < 0)	1364	if (trace_array_get(tr) < 0)
1365	return -ENODEV;	1365	return -ENODEV;
1366		1366
1367	if ((file->f_mode & FMODE_WRITE) &&	1367	if ((file->f_mode & FMODE_WRITE) &&
1368	(file->f_flags & O_TRUNC))	1368	(file->f_flags & O_TRUNC))
1369	ftrace_clear_events(tr);	1369	ftrace_clear_events(tr);
1370		1370
1371	ret = ftrace_event_open(inode, file, seq_ops);	1371	ret = ftrace_event_open(inode, file, seq_ops);
1372	if (ret < 0)	1372	if (ret < 0)
1373	trace_array_put(tr);	1373	trace_array_put(tr);
1374	return ret;	1374	return ret;
1375	}	1375	}
1376		1376
1377	static struct event_subsystem *	1377	static struct event_subsystem *
1378	create_new_subsystem(const char *name)	1378	create_new_subsystem(const char *name)
1379	{	1379	{
1380	struct event_subsystem *system;	1380	struct event_subsystem *system;
1381		1381
1382	/* need to create new entry */	1382	/* need to create new entry */
1383	system = kmalloc(sizeof(*system), GFP_KERNEL);	1383	system = kmalloc(sizeof(*system), GFP_KERNEL);
1384	if (!system)	1384	if (!system)
1385	return NULL;	1385	return NULL;
1386		1386
1387	system->ref_count = 1;	1387	system->ref_count = 1;
1388		1388
1389	/* Only allocate if dynamic (kprobes and modules) */	1389	/* Only allocate if dynamic (kprobes and modules) */
1390	if (!core_kernel_data((unsigned long)name)) {	1390	if (!core_kernel_data((unsigned long)name)) {
1391	system->ref_count \|= SYSTEM_FL_FREE_NAME;	1391	system->ref_count \|= SYSTEM_FL_FREE_NAME;
1392	system->name = kstrdup(name, GFP_KERNEL);	1392	system->name = kstrdup(name, GFP_KERNEL);
1393	if (!system->name)	1393	if (!system->name)
1394	goto out_free;	1394	goto out_free;
1395	} else	1395	} else
1396	system->name = name;	1396	system->name = name;
1397		1397
1398	system->filter = NULL;	1398	system->filter = NULL;
1399		1399
1400	system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);	1400	system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
1401	if (!system->filter)	1401	if (!system->filter)
1402	goto out_free;	1402	goto out_free;
1403		1403
1404	list_add(&system->list, &event_subsystems);	1404	list_add(&system->list, &event_subsystems);
1405		1405
1406	return system;	1406	return system;
1407		1407
1408	out_free:	1408	out_free:
1409	if (system->ref_count & SYSTEM_FL_FREE_NAME)	1409	if (system->ref_count & SYSTEM_FL_FREE_NAME)
1410	kfree(system->name);	1410	kfree(system->name);
1411	kfree(system);	1411	kfree(system);
1412	return NULL;	1412	return NULL;
1413	}	1413	}
1414		1414
1415	static struct dentry *	1415	static struct dentry *
1416	event_subsystem_dir(struct trace_array tr, const char name,	1416	event_subsystem_dir(struct trace_array tr, const char name,
1417	struct ftrace_event_file file, struct dentry parent)	1417	struct ftrace_event_file file, struct dentry parent)
1418	{	1418	{
1419	struct ftrace_subsystem_dir *dir;	1419	struct ftrace_subsystem_dir *dir;
1420	struct event_subsystem *system;	1420	struct event_subsystem *system;
1421	struct dentry *entry;	1421	struct dentry *entry;
1422		1422
1423	/* First see if we did not already create this dir */	1423	/* First see if we did not already create this dir */
1424	list_for_each_entry(dir, &tr->systems, list) {	1424	list_for_each_entry(dir, &tr->systems, list) {
1425	system = dir->subsystem;	1425	system = dir->subsystem;
1426	if (strcmp(system->name, name) == 0) {	1426	if (strcmp(system->name, name) == 0) {
1427	dir->nr_events++;	1427	dir->nr_events++;
1428	file->system = dir;	1428	file->system = dir;
1429	return dir->entry;	1429	return dir->entry;
1430	}	1430	}
1431	}	1431	}
1432		1432
1433	/* Now see if the system itself exists. */	1433	/* Now see if the system itself exists. */
1434	list_for_each_entry(system, &event_subsystems, list) {	1434	list_for_each_entry(system, &event_subsystems, list) {
1435	if (strcmp(system->name, name) == 0)	1435	if (strcmp(system->name, name) == 0)
1436	break;	1436	break;
1437	}	1437	}
1438	/* Reset system variable when not found */	1438	/* Reset system variable when not found */
1439	if (&system->list == &event_subsystems)	1439	if (&system->list == &event_subsystems)
1440	system = NULL;	1440	system = NULL;
1441		1441
1442	dir = kmalloc(sizeof(*dir), GFP_KERNEL);	1442	dir = kmalloc(sizeof(*dir), GFP_KERNEL);
1443	if (!dir)	1443	if (!dir)
1444	goto out_fail;	1444	goto out_fail;
1445		1445
1446	if (!system) {	1446	if (!system) {
1447	system = create_new_subsystem(name);	1447	system = create_new_subsystem(name);
1448	if (!system)	1448	if (!system)
1449	goto out_free;	1449	goto out_free;
1450	} else	1450	} else
1451	__get_system(system);	1451	__get_system(system);
1452		1452
1453	dir->entry = debugfs_create_dir(name, parent);	1453	dir->entry = debugfs_create_dir(name, parent);
1454	if (!dir->entry) {	1454	if (!dir->entry) {
1455	pr_warning("Failed to create system directory %s\n", name);	1455	pr_warning("Failed to create system directory %s\n", name);
1456	__put_system(system);	1456	__put_system(system);
1457	goto out_free;	1457	goto out_free;
1458	}	1458	}
1459		1459
1460	dir->tr = tr;	1460	dir->tr = tr;
1461	dir->ref_count = 1;	1461	dir->ref_count = 1;
1462	dir->nr_events = 1;	1462	dir->nr_events = 1;
1463	dir->subsystem = system;	1463	dir->subsystem = system;
1464	file->system = dir;	1464	file->system = dir;
1465		1465
1466	entry = debugfs_create_file("filter", 0644, dir->entry, dir,	1466	entry = debugfs_create_file("filter", 0644, dir->entry, dir,
1467	&ftrace_subsystem_filter_fops);	1467	&ftrace_subsystem_filter_fops);
1468	if (!entry) {	1468	if (!entry) {
1469	kfree(system->filter);	1469	kfree(system->filter);
1470	system->filter = NULL;	1470	system->filter = NULL;
1471	pr_warning("Could not create debugfs '%s/filter' entry\n", name);	1471	pr_warning("Could not create debugfs '%s/filter' entry\n", name);
1472	}	1472	}
1473		1473
1474	trace_create_file("enable", 0644, dir->entry, dir,	1474	trace_create_file("enable", 0644, dir->entry, dir,
1475	&ftrace_system_enable_fops);	1475	&ftrace_system_enable_fops);
1476		1476
1477	list_add(&dir->list, &tr->systems);	1477	list_add(&dir->list, &tr->systems);
1478		1478
1479	return dir->entry;	1479	return dir->entry;
1480		1480
1481	out_free:	1481	out_free:
1482	kfree(dir);	1482	kfree(dir);
1483	out_fail:	1483	out_fail:
1484	/* Only print this message if failed on memory allocation */	1484	/* Only print this message if failed on memory allocation */
1485	if (!dir \|\| !system)	1485	if (!dir \|\| !system)
1486	pr_warning("No memory to create event subsystem %s\n",	1486	pr_warning("No memory to create event subsystem %s\n",
1487	name);	1487	name);
1488	return NULL;	1488	return NULL;
1489	}	1489	}
1490		1490
1491	static int	1491	static int
1492	event_create_dir(struct dentry *parent,	1492	event_create_dir(struct dentry parent, struct ftrace_event_file file)
1493	struct ftrace_event_file *file,
1494	const struct file_operations *id,
1495	const struct file_operations *enable,
1496	const struct file_operations *filter,
1497	const struct file_operations *format)
1498	{	1493	{
1499	struct ftrace_event_call *call = file->event_call;	1494	struct ftrace_event_call *call = file->event_call;
1500	struct trace_array *tr = file->tr;	1495	struct trace_array *tr = file->tr;
1501	struct list_head *head;	1496	struct list_head *head;
1502	struct dentry *d_events;	1497	struct dentry *d_events;
1503	int ret;	1498	int ret;
1504		1499
1505	/*	1500	/*
1506	* If the trace point header did not define TRACE_SYSTEM	1501	* If the trace point header did not define TRACE_SYSTEM
1507	* then the system would be called "TRACE_SYSTEM".	1502	* then the system would be called "TRACE_SYSTEM".
1508	*/	1503	*/
1509	if (strcmp(call->class->system, TRACE_SYSTEM) != 0) {	1504	if (strcmp(call->class->system, TRACE_SYSTEM) != 0) {
1510	d_events = event_subsystem_dir(tr, call->class->system, file, parent);	1505	d_events = event_subsystem_dir(tr, call->class->system, file, parent);
1511	if (!d_events)	1506	if (!d_events)
1512	return -ENOMEM;	1507	return -ENOMEM;
1513	} else	1508	} else
1514	d_events = parent;	1509	d_events = parent;
1515		1510
1516	file->dir = debugfs_create_dir(call->name, d_events);	1511	file->dir = debugfs_create_dir(call->name, d_events);
1517	if (!file->dir) {	1512	if (!file->dir) {
1518	pr_warning("Could not create debugfs '%s' directory\n",	1513	pr_warning("Could not create debugfs '%s' directory\n",
1519	call->name);	1514	call->name);
1520	return -1;	1515	return -1;
1521	}	1516	}
1522		1517
1523	if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))	1518	if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1524	trace_create_file("enable", 0644, file->dir, file,	1519	trace_create_file("enable", 0644, file->dir, file,
1525	enable);	1520	&ftrace_enable_fops);
1526		1521
1527	#ifdef CONFIG_PERF_EVENTS	1522	#ifdef CONFIG_PERF_EVENTS
1528	if (call->event.type && call->class->reg)	1523	if (call->event.type && call->class->reg)
1529	trace_create_file("id", 0444, file->dir,	1524	trace_create_file("id", 0444, file->dir,
1530	(void *)(long)call->event.type, id);	1525	(void *)(long)call->event.type,
		1526	&ftrace_event_id_fops);
1531	#endif	1527	#endif
1532		1528
1533	/*	1529	/*
1534	* Other events may have the same class. Only update	1530	* Other events may have the same class. Only update
1535	* the fields if they are not already defined.	1531	* the fields if they are not already defined.
1536	*/	1532	*/
1537	head = trace_get_fields(call);	1533	head = trace_get_fields(call);
1538	if (list_empty(head)) {	1534	if (list_empty(head)) {
1539	ret = call->class->define_fields(call);	1535	ret = call->class->define_fields(call);
1540	if (ret < 0) {	1536	if (ret < 0) {
1541	pr_warning("Could not initialize trace point"	1537	pr_warning("Could not initialize trace point"
1542	" events/%s\n", call->name);	1538	" events/%s\n", call->name);
1543	return -1;	1539	return -1;
1544	}	1540	}
1545	}	1541	}
1546	trace_create_file("filter", 0644, file->dir, call,	1542	trace_create_file("filter", 0644, file->dir, call,
1547	filter);	1543	&ftrace_event_filter_fops);
1548		1544
1549	trace_create_file("format", 0444, file->dir, call,	1545	trace_create_file("format", 0444, file->dir, call,
1550	format);	1546	&ftrace_event_format_fops);
1551		1547
1552	return 0;	1548	return 0;
1553	}	1549	}
1554		1550
1555	static void remove_event_from_tracers(struct ftrace_event_call *call)	1551	static void remove_event_from_tracers(struct ftrace_event_call *call)
1556	{	1552	{
1557	struct ftrace_event_file *file;	1553	struct ftrace_event_file *file;
1558	struct trace_array *tr;	1554	struct trace_array *tr;
1559		1555
1560	do_for_each_event_file_safe(tr, file) {	1556	do_for_each_event_file_safe(tr, file) {
1561	if (file->event_call != call)	1557	if (file->event_call != call)
1562	continue;	1558	continue;
1563		1559
1564	remove_event_file_dir(file);	1560	remove_event_file_dir(file);
1565	/*	1561	/*
1566	* The do_for_each_event_file_safe() is	1562	* The do_for_each_event_file_safe() is
1567	* a double loop. After finding the call for this	1563	* a double loop. After finding the call for this
1568	* trace_array, we use break to jump to the next	1564	* trace_array, we use break to jump to the next
1569	* trace_array.	1565	* trace_array.
1570	*/	1566	*/
1571	break;	1567	break;
1572	} while_for_each_event_file();	1568	} while_for_each_event_file();
1573	}	1569	}
1574		1570
1575	static void event_remove(struct ftrace_event_call *call)	1571	static void event_remove(struct ftrace_event_call *call)
1576	{	1572	{
1577	struct trace_array *tr;	1573	struct trace_array *tr;
1578	struct ftrace_event_file *file;	1574	struct ftrace_event_file *file;
1579		1575
1580	do_for_each_event_file(tr, file) {	1576	do_for_each_event_file(tr, file) {
1581	if (file->event_call != call)	1577	if (file->event_call != call)
1582	continue;	1578	continue;
1583	ftrace_event_enable_disable(file, 0);	1579	ftrace_event_enable_disable(file, 0);
1584	/*	1580	/*
1585	* The do_for_each_event_file() is	1581	* The do_for_each_event_file() is
1586	* a double loop. After finding the call for this	1582	* a double loop. After finding the call for this
1587	* trace_array, we use break to jump to the next	1583	* trace_array, we use break to jump to the next
1588	* trace_array.	1584	* trace_array.
1589	*/	1585	*/
1590	break;	1586	break;
1591	} while_for_each_event_file();	1587	} while_for_each_event_file();
1592		1588
1593	if (call->event.funcs)	1589	if (call->event.funcs)
1594	__unregister_ftrace_event(&call->event);	1590	__unregister_ftrace_event(&call->event);
1595	remove_event_from_tracers(call);	1591	remove_event_from_tracers(call);
1596	list_del(&call->list);	1592	list_del(&call->list);
1597	}	1593	}
1598		1594
1599	static int event_init(struct ftrace_event_call *call)	1595	static int event_init(struct ftrace_event_call *call)
1600	{	1596	{
1601	int ret = 0;	1597	int ret = 0;
1602		1598
1603	if (WARN_ON(!call->name))	1599	if (WARN_ON(!call->name))
1604	return -EINVAL;	1600	return -EINVAL;
1605		1601
1606	if (call->class->raw_init) {	1602	if (call->class->raw_init) {
1607	ret = call->class->raw_init(call);	1603	ret = call->class->raw_init(call);
1608	if (ret < 0 && ret != -ENOSYS)	1604	if (ret < 0 && ret != -ENOSYS)
1609	pr_warn("Could not initialize trace events/%s\n",	1605	pr_warn("Could not initialize trace events/%s\n",
1610	call->name);	1606	call->name);
1611	}	1607	}
1612		1608
1613	return ret;	1609	return ret;
1614	}	1610	}
1615		1611
1616	static int	1612	static int
1617	__register_event(struct ftrace_event_call call, struct module mod)	1613	__register_event(struct ftrace_event_call call, struct module mod)
1618	{	1614	{
1619	int ret;	1615	int ret;
1620		1616
1621	ret = event_init(call);	1617	ret = event_init(call);
1622	if (ret < 0)	1618	if (ret < 0)
1623	return ret;	1619	return ret;
1624		1620
1625	list_add(&call->list, &ftrace_events);	1621	list_add(&call->list, &ftrace_events);
1626	call->mod = mod;	1622	call->mod = mod;
1627		1623
1628	return 0;	1624	return 0;
1629	}	1625	}
1630		1626
1631	static struct ftrace_event_file *	1627	static struct ftrace_event_file *
1632	trace_create_new_event(struct ftrace_event_call *call,	1628	trace_create_new_event(struct ftrace_event_call *call,
1633	struct trace_array *tr)	1629	struct trace_array *tr)
1634	{	1630	{
1635	struct ftrace_event_file *file;	1631	struct ftrace_event_file *file;
1636		1632
1637	file = kmem_cache_alloc(file_cachep, GFP_TRACE);	1633	file = kmem_cache_alloc(file_cachep, GFP_TRACE);
1638	if (!file)	1634	if (!file)
1639	return NULL;	1635	return NULL;
1640		1636
1641	file->event_call = call;	1637	file->event_call = call;
1642	file->tr = tr;	1638	file->tr = tr;
1643	atomic_set(&file->sm_ref, 0);	1639	atomic_set(&file->sm_ref, 0);
1644	list_add(&file->list, &tr->events);	1640	list_add(&file->list, &tr->events);
1645		1641
1646	return file;	1642	return file;
1647	}	1643	}
1648		1644
1649	/* Add an event to a trace directory */	1645	/* Add an event to a trace directory */
1650	static int	1646	static int
1651	__trace_add_new_event(struct ftrace_event_call *call,	1647	__trace_add_new_event(struct ftrace_event_call call, struct trace_array tr)
1652	struct trace_array *tr,
1653	const struct file_operations *id,
1654	const struct file_operations *enable,
1655	const struct file_operations *filter,
1656	const struct file_operations *format)
1657	{	1648	{
1658	struct ftrace_event_file *file;	1649	struct ftrace_event_file *file;
1659		1650
1660	file = trace_create_new_event(call, tr);	1651	file = trace_create_new_event(call, tr);
1661	if (!file)	1652	if (!file)
1662	return -ENOMEM;	1653	return -ENOMEM;
1663		1654
1664	return event_create_dir(tr->event_dir, file, id, enable, filter, format);	1655	return event_create_dir(tr->event_dir, file);
1665	}	1656	}
1666		1657
1667	/*	1658	/*
1668	* Just create a decriptor for early init. A descriptor is required	1659	* Just create a decriptor for early init. A descriptor is required
1669	* for enabling events at boot. We want to enable events before	1660	* for enabling events at boot. We want to enable events before
1670	* the filesystem is initialized.	1661	* the filesystem is initialized.
1671	*/	1662	*/
1672	static __init int	1663	static __init int
1673	__trace_early_add_new_event(struct ftrace_event_call *call,	1664	__trace_early_add_new_event(struct ftrace_event_call *call,
1674	struct trace_array *tr)	1665	struct trace_array *tr)
1675	{	1666	{
1676	struct ftrace_event_file *file;	1667	struct ftrace_event_file *file;
1677		1668
1678	file = trace_create_new_event(call, tr);	1669	file = trace_create_new_event(call, tr);
1679	if (!file)	1670	if (!file)
1680	return -ENOMEM;	1671	return -ENOMEM;
1681		1672
1682	return 0;	1673	return 0;
1683	}	1674	}
1684		1675
1685	struct ftrace_module_file_ops;	1676	struct ftrace_module_file_ops;
1686	static void __add_event_to_tracers(struct ftrace_event_call *call,	1677	static void __add_event_to_tracers(struct ftrace_event_call *call);
1687	struct ftrace_module_file_ops *file_ops);
1688		1678
1689	/* Add an additional event_call dynamically */	1679	/* Add an additional event_call dynamically */
1690	int trace_add_event_call(struct ftrace_event_call *call)	1680	int trace_add_event_call(struct ftrace_event_call *call)
1691	{	1681	{
1692	int ret;	1682	int ret;
1693	mutex_lock(&trace_types_lock);	1683	mutex_lock(&trace_types_lock);
1694	mutex_lock(&event_mutex);	1684	mutex_lock(&event_mutex);
1695		1685
1696	ret = __register_event(call, NULL);	1686	ret = __register_event(call, NULL);
1697	if (ret >= 0)	1687	if (ret >= 0)
1698	__add_event_to_tracers(call, NULL);	1688	__add_event_to_tracers(call);
1699		1689
1700	mutex_unlock(&event_mutex);	1690	mutex_unlock(&event_mutex);
1701	mutex_unlock(&trace_types_lock);	1691	mutex_unlock(&trace_types_lock);
1702	return ret;	1692	return ret;
1703	}	1693	}
1704		1694
1705	/*	1695	/*
1706	* Must be called under locking of trace_types_lock, event_mutex and	1696	* Must be called under locking of trace_types_lock, event_mutex and
1707	* trace_event_sem.	1697	* trace_event_sem.
1708	*/	1698	*/
1709	static void __trace_remove_event_call(struct ftrace_event_call *call)	1699	static void __trace_remove_event_call(struct ftrace_event_call *call)
1710	{	1700	{
1711	event_remove(call);	1701	event_remove(call);
1712	trace_destroy_fields(call);	1702	trace_destroy_fields(call);
1713	destroy_preds(call);	1703	destroy_preds(call);
1714	}	1704	}
1715		1705
1716	static int probe_remove_event_call(struct ftrace_event_call *call)	1706	static int probe_remove_event_call(struct ftrace_event_call *call)
1717	{	1707	{
1718	struct trace_array *tr;	1708	struct trace_array *tr;
1719	struct ftrace_event_file *file;	1709	struct ftrace_event_file *file;
1720		1710
1721	#ifdef CONFIG_PERF_EVENTS	1711	#ifdef CONFIG_PERF_EVENTS
1722	if (call->perf_refcount)	1712	if (call->perf_refcount)
1723	return -EBUSY;	1713	return -EBUSY;
1724	#endif	1714	#endif
1725	do_for_each_event_file(tr, file) {	1715	do_for_each_event_file(tr, file) {
1726	if (file->event_call != call)	1716	if (file->event_call != call)
1727	continue;	1717	continue;
1728	/*	1718	/*
1729	* We can't rely on ftrace_event_enable_disable(enable => 0)	1719	* We can't rely on ftrace_event_enable_disable(enable => 0)
1730	* we are going to do, FTRACE_EVENT_FL_SOFT_MODE can suppress	1720	* we are going to do, FTRACE_EVENT_FL_SOFT_MODE can suppress
1731	* TRACE_REG_UNREGISTER.	1721	* TRACE_REG_UNREGISTER.
1732	*/	1722	*/
1733	if (file->flags & FTRACE_EVENT_FL_ENABLED)	1723	if (file->flags & FTRACE_EVENT_FL_ENABLED)
1734	return -EBUSY;	1724	return -EBUSY;
1735	/*	1725	/*
1736	* The do_for_each_event_file_safe() is	1726	* The do_for_each_event_file_safe() is
1737	* a double loop. After finding the call for this	1727	* a double loop. After finding the call for this
1738	* trace_array, we use break to jump to the next	1728	* trace_array, we use break to jump to the next
1739	* trace_array.	1729	* trace_array.
1740	*/	1730	*/
1741	break;	1731	break;
1742	} while_for_each_event_file();	1732	} while_for_each_event_file();
1743		1733
1744	__trace_remove_event_call(call);	1734	__trace_remove_event_call(call);
1745		1735
1746	return 0;	1736	return 0;
1747	}	1737	}
1748		1738
1749	/* Remove an event_call */	1739	/* Remove an event_call */
1750	int trace_remove_event_call(struct ftrace_event_call *call)	1740	int trace_remove_event_call(struct ftrace_event_call *call)
1751	{	1741	{
1752	int ret;	1742	int ret;
1753		1743
1754	mutex_lock(&trace_types_lock);	1744	mutex_lock(&trace_types_lock);
1755	mutex_lock(&event_mutex);	1745	mutex_lock(&event_mutex);
1756	down_write(&trace_event_sem);	1746	down_write(&trace_event_sem);
1757	ret = probe_remove_event_call(call);	1747	ret = probe_remove_event_call(call);
1758	up_write(&trace_event_sem);	1748	up_write(&trace_event_sem);
1759	mutex_unlock(&event_mutex);	1749	mutex_unlock(&event_mutex);
1760	mutex_unlock(&trace_types_lock);	1750	mutex_unlock(&trace_types_lock);
1761		1751
1762	return ret;	1752	return ret;
1763	}	1753	}
1764		1754
1765	#define for_each_event(event, start, end) \	1755	#define for_each_event(event, start, end) \
1766	for (event = start; \	1756	for (event = start; \
1767	(unsigned long)event < (unsigned long)end; \	1757	(unsigned long)event < (unsigned long)end; \
1768	event++)	1758	event++)
1769		1759
1770	#ifdef CONFIG_MODULES	1760	#ifdef CONFIG_MODULES
1771		1761
1772	static LIST_HEAD(ftrace_module_file_list);
1773
1774	/*
1775	* Modules must own their file_operations to keep up with
1776	* reference counting.
1777	*/
1778	struct ftrace_module_file_ops {
1779	struct list_head list;
1780	struct module *mod;
1781	struct file_operations id;
1782	struct file_operations enable;
1783	struct file_operations format;
1784	struct file_operations filter;
1785	};
1786
1787	static struct ftrace_module_file_ops *
1788	find_ftrace_file_ops(struct ftrace_module_file_ops file_ops, struct module mod)
1789	{
1790	/*
1791	* As event_calls are added in groups by module,
1792	* when we find one file_ops, we don't need to search for
1793	* each call in that module, as the rest should be the
1794	* same. Only search for a new one if the last one did
1795	* not match.
1796	*/
1797	if (file_ops && mod == file_ops->mod)
1798	return file_ops;
1799
1800	list_for_each_entry(file_ops, &ftrace_module_file_list, list) {
1801	if (file_ops->mod == mod)
1802	return file_ops;
1803	}
1804	return NULL;
1805	}
1806
1807	static struct ftrace_module_file_ops *
1808	trace_create_file_ops(struct module *mod)
1809	{
1810	struct ftrace_module_file_ops *file_ops;
1811
1812	/*
1813	* This is a bit of a PITA. To allow for correct reference
1814	* counting, modules must "own" their file_operations.
1815	* To do this, we allocate the file operations that will be
1816	* used in the event directory.
1817	*/
1818
1819	file_ops = kmalloc(sizeof(*file_ops), GFP_KERNEL);
1820	if (!file_ops)
1821	return NULL;
1822
1823	file_ops->mod = mod;
1824
1825	file_ops->id = ftrace_event_id_fops;
1826	file_ops->id.owner = mod;
1827
1828	file_ops->enable = ftrace_enable_fops;
1829	file_ops->enable.owner = mod;
1830
1831	file_ops->filter = ftrace_event_filter_fops;
1832	file_ops->filter.owner = mod;
1833
1834	file_ops->format = ftrace_event_format_fops;
1835	file_ops->format.owner = mod;
1836
1837	list_add(&file_ops->list, &ftrace_module_file_list);
1838
1839	return file_ops;
1840	}
1841
1842	static void trace_module_add_events(struct module *mod)	1762	static void trace_module_add_events(struct module *mod)
1843	{	1763	{
1844	struct ftrace_module_file_ops *file_ops = NULL;
1845	struct ftrace_event_call call, start, **end;	1764	struct ftrace_event_call call, start, **end;
1846		1765
1847	start = mod->trace_events;	1766	start = mod->trace_events;
1848	end = mod->trace_events + mod->num_trace_events;	1767	end = mod->trace_events + mod->num_trace_events;
1849		1768
1850	if (start == end)
1851	return;
1852
1853	file_ops = trace_create_file_ops(mod);
1854	if (!file_ops)
1855	return;
1856
1857	for_each_event(call, start, end) {	1769	for_each_event(call, start, end) {
1858	__register_event(*call, mod);	1770	__register_event(*call, mod);
1859	__add_event_to_tracers(*call, file_ops);	1771	__add_event_to_tracers(*call);
1860	}	1772	}
1861	}	1773	}
1862		1774
1863	static void trace_module_remove_events(struct module *mod)	1775	static void trace_module_remove_events(struct module *mod)
1864	{	1776	{
1865	struct ftrace_module_file_ops *file_ops;
1866	struct ftrace_event_call call, p;	1777	struct ftrace_event_call call, p;
1867	bool clear_trace = false;	1778	bool clear_trace = false;
1868		1779
1869	down_write(&trace_event_sem);	1780	down_write(&trace_event_sem);
1870	list_for_each_entry_safe(call, p, &ftrace_events, list) {	1781	list_for_each_entry_safe(call, p, &ftrace_events, list) {
1871	if (call->mod == mod) {	1782	if (call->mod == mod) {
1872	if (call->flags & TRACE_EVENT_FL_WAS_ENABLED)	1783	if (call->flags & TRACE_EVENT_FL_WAS_ENABLED)
1873	clear_trace = true;	1784	clear_trace = true;
1874	__trace_remove_event_call(call);	1785	__trace_remove_event_call(call);
1875	}	1786	}
1876	}	1787	}
1877
1878	/* Now free the file_operations */
1879	list_for_each_entry(file_ops, &ftrace_module_file_list, list) {
1880	if (file_ops->mod == mod)
1881	break;
1882	}
1883	if (&file_ops->list != &ftrace_module_file_list) {
1884	list_del(&file_ops->list);
1885	kfree(file_ops);
1886	}
1887	up_write(&trace_event_sem);	1788	up_write(&trace_event_sem);
1888		1789
1889	/*	1790	/*
1890	* It is safest to reset the ring buffer if the module being unloaded	1791	* It is safest to reset the ring buffer if the module being unloaded
1891	* registered any events that were used. The only worry is if	1792	* registered any events that were used. The only worry is if
1892	* a new module gets loaded, and takes on the same id as the events	1793	* a new module gets loaded, and takes on the same id as the events
1893	* of this module. When printing out the buffer, traced events left	1794	* of this module. When printing out the buffer, traced events left
1894	* over from this module may be passed to the new module events and	1795	* over from this module may be passed to the new module events and
1895	* unexpected results may occur.	1796	* unexpected results may occur.
1896	*/	1797	*/
1897	if (clear_trace)	1798	if (clear_trace)
1898	tracing_reset_all_online_cpus();	1799	tracing_reset_all_online_cpus();
1899	}	1800	}
1900		1801
1901	static int trace_module_notify(struct notifier_block *self,	1802	static int trace_module_notify(struct notifier_block *self,
1902	unsigned long val, void *data)	1803	unsigned long val, void *data)
1903	{	1804	{
1904	struct module *mod = data;	1805	struct module *mod = data;
1905		1806
1906	mutex_lock(&trace_types_lock);	1807	mutex_lock(&trace_types_lock);
1907	mutex_lock(&event_mutex);	1808	mutex_lock(&event_mutex);
1908	switch (val) {	1809	switch (val) {
1909	case MODULE_STATE_COMING:	1810	case MODULE_STATE_COMING:
1910	trace_module_add_events(mod);	1811	trace_module_add_events(mod);
1911	break;	1812	break;
1912	case MODULE_STATE_GOING:	1813	case MODULE_STATE_GOING:
1913	trace_module_remove_events(mod);	1814	trace_module_remove_events(mod);
1914	break;	1815	break;
1915	}	1816	}
1916	mutex_unlock(&event_mutex);	1817	mutex_unlock(&event_mutex);
1917	mutex_unlock(&trace_types_lock);	1818	mutex_unlock(&trace_types_lock);
1918		1819
1919	return 0;	1820	return 0;
1920	}	1821	}
1921		1822
1922	static int	1823	static struct notifier_block trace_module_nb = {
1923	__trace_add_new_mod_event(struct ftrace_event_call *call,	1824	.notifier_call = trace_module_notify,
1924	struct trace_array *tr,	1825	.priority = 0,
1925	struct ftrace_module_file_ops *file_ops)	1826	};
1926	{
1927	return __trace_add_new_event(call, tr,
1928	&file_ops->id, &file_ops->enable,
1929	&file_ops->filter, &file_ops->format);
1930	}
1931
1932	#else
1933	static inline struct ftrace_module_file_ops *
1934	find_ftrace_file_ops(struct ftrace_module_file_ops file_ops, struct module mod)
1935	{
1936	return NULL;
1937	}
1938	static inline int trace_module_notify(struct notifier_block *self,
1939	unsigned long val, void *data)
1940	{
1941	return 0;
1942	}
1943	static inline int
1944	__trace_add_new_mod_event(struct ftrace_event_call *call,
1945	struct trace_array *tr,
1946	struct ftrace_module_file_ops *file_ops)
1947	{
1948	return -ENODEV;
1949	}
1950	#endif /* CONFIG_MODULES */	1827	#endif /* CONFIG_MODULES */
1951		1828
1952	/* Create a new event directory structure for a trace directory. */	1829	/* Create a new event directory structure for a trace directory. */
1953	static void	1830	static void
1954	__trace_add_event_dirs(struct trace_array *tr)	1831	__trace_add_event_dirs(struct trace_array *tr)
1955	{	1832	{
1956	struct ftrace_module_file_ops *file_ops = NULL;
1957	struct ftrace_event_call *call;	1833	struct ftrace_event_call *call;
1958	int ret;	1834	int ret;
1959		1835
1960	list_for_each_entry(call, &ftrace_events, list) {	1836	list_for_each_entry(call, &ftrace_events, list) {
1961	if (call->mod) {	1837	ret = __trace_add_new_event(call, tr);
1962	/*
1963	* Directories for events by modules need to
1964	* keep module ref counts when opened (as we don't
1965	* want the module to disappear when reading one
1966	* of these files). The file_ops keep account of
1967	* the module ref count.
1968	*/
1969	file_ops = find_ftrace_file_ops(file_ops, call->mod);
1970	if (!file_ops)
1971	continue; /* Warn? */
1972	ret = __trace_add_new_mod_event(call, tr, file_ops);
1973	if (ret < 0)
1974	pr_warning("Could not create directory for event %s\n",
1975	call->name);
1976	continue;
1977	}
1978	ret = __trace_add_new_event(call, tr,
1979	&ftrace_event_id_fops,
1980	&ftrace_enable_fops,
1981	&ftrace_event_filter_fops,
1982	&ftrace_event_format_fops);
1983	if (ret < 0)	1838	if (ret < 0)
1984	pr_warning("Could not create directory for event %s\n",	1839	pr_warning("Could not create directory for event %s\n",
1985	call->name);	1840	call->name);
1986	}	1841	}
1987	}	1842	}
1988		1843
1989	#ifdef CONFIG_DYNAMIC_FTRACE	1844	#ifdef CONFIG_DYNAMIC_FTRACE
1990		1845
1991	/* Avoid typos */	1846	/* Avoid typos */
1992	#define ENABLE_EVENT_STR "enable_event"	1847	#define ENABLE_EVENT_STR "enable_event"
1993	#define DISABLE_EVENT_STR "disable_event"	1848	#define DISABLE_EVENT_STR "disable_event"
1994		1849
1995	struct event_probe_data {	1850	struct event_probe_data {
1996	struct ftrace_event_file *file;	1851	struct ftrace_event_file *file;
1997	unsigned long count;	1852	unsigned long count;
1998	int ref;	1853	int ref;
1999	bool enable;	1854	bool enable;
2000	};	1855	};
2001		1856
2002	static struct ftrace_event_file *	1857	static struct ftrace_event_file *
2003	find_event_file(struct trace_array tr, const char system, const char *event)	1858	find_event_file(struct trace_array tr, const char system, const char *event)
2004	{	1859	{
2005	struct ftrace_event_file *file;	1860	struct ftrace_event_file *file;
2006	struct ftrace_event_call *call;	1861	struct ftrace_event_call *call;
2007		1862
2008	list_for_each_entry(file, &tr->events, list) {	1863	list_for_each_entry(file, &tr->events, list) {
2009		1864
2010	call = file->event_call;	1865	call = file->event_call;
2011		1866
2012	if (!call->name \|\| !call->class \|\| !call->class->reg)	1867	if (!call->name \|\| !call->class \|\| !call->class->reg)
2013	continue;	1868	continue;
2014		1869
2015	if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)	1870	if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
2016	continue;	1871	continue;
2017		1872
2018	if (strcmp(event, call->name) == 0 &&	1873	if (strcmp(event, call->name) == 0 &&
2019	strcmp(system, call->class->system) == 0)	1874	strcmp(system, call->class->system) == 0)
2020	return file;	1875	return file;
2021	}	1876	}
2022	return NULL;	1877	return NULL;
2023	}	1878	}
2024		1879
2025	static void	1880	static void
2026	event_enable_probe(unsigned long ip, unsigned long parent_ip, void **_data)	1881	event_enable_probe(unsigned long ip, unsigned long parent_ip, void **_data)
2027	{	1882	{
2028	struct event_probe_data pdata = (struct event_probe_data )_data;	1883	struct event_probe_data pdata = (struct event_probe_data )_data;
2029	struct event_probe_data data = pdata;	1884	struct event_probe_data data = pdata;
2030		1885
2031	if (!data)	1886	if (!data)
2032	return;	1887	return;
2033		1888
2034	if (data->enable)	1889	if (data->enable)
2035	clear_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &data->file->flags);	1890	clear_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &data->file->flags);
2036	else	1891	else
2037	set_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &data->file->flags);	1892	set_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &data->file->flags);
2038	}	1893	}
2039		1894
2040	static void	1895	static void
2041	event_enable_count_probe(unsigned long ip, unsigned long parent_ip, void **_data)	1896	event_enable_count_probe(unsigned long ip, unsigned long parent_ip, void **_data)
2042	{	1897	{
2043	struct event_probe_data pdata = (struct event_probe_data )_data;	1898	struct event_probe_data pdata = (struct event_probe_data )_data;
2044	struct event_probe_data data = pdata;	1899	struct event_probe_data data = pdata;
2045		1900
2046	if (!data)	1901	if (!data)
2047	return;	1902	return;
2048		1903
2049	if (!data->count)	1904	if (!data->count)
2050	return;	1905	return;
2051		1906
2052	/* Skip if the event is in a state we want to switch to */	1907	/* Skip if the event is in a state we want to switch to */
2053	if (data->enable == !(data->file->flags & FTRACE_EVENT_FL_SOFT_DISABLED))	1908	if (data->enable == !(data->file->flags & FTRACE_EVENT_FL_SOFT_DISABLED))
2054	return;	1909	return;
2055		1910
2056	if (data->count != -1)	1911	if (data->count != -1)
2057	(data->count)--;	1912	(data->count)--;
2058		1913
2059	event_enable_probe(ip, parent_ip, _data);	1914	event_enable_probe(ip, parent_ip, _data);
2060	}	1915	}
2061		1916
2062	static int	1917	static int
2063	event_enable_print(struct seq_file *m, unsigned long ip,	1918	event_enable_print(struct seq_file *m, unsigned long ip,
2064	struct ftrace_probe_ops ops, void _data)	1919	struct ftrace_probe_ops ops, void _data)
2065	{	1920	{
2066	struct event_probe_data *data = _data;	1921	struct event_probe_data *data = _data;
2067		1922
2068	seq_printf(m, "%ps:", (void *)ip);	1923	seq_printf(m, "%ps:", (void *)ip);
2069		1924
2070	seq_printf(m, "%s:%s:%s",	1925	seq_printf(m, "%s:%s:%s",
2071	data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,	1926	data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
2072	data->file->event_call->class->system,	1927	data->file->event_call->class->system,
2073	data->file->event_call->name);	1928	data->file->event_call->name);
2074		1929
2075	if (data->count == -1)	1930	if (data->count == -1)
2076	seq_printf(m, ":unlimited\n");	1931	seq_printf(m, ":unlimited\n");
2077	else	1932	else
2078	seq_printf(m, ":count=%ld\n", data->count);	1933	seq_printf(m, ":count=%ld\n", data->count);
2079		1934
2080	return 0;	1935	return 0;
2081	}	1936	}
2082		1937
2083	static int	1938	static int
2084	event_enable_init(struct ftrace_probe_ops *ops, unsigned long ip,	1939	event_enable_init(struct ftrace_probe_ops *ops, unsigned long ip,
2085	void **_data)	1940	void **_data)
2086	{	1941	{
2087	struct event_probe_data pdata = (struct event_probe_data )_data;	1942	struct event_probe_data pdata = (struct event_probe_data )_data;
2088	struct event_probe_data data = pdata;	1943	struct event_probe_data data = pdata;
2089		1944
2090	data->ref++;	1945	data->ref++;
2091	return 0;	1946	return 0;
2092	}	1947	}
2093		1948
2094	static void	1949	static void
2095	event_enable_free(struct ftrace_probe_ops *ops, unsigned long ip,	1950	event_enable_free(struct ftrace_probe_ops *ops, unsigned long ip,
2096	void **_data)	1951	void **_data)
2097	{	1952	{
2098	struct event_probe_data pdata = (struct event_probe_data )_data;	1953	struct event_probe_data pdata = (struct event_probe_data )_data;
2099	struct event_probe_data data = pdata;	1954	struct event_probe_data data = pdata;
2100		1955
2101	if (WARN_ON_ONCE(data->ref <= 0))	1956	if (WARN_ON_ONCE(data->ref <= 0))
2102	return;	1957	return;
2103		1958
2104	data->ref--;	1959	data->ref--;
2105	if (!data->ref) {	1960	if (!data->ref) {
2106	/* Remove the SOFT_MODE flag */	1961	/* Remove the SOFT_MODE flag */
2107	__ftrace_event_enable_disable(data->file, 0, 1);	1962	__ftrace_event_enable_disable(data->file, 0, 1);
2108	module_put(data->file->event_call->mod);	1963	module_put(data->file->event_call->mod);
2109	kfree(data);	1964	kfree(data);
2110	}	1965	}
2111	*pdata = NULL;	1966	*pdata = NULL;
2112	}	1967	}
2113		1968
2114	static struct ftrace_probe_ops event_enable_probe_ops = {	1969	static struct ftrace_probe_ops event_enable_probe_ops = {
2115	.func = event_enable_probe,	1970	.func = event_enable_probe,
2116	.print = event_enable_print,	1971	.print = event_enable_print,
2117	.init = event_enable_init,	1972	.init = event_enable_init,
2118	.free = event_enable_free,	1973	.free = event_enable_free,
2119	};	1974	};
2120		1975
2121	static struct ftrace_probe_ops event_enable_count_probe_ops = {	1976	static struct ftrace_probe_ops event_enable_count_probe_ops = {
2122	.func = event_enable_count_probe,	1977	.func = event_enable_count_probe,
2123	.print = event_enable_print,	1978	.print = event_enable_print,
2124	.init = event_enable_init,	1979	.init = event_enable_init,
2125	.free = event_enable_free,	1980	.free = event_enable_free,
2126	};	1981	};
2127		1982
2128	static struct ftrace_probe_ops event_disable_probe_ops = {	1983	static struct ftrace_probe_ops event_disable_probe_ops = {
2129	.func = event_enable_probe,	1984	.func = event_enable_probe,
2130	.print = event_enable_print,	1985	.print = event_enable_print,
2131	.init = event_enable_init,	1986	.init = event_enable_init,
2132	.free = event_enable_free,	1987	.free = event_enable_free,
2133	};	1988	};
2134		1989
2135	static struct ftrace_probe_ops event_disable_count_probe_ops = {	1990	static struct ftrace_probe_ops event_disable_count_probe_ops = {
2136	.func = event_enable_count_probe,	1991	.func = event_enable_count_probe,
2137	.print = event_enable_print,	1992	.print = event_enable_print,
2138	.init = event_enable_init,	1993	.init = event_enable_init,
2139	.free = event_enable_free,	1994	.free = event_enable_free,
2140	};	1995	};
2141		1996
2142	static int	1997	static int
2143	event_enable_func(struct ftrace_hash *hash,	1998	event_enable_func(struct ftrace_hash *hash,
2144	char glob, char cmd, char *param, int enabled)	1999	char glob, char cmd, char *param, int enabled)
2145	{	2000	{
2146	struct trace_array *tr = top_trace_array();	2001	struct trace_array *tr = top_trace_array();
2147	struct ftrace_event_file *file;	2002	struct ftrace_event_file *file;
2148	struct ftrace_probe_ops *ops;	2003	struct ftrace_probe_ops *ops;
2149	struct event_probe_data *data;	2004	struct event_probe_data *data;
2150	const char *system;	2005	const char *system;
2151	const char *event;	2006	const char *event;
2152	char *number;	2007	char *number;
2153	bool enable;	2008	bool enable;
2154	int ret;	2009	int ret;
2155		2010
2156	/* hash funcs only work with set_ftrace_filter */	2011	/* hash funcs only work with set_ftrace_filter */
2157	if (!enabled \|\| !param)	2012	if (!enabled \|\| !param)
2158	return -EINVAL;	2013	return -EINVAL;
2159		2014
2160	system = strsep(&param, ":");	2015	system = strsep(&param, ":");
2161	if (!param)	2016	if (!param)
2162	return -EINVAL;	2017	return -EINVAL;
2163		2018
2164	event = strsep(&param, ":");	2019	event = strsep(&param, ":");
2165		2020
2166	mutex_lock(&event_mutex);	2021	mutex_lock(&event_mutex);
2167		2022
2168	ret = -EINVAL;	2023	ret = -EINVAL;
2169	file = find_event_file(tr, system, event);	2024	file = find_event_file(tr, system, event);
2170	if (!file)	2025	if (!file)
2171	goto out;	2026	goto out;
2172		2027
2173	enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;	2028	enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
2174		2029
2175	if (enable)	2030	if (enable)
2176	ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;	2031	ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
2177	else	2032	else
2178	ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;	2033	ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
2179		2034
2180	if (glob[0] == '!') {	2035	if (glob[0] == '!') {
2181	unregister_ftrace_function_probe_func(glob+1, ops);	2036	unregister_ftrace_function_probe_func(glob+1, ops);
2182	ret = 0;	2037	ret = 0;
2183	goto out;	2038	goto out;
2184	}	2039	}
2185		2040
2186	ret = -ENOMEM;	2041	ret = -ENOMEM;
2187	data = kzalloc(sizeof(*data), GFP_KERNEL);	2042	data = kzalloc(sizeof(*data), GFP_KERNEL);
2188	if (!data)	2043	if (!data)
2189	goto out;	2044	goto out;
2190		2045
2191	data->enable = enable;	2046	data->enable = enable;
2192	data->count = -1;	2047	data->count = -1;
2193	data->file = file;	2048	data->file = file;
2194		2049
2195	if (!param)	2050	if (!param)
2196	goto out_reg;	2051	goto out_reg;
2197		2052
2198	number = strsep(&param, ":");	2053	number = strsep(&param, ":");
2199		2054
2200	ret = -EINVAL;	2055	ret = -EINVAL;
2201	if (!strlen(number))	2056	if (!strlen(number))
2202	goto out_free;	2057	goto out_free;
2203		2058
2204	/*	2059	/*
2205	* We use the callback data field (which is a pointer)	2060	* We use the callback data field (which is a pointer)
2206	* as our counter.	2061	* as our counter.
2207	*/	2062	*/
2208	ret = kstrtoul(number, 0, &data->count);	2063	ret = kstrtoul(number, 0, &data->count);
2209	if (ret)	2064	if (ret)
2210	goto out_free;	2065	goto out_free;
2211		2066
2212	out_reg:	2067	out_reg:
2213	/* Don't let event modules unload while probe registered */	2068	/* Don't let event modules unload while probe registered */
2214	ret = try_module_get(file->event_call->mod);	2069	ret = try_module_get(file->event_call->mod);
2215	if (!ret) {	2070	if (!ret) {
2216	ret = -EBUSY;	2071	ret = -EBUSY;
2217	goto out_free;	2072	goto out_free;
2218	}	2073	}
2219		2074
2220	ret = __ftrace_event_enable_disable(file, 1, 1);	2075	ret = __ftrace_event_enable_disable(file, 1, 1);
2221	if (ret < 0)	2076	if (ret < 0)
2222	goto out_put;	2077	goto out_put;
2223	ret = register_ftrace_function_probe(glob, ops, data);	2078	ret = register_ftrace_function_probe(glob, ops, data);
2224	/*	2079	/*
2225	* The above returns on success the # of functions enabled,	2080	* The above returns on success the # of functions enabled,
2226	* but if it didn't find any functions it returns zero.	2081	* but if it didn't find any functions it returns zero.
2227	* Consider no functions a failure too.	2082	* Consider no functions a failure too.
2228	*/	2083	*/
2229	if (!ret) {	2084	if (!ret) {
2230	ret = -ENOENT;	2085	ret = -ENOENT;
2231	goto out_disable;	2086	goto out_disable;
2232	} else if (ret < 0)	2087	} else if (ret < 0)
2233	goto out_disable;	2088	goto out_disable;
2234	/* Just return zero, not the number of enabled functions */	2089	/* Just return zero, not the number of enabled functions */
2235	ret = 0;	2090	ret = 0;
2236	out:	2091	out:
2237	mutex_unlock(&event_mutex);	2092	mutex_unlock(&event_mutex);
2238	return ret;	2093	return ret;
2239		2094
2240	out_disable:	2095	out_disable:
2241	__ftrace_event_enable_disable(file, 0, 1);	2096	__ftrace_event_enable_disable(file, 0, 1);
2242	out_put:	2097	out_put:
2243	module_put(file->event_call->mod);	2098	module_put(file->event_call->mod);
2244	out_free:	2099	out_free:
2245	kfree(data);	2100	kfree(data);
2246	goto out;	2101	goto out;
2247	}	2102	}
2248		2103
2249	static struct ftrace_func_command event_enable_cmd = {	2104	static struct ftrace_func_command event_enable_cmd = {
2250	.name = ENABLE_EVENT_STR,	2105	.name = ENABLE_EVENT_STR,
2251	.func = event_enable_func,	2106	.func = event_enable_func,
2252	};	2107	};
2253		2108
2254	static struct ftrace_func_command event_disable_cmd = {	2109	static struct ftrace_func_command event_disable_cmd = {
2255	.name = DISABLE_EVENT_STR,	2110	.name = DISABLE_EVENT_STR,
2256	.func = event_enable_func,	2111	.func = event_enable_func,
2257	};	2112	};
2258		2113
2259	static __init int register_event_cmds(void)	2114	static __init int register_event_cmds(void)
2260	{	2115	{
2261	int ret;	2116	int ret;
2262		2117
2263	ret = register_ftrace_command(&event_enable_cmd);	2118	ret = register_ftrace_command(&event_enable_cmd);
2264	if (WARN_ON(ret < 0))	2119	if (WARN_ON(ret < 0))
2265	return ret;	2120	return ret;
2266	ret = register_ftrace_command(&event_disable_cmd);	2121	ret = register_ftrace_command(&event_disable_cmd);
2267	if (WARN_ON(ret < 0))	2122	if (WARN_ON(ret < 0))
2268	unregister_ftrace_command(&event_enable_cmd);	2123	unregister_ftrace_command(&event_enable_cmd);
2269	return ret;	2124	return ret;
2270	}	2125	}
2271	#else	2126	#else
2272	static inline int register_event_cmds(void) { return 0; }	2127	static inline int register_event_cmds(void) { return 0; }
2273	#endif /* CONFIG_DYNAMIC_FTRACE */	2128	#endif /* CONFIG_DYNAMIC_FTRACE */
2274		2129
2275	/*	2130	/*
2276	* The top level array has already had its ftrace_event_file	2131	* The top level array has already had its ftrace_event_file
2277	* descriptors created in order to allow for early events to	2132	* descriptors created in order to allow for early events to
2278	* be recorded. This function is called after the debugfs has been	2133	* be recorded. This function is called after the debugfs has been
2279	* initialized, and we now have to create the files associated	2134	* initialized, and we now have to create the files associated
2280	* to the events.	2135	* to the events.
2281	*/	2136	*/
2282	static __init void	2137	static __init void
2283	__trace_early_add_event_dirs(struct trace_array *tr)	2138	__trace_early_add_event_dirs(struct trace_array *tr)
2284	{	2139	{
2285	struct ftrace_event_file *file;	2140	struct ftrace_event_file *file;
2286	int ret;	2141	int ret;
2287		2142
2288		2143
2289	list_for_each_entry(file, &tr->events, list) {	2144	list_for_each_entry(file, &tr->events, list) {
2290	ret = event_create_dir(tr->event_dir, file,	2145	ret = event_create_dir(tr->event_dir, file);
2291	&ftrace_event_id_fops,
2292	&ftrace_enable_fops,
2293	&ftrace_event_filter_fops,
2294	&ftrace_event_format_fops);
2295	if (ret < 0)	2146	if (ret < 0)
2296	pr_warning("Could not create directory for event %s\n",	2147	pr_warning("Could not create directory for event %s\n",
2297	file->event_call->name);	2148	file->event_call->name);
2298	}	2149	}
2299	}	2150	}
2300		2151
2301	/*	2152	/*
2302	* For early boot up, the top trace array requires to have	2153	* For early boot up, the top trace array requires to have
2303	* a list of events that can be enabled. This must be done before	2154	* a list of events that can be enabled. This must be done before
2304	* the filesystem is set up in order to allow events to be traced	2155	* the filesystem is set up in order to allow events to be traced
2305	* early.	2156	* early.
2306	*/	2157	*/
2307	static __init void	2158	static __init void
2308	__trace_early_add_events(struct trace_array *tr)	2159	__trace_early_add_events(struct trace_array *tr)
2309	{	2160	{
2310	struct ftrace_event_call *call;	2161	struct ftrace_event_call *call;
2311	int ret;	2162	int ret;
2312		2163
2313	list_for_each_entry(call, &ftrace_events, list) {	2164	list_for_each_entry(call, &ftrace_events, list) {
2314	/* Early boot up should not have any modules loaded */	2165	/* Early boot up should not have any modules loaded */
2315	if (WARN_ON_ONCE(call->mod))	2166	if (WARN_ON_ONCE(call->mod))
2316	continue;	2167	continue;
2317		2168
2318	ret = __trace_early_add_new_event(call, tr);	2169	ret = __trace_early_add_new_event(call, tr);
2319	if (ret < 0)	2170	if (ret < 0)
2320	pr_warning("Could not create early event %s\n",	2171	pr_warning("Could not create early event %s\n",
2321	call->name);	2172	call->name);
2322	}	2173	}
2323	}	2174	}
2324		2175
2325	/* Remove the event directory structure for a trace directory. */	2176	/* Remove the event directory structure for a trace directory. */
2326	static void	2177	static void
2327	__trace_remove_event_dirs(struct trace_array *tr)	2178	__trace_remove_event_dirs(struct trace_array *tr)
2328	{	2179	{
2329	struct ftrace_event_file file, next;	2180	struct ftrace_event_file file, next;
2330		2181
2331	list_for_each_entry_safe(file, next, &tr->events, list)	2182	list_for_each_entry_safe(file, next, &tr->events, list)
2332	remove_event_file_dir(file);	2183	remove_event_file_dir(file);
2333	}	2184	}
2334		2185
2335	static void	2186	static void __add_event_to_tracers(struct ftrace_event_call *call)
2336	__add_event_to_tracers(struct ftrace_event_call *call,
2337	struct ftrace_module_file_ops *file_ops)
2338	{	2187	{
2339	struct trace_array *tr;	2188	struct trace_array *tr;
2340		2189
2341	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	2190	list_for_each_entry(tr, &ftrace_trace_arrays, list)
2342	if (file_ops)	2191	__trace_add_new_event(call, tr);
2343	__trace_add_new_mod_event(call, tr, file_ops);
2344	else
2345	__trace_add_new_event(call, tr,
2346	&ftrace_event_id_fops,
2347	&ftrace_enable_fops,
2348	&ftrace_event_filter_fops,
2349	&ftrace_event_format_fops);
2350	}
2351	}	2192	}
2352		2193
2353	static struct notifier_block trace_module_nb = {
2354	.notifier_call = trace_module_notify,
2355	.priority = 0,
2356	};
2357
2358	extern struct ftrace_event_call *__start_ftrace_events[];	2194	extern struct ftrace_event_call *__start_ftrace_events[];
2359	extern struct ftrace_event_call *__stop_ftrace_events[];	2195	extern struct ftrace_event_call *__stop_ftrace_events[];
2360		2196
2361	static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;	2197	static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
2362		2198
2363	static __init int setup_trace_event(char *str)	2199	static __init int setup_trace_event(char *str)
2364	{	2200	{
2365	strlcpy(bootup_event_buf, str, COMMAND_LINE_SIZE);	2201	strlcpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
2366	ring_buffer_expanded = true;	2202	ring_buffer_expanded = true;
2367	tracing_selftest_disabled = true;	2203	tracing_selftest_disabled = true;
2368		2204
2369	return 1;	2205	return 1;
2370	}	2206	}
2371	__setup("trace_event=", setup_trace_event);	2207	__setup("trace_event=", setup_trace_event);
2372		2208
2373	/* Expects to have event_mutex held when called */	2209	/* Expects to have event_mutex held when called */
2374	static int	2210	static int
2375	create_event_toplevel_files(struct dentry parent, struct trace_array tr)	2211	create_event_toplevel_files(struct dentry parent, struct trace_array tr)
2376	{	2212	{
2377	struct dentry *d_events;	2213	struct dentry *d_events;
2378	struct dentry *entry;	2214	struct dentry *entry;
2379		2215
2380	entry = debugfs_create_file("set_event", 0644, parent,	2216	entry = debugfs_create_file("set_event", 0644, parent,
2381	tr, &ftrace_set_event_fops);	2217	tr, &ftrace_set_event_fops);
2382	if (!entry) {	2218	if (!entry) {
2383	pr_warning("Could not create debugfs 'set_event' entry\n");	2219	pr_warning("Could not create debugfs 'set_event' entry\n");
2384	return -ENOMEM;	2220	return -ENOMEM;
2385	}	2221	}
2386		2222
2387	d_events = debugfs_create_dir("events", parent);	2223	d_events = debugfs_create_dir("events", parent);
2388	if (!d_events) {	2224	if (!d_events) {
2389	pr_warning("Could not create debugfs 'events' directory\n");	2225	pr_warning("Could not create debugfs 'events' directory\n");
2390	return -ENOMEM;	2226	return -ENOMEM;
2391	}	2227	}
2392		2228
2393	/* ring buffer internal formats */	2229	/* ring buffer internal formats */
2394	trace_create_file("header_page", 0444, d_events,	2230	trace_create_file("header_page", 0444, d_events,
2395	ring_buffer_print_page_header,	2231	ring_buffer_print_page_header,
2396	&ftrace_show_header_fops);	2232	&ftrace_show_header_fops);
2397		2233
2398	trace_create_file("header_event", 0444, d_events,	2234	trace_create_file("header_event", 0444, d_events,
2399	ring_buffer_print_entry_header,	2235	ring_buffer_print_entry_header,
2400	&ftrace_show_header_fops);	2236	&ftrace_show_header_fops);
2401		2237
2402	trace_create_file("enable", 0644, d_events,	2238	trace_create_file("enable", 0644, d_events,
2403	tr, &ftrace_tr_enable_fops);	2239	tr, &ftrace_tr_enable_fops);
2404		2240
2405	tr->event_dir = d_events;	2241	tr->event_dir = d_events;
2406		2242
2407	return 0;	2243	return 0;
2408	}	2244	}
2409		2245
2410	/**	2246	/**
2411	* event_trace_add_tracer - add a instance of a trace_array to events	2247	* event_trace_add_tracer - add a instance of a trace_array to events
2412	* @parent: The parent dentry to place the files/directories for events in	2248	* @parent: The parent dentry to place the files/directories for events in
2413	* @tr: The trace array associated with these events	2249	* @tr: The trace array associated with these events
2414	*	2250	*
2415	* When a new instance is created, it needs to set up its events	2251	* When a new instance is created, it needs to set up its events
2416	* directory, as well as other files associated with events. It also	2252	* directory, as well as other files associated with events. It also
2417	* creates the event hierachry in the @parent/events directory.	2253	* creates the event hierachry in the @parent/events directory.
2418	*	2254	*
2419	* Returns 0 on success.	2255	* Returns 0 on success.
2420	*/	2256	*/
2421	int event_trace_add_tracer(struct dentry parent, struct trace_array tr)	2257	int event_trace_add_tracer(struct dentry parent, struct trace_array tr)
2422	{	2258	{
2423	int ret;	2259	int ret;
2424		2260
2425	mutex_lock(&event_mutex);	2261	mutex_lock(&event_mutex);
2426		2262
2427	ret = create_event_toplevel_files(parent, tr);	2263	ret = create_event_toplevel_files(parent, tr);
2428	if (ret)	2264	if (ret)
2429	goto out_unlock;	2265	goto out_unlock;
2430		2266
2431	down_write(&trace_event_sem);	2267	down_write(&trace_event_sem);
2432	__trace_add_event_dirs(tr);	2268	__trace_add_event_dirs(tr);
2433	up_write(&trace_event_sem);	2269	up_write(&trace_event_sem);
2434		2270
2435	out_unlock:	2271	out_unlock:
2436	mutex_unlock(&event_mutex);	2272	mutex_unlock(&event_mutex);
2437		2273
2438	return ret;	2274	return ret;
2439	}	2275	}
2440		2276
2441	/*	2277	/*
2442	* The top trace array already had its file descriptors created.	2278	* The top trace array already had its file descriptors created.
2443	* Now the files themselves need to be created.	2279	* Now the files themselves need to be created.
2444	*/	2280	*/
2445	static __init int	2281	static __init int
2446	early_event_add_tracer(struct dentry parent, struct trace_array tr)	2282	early_event_add_tracer(struct dentry parent, struct trace_array tr)
2447	{	2283	{
2448	int ret;	2284	int ret;
2449		2285
2450	mutex_lock(&event_mutex);	2286	mutex_lock(&event_mutex);
2451		2287
2452	ret = create_event_toplevel_files(parent, tr);	2288	ret = create_event_toplevel_files(parent, tr);
2453	if (ret)	2289	if (ret)
2454	goto out_unlock;	2290	goto out_unlock;
2455		2291
2456	down_write(&trace_event_sem);	2292	down_write(&trace_event_sem);
2457	__trace_early_add_event_dirs(tr);	2293	__trace_early_add_event_dirs(tr);
2458	up_write(&trace_event_sem);	2294	up_write(&trace_event_sem);
2459		2295
2460	out_unlock:	2296	out_unlock:
2461	mutex_unlock(&event_mutex);	2297	mutex_unlock(&event_mutex);
2462		2298
2463	return ret;	2299	return ret;
2464	}	2300	}
2465		2301
2466	int event_trace_del_tracer(struct trace_array *tr)	2302	int event_trace_del_tracer(struct trace_array *tr)
2467	{	2303	{
2468	mutex_lock(&event_mutex);	2304	mutex_lock(&event_mutex);
2469		2305
2470	/* Disable any running events */	2306	/* Disable any running events */
2471	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);	2307	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);
2472		2308
2473	down_write(&trace_event_sem);	2309	down_write(&trace_event_sem);
2474	__trace_remove_event_dirs(tr);	2310	__trace_remove_event_dirs(tr);
2475	debugfs_remove_recursive(tr->event_dir);	2311	debugfs_remove_recursive(tr->event_dir);
2476	up_write(&trace_event_sem);	2312	up_write(&trace_event_sem);
2477		2313
2478	tr->event_dir = NULL;	2314	tr->event_dir = NULL;
2479		2315
2480	mutex_unlock(&event_mutex);	2316	mutex_unlock(&event_mutex);
2481		2317
2482	return 0;	2318	return 0;
2483	}	2319	}
2484		2320
2485	static __init int event_trace_memsetup(void)	2321	static __init int event_trace_memsetup(void)
2486	{	2322	{
2487	field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);	2323	field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
2488	file_cachep = KMEM_CACHE(ftrace_event_file, SLAB_PANIC);	2324	file_cachep = KMEM_CACHE(ftrace_event_file, SLAB_PANIC);
2489	return 0;	2325	return 0;
2490	}	2326	}
2491		2327
2492	static __init int event_trace_enable(void)	2328	static __init int event_trace_enable(void)
2493	{	2329	{
2494	struct trace_array *tr = top_trace_array();	2330	struct trace_array *tr = top_trace_array();
2495	struct ftrace_event_call *iter, call;	2331	struct ftrace_event_call *iter, call;
2496	char *buf = bootup_event_buf;	2332	char *buf = bootup_event_buf;
2497	char *token;	2333	char *token;
2498	int ret;	2334	int ret;
2499		2335
2500	for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {	2336	for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
2501		2337
2502	call = *iter;	2338	call = *iter;
2503	ret = event_init(call);	2339	ret = event_init(call);
2504	if (!ret)	2340	if (!ret)
2505	list_add(&call->list, &ftrace_events);	2341	list_add(&call->list, &ftrace_events);
2506	}	2342	}
2507		2343
2508	/*	2344	/*
2509	* We need the top trace array to have a working set of trace	2345	* We need the top trace array to have a working set of trace
2510	* points at early init, before the debug files and directories	2346	* points at early init, before the debug files and directories
2511	* are created. Create the file entries now, and attach them	2347	* are created. Create the file entries now, and attach them
2512	* to the actual file dentries later.	2348	* to the actual file dentries later.
2513	*/	2349	*/
2514	__trace_early_add_events(tr);	2350	__trace_early_add_events(tr);
2515		2351
2516	while (true) {	2352	while (true) {
2517	token = strsep(&buf, ",");	2353	token = strsep(&buf, ",");
2518		2354
2519	if (!token)	2355	if (!token)
2520	break;	2356	break;
2521	if (!*token)	2357	if (!*token)
2522	continue;	2358	continue;
2523		2359
2524	ret = ftrace_set_clr_event(tr, token, 1);	2360	ret = ftrace_set_clr_event(tr, token, 1);
2525	if (ret)	2361	if (ret)
2526	pr_warn("Failed to enable trace event: %s\n", token);	2362	pr_warn("Failed to enable trace event: %s\n", token);
2527	}	2363	}
2528		2364
2529	trace_printk_start_comm();	2365	trace_printk_start_comm();
2530		2366
2531	register_event_cmds();	2367	register_event_cmds();
2532		2368
2533	return 0;	2369	return 0;
2534	}	2370	}
2535		2371
2536	static __init int event_trace_init(void)	2372	static __init int event_trace_init(void)
2537	{	2373	{
2538	struct trace_array *tr;	2374	struct trace_array *tr;
2539	struct dentry *d_tracer;	2375	struct dentry *d_tracer;
2540	struct dentry *entry;	2376	struct dentry *entry;
2541	int ret;	2377	int ret;
2542		2378
2543	tr = top_trace_array();	2379	tr = top_trace_array();
2544		2380
2545	d_tracer = tracing_init_dentry();	2381	d_tracer = tracing_init_dentry();
2546	if (!d_tracer)	2382	if (!d_tracer)
2547	return 0;	2383	return 0;
2548		2384
2549	entry = debugfs_create_file("available_events", 0444, d_tracer,	2385	entry = debugfs_create_file("available_events", 0444, d_tracer,
2550	tr, &ftrace_avail_fops);	2386	tr, &ftrace_avail_fops);
2551	if (!entry)	2387	if (!entry)
2552	pr_warning("Could not create debugfs "	2388	pr_warning("Could not create debugfs "
2553	"'available_events' entry\n");	2389	"'available_events' entry\n");
2554		2390
2555	if (trace_define_common_fields())	2391	if (trace_define_common_fields())
2556	pr_warning("tracing: Failed to allocate common fields");	2392	pr_warning("tracing: Failed to allocate common fields");
2557		2393
2558	ret = early_event_add_tracer(d_tracer, tr);	2394	ret = early_event_add_tracer(d_tracer, tr);
2559	if (ret)	2395	if (ret)
2560	return ret;	2396	return ret;
2561		2397
		2398	#ifdef CONFIG_MODULES
2562	ret = register_module_notifier(&trace_module_nb);	2399	ret = register_module_notifier(&trace_module_nb);
2563	if (ret)	2400	if (ret)
2564	pr_warning("Failed to register trace events module notifier\n");	2401	pr_warning("Failed to register trace events module notifier\n");
2565		2402	#endif
2566	return 0;	2403	return 0;
2567	}	2404	}
2568	early_initcall(event_trace_memsetup);	2405	early_initcall(event_trace_memsetup);
2569	core_initcall(event_trace_enable);	2406	core_initcall(event_trace_enable);
2570	fs_initcall(event_trace_init);	2407	fs_initcall(event_trace_init);
2571		2408
2572	#ifdef CONFIG_FTRACE_STARTUP_TEST	2409	#ifdef CONFIG_FTRACE_STARTUP_TEST
2573		2410
2574	static DEFINE_SPINLOCK(test_spinlock);	2411	static DEFINE_SPINLOCK(test_spinlock);
2575	static DEFINE_SPINLOCK(test_spinlock_irq);	2412	static DEFINE_SPINLOCK(test_spinlock_irq);
2576	static DEFINE_MUTEX(test_mutex);	2413	static DEFINE_MUTEX(test_mutex);
2577		2414
2578	static __init void test_work(struct work_struct *dummy)	2415	static __init void test_work(struct work_struct *dummy)
2579	{	2416	{
2580	spin_lock(&test_spinlock);	2417	spin_lock(&test_spinlock);
2581	spin_lock_irq(&test_spinlock_irq);	2418	spin_lock_irq(&test_spinlock_irq);
2582	udelay(1);	2419	udelay(1);
2583	spin_unlock_irq(&test_spinlock_irq);	2420	spin_unlock_irq(&test_spinlock_irq);
2584	spin_unlock(&test_spinlock);	2421	spin_unlock(&test_spinlock);
2585		2422
2586	mutex_lock(&test_mutex);	2423	mutex_lock(&test_mutex);
2587	msleep(1);	2424	msleep(1);
2588	mutex_unlock(&test_mutex);	2425	mutex_unlock(&test_mutex);
2589	}	2426	}
2590		2427
2591	static __init int event_test_thread(void *unused)	2428	static __init int event_test_thread(void *unused)
2592	{	2429	{
2593	void *test_malloc;	2430	void *test_malloc;
2594		2431
2595	test_malloc = kmalloc(1234, GFP_KERNEL);	2432	test_malloc = kmalloc(1234, GFP_KERNEL);
2596	if (!test_malloc)	2433	if (!test_malloc)
2597	pr_info("failed to kmalloc\n");	2434	pr_info("failed to kmalloc\n");
2598		2435
2599	schedule_on_each_cpu(test_work);	2436	schedule_on_each_cpu(test_work);
2600		2437
2601	kfree(test_malloc);	2438	kfree(test_malloc);
2602		2439
2603	set_current_state(TASK_INTERRUPTIBLE);	2440	set_current_state(TASK_INTERRUPTIBLE);
2604	while (!kthread_should_stop())	2441	while (!kthread_should_stop())
2605	schedule();	2442	schedule();
2606		2443
2607	return 0;	2444	return 0;
2608	}	2445	}
2609		2446
2610	/*	2447	/*
2611	* Do various things that may trigger events.	2448	* Do various things that may trigger events.
2612	*/	2449	*/
2613	static __init void event_test_stuff(void)	2450	static __init void event_test_stuff(void)
2614	{	2451	{
2615	struct task_struct *test_thread;	2452	struct task_struct *test_thread;
2616		2453
2617	test_thread = kthread_run(event_test_thread, NULL, "test-events");	2454	test_thread = kthread_run(event_test_thread, NULL, "test-events");
2618	msleep(1);	2455	msleep(1);
2619	kthread_stop(test_thread);	2456	kthread_stop(test_thread);
2620	}	2457	}
2621		2458
2622	/*	2459	/*
2623	* For every trace event defined, we will test each trace point separately,	2460	* For every trace event defined, we will test each trace point separately,
2624	* and then by groups, and finally all trace points.	2461	* and then by groups, and finally all trace points.
2625	*/	2462	*/
2626	static __init void event_trace_self_tests(void)	2463	static __init void event_trace_self_tests(void)
2627	{	2464	{
2628	struct ftrace_subsystem_dir *dir;	2465	struct ftrace_subsystem_dir *dir;
2629	struct ftrace_event_file *file;	2466	struct ftrace_event_file *file;
2630	struct ftrace_event_call *call;	2467	struct ftrace_event_call *call;
2631	struct event_subsystem *system;	2468	struct event_subsystem *system;
2632	struct trace_array *tr;	2469	struct trace_array *tr;
2633	int ret;	2470	int ret;
2634		2471
2635	tr = top_trace_array();	2472	tr = top_trace_array();
2636		2473
2637	pr_info("Running tests on trace events:\n");	2474	pr_info("Running tests on trace events:\n");
2638		2475
2639	list_for_each_entry(file, &tr->events, list) {	2476	list_for_each_entry(file, &tr->events, list) {
2640		2477
2641	call = file->event_call;	2478	call = file->event_call;
2642		2479
2643	/* Only test those that have a probe */	2480	/* Only test those that have a probe */
2644	if (!call->class \|\| !call->class->probe)	2481	if (!call->class \|\| !call->class->probe)
2645	continue;	2482	continue;
2646		2483
2647	/*	2484	/*
2648	* Testing syscall events here is pretty useless, but	2485	* Testing syscall events here is pretty useless, but
2649	* we still do it if configured. But this is time consuming.	2486	* we still do it if configured. But this is time consuming.
2650	* What we really need is a user thread to perform the	2487	* What we really need is a user thread to perform the
2651	* syscalls as we test.	2488	* syscalls as we test.
2652	*/	2489	*/
2653	#ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS	2490	#ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
2654	if (call->class->system &&	2491	if (call->class->system &&
2655	strcmp(call->class->system, "syscalls") == 0)	2492	strcmp(call->class->system, "syscalls") == 0)
2656	continue;	2493	continue;
2657	#endif	2494	#endif
2658		2495
2659	pr_info("Testing event %s: ", call->name);	2496	pr_info("Testing event %s: ", call->name);
2660		2497
2661	/*	2498	/*
2662	* If an event is already enabled, someone is using	2499	* If an event is already enabled, someone is using
2663	* it and the self test should not be on.	2500	* it and the self test should not be on.
2664	*/	2501	*/
2665	if (file->flags & FTRACE_EVENT_FL_ENABLED) {	2502	if (file->flags & FTRACE_EVENT_FL_ENABLED) {
2666	pr_warning("Enabled event during self test!\n");	2503	pr_warning("Enabled event during self test!\n");
2667	WARN_ON_ONCE(1);	2504	WARN_ON_ONCE(1);
2668	continue;	2505	continue;
2669	}	2506	}
2670		2507
2671	ftrace_event_enable_disable(file, 1);	2508	ftrace_event_enable_disable(file, 1);
2672	event_test_stuff();	2509	event_test_stuff();
2673	ftrace_event_enable_disable(file, 0);	2510	ftrace_event_enable_disable(file, 0);
2674		2511
2675	pr_cont("OK\n");	2512	pr_cont("OK\n");
2676	}	2513	}
2677		2514
2678	/* Now test at the sub system level */	2515	/* Now test at the sub system level */
2679		2516
2680	pr_info("Running tests on trace event systems:\n");	2517	pr_info("Running tests on trace event systems:\n");
2681		2518
2682	list_for_each_entry(dir, &tr->systems, list) {	2519	list_for_each_entry(dir, &tr->systems, list) {
2683		2520
2684	system = dir->subsystem;	2521	system = dir->subsystem;
2685		2522
2686	/* the ftrace system is special, skip it */	2523	/* the ftrace system is special, skip it */
2687	if (strcmp(system->name, "ftrace") == 0)	2524	if (strcmp(system->name, "ftrace") == 0)
2688	continue;	2525	continue;
2689		2526
2690	pr_info("Testing event system %s: ", system->name);	2527	pr_info("Testing event system %s: ", system->name);
2691		2528
2692	ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);	2529	ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);
2693	if (WARN_ON_ONCE(ret)) {	2530	if (WARN_ON_ONCE(ret)) {
2694	pr_warning("error enabling system %s\n",	2531	pr_warning("error enabling system %s\n",
2695	system->name);	2532	system->name);
2696	continue;	2533	continue;
2697	}	2534	}
2698		2535
2699	event_test_stuff();	2536	event_test_stuff();
2700		2537
2701	ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);	2538	ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);
2702	if (WARN_ON_ONCE(ret)) {	2539	if (WARN_ON_ONCE(ret)) {
2703	pr_warning("error disabling system %s\n",	2540	pr_warning("error disabling system %s\n",
2704	system->name);	2541	system->name);
2705	continue;	2542	continue;
2706	}	2543	}
2707		2544
2708	pr_cont("OK\n");	2545	pr_cont("OK\n");
2709	}	2546	}
2710		2547
2711	/* Test with all events enabled */	2548	/* Test with all events enabled */
2712		2549
2713	pr_info("Running tests on all trace events:\n");	2550	pr_info("Running tests on all trace events:\n");
2714	pr_info("Testing all events: ");	2551	pr_info("Testing all events: ");
2715		2552
2716	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);	2553	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);
2717	if (WARN_ON_ONCE(ret)) {	2554	if (WARN_ON_ONCE(ret)) {
2718	pr_warning("error enabling all events\n");	2555	pr_warning("error enabling all events\n");
2719	return;	2556	return;
2720	}	2557	}
2721		2558
2722	event_test_stuff();	2559	event_test_stuff();
2723		2560
2724	/* reset sysname */	2561	/* reset sysname */
2725	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);	2562	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);
2726	if (WARN_ON_ONCE(ret)) {	2563	if (WARN_ON_ONCE(ret)) {
2727	pr_warning("error disabling all events\n");	2564	pr_warning("error disabling all events\n");
2728	return;	2565	return;
2729	}	2566	}
2730		2567
2731	pr_cont("OK\n");	2568	pr_cont("OK\n");
2732	}	2569	}
2733		2570
2734	#ifdef CONFIG_FUNCTION_TRACER	2571	#ifdef CONFIG_FUNCTION_TRACER
2735		2572
2736	static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);	2573	static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
2737		2574
2738	static void	2575	static void
2739	function_test_events_call(unsigned long ip, unsigned long parent_ip,	2576	function_test_events_call(unsigned long ip, unsigned long parent_ip,
2740	struct ftrace_ops op, struct pt_regs pt_regs)	2577	struct ftrace_ops op, struct pt_regs pt_regs)
2741	{	2578	{
2742	struct ring_buffer_event *event;	2579	struct ring_buffer_event *event;
2743	struct ring_buffer *buffer;	2580	struct ring_buffer *buffer;
2744	struct ftrace_entry *entry;	2581	struct ftrace_entry *entry;
2745	unsigned long flags;	2582	unsigned long flags;
2746	long disabled;	2583	long disabled;
2747	int cpu;	2584	int cpu;
2748	int pc;	2585	int pc;
2749		2586
2750	pc = preempt_count();	2587	pc = preempt_count();
2751	preempt_disable_notrace();	2588	preempt_disable_notrace();
2752	cpu = raw_smp_processor_id();	2589	cpu = raw_smp_processor_id();
2753	disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));	2590	disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
2754		2591
2755	if (disabled != 1)	2592	if (disabled != 1)
2756	goto out;	2593	goto out;
2757		2594
2758	local_save_flags(flags);	2595	local_save_flags(flags);
2759		2596
2760	event = trace_current_buffer_lock_reserve(&buffer,	2597	event = trace_current_buffer_lock_reserve(&buffer,
2761	TRACE_FN, sizeof(*entry),	2598	TRACE_FN, sizeof(*entry),
2762	flags, pc);	2599	flags, pc);
2763	if (!event)	2600	if (!event)
2764	goto out;	2601	goto out;
2765	entry = ring_buffer_event_data(event);	2602	entry = ring_buffer_event_data(event);
2766	entry->ip = ip;	2603	entry->ip = ip;
2767	entry->parent_ip = parent_ip;	2604	entry->parent_ip = parent_ip;
2768		2605
2769	trace_buffer_unlock_commit(buffer, event, flags, pc);	2606	trace_buffer_unlock_commit(buffer, event, flags, pc);
2770		2607
2771	out:	2608	out:
2772	atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));	2609	atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
2773	preempt_enable_notrace();	2610	preempt_enable_notrace();
2774	}	2611	}
2775		2612
2776	static struct ftrace_ops trace_ops __initdata =	2613	static struct ftrace_ops trace_ops __initdata =
2777	{	2614	{
2778	.func = function_test_events_call,	2615	.func = function_test_events_call,
2779	.flags = FTRACE_OPS_FL_RECURSION_SAFE,	2616	.flags = FTRACE_OPS_FL_RECURSION_SAFE,
2780	};	2617	};
2781		2618
2782	static __init void event_trace_self_test_with_function(void)	2619	static __init void event_trace_self_test_with_function(void)
2783	{	2620	{
2784	int ret;	2621	int ret;
2785	ret = register_ftrace_function(&trace_ops);	2622	ret = register_ftrace_function(&trace_ops);
2786	if (WARN_ON(ret < 0)) {	2623	if (WARN_ON(ret < 0)) {
2787	pr_info("Failed to enable function tracer for event tests\n");	2624	pr_info("Failed to enable function tracer for event tests\n");
2788	return;	2625	return;
2789	}	2626	}
2790	pr_info("Running tests again, along with the function tracer\n");	2627	pr_info("Running tests again, along with the function tracer\n");
2791	event_trace_self_tests();	2628	event_trace_self_tests();
2792	unregister_ftrace_function(&trace_ops);	2629	unregister_ftrace_function(&trace_ops);
2793	}	2630	}
2794	#else	2631	#else
2795	static __init void event_trace_self_test_with_function(void)	2632	static __init void event_trace_self_test_with_function(void)
2796	{	2633	{
2797	}	2634	}
2798	#endif	2635	#endif
2799		2636
2800	static __init int event_trace_self_tests_init(void)	2637	static __init int event_trace_self_tests_init(void)
2801	{	2638	{
2802	if (!tracing_selftest_disabled) {	2639	if (!tracing_selftest_disabled) {
2803	event_trace_self_tests();	2640	event_trace_self_tests();
2804	event_trace_self_test_with_function();	2641	event_trace_self_test_with_function();
2805	}	2642	}
2806		2643
2807	return 0;	2644	return 0;
2808	}	2645	}
2809		2646
2810	late_initcall(event_trace_self_tests_init);	2647	late_initcall(event_trace_self_tests_init);
2811		2648

kernel/trace/trace_syscalls.c

Diff comments View file @ 7eb6952

1	#include <trace/syscall.h>	1	#include <trace/syscall.h>
2	#include <trace/events/syscalls.h>	2	#include <trace/events/syscalls.h>
3	#include <linux/syscalls.h>	3	#include <linux/syscalls.h>
4	#include <linux/slab.h>	4	#include <linux/slab.h>
5	#include <linux/kernel.h>	5	#include <linux/kernel.h>
6	#include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */	6	#include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */
7	#include <linux/ftrace.h>	7	#include <linux/ftrace.h>
8	#include <linux/perf_event.h>	8	#include <linux/perf_event.h>
9	#include <asm/syscall.h>	9	#include <asm/syscall.h>
10		10
11	#include "trace_output.h"	11	#include "trace_output.h"
12	#include "trace.h"	12	#include "trace.h"
13		13
14	static DEFINE_MUTEX(syscall_trace_lock);	14	static DEFINE_MUTEX(syscall_trace_lock);
15		15
16	static int syscall_enter_register(struct ftrace_event_call *event,	16	static int syscall_enter_register(struct ftrace_event_call *event,
17	enum trace_reg type, void *data);	17	enum trace_reg type, void *data);
18	static int syscall_exit_register(struct ftrace_event_call *event,	18	static int syscall_exit_register(struct ftrace_event_call *event,
19	enum trace_reg type, void *data);	19	enum trace_reg type, void *data);
20		20
21	static struct list_head *	21	static struct list_head *
22	syscall_get_enter_fields(struct ftrace_event_call *call)	22	syscall_get_enter_fields(struct ftrace_event_call *call)
23	{	23	{
24	struct syscall_metadata *entry = call->data;	24	struct syscall_metadata *entry = call->data;
25		25
26	return &entry->enter_fields;	26	return &entry->enter_fields;
27	}	27	}
28		28
29	extern struct syscall_metadata *__start_syscalls_metadata[];	29	extern struct syscall_metadata *__start_syscalls_metadata[];
30	extern struct syscall_metadata *__stop_syscalls_metadata[];	30	extern struct syscall_metadata *__stop_syscalls_metadata[];
31		31
32	static struct syscall_metadata **syscalls_metadata;	32	static struct syscall_metadata **syscalls_metadata;
33		33
34	#ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME	34	#ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME
35	static inline bool arch_syscall_match_sym_name(const char sym, const char name)	35	static inline bool arch_syscall_match_sym_name(const char sym, const char name)
36	{	36	{
37	/*	37	/*
38	* Only compare after the "sys" prefix. Archs that use	38	* Only compare after the "sys" prefix. Archs that use
39	* syscall wrappers may have syscalls symbols aliases prefixed	39	* syscall wrappers may have syscalls symbols aliases prefixed
40	* with ".SyS" or ".sys" instead of "sys", leading to an unwanted	40	* with ".SyS" or ".sys" instead of "sys", leading to an unwanted
41	* mismatch.	41	* mismatch.
42	*/	42	*/
43	return !strcmp(sym + 3, name + 3);	43	return !strcmp(sym + 3, name + 3);
44	}	44	}
45	#endif	45	#endif
46		46
47	#ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS	47	#ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
48	/*	48	/*
49	* Some architectures that allow for 32bit applications	49	* Some architectures that allow for 32bit applications
50	* to run on a 64bit kernel, do not map the syscalls for	50	* to run on a 64bit kernel, do not map the syscalls for
51	* the 32bit tasks the same as they do for 64bit tasks.	51	* the 32bit tasks the same as they do for 64bit tasks.
52	*	52	*
53	* coughx86cough	53	* coughx86cough
54	*	54	*
55	* In such a case, instead of reporting the wrong syscalls,	55	* In such a case, instead of reporting the wrong syscalls,
56	* simply ignore them.	56	* simply ignore them.
57	*	57	*
58	* For an arch to ignore the compat syscalls it needs to	58	* For an arch to ignore the compat syscalls it needs to
59	* define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS as well as	59	* define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS as well as
60	* define the function arch_trace_is_compat_syscall() to let	60	* define the function arch_trace_is_compat_syscall() to let
61	* the tracing system know that it should ignore it.	61	* the tracing system know that it should ignore it.
62	*/	62	*/
63	static int	63	static int
64	trace_get_syscall_nr(struct task_struct task, struct pt_regs regs)	64	trace_get_syscall_nr(struct task_struct task, struct pt_regs regs)
65	{	65	{
66	if (unlikely(arch_trace_is_compat_syscall(regs)))	66	if (unlikely(arch_trace_is_compat_syscall(regs)))
67	return -1;	67	return -1;
68		68
69	return syscall_get_nr(task, regs);	69	return syscall_get_nr(task, regs);
70	}	70	}
71	#else	71	#else
72	static inline int	72	static inline int
73	trace_get_syscall_nr(struct task_struct task, struct pt_regs regs)	73	trace_get_syscall_nr(struct task_struct task, struct pt_regs regs)
74	{	74	{
75	return syscall_get_nr(task, regs);	75	return syscall_get_nr(task, regs);
76	}	76	}
77	#endif /* ARCH_TRACE_IGNORE_COMPAT_SYSCALLS */	77	#endif /* ARCH_TRACE_IGNORE_COMPAT_SYSCALLS */
78		78
79	static __init struct syscall_metadata *	79	static __init struct syscall_metadata *
80	find_syscall_meta(unsigned long syscall)	80	find_syscall_meta(unsigned long syscall)
81	{	81	{
82	struct syscall_metadata **start;	82	struct syscall_metadata **start;
83	struct syscall_metadata **stop;	83	struct syscall_metadata **stop;
84	char str[KSYM_SYMBOL_LEN];	84	char str[KSYM_SYMBOL_LEN];
85		85
86		86
87	start = __start_syscalls_metadata;	87	start = __start_syscalls_metadata;
88	stop = __stop_syscalls_metadata;	88	stop = __stop_syscalls_metadata;
89	kallsyms_lookup(syscall, NULL, NULL, NULL, str);	89	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
90		90
91	if (arch_syscall_match_sym_name(str, "sys_ni_syscall"))	91	if (arch_syscall_match_sym_name(str, "sys_ni_syscall"))
92	return NULL;	92	return NULL;
93		93
94	for ( ; start < stop; start++) {	94	for ( ; start < stop; start++) {
95	if ((start)->name && arch_syscall_match_sym_name(str, (start)->name))	95	if ((start)->name && arch_syscall_match_sym_name(str, (start)->name))
96	return *start;	96	return *start;
97	}	97	}
98	return NULL;	98	return NULL;
99	}	99	}
100		100
101	static struct syscall_metadata *syscall_nr_to_meta(int nr)	101	static struct syscall_metadata *syscall_nr_to_meta(int nr)
102	{	102	{
103	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)	103	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
104	return NULL;	104	return NULL;
105		105
106	return syscalls_metadata[nr];	106	return syscalls_metadata[nr];
107	}	107	}
108		108
109	static enum print_line_t	109	static enum print_line_t
110	print_syscall_enter(struct trace_iterator *iter, int flags,	110	print_syscall_enter(struct trace_iterator *iter, int flags,
111	struct trace_event *event)	111	struct trace_event *event)
112	{	112	{
113	struct trace_seq *s = &iter->seq;	113	struct trace_seq *s = &iter->seq;
114	struct trace_entry *ent = iter->ent;	114	struct trace_entry *ent = iter->ent;
115	struct syscall_trace_enter *trace;	115	struct syscall_trace_enter *trace;
116	struct syscall_metadata *entry;	116	struct syscall_metadata *entry;
117	int i, ret, syscall;	117	int i, ret, syscall;
118		118
119	trace = (typeof(trace))ent;	119	trace = (typeof(trace))ent;
120	syscall = trace->nr;	120	syscall = trace->nr;
121	entry = syscall_nr_to_meta(syscall);	121	entry = syscall_nr_to_meta(syscall);
122		122
123	if (!entry)	123	if (!entry)
124	goto end;	124	goto end;
125		125
126	if (entry->enter_event->event.type != ent->type) {	126	if (entry->enter_event->event.type != ent->type) {
127	WARN_ON_ONCE(1);	127	WARN_ON_ONCE(1);
128	goto end;	128	goto end;
129	}	129	}
130		130
131	ret = trace_seq_printf(s, "%s(", entry->name);	131	ret = trace_seq_printf(s, "%s(", entry->name);
132	if (!ret)	132	if (!ret)
133	return TRACE_TYPE_PARTIAL_LINE;	133	return TRACE_TYPE_PARTIAL_LINE;
134		134
135	for (i = 0; i < entry->nb_args; i++) {	135	for (i = 0; i < entry->nb_args; i++) {
136	/* parameter types */	136	/* parameter types */
137	if (trace_flags & TRACE_ITER_VERBOSE) {	137	if (trace_flags & TRACE_ITER_VERBOSE) {
138	ret = trace_seq_printf(s, "%s ", entry->types[i]);	138	ret = trace_seq_printf(s, "%s ", entry->types[i]);
139	if (!ret)	139	if (!ret)
140	return TRACE_TYPE_PARTIAL_LINE;	140	return TRACE_TYPE_PARTIAL_LINE;
141	}	141	}
142	/* parameter values */	142	/* parameter values */
143	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],	143	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
144	trace->args[i],	144	trace->args[i],
145	i == entry->nb_args - 1 ? "" : ", ");	145	i == entry->nb_args - 1 ? "" : ", ");
146	if (!ret)	146	if (!ret)
147	return TRACE_TYPE_PARTIAL_LINE;	147	return TRACE_TYPE_PARTIAL_LINE;
148	}	148	}
149		149
150	ret = trace_seq_putc(s, ')');	150	ret = trace_seq_putc(s, ')');
151	if (!ret)	151	if (!ret)
152	return TRACE_TYPE_PARTIAL_LINE;	152	return TRACE_TYPE_PARTIAL_LINE;
153		153
154	end:	154	end:
155	ret = trace_seq_putc(s, '\n');	155	ret = trace_seq_putc(s, '\n');
156	if (!ret)	156	if (!ret)
157	return TRACE_TYPE_PARTIAL_LINE;	157	return TRACE_TYPE_PARTIAL_LINE;
158		158
159	return TRACE_TYPE_HANDLED;	159	return TRACE_TYPE_HANDLED;
160	}	160	}
161		161
162	static enum print_line_t	162	static enum print_line_t
163	print_syscall_exit(struct trace_iterator *iter, int flags,	163	print_syscall_exit(struct trace_iterator *iter, int flags,
164	struct trace_event *event)	164	struct trace_event *event)
165	{	165	{
166	struct trace_seq *s = &iter->seq;	166	struct trace_seq *s = &iter->seq;
167	struct trace_entry *ent = iter->ent;	167	struct trace_entry *ent = iter->ent;
168	struct syscall_trace_exit *trace;	168	struct syscall_trace_exit *trace;
169	int syscall;	169	int syscall;
170	struct syscall_metadata *entry;	170	struct syscall_metadata *entry;
171	int ret;	171	int ret;
172		172
173	trace = (typeof(trace))ent;	173	trace = (typeof(trace))ent;
174	syscall = trace->nr;	174	syscall = trace->nr;
175	entry = syscall_nr_to_meta(syscall);	175	entry = syscall_nr_to_meta(syscall);
176		176
177	if (!entry) {	177	if (!entry) {
178	trace_seq_putc(s, '\n');	178	trace_seq_putc(s, '\n');
179	return TRACE_TYPE_HANDLED;	179	return TRACE_TYPE_HANDLED;
180	}	180	}
181		181
182	if (entry->exit_event->event.type != ent->type) {	182	if (entry->exit_event->event.type != ent->type) {
183	WARN_ON_ONCE(1);	183	WARN_ON_ONCE(1);
184	return TRACE_TYPE_UNHANDLED;	184	return TRACE_TYPE_UNHANDLED;
185	}	185	}
186		186
187	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,	187	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
188	trace->ret);	188	trace->ret);
189	if (!ret)	189	if (!ret)
190	return TRACE_TYPE_PARTIAL_LINE;	190	return TRACE_TYPE_PARTIAL_LINE;
191		191
192	return TRACE_TYPE_HANDLED;	192	return TRACE_TYPE_HANDLED;
193	}	193	}
194		194
195	extern char *__bad_type_size(void);	195	extern char *__bad_type_size(void);
196		196
197	#define SYSCALL_FIELD(type, name) \	197	#define SYSCALL_FIELD(type, name) \
198	sizeof(type) != sizeof(trace.name) ? \	198	sizeof(type) != sizeof(trace.name) ? \
199	__bad_type_size() : \	199	__bad_type_size() : \
200	#type, #name, offsetof(typeof(trace), name), \	200	#type, #name, offsetof(typeof(trace), name), \
201	sizeof(trace.name), is_signed_type(type)	201	sizeof(trace.name), is_signed_type(type)
202		202
203	static	203	static int __init
204	int __set_enter_print_fmt(struct syscall_metadata entry, char buf, int len)	204	__set_enter_print_fmt(struct syscall_metadata entry, char buf, int len)
205	{	205	{
206	int i;	206	int i;
207	int pos = 0;	207	int pos = 0;
208		208
209	/* When len=0, we just calculate the needed length */	209	/* When len=0, we just calculate the needed length */
210	#define LEN_OR_ZERO (len ? len - pos : 0)	210	#define LEN_OR_ZERO (len ? len - pos : 0)
211		211
212	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");	212	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
213	for (i = 0; i < entry->nb_args; i++) {	213	for (i = 0; i < entry->nb_args; i++) {
214	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",	214	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
215	entry->args[i], sizeof(unsigned long),	215	entry->args[i], sizeof(unsigned long),
216	i == entry->nb_args - 1 ? "" : ", ");	216	i == entry->nb_args - 1 ? "" : ", ");
217	}	217	}
218	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");	218	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
219		219
220	for (i = 0; i < entry->nb_args; i++) {	220	for (i = 0; i < entry->nb_args; i++) {
221	pos += snprintf(buf + pos, LEN_OR_ZERO,	221	pos += snprintf(buf + pos, LEN_OR_ZERO,
222	", ((unsigned long)(REC->%s))", entry->args[i]);	222	", ((unsigned long)(REC->%s))", entry->args[i]);
223	}	223	}
224		224
225	#undef LEN_OR_ZERO	225	#undef LEN_OR_ZERO
226		226
227	/* return the length of print_fmt */	227	/* return the length of print_fmt */
228	return pos;	228	return pos;
229	}	229	}
230		230
231	static int set_syscall_print_fmt(struct ftrace_event_call *call)	231	static int __init set_syscall_print_fmt(struct ftrace_event_call *call)
232	{	232	{
233	char *print_fmt;	233	char *print_fmt;
234	int len;	234	int len;
235	struct syscall_metadata *entry = call->data;	235	struct syscall_metadata *entry = call->data;
236		236
237	if (entry->enter_event != call) {	237	if (entry->enter_event != call) {
238	call->print_fmt = "\"0x%lx\", REC->ret";	238	call->print_fmt = "\"0x%lx\", REC->ret";
239	return 0;	239	return 0;
240	}	240	}
241		241
242	/* First: called with 0 length to calculate the needed length */	242	/* First: called with 0 length to calculate the needed length */
243	len = __set_enter_print_fmt(entry, NULL, 0);	243	len = __set_enter_print_fmt(entry, NULL, 0);
244		244
245	print_fmt = kmalloc(len + 1, GFP_KERNEL);	245	print_fmt = kmalloc(len + 1, GFP_KERNEL);
246	if (!print_fmt)	246	if (!print_fmt)
247	return -ENOMEM;	247	return -ENOMEM;
248		248
249	/* Second: actually write the @print_fmt */	249	/* Second: actually write the @print_fmt */
250	__set_enter_print_fmt(entry, print_fmt, len + 1);	250	__set_enter_print_fmt(entry, print_fmt, len + 1);
251	call->print_fmt = print_fmt;	251	call->print_fmt = print_fmt;
252		252
253	return 0;	253	return 0;
254	}	254	}
255		255
256	static void free_syscall_print_fmt(struct ftrace_event_call *call)	256	static void __init free_syscall_print_fmt(struct ftrace_event_call *call)
257	{	257	{
258	struct syscall_metadata *entry = call->data;	258	struct syscall_metadata *entry = call->data;
259		259
260	if (entry->enter_event == call)	260	if (entry->enter_event == call)
261	kfree(call->print_fmt);	261	kfree(call->print_fmt);
262	}	262	}
263		263
264	static int __init syscall_enter_define_fields(struct ftrace_event_call *call)	264	static int __init syscall_enter_define_fields(struct ftrace_event_call *call)
265	{	265	{
266	struct syscall_trace_enter trace;	266	struct syscall_trace_enter trace;
267	struct syscall_metadata *meta = call->data;	267	struct syscall_metadata *meta = call->data;
268	int ret;	268	int ret;
269	int i;	269	int i;
270	int offset = offsetof(typeof(trace), args);	270	int offset = offsetof(typeof(trace), args);
271		271
272	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);	272	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
273	if (ret)	273	if (ret)
274	return ret;	274	return ret;
275		275
276	for (i = 0; i < meta->nb_args; i++) {	276	for (i = 0; i < meta->nb_args; i++) {
277	ret = trace_define_field(call, meta->types[i],	277	ret = trace_define_field(call, meta->types[i],
278	meta->args[i], offset,	278	meta->args[i], offset,
279	sizeof(unsigned long), 0,	279	sizeof(unsigned long), 0,
280	FILTER_OTHER);	280	FILTER_OTHER);
281	offset += sizeof(unsigned long);	281	offset += sizeof(unsigned long);
282	}	282	}
283		283
284	return ret;	284	return ret;
285	}	285	}
286		286
287	static int __init syscall_exit_define_fields(struct ftrace_event_call *call)	287	static int __init syscall_exit_define_fields(struct ftrace_event_call *call)
288	{	288	{
289	struct syscall_trace_exit trace;	289	struct syscall_trace_exit trace;
290	int ret;	290	int ret;
291		291
292	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);	292	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
293	if (ret)	293	if (ret)
294	return ret;	294	return ret;
295		295
296	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),	296	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
297	FILTER_OTHER);	297	FILTER_OTHER);
298		298
299	return ret;	299	return ret;
300	}	300	}
301		301
302	static void ftrace_syscall_enter(void data, struct pt_regs regs, long id)	302	static void ftrace_syscall_enter(void data, struct pt_regs regs, long id)
303	{	303	{
304	struct trace_array *tr = data;	304	struct trace_array *tr = data;
305	struct syscall_trace_enter *entry;	305	struct syscall_trace_enter *entry;
306	struct syscall_metadata *sys_data;	306	struct syscall_metadata *sys_data;
307	struct ring_buffer_event *event;	307	struct ring_buffer_event *event;
308	struct ring_buffer *buffer;	308	struct ring_buffer *buffer;
309	unsigned long irq_flags;	309	unsigned long irq_flags;
310	int pc;	310	int pc;
311	int syscall_nr;	311	int syscall_nr;
312	int size;	312	int size;
313		313
314	syscall_nr = trace_get_syscall_nr(current, regs);	314	syscall_nr = trace_get_syscall_nr(current, regs);
315	if (syscall_nr < 0)	315	if (syscall_nr < 0)
316	return;	316	return;
317	if (!test_bit(syscall_nr, tr->enabled_enter_syscalls))	317	if (!test_bit(syscall_nr, tr->enabled_enter_syscalls))
318	return;	318	return;
319		319
320	sys_data = syscall_nr_to_meta(syscall_nr);	320	sys_data = syscall_nr_to_meta(syscall_nr);
321	if (!sys_data)	321	if (!sys_data)
322	return;	322	return;
323		323
324	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;	324	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
325		325
326	local_save_flags(irq_flags);	326	local_save_flags(irq_flags);
327	pc = preempt_count();	327	pc = preempt_count();
328		328
329	buffer = tr->trace_buffer.buffer;	329	buffer = tr->trace_buffer.buffer;
330	event = trace_buffer_lock_reserve(buffer,	330	event = trace_buffer_lock_reserve(buffer,
331	sys_data->enter_event->event.type, size, irq_flags, pc);	331	sys_data->enter_event->event.type, size, irq_flags, pc);
332	if (!event)	332	if (!event)
333	return;	333	return;
334		334
335	entry = ring_buffer_event_data(event);	335	entry = ring_buffer_event_data(event);
336	entry->nr = syscall_nr;	336	entry->nr = syscall_nr;
337	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);	337	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
338		338
339	if (!filter_current_check_discard(buffer, sys_data->enter_event,	339	if (!filter_current_check_discard(buffer, sys_data->enter_event,
340	entry, event))	340	entry, event))
341	trace_current_buffer_unlock_commit(buffer, event,	341	trace_current_buffer_unlock_commit(buffer, event,
342	irq_flags, pc);	342	irq_flags, pc);
343	}	343	}
344		344
345	static void ftrace_syscall_exit(void data, struct pt_regs regs, long ret)	345	static void ftrace_syscall_exit(void data, struct pt_regs regs, long ret)
346	{	346	{
347	struct trace_array *tr = data;	347	struct trace_array *tr = data;
348	struct syscall_trace_exit *entry;	348	struct syscall_trace_exit *entry;
349	struct syscall_metadata *sys_data;	349	struct syscall_metadata *sys_data;
350	struct ring_buffer_event *event;	350	struct ring_buffer_event *event;
351	struct ring_buffer *buffer;	351	struct ring_buffer *buffer;
352	unsigned long irq_flags;	352	unsigned long irq_flags;
353	int pc;	353	int pc;
354	int syscall_nr;	354	int syscall_nr;
355		355
356	syscall_nr = trace_get_syscall_nr(current, regs);	356	syscall_nr = trace_get_syscall_nr(current, regs);
357	if (syscall_nr < 0)	357	if (syscall_nr < 0)
358	return;	358	return;
359	if (!test_bit(syscall_nr, tr->enabled_exit_syscalls))	359	if (!test_bit(syscall_nr, tr->enabled_exit_syscalls))
360	return;	360	return;
361		361
362	sys_data = syscall_nr_to_meta(syscall_nr);	362	sys_data = syscall_nr_to_meta(syscall_nr);
363	if (!sys_data)	363	if (!sys_data)
364	return;	364	return;
365		365
366	local_save_flags(irq_flags);	366	local_save_flags(irq_flags);
367	pc = preempt_count();	367	pc = preempt_count();
368		368
369	buffer = tr->trace_buffer.buffer;	369	buffer = tr->trace_buffer.buffer;
370	event = trace_buffer_lock_reserve(buffer,	370	event = trace_buffer_lock_reserve(buffer,
371	sys_data->exit_event->event.type, sizeof(*entry),	371	sys_data->exit_event->event.type, sizeof(*entry),
372	irq_flags, pc);	372	irq_flags, pc);
373	if (!event)	373	if (!event)
374	return;	374	return;
375		375
376	entry = ring_buffer_event_data(event);	376	entry = ring_buffer_event_data(event);
377	entry->nr = syscall_nr;	377	entry->nr = syscall_nr;
378	entry->ret = syscall_get_return_value(current, regs);	378	entry->ret = syscall_get_return_value(current, regs);
379		379
380	if (!filter_current_check_discard(buffer, sys_data->exit_event,	380	if (!filter_current_check_discard(buffer, sys_data->exit_event,
381	entry, event))	381	entry, event))
382	trace_current_buffer_unlock_commit(buffer, event,	382	trace_current_buffer_unlock_commit(buffer, event,
383	irq_flags, pc);	383	irq_flags, pc);
384	}	384	}
385		385
386	static int reg_event_syscall_enter(struct ftrace_event_file *file,	386	static int reg_event_syscall_enter(struct ftrace_event_file *file,
387	struct ftrace_event_call *call)	387	struct ftrace_event_call *call)
388	{	388	{
389	struct trace_array *tr = file->tr;	389	struct trace_array *tr = file->tr;
390	int ret = 0;	390	int ret = 0;
391	int num;	391	int num;
392		392
393	num = ((struct syscall_metadata *)call->data)->syscall_nr;	393	num = ((struct syscall_metadata *)call->data)->syscall_nr;
394	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))	394	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))
395	return -ENOSYS;	395	return -ENOSYS;
396	mutex_lock(&syscall_trace_lock);	396	mutex_lock(&syscall_trace_lock);
397	if (!tr->sys_refcount_enter)	397	if (!tr->sys_refcount_enter)
398	ret = register_trace_sys_enter(ftrace_syscall_enter, tr);	398	ret = register_trace_sys_enter(ftrace_syscall_enter, tr);
399	if (!ret) {	399	if (!ret) {
400	set_bit(num, tr->enabled_enter_syscalls);	400	set_bit(num, tr->enabled_enter_syscalls);
401	tr->sys_refcount_enter++;	401	tr->sys_refcount_enter++;
402	}	402	}
403	mutex_unlock(&syscall_trace_lock);	403	mutex_unlock(&syscall_trace_lock);
404	return ret;	404	return ret;
405	}	405	}
406		406
407	static void unreg_event_syscall_enter(struct ftrace_event_file *file,	407	static void unreg_event_syscall_enter(struct ftrace_event_file *file,
408	struct ftrace_event_call *call)	408	struct ftrace_event_call *call)
409	{	409	{
410	struct trace_array *tr = file->tr;	410	struct trace_array *tr = file->tr;
411	int num;	411	int num;
412		412
413	num = ((struct syscall_metadata *)call->data)->syscall_nr;	413	num = ((struct syscall_metadata *)call->data)->syscall_nr;
414	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))	414	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))
415	return;	415	return;
416	mutex_lock(&syscall_trace_lock);	416	mutex_lock(&syscall_trace_lock);
417	tr->sys_refcount_enter--;	417	tr->sys_refcount_enter--;
418	clear_bit(num, tr->enabled_enter_syscalls);	418	clear_bit(num, tr->enabled_enter_syscalls);
419	if (!tr->sys_refcount_enter)	419	if (!tr->sys_refcount_enter)
420	unregister_trace_sys_enter(ftrace_syscall_enter, tr);	420	unregister_trace_sys_enter(ftrace_syscall_enter, tr);
421	mutex_unlock(&syscall_trace_lock);	421	mutex_unlock(&syscall_trace_lock);
422	}	422	}
423		423
424	static int reg_event_syscall_exit(struct ftrace_event_file *file,	424	static int reg_event_syscall_exit(struct ftrace_event_file *file,
425	struct ftrace_event_call *call)	425	struct ftrace_event_call *call)
426	{	426	{
427	struct trace_array *tr = file->tr;	427	struct trace_array *tr = file->tr;
428	int ret = 0;	428	int ret = 0;
429	int num;	429	int num;
430		430
431	num = ((struct syscall_metadata *)call->data)->syscall_nr;	431	num = ((struct syscall_metadata *)call->data)->syscall_nr;
432	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))	432	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))
433	return -ENOSYS;	433	return -ENOSYS;
434	mutex_lock(&syscall_trace_lock);	434	mutex_lock(&syscall_trace_lock);
435	if (!tr->sys_refcount_exit)	435	if (!tr->sys_refcount_exit)
436	ret = register_trace_sys_exit(ftrace_syscall_exit, tr);	436	ret = register_trace_sys_exit(ftrace_syscall_exit, tr);
437	if (!ret) {	437	if (!ret) {
438	set_bit(num, tr->enabled_exit_syscalls);	438	set_bit(num, tr->enabled_exit_syscalls);
439	tr->sys_refcount_exit++;	439	tr->sys_refcount_exit++;
440	}	440	}
441	mutex_unlock(&syscall_trace_lock);	441	mutex_unlock(&syscall_trace_lock);
442	return ret;	442	return ret;
443	}	443	}
444		444
445	static void unreg_event_syscall_exit(struct ftrace_event_file *file,	445	static void unreg_event_syscall_exit(struct ftrace_event_file *file,
446	struct ftrace_event_call *call)	446	struct ftrace_event_call *call)
447	{	447	{
448	struct trace_array *tr = file->tr;	448	struct trace_array *tr = file->tr;
449	int num;	449	int num;
450		450
451	num = ((struct syscall_metadata *)call->data)->syscall_nr;	451	num = ((struct syscall_metadata *)call->data)->syscall_nr;
452	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))	452	if (WARN_ON_ONCE(num < 0 \|\| num >= NR_syscalls))
453	return;	453	return;
454	mutex_lock(&syscall_trace_lock);	454	mutex_lock(&syscall_trace_lock);
455	tr->sys_refcount_exit--;	455	tr->sys_refcount_exit--;
456	clear_bit(num, tr->enabled_exit_syscalls);	456	clear_bit(num, tr->enabled_exit_syscalls);
457	if (!tr->sys_refcount_exit)	457	if (!tr->sys_refcount_exit)
458	unregister_trace_sys_exit(ftrace_syscall_exit, tr);	458	unregister_trace_sys_exit(ftrace_syscall_exit, tr);
459	mutex_unlock(&syscall_trace_lock);	459	mutex_unlock(&syscall_trace_lock);
460	}	460	}
461		461
462	static int init_syscall_trace(struct ftrace_event_call *call)	462	static int __init init_syscall_trace(struct ftrace_event_call *call)
463	{	463	{
464	int id;	464	int id;
465	int num;	465	int num;
466		466
467	num = ((struct syscall_metadata *)call->data)->syscall_nr;	467	num = ((struct syscall_metadata *)call->data)->syscall_nr;
468	if (num < 0 \|\| num >= NR_syscalls) {	468	if (num < 0 \|\| num >= NR_syscalls) {
469	pr_debug("syscall %s metadata not mapped, disabling ftrace event\n",	469	pr_debug("syscall %s metadata not mapped, disabling ftrace event\n",
470	((struct syscall_metadata *)call->data)->name);	470	((struct syscall_metadata *)call->data)->name);
471	return -ENOSYS;	471	return -ENOSYS;
472	}	472	}
473		473
474	if (set_syscall_print_fmt(call) < 0)	474	if (set_syscall_print_fmt(call) < 0)
475	return -ENOMEM;	475	return -ENOMEM;
476		476
477	id = trace_event_raw_init(call);	477	id = trace_event_raw_init(call);
478		478
479	if (id < 0) {	479	if (id < 0) {
480	free_syscall_print_fmt(call);	480	free_syscall_print_fmt(call);
481	return id;	481	return id;
482	}	482	}
483		483
484	return id;	484	return id;
485	}	485	}
486		486
487	struct trace_event_functions enter_syscall_print_funcs = {	487	struct trace_event_functions enter_syscall_print_funcs = {
488	.trace = print_syscall_enter,	488	.trace = print_syscall_enter,
489	};	489	};
490		490
491	struct trace_event_functions exit_syscall_print_funcs = {	491	struct trace_event_functions exit_syscall_print_funcs = {
492	.trace = print_syscall_exit,	492	.trace = print_syscall_exit,
493	};	493	};
494		494
495	struct ftrace_event_class __refdata event_class_syscall_enter = {	495	struct ftrace_event_class __refdata event_class_syscall_enter = {
496	.system = "syscalls",	496	.system = "syscalls",
497	.reg = syscall_enter_register,	497	.reg = syscall_enter_register,
498	.define_fields = syscall_enter_define_fields,	498	.define_fields = syscall_enter_define_fields,
499	.get_fields = syscall_get_enter_fields,	499	.get_fields = syscall_get_enter_fields,
500	.raw_init = init_syscall_trace,	500	.raw_init = init_syscall_trace,
501	};	501	};
502		502
503	struct ftrace_event_class __refdata event_class_syscall_exit = {	503	struct ftrace_event_class __refdata event_class_syscall_exit = {
504	.system = "syscalls",	504	.system = "syscalls",
505	.reg = syscall_exit_register,	505	.reg = syscall_exit_register,
506	.define_fields = syscall_exit_define_fields,	506	.define_fields = syscall_exit_define_fields,
507	.fields = LIST_HEAD_INIT(event_class_syscall_exit.fields),	507	.fields = LIST_HEAD_INIT(event_class_syscall_exit.fields),
508	.raw_init = init_syscall_trace,	508	.raw_init = init_syscall_trace,
509	};	509	};
510		510
511	unsigned long __init __weak arch_syscall_addr(int nr)	511	unsigned long __init __weak arch_syscall_addr(int nr)
512	{	512	{
513	return (unsigned long)sys_call_table[nr];	513	return (unsigned long)sys_call_table[nr];
514	}	514	}
515		515
516	static int __init init_ftrace_syscalls(void)	516	static int __init init_ftrace_syscalls(void)
517	{	517	{
518	struct syscall_metadata *meta;	518	struct syscall_metadata *meta;
519	unsigned long addr;	519	unsigned long addr;
520	int i;	520	int i;
521		521
522	syscalls_metadata = kcalloc(NR_syscalls, sizeof(*syscalls_metadata),	522	syscalls_metadata = kcalloc(NR_syscalls, sizeof(*syscalls_metadata),
523	GFP_KERNEL);	523	GFP_KERNEL);
524	if (!syscalls_metadata) {	524	if (!syscalls_metadata) {
525	WARN_ON(1);	525	WARN_ON(1);
526	return -ENOMEM;	526	return -ENOMEM;
527	}	527	}
528		528
529	for (i = 0; i < NR_syscalls; i++) {	529	for (i = 0; i < NR_syscalls; i++) {
530	addr = arch_syscall_addr(i);	530	addr = arch_syscall_addr(i);
531	meta = find_syscall_meta(addr);	531	meta = find_syscall_meta(addr);
532	if (!meta)	532	if (!meta)
533	continue;	533	continue;
534		534
535	meta->syscall_nr = i;	535	meta->syscall_nr = i;
536	syscalls_metadata[i] = meta;	536	syscalls_metadata[i] = meta;
537	}	537	}
538		538
539	return 0;	539	return 0;
540	}	540	}
541	early_initcall(init_ftrace_syscalls);	541	early_initcall(init_ftrace_syscalls);
542		542
543	#ifdef CONFIG_PERF_EVENTS	543	#ifdef CONFIG_PERF_EVENTS
544		544
545	static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);	545	static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
546	static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);	546	static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
547	static int sys_perf_refcount_enter;	547	static int sys_perf_refcount_enter;
548	static int sys_perf_refcount_exit;	548	static int sys_perf_refcount_exit;
549		549
550	static void perf_syscall_enter(void ignore, struct pt_regs regs, long id)	550	static void perf_syscall_enter(void ignore, struct pt_regs regs, long id)
551	{	551	{
552	struct syscall_metadata *sys_data;	552	struct syscall_metadata *sys_data;
553	struct syscall_trace_enter *rec;	553	struct syscall_trace_enter *rec;
554	struct hlist_head *head;	554	struct hlist_head *head;
555	int syscall_nr;	555	int syscall_nr;
556	int rctx;	556	int rctx;
557	int size;	557	int size;
558		558
559	syscall_nr = trace_get_syscall_nr(current, regs);	559	syscall_nr = trace_get_syscall_nr(current, regs);
560	if (syscall_nr < 0)	560	if (syscall_nr < 0)
561	return;	561	return;
562	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))	562	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
563	return;	563	return;
564		564
565	sys_data = syscall_nr_to_meta(syscall_nr);	565	sys_data = syscall_nr_to_meta(syscall_nr);
566	if (!sys_data)	566	if (!sys_data)
567	return;	567	return;
568		568
569	head = this_cpu_ptr(sys_data->enter_event->perf_events);	569	head = this_cpu_ptr(sys_data->enter_event->perf_events);
570	if (hlist_empty(head))	570	if (hlist_empty(head))
571	return;	571	return;
572		572
573	/* get the size after alignment with the u32 buffer size field */	573	/* get the size after alignment with the u32 buffer size field */
574	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);	574	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
575	size = ALIGN(size + sizeof(u32), sizeof(u64));	575	size = ALIGN(size + sizeof(u32), sizeof(u64));
576	size -= sizeof(u32);	576	size -= sizeof(u32);
577		577
578	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,	578	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
579	sys_data->enter_event->event.type, regs, &rctx);	579	sys_data->enter_event->event.type, regs, &rctx);
580	if (!rec)	580	if (!rec)
581	return;	581	return;
582		582
583	rec->nr = syscall_nr;	583	rec->nr = syscall_nr;
584	syscall_get_arguments(current, regs, 0, sys_data->nb_args,	584	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
585	(unsigned long *)&rec->args);	585	(unsigned long *)&rec->args);
586	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);	586	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
587	}	587	}
588		588
589	static int perf_sysenter_enable(struct ftrace_event_call *call)	589	static int perf_sysenter_enable(struct ftrace_event_call *call)
590	{	590	{
591	int ret = 0;	591	int ret = 0;
592	int num;	592	int num;
593		593
594	num = ((struct syscall_metadata *)call->data)->syscall_nr;	594	num = ((struct syscall_metadata *)call->data)->syscall_nr;
595		595
596	mutex_lock(&syscall_trace_lock);	596	mutex_lock(&syscall_trace_lock);
597	if (!sys_perf_refcount_enter)	597	if (!sys_perf_refcount_enter)
598	ret = register_trace_sys_enter(perf_syscall_enter, NULL);	598	ret = register_trace_sys_enter(perf_syscall_enter, NULL);
599	if (ret) {	599	if (ret) {
600	pr_info("event trace: Could not activate"	600	pr_info("event trace: Could not activate"
601	"syscall entry trace point");	601	"syscall entry trace point");
602	} else {	602	} else {
603	set_bit(num, enabled_perf_enter_syscalls);	603	set_bit(num, enabled_perf_enter_syscalls);
604	sys_perf_refcount_enter++;	604	sys_perf_refcount_enter++;
605	}	605	}
606	mutex_unlock(&syscall_trace_lock);	606	mutex_unlock(&syscall_trace_lock);
607	return ret;	607	return ret;
608	}	608	}
609		609
610	static void perf_sysenter_disable(struct ftrace_event_call *call)	610	static void perf_sysenter_disable(struct ftrace_event_call *call)
611	{	611	{
612	int num;	612	int num;
613		613
614	num = ((struct syscall_metadata *)call->data)->syscall_nr;	614	num = ((struct syscall_metadata *)call->data)->syscall_nr;
615		615
616	mutex_lock(&syscall_trace_lock);	616	mutex_lock(&syscall_trace_lock);
617	sys_perf_refcount_enter--;	617	sys_perf_refcount_enter--;
618	clear_bit(num, enabled_perf_enter_syscalls);	618	clear_bit(num, enabled_perf_enter_syscalls);
619	if (!sys_perf_refcount_enter)	619	if (!sys_perf_refcount_enter)
620	unregister_trace_sys_enter(perf_syscall_enter, NULL);	620	unregister_trace_sys_enter(perf_syscall_enter, NULL);
621	mutex_unlock(&syscall_trace_lock);	621	mutex_unlock(&syscall_trace_lock);
622	}	622	}
623		623
624	static void perf_syscall_exit(void ignore, struct pt_regs regs, long ret)	624	static void perf_syscall_exit(void ignore, struct pt_regs regs, long ret)
625	{	625	{
626	struct syscall_metadata *sys_data;	626	struct syscall_metadata *sys_data;
627	struct syscall_trace_exit *rec;	627	struct syscall_trace_exit *rec;
628	struct hlist_head *head;	628	struct hlist_head *head;
629	int syscall_nr;	629	int syscall_nr;
630	int rctx;	630	int rctx;
631	int size;	631	int size;
632		632
633	syscall_nr = trace_get_syscall_nr(current, regs);	633	syscall_nr = trace_get_syscall_nr(current, regs);
634	if (syscall_nr < 0)	634	if (syscall_nr < 0)
635	return;	635	return;
636	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))	636	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
637	return;	637	return;
638		638
639	sys_data = syscall_nr_to_meta(syscall_nr);	639	sys_data = syscall_nr_to_meta(syscall_nr);
640	if (!sys_data)	640	if (!sys_data)
641	return;	641	return;
642		642
643	head = this_cpu_ptr(sys_data->exit_event->perf_events);	643	head = this_cpu_ptr(sys_data->exit_event->perf_events);
644	if (hlist_empty(head))	644	if (hlist_empty(head))
645	return;	645	return;
646		646
647	/* We can probably do that at build time */	647	/* We can probably do that at build time */
648	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));	648	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
649	size -= sizeof(u32);	649	size -= sizeof(u32);
650		650
651	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,	651	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
652	sys_data->exit_event->event.type, regs, &rctx);	652	sys_data->exit_event->event.type, regs, &rctx);
653	if (!rec)	653	if (!rec)
654	return;	654	return;
655		655
656	rec->nr = syscall_nr;	656	rec->nr = syscall_nr;
657	rec->ret = syscall_get_return_value(current, regs);	657	rec->ret = syscall_get_return_value(current, regs);
658	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);	658	perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL);
659	}	659	}
660		660
661	static int perf_sysexit_enable(struct ftrace_event_call *call)	661	static int perf_sysexit_enable(struct ftrace_event_call *call)
662	{	662	{
663	int ret = 0;	663	int ret = 0;
664	int num;	664	int num;
665		665
666	num = ((struct syscall_metadata *)call->data)->syscall_nr;	666	num = ((struct syscall_metadata *)call->data)->syscall_nr;
667		667
668	mutex_lock(&syscall_trace_lock);	668	mutex_lock(&syscall_trace_lock);
669	if (!sys_perf_refcount_exit)	669	if (!sys_perf_refcount_exit)
670	ret = register_trace_sys_exit(perf_syscall_exit, NULL);	670	ret = register_trace_sys_exit(perf_syscall_exit, NULL);
671	if (ret) {	671	if (ret) {
672	pr_info("event trace: Could not activate"	672	pr_info("event trace: Could not activate"
673	"syscall exit trace point");	673	"syscall exit trace point");
674	} else {	674	} else {
675	set_bit(num, enabled_perf_exit_syscalls);	675	set_bit(num, enabled_perf_exit_syscalls);
676	sys_perf_refcount_exit++;	676	sys_perf_refcount_exit++;
677	}	677	}
678	mutex_unlock(&syscall_trace_lock);	678	mutex_unlock(&syscall_trace_lock);
679	return ret;	679	return ret;
680	}	680	}
681		681
682	static void perf_sysexit_disable(struct ftrace_event_call *call)	682	static void perf_sysexit_disable(struct ftrace_event_call *call)
683	{	683	{
684	int num;	684	int num;
685		685
686	num = ((struct syscall_metadata *)call->data)->syscall_nr;	686	num = ((struct syscall_metadata *)call->data)->syscall_nr;
687		687
688	mutex_lock(&syscall_trace_lock);	688	mutex_lock(&syscall_trace_lock);
689	sys_perf_refcount_exit--;	689	sys_perf_refcount_exit--;
690	clear_bit(num, enabled_perf_exit_syscalls);	690	clear_bit(num, enabled_perf_exit_syscalls);
691	if (!sys_perf_refcount_exit)	691	if (!sys_perf_refcount_exit)
692	unregister_trace_sys_exit(perf_syscall_exit, NULL);	692	unregister_trace_sys_exit(perf_syscall_exit, NULL);
693	mutex_unlock(&syscall_trace_lock);	693	mutex_unlock(&syscall_trace_lock);
694	}	694	}
695		695
696	#endif /* CONFIG_PERF_EVENTS */	696	#endif /* CONFIG_PERF_EVENTS */
697		697
698	static int syscall_enter_register(struct ftrace_event_call *event,	698	static int syscall_enter_register(struct ftrace_event_call *event,
699	enum trace_reg type, void *data)	699	enum trace_reg type, void *data)
700	{	700	{
701	struct ftrace_event_file *file = data;	701	struct ftrace_event_file *file = data;
702		702
703	switch (type) {	703	switch (type) {
704	case TRACE_REG_REGISTER:	704	case TRACE_REG_REGISTER:
705	return reg_event_syscall_enter(file, event);	705	return reg_event_syscall_enter(file, event);
706	case TRACE_REG_UNREGISTER:	706	case TRACE_REG_UNREGISTER:
707	unreg_event_syscall_enter(file, event);	707	unreg_event_syscall_enter(file, event);
708	return 0;	708	return 0;
709		709
710	#ifdef CONFIG_PERF_EVENTS	710	#ifdef CONFIG_PERF_EVENTS
711	case TRACE_REG_PERF_REGISTER:	711	case TRACE_REG_PERF_REGISTER:
712	return perf_sysenter_enable(event);	712	return perf_sysenter_enable(event);
713	case TRACE_REG_PERF_UNREGISTER:	713	case TRACE_REG_PERF_UNREGISTER:
714	perf_sysenter_disable(event);	714	perf_sysenter_disable(event);
715	return 0;	715	return 0;
716	case TRACE_REG_PERF_OPEN:	716	case TRACE_REG_PERF_OPEN:
717	case TRACE_REG_PERF_CLOSE:	717	case TRACE_REG_PERF_CLOSE:
718	case TRACE_REG_PERF_ADD:	718	case TRACE_REG_PERF_ADD:
719	case TRACE_REG_PERF_DEL:	719	case TRACE_REG_PERF_DEL:
720	return 0;	720	return 0;
721	#endif	721	#endif
722	}	722	}
723	return 0;	723	return 0;
724	}	724	}
725		725
726	static int syscall_exit_register(struct ftrace_event_call *event,	726	static int syscall_exit_register(struct ftrace_event_call *event,
727	enum trace_reg type, void *data)	727	enum trace_reg type, void *data)
728	{	728	{
729	struct ftrace_event_file *file = data;	729	struct ftrace_event_file *file = data;
730		730
731	switch (type) {	731	switch (type) {
732	case TRACE_REG_REGISTER:	732	case TRACE_REG_REGISTER:
733	return reg_event_syscall_exit(file, event);	733	return reg_event_syscall_exit(file, event);
734	case TRACE_REG_UNREGISTER:	734	case TRACE_REG_UNREGISTER:
735	unreg_event_syscall_exit(file, event);	735	unreg_event_syscall_exit(file, event);
736	return 0;	736	return 0;
737		737
738	#ifdef CONFIG_PERF_EVENTS	738	#ifdef CONFIG_PERF_EVENTS
739	case TRACE_REG_PERF_REGISTER:	739	case TRACE_REG_PERF_REGISTER:
740	return perf_sysexit_enable(event);	740	return perf_sysexit_enable(event);
741	case TRACE_REG_PERF_UNREGISTER:	741	case TRACE_REG_PERF_UNREGISTER:
742	perf_sysexit_disable(event);	742	perf_sysexit_disable(event);
743	return 0;	743	return 0;
744	case TRACE_REG_PERF_OPEN:	744	case TRACE_REG_PERF_OPEN:
745	case TRACE_REG_PERF_CLOSE:	745	case TRACE_REG_PERF_CLOSE:
746	case TRACE_REG_PERF_ADD:	746	case TRACE_REG_PERF_ADD:
747	case TRACE_REG_PERF_DEL:	747	case TRACE_REG_PERF_DEL:
748	return 0;	748	return 0;
749	#endif	749	#endif
750	}	750	}
751	return 0;	751	return 0;
752	}	752	}
753		753