Eric Lee / smarc-ti-linux-kernel | Embedian Git Server

Commit 2aafe1a4d451866e3e7b476e2fa0813b69b313c1

Authored by Linus Torvalds 2014-04-29 07:57:51 +0800

Exists in master and in 13 other branches

Merge tag 'trace-fixes-v3.15-rc2' of git://git.kernel.org/pub/scm/linux/kernel/g…

…it/rostedt/linux-trace

Pull ftrace bugfix from Steven Rostedt:
 "Takao Indoh reported that he was able to cause a ftrace bug while
  loading a module and enabling function tracing at the same time.

  He uncovered a race where the module when loaded will convert the
  calls to mcount into nops, and expects the module's text to be RW.
  But when function tracing is enabled, it will convert all kernel text
  (core and module) from RO to RW to convert the nops to calls to ftrace
  to record the function.  After the convertion, it will convert all the
  text back from RW to RO.

  The issue is, it will also convert the module's text that is loading.
  If it converts it to RO before ftrace does its conversion, it will
  cause ftrace to fail and require a reboot to fix it again.

  This patch moves the ftrace module update that converts calls to
  mcount into nops to be done when the module state is still
  MODULE_STATE_UNFORMED.  This will ignore the module when the text is
  being converted from RW back to RO"

* tag 'trace-fixes-v3.15-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace:
  ftrace/module: Hardcode ftrace_module_init() call into load_module()

Showing 3 changed files Inline Diff

include/linux/ftrace.h
kernel/module.c
kernel/trace/ftrace.c

include/linux/ftrace.h

Diff comments View file @ 2aafe1a

 /*
  * Ftrace header.  For implementation details beyond the random comments
  * scattered below, see: Documentation/trace/ftrace-design.txt
  */
 #ifndef _LINUX_FTRACE_H
 #define _LINUX_FTRACE_H
 #include <linux/trace_clock.h>
 #include <linux/kallsyms.h>
 #include <linux/linkage.h>
 #include <linux/bitops.h>
 #include <linux/ptrace.h>
 #include <linux/ktime.h>
 #include <linux/sched.h>
 #include <linux/types.h>
 #include <linux/init.h>
 #include <linux/fs.h>
 #include <asm/ftrace.h>
 /*
  * If the arch supports passing the variable contents of
  * function_trace_op as the third parameter back from the
  * mcount call, then the arch should define this as 1.
  */
 #ifndef ARCH_SUPPORTS_FTRACE_OPS
 #define ARCH_SUPPORTS_FTRACE_OPS 0
 #endif
 /*
  * If the arch's mcount caller does not support all of ftrace's
  * features, then it must call an indirect function that
  * does. Or at least does enough to prevent any unwelcomed side effects.
  */
 #if !defined(CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST) || \
 	!ARCH_SUPPORTS_FTRACE_OPS
 # define FTRACE_FORCE_LIST_FUNC 1
 #else
 # define FTRACE_FORCE_LIST_FUNC 0
 #endif
 struct module;
 struct ftrace_hash;
 #ifdef CONFIG_FUNCTION_TRACER
 extern int ftrace_enabled;
 extern int
 ftrace_enable_sysctl(struct ctl_table *table, int write,
 		     void __user *buffer, size_t *lenp,
 		     loff_t *ppos);
 struct ftrace_ops;
 typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,
 			      struct ftrace_ops *op, struct pt_regs *regs);
 /*
  * FTRACE_OPS_FL_* bits denote the state of ftrace_ops struct and are
  * set in the flags member.
  *
  * ENABLED - set/unset when ftrace_ops is registered/unregistered
  * GLOBAL  - set manualy by ftrace_ops user to denote the ftrace_ops
  *           is part of the global tracers sharing the same filter
  *           via set_ftrace_* debugfs files.
  * DYNAMIC - set when ftrace_ops is registered to denote dynamically
  *           allocated ftrace_ops which need special care
  * CONTROL - set manualy by ftrace_ops user to denote the ftrace_ops
  *           could be controled by following calls:
  *             ftrace_function_local_enable
  *             ftrace_function_local_disable
  * SAVE_REGS - The ftrace_ops wants regs saved at each function called
  *            and passed to the callback. If this flag is set, but the
  *            architecture does not support passing regs
  *            (CONFIG_DYNAMIC_FTRACE_WITH_REGS is not defined), then the
  *            ftrace_ops will fail to register, unless the next flag
  *            is set.
  * SAVE_REGS_IF_SUPPORTED - This is the same as SAVE_REGS, but if the
  *            handler can handle an arch that does not save regs
  *            (the handler tests if regs == NULL), then it can set
  *            this flag instead. It will not fail registering the ftrace_ops
  *            but, the regs field will be NULL if the arch does not support
  *            passing regs to the handler.
  *            Note, if this flag is set, the SAVE_REGS flag will automatically
  *            get set upon registering the ftrace_ops, if the arch supports it.
  * RECURSION_SAFE - The ftrace_ops can set this to tell the ftrace infrastructure
  *            that the call back has its own recursion protection. If it does
  *            not set this, then the ftrace infrastructure will add recursion
  *            protection for the caller.
  * STUB   - The ftrace_ops is just a place holder.
  * INITIALIZED - The ftrace_ops has already been initialized (first use time
  *            register_ftrace_function() is called, it will initialized the ops)
  * DELETED - The ops are being deleted, do not let them be registered again.
  */
 enum {
 	FTRACE_OPS_FL_ENABLED			= 1 << 0,
 	FTRACE_OPS_FL_GLOBAL			= 1 << 1,
 	FTRACE_OPS_FL_DYNAMIC			= 1 << 2,
 	FTRACE_OPS_FL_CONTROL			= 1 << 3,
 	FTRACE_OPS_FL_SAVE_REGS			= 1 << 4,
 	FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED	= 1 << 5,
 	FTRACE_OPS_FL_RECURSION_SAFE		= 1 << 6,
 	FTRACE_OPS_FL_STUB			= 1 << 7,
 	FTRACE_OPS_FL_INITIALIZED		= 1 << 8,
 	FTRACE_OPS_FL_DELETED			= 1 << 9,
 };
 /*
  * Note, ftrace_ops can be referenced outside of RCU protection.
  * (Although, for perf, the control ops prevent that). If ftrace_ops is
  * allocated and not part of kernel core data, the unregistering of it will
  * perform a scheduling on all CPUs to make sure that there are no more users.
  * Depending on the load of the system that may take a bit of time.
  *
  * Any private data added must also take care not to be freed and if private
  * data is added to a ftrace_ops that is in core code, the user of the
  * ftrace_ops must perform a schedule_on_each_cpu() before freeing it.
  */
 struct ftrace_ops {
 	ftrace_func_t			func;
 	struct ftrace_ops		*next;
 	unsigned long			flags;
 	int __percpu			*disabled;
 	void				*private;
 #ifdef CONFIG_DYNAMIC_FTRACE
 	struct ftrace_hash		*notrace_hash;
 	struct ftrace_hash		*filter_hash;
 	struct mutex			regex_lock;
 #endif
 };
 extern int function_trace_stop;
 /*
  * Type of the current tracing.
  */
 enum ftrace_tracing_type_t {
 	FTRACE_TYPE_ENTER = 0, /* Hook the call of the function */
 	FTRACE_TYPE_RETURN,	/* Hook the return of the function */
 };
 /* Current tracing type, default is FTRACE_TYPE_ENTER */
 extern enum ftrace_tracing_type_t ftrace_tracing_type;
 /**
  * ftrace_stop - stop function tracer.
  *
  * A quick way to stop the function tracer. Note this an on off switch,
  * it is not something that is recursive like preempt_disable.
  * This does not disable the calling of mcount, it only stops the
  * calling of functions from mcount.
  */
 static inline void ftrace_stop(void)
 {
 	function_trace_stop = 1;
 }
 /**
  * ftrace_start - start the function tracer.
  *
  * This function is the inverse of ftrace_stop. This does not enable
  * the function tracing if the function tracer is disabled. This only
  * sets the function tracer flag to continue calling the functions
  * from mcount.
  */
 static inline void ftrace_start(void)
 {
 	function_trace_stop = 0;
 }
 /*
  * The ftrace_ops must be a static and should also
  * be read_mostly.  These functions do modify read_mostly variables
  * so use them sparely. Never free an ftrace_op or modify the
  * next pointer after it has been registered. Even after unregistering
  * it, the next pointer may still be used internally.
  */
 int register_ftrace_function(struct ftrace_ops *ops);
 int unregister_ftrace_function(struct ftrace_ops *ops);
 void clear_ftrace_function(void);
 /**
  * ftrace_function_local_enable - enable controlled ftrace_ops on current cpu
  *
  * This function enables tracing on current cpu by decreasing
  * the per cpu control variable.
  * It must be called with preemption disabled and only on ftrace_ops
  * registered with FTRACE_OPS_FL_CONTROL. If called without preemption
  * disabled, this_cpu_ptr will complain when CONFIG_DEBUG_PREEMPT is enabled.
  */
 static inline void ftrace_function_local_enable(struct ftrace_ops *ops)
 {
 	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_CONTROL)))
 		return;
 	(*this_cpu_ptr(ops->disabled))--;
 }
 /**
  * ftrace_function_local_disable - enable controlled ftrace_ops on current cpu
  *
  * This function enables tracing on current cpu by decreasing
  * the per cpu control variable.
  * It must be called with preemption disabled and only on ftrace_ops
  * registered with FTRACE_OPS_FL_CONTROL. If called without preemption
  * disabled, this_cpu_ptr will complain when CONFIG_DEBUG_PREEMPT is enabled.
  */
 static inline void ftrace_function_local_disable(struct ftrace_ops *ops)
 {
 	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_CONTROL)))
 		return;
 	(*this_cpu_ptr(ops->disabled))++;
 }
 /**
  * ftrace_function_local_disabled - returns ftrace_ops disabled value
  *                                  on current cpu
  *
  * This function returns value of ftrace_ops::disabled on current cpu.
  * It must be called with preemption disabled and only on ftrace_ops
  * registered with FTRACE_OPS_FL_CONTROL. If called without preemption
  * disabled, this_cpu_ptr will complain when CONFIG_DEBUG_PREEMPT is enabled.
  */
 static inline int ftrace_function_local_disabled(struct ftrace_ops *ops)
 {
 	WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_CONTROL));
 	return *this_cpu_ptr(ops->disabled);
 }
 extern void ftrace_stub(unsigned long a0, unsigned long a1,
 			struct ftrace_ops *op, struct pt_regs *regs);
 #else /* !CONFIG_FUNCTION_TRACER */
 /*
  * (un)register_ftrace_function must be a macro since the ops parameter
  * must not be evaluated.
  */
 #define register_ftrace_function(ops) ({ 0; })
 #define unregister_ftrace_function(ops) ({ 0; })
 static inline int ftrace_nr_registered_ops(void)
 {
 	return 0;
 }
 static inline void clear_ftrace_function(void) { }
 static inline void ftrace_kill(void) { }
 static inline void ftrace_stop(void) { }
 static inline void ftrace_start(void) { }
 #endif /* CONFIG_FUNCTION_TRACER */
 #ifdef CONFIG_STACK_TRACER
 extern int stack_tracer_enabled;
 int
 stack_trace_sysctl(struct ctl_table *table, int write,
 		   void __user *buffer, size_t *lenp,
 		   loff_t *ppos);
 #endif
 struct ftrace_func_command {
 	struct list_head	list;
 	char			*name;
 	int			(*func)(struct ftrace_hash *hash,
 					char *func, char *cmd,
 					char *params, int enable);
 };
 #ifdef CONFIG_DYNAMIC_FTRACE
 int ftrace_arch_code_modify_prepare(void);
 int ftrace_arch_code_modify_post_process(void);
 void ftrace_bug(int err, unsigned long ip);
 struct seq_file;
 struct ftrace_probe_ops {
 	void			(*func)(unsigned long ip,
 					unsigned long parent_ip,
 					void **data);
 	int			(*init)(struct ftrace_probe_ops *ops,
 					unsigned long ip, void **data);
 	void			(*free)(struct ftrace_probe_ops *ops,
 					unsigned long ip, void **data);
 	int			(*print)(struct seq_file *m,
 					 unsigned long ip,
 					 struct ftrace_probe_ops *ops,
 					 void *data);
 };
 extern int
 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 			      void *data);
 extern void
 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
 				void *data);
 extern void
 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops);
 extern void unregister_ftrace_function_probe_all(char *glob);
 extern int ftrace_text_reserved(const void *start, const void *end);
 extern int ftrace_nr_registered_ops(void);
 /*
  * The dyn_ftrace record's flags field is split into two parts.
  * the first part which is '0-FTRACE_REF_MAX' is a counter of
  * the number of callbacks that have registered the function that
  * the dyn_ftrace descriptor represents.
  *
  * The second part is a mask:
  *  ENABLED - the function is being traced
  *  REGS    - the record wants the function to save regs
  *  REGS_EN - the function is set up to save regs.
  *
  * When a new ftrace_ops is registered and wants a function to save
  * pt_regs, the rec->flag REGS is set. When the function has been
  * set up to save regs, the REG_EN flag is set. Once a function
  * starts saving regs it will do so until all ftrace_ops are removed
  * from tracing that function.
  */
 enum {
 	FTRACE_FL_ENABLED	= (1UL << 29),
 	FTRACE_FL_REGS		= (1UL << 30),
 	FTRACE_FL_REGS_EN	= (1UL << 31)
 };
 #define FTRACE_FL_MASK		(0x7UL << 29)
 #define FTRACE_REF_MAX		((1UL << 29) - 1)
 struct dyn_ftrace {
 	unsigned long		ip; /* address of mcount call-site */
 	unsigned long		flags;
 	struct dyn_arch_ftrace	arch;
 };
 int ftrace_force_update(void);
 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
 			 int remove, int reset);
 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
 		       int len, int reset);
 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
 			int len, int reset);
 void ftrace_set_global_filter(unsigned char *buf, int len, int reset);
 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
 void ftrace_free_filter(struct ftrace_ops *ops);
 int register_ftrace_command(struct ftrace_func_command *cmd);
 int unregister_ftrace_command(struct ftrace_func_command *cmd);
 enum {
 	FTRACE_UPDATE_CALLS		= (1 << 0),
 	FTRACE_DISABLE_CALLS		= (1 << 1),
 	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
 	FTRACE_START_FUNC_RET		= (1 << 3),
 	FTRACE_STOP_FUNC_RET		= (1 << 4),
 };
 /*
  * The FTRACE_UPDATE_* enum is used to pass information back
  * from the ftrace_update_record() and ftrace_test_record()
  * functions. These are called by the code update routines
  * to find out what is to be done for a given function.
  *
  *  IGNORE           - The function is already what we want it to be
  *  MAKE_CALL        - Start tracing the function
  *  MODIFY_CALL      - Stop saving regs for the function
  *  MODIFY_CALL_REGS - Start saving regs for the function
  *  MAKE_NOP         - Stop tracing the function
  */
 enum {
 	FTRACE_UPDATE_IGNORE,
 	FTRACE_UPDATE_MAKE_CALL,
 	FTRACE_UPDATE_MODIFY_CALL,
 	FTRACE_UPDATE_MODIFY_CALL_REGS,
 	FTRACE_UPDATE_MAKE_NOP,
 };
 enum {
 	FTRACE_ITER_FILTER	= (1 << 0),
 	FTRACE_ITER_NOTRACE	= (1 << 1),
 	FTRACE_ITER_PRINTALL	= (1 << 2),
 	FTRACE_ITER_DO_HASH	= (1 << 3),
 	FTRACE_ITER_HASH	= (1 << 4),
 	FTRACE_ITER_ENABLED	= (1 << 5),
 };
 void arch_ftrace_update_code(int command);
 struct ftrace_rec_iter;
 struct ftrace_rec_iter *ftrace_rec_iter_start(void);
 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter);
 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter);
 #define for_ftrace_rec_iter(iter)		\
 	for (iter = ftrace_rec_iter_start();	\
 	     iter;				\
 	     iter = ftrace_rec_iter_next(iter))
 int ftrace_update_record(struct dyn_ftrace *rec, int enable);
 int ftrace_test_record(struct dyn_ftrace *rec, int enable);
 void ftrace_run_stop_machine(int command);
 unsigned long ftrace_location(unsigned long ip);
 extern ftrace_func_t ftrace_trace_function;
 int ftrace_regex_open(struct ftrace_ops *ops, int flag,
 		  struct inode *inode, struct file *file);
 ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
 			    size_t cnt, loff_t *ppos);
 ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
 			     size_t cnt, loff_t *ppos);
 int ftrace_regex_release(struct inode *inode, struct file *file);
 void __init
 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable);
 /* defined in arch */
 extern int ftrace_ip_converted(unsigned long ip);
 extern int ftrace_dyn_arch_init(void);
 extern void ftrace_replace_code(int enable);
 extern int ftrace_update_ftrace_func(ftrace_func_t func);
 extern void ftrace_caller(void);
 extern void ftrace_regs_caller(void);
 extern void ftrace_call(void);
 extern void ftrace_regs_call(void);
 extern void mcount_call(void);
 void ftrace_modify_all_code(int command);
 #ifndef FTRACE_ADDR
 #define FTRACE_ADDR ((unsigned long)ftrace_caller)
 #endif
 #ifndef FTRACE_REGS_ADDR
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 # define FTRACE_REGS_ADDR ((unsigned long)ftrace_regs_caller)
 #else
 # define FTRACE_REGS_ADDR FTRACE_ADDR
 #endif
 #endif
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 extern void ftrace_graph_caller(void);
 extern int ftrace_enable_ftrace_graph_caller(void);
 extern int ftrace_disable_ftrace_graph_caller(void);
 #else
 static inline int ftrace_enable_ftrace_graph_caller(void) { return 0; }
 static inline int ftrace_disable_ftrace_graph_caller(void) { return 0; }
 #endif
 /**
  * ftrace_make_nop - convert code into nop
  * @mod: module structure if called by module load initialization
  * @rec: the mcount call site record
  * @addr: the address that the call site should be calling
  *
  * This is a very sensitive operation and great care needs
  * to be taken by the arch.  The operation should carefully
  * read the location, check to see if what is read is indeed
  * what we expect it to be, and then on success of the compare,
  * it should write to the location.
  *
  * The code segment at @rec->ip should be a caller to @addr
  *
  * Return must be:
  *  0 on success
  *  -EFAULT on error reading the location
  *  -EINVAL on a failed compare of the contents
  *  -EPERM  on error writing to the location
  * Any other value will be considered a failure.
  */
 extern int ftrace_make_nop(struct module *mod,
 			   struct dyn_ftrace *rec, unsigned long addr);
 /**
  * ftrace_make_call - convert a nop call site into a call to addr
  * @rec: the mcount call site record
  * @addr: the address that the call site should call
  *
  * This is a very sensitive operation and great care needs
  * to be taken by the arch.  The operation should carefully
  * read the location, check to see if what is read is indeed
  * what we expect it to be, and then on success of the compare,
  * it should write to the location.
  *
  * The code segment at @rec->ip should be a nop
  *
  * Return must be:
  *  0 on success
  *  -EFAULT on error reading the location
  *  -EINVAL on a failed compare of the contents
  *  -EPERM  on error writing to the location
  * Any other value will be considered a failure.
  */
 extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 /**
  * ftrace_modify_call - convert from one addr to another (no nop)
  * @rec: the mcount call site record
  * @old_addr: the address expected to be currently called to
  * @addr: the address to change to
  *
  * This is a very sensitive operation and great care needs
  * to be taken by the arch.  The operation should carefully
  * read the location, check to see if what is read is indeed
  * what we expect it to be, and then on success of the compare,
  * it should write to the location.
  *
  * The code segment at @rec->ip should be a caller to @old_addr
  *
  * Return must be:
  *  0 on success
  *  -EFAULT on error reading the location
  *  -EINVAL on a failed compare of the contents
  *  -EPERM  on error writing to the location
  * Any other value will be considered a failure.
  */
 extern int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 			      unsigned long addr);
 #else
 /* Should never be called */
 static inline int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 				     unsigned long addr)
 {
 	return -EINVAL;
 }
 #endif
 /* May be defined in arch */
 extern int ftrace_arch_read_dyn_info(char *buf, int size);
 extern int skip_trace(unsigned long ip);
+extern void ftrace_module_init(struct module *mod);
 extern void ftrace_disable_daemon(void);
 extern void ftrace_enable_daemon(void);
 #else /* CONFIG_DYNAMIC_FTRACE */
 static inline int skip_trace(unsigned long ip) { return 0; }
 static inline int ftrace_force_update(void) { return 0; }
 static inline void ftrace_disable_daemon(void) { }
 static inline void ftrace_enable_daemon(void) { }
 static inline void ftrace_release_mod(struct module *mod) {}
+static inline void ftrace_module_init(struct module *mod) {}
 static inline __init int register_ftrace_command(struct ftrace_func_command *cmd)
 {
 	return -EINVAL;
 }
 static inline __init int unregister_ftrace_command(char *cmd_name)
 {
 	return -EINVAL;
 }
 static inline int ftrace_text_reserved(const void *start, const void *end)
 {
 	return 0;
 }
 static inline unsigned long ftrace_location(unsigned long ip)
 {
 	return 0;
 }
 /*
  * Again users of functions that have ftrace_ops may not
  * have them defined when ftrace is not enabled, but these
  * functions may still be called. Use a macro instead of inline.
  */
 #define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; })
 #define ftrace_set_early_filter(ops, buf, enable) do { } while (0)
 #define ftrace_set_filter_ip(ops, ip, remove, reset) ({ -ENODEV; })
 #define ftrace_set_filter(ops, buf, len, reset) ({ -ENODEV; })
 #define ftrace_set_notrace(ops, buf, len, reset) ({ -ENODEV; })
 #define ftrace_free_filter(ops) do { } while (0)
 static inline ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
 			    size_t cnt, loff_t *ppos) { return -ENODEV; }
 static inline ssize_t ftrace_notrace_write(struct file *file, const char __user *ubuf,
 			     size_t cnt, loff_t *ppos) { return -ENODEV; }
 static inline int
 ftrace_regex_release(struct inode *inode, struct file *file) { return -ENODEV; }
 #endif /* CONFIG_DYNAMIC_FTRACE */
 /* totally disable ftrace - can not re-enable after this */
 void ftrace_kill(void);
 static inline void tracer_disable(void)
 {
 #ifdef CONFIG_FUNCTION_TRACER
 	ftrace_enabled = 0;
 #endif
 }
 /*
  * Ftrace disable/restore without lock. Some synchronization mechanism
  * must be used to prevent ftrace_enabled to be changed between
  * disable/restore.
  */
 static inline int __ftrace_enabled_save(void)
 {
 #ifdef CONFIG_FUNCTION_TRACER
 	int saved_ftrace_enabled = ftrace_enabled;
 	ftrace_enabled = 0;
 	return saved_ftrace_enabled;
 #else
 	return 0;
 #endif
 }
 static inline void __ftrace_enabled_restore(int enabled)
 {
 #ifdef CONFIG_FUNCTION_TRACER
 	ftrace_enabled = enabled;
 #endif
 }
 #ifndef HAVE_ARCH_CALLER_ADDR
 # ifdef CONFIG_FRAME_POINTER
 #  define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0))
 #  define CALLER_ADDR1 ((unsigned long)__builtin_return_address(1))
 #  define CALLER_ADDR2 ((unsigned long)__builtin_return_address(2))
 #  define CALLER_ADDR3 ((unsigned long)__builtin_return_address(3))
 #  define CALLER_ADDR4 ((unsigned long)__builtin_return_address(4))
 #  define CALLER_ADDR5 ((unsigned long)__builtin_return_address(5))
 #  define CALLER_ADDR6 ((unsigned long)__builtin_return_address(6))
 # else
 #  define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0))
 #  define CALLER_ADDR1 0UL
 #  define CALLER_ADDR2 0UL
 #  define CALLER_ADDR3 0UL
 #  define CALLER_ADDR4 0UL
 #  define CALLER_ADDR5 0UL
 #  define CALLER_ADDR6 0UL
 # endif
 #endif /* ifndef HAVE_ARCH_CALLER_ADDR */
 #ifdef CONFIG_IRQSOFF_TRACER
   extern void time_hardirqs_on(unsigned long a0, unsigned long a1);
   extern void time_hardirqs_off(unsigned long a0, unsigned long a1);
 #else
   static inline void time_hardirqs_on(unsigned long a0, unsigned long a1) { }
   static inline void time_hardirqs_off(unsigned long a0, unsigned long a1) { }
 #endif
 #ifdef CONFIG_PREEMPT_TRACER
   extern void trace_preempt_on(unsigned long a0, unsigned long a1);
   extern void trace_preempt_off(unsigned long a0, unsigned long a1);
 #else
 /*
  * Use defines instead of static inlines because some arches will make code out
  * of the CALLER_ADDR, when we really want these to be a real nop.
  */
 # define trace_preempt_on(a0, a1) do { } while (0)
 # define trace_preempt_off(a0, a1) do { } while (0)
 #endif
 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
 extern void ftrace_init(void);
 #else
 static inline void ftrace_init(void) { }
 #endif
 /*
  * Structure that defines an entry function trace.
  */
 struct ftrace_graph_ent {
 	unsigned long func; /* Current function */
 	int depth;
 };
 /*
  * Structure that defines a return function trace.
  */
 struct ftrace_graph_ret {
 	unsigned long func; /* Current function */
 	unsigned long long calltime;
 	unsigned long long rettime;
 	/* Number of functions that overran the depth limit for current task */
 	unsigned long overrun;
 	int depth;
 };
 /* Type of the callback handlers for tracing function graph*/
 typedef void (*trace_func_graph_ret_t)(struct ftrace_graph_ret *); /* return */
 typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *); /* entry */
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 /* for init task */
 #define INIT_FTRACE_GRAPH		.ret_stack = NULL,
 /*
  * Stack of return addresses for functions
  * of a thread.
  * Used in struct thread_info
  */
 struct ftrace_ret_stack {
 	unsigned long ret;
 	unsigned long func;
 	unsigned long long calltime;
 	unsigned long long subtime;
 	unsigned long fp;
 };
 /*
  * Primary handler of a function return.
  * It relays on ftrace_return_to_handler.
  * Defined in entry_32/64.S
  */
 extern void return_to_handler(void);
 extern int
 ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
 			 unsigned long frame_pointer);
 /*
  * Sometimes we don't want to trace a function with the function
  * graph tracer but we want them to keep traced by the usual function
  * tracer if the function graph tracer is not configured.
  */
 #define __notrace_funcgraph		notrace
 /*
  * We want to which function is an entrypoint of a hardirq.
  * That will help us to put a signal on output.
  */
 #define __irq_entry		 __attribute__((__section__(".irqentry.text")))
 /* Limits of hardirq entrypoints */
 extern char __irqentry_text_start[];
 extern char __irqentry_text_end[];
 #define FTRACE_NOTRACE_DEPTH 65536
 #define FTRACE_RETFUNC_DEPTH 50
 #define FTRACE_RETSTACK_ALLOC_SIZE 32
 extern int register_ftrace_graph(trace_func_graph_ret_t retfunc,
 				trace_func_graph_ent_t entryfunc);
 extern void ftrace_graph_stop(void);
 /* The current handlers in use */
 extern trace_func_graph_ret_t ftrace_graph_return;
 extern trace_func_graph_ent_t ftrace_graph_entry;
 extern void unregister_ftrace_graph(void);
 extern void ftrace_graph_init_task(struct task_struct *t);
 extern void ftrace_graph_exit_task(struct task_struct *t);
 extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu);
 static inline int task_curr_ret_stack(struct task_struct *t)
 {
 	return t->curr_ret_stack;
 }
 static inline void pause_graph_tracing(void)
 {
 	atomic_inc(&current->tracing_graph_pause);
 }
 static inline void unpause_graph_tracing(void)
 {
 	atomic_dec(&current->tracing_graph_pause);
 }
 #else /* !CONFIG_FUNCTION_GRAPH_TRACER */
 #define __notrace_funcgraph
 #define __irq_entry
 #define INIT_FTRACE_GRAPH
 static inline void ftrace_graph_init_task(struct task_struct *t) { }
 static inline void ftrace_graph_exit_task(struct task_struct *t) { }
 static inline void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { }
 static inline int register_ftrace_graph(trace_func_graph_ret_t retfunc,
 			  trace_func_graph_ent_t entryfunc)
 {
 	return -1;
 }
 static inline void unregister_ftrace_graph(void) { }
 static inline int task_curr_ret_stack(struct task_struct *tsk)
 {
 	return -1;
 }
 static inline void pause_graph_tracing(void) { }
 static inline void unpause_graph_tracing(void) { }
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 #ifdef CONFIG_TRACING
 /* flags for current->trace */
 enum {
 	TSK_TRACE_FL_TRACE_BIT	= 0,
 	TSK_TRACE_FL_GRAPH_BIT	= 1,
 };
 enum {
 	TSK_TRACE_FL_TRACE	= 1 << TSK_TRACE_FL_TRACE_BIT,
 	TSK_TRACE_FL_GRAPH	= 1 << TSK_TRACE_FL_GRAPH_BIT,
 };
 static inline void set_tsk_trace_trace(struct task_struct *tsk)
 {
 	set_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace);
 }
 static inline void clear_tsk_trace_trace(struct task_struct *tsk)
 {
 	clear_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace);
 }
 static inline int test_tsk_trace_trace(struct task_struct *tsk)
 {
 	return tsk->trace & TSK_TRACE_FL_TRACE;
 }
 static inline void set_tsk_trace_graph(struct task_struct *tsk)
 {
 	set_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace);
 }
 static inline void clear_tsk_trace_graph(struct task_struct *tsk)
 {
 	clear_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace);
 }
 static inline int test_tsk_trace_graph(struct task_struct *tsk)
 {
 	return tsk->trace & TSK_TRACE_FL_GRAPH;
 }
 enum ftrace_dump_mode;
 extern enum ftrace_dump_mode ftrace_dump_on_oops;
 extern void disable_trace_on_warning(void);
 extern int __disable_trace_on_warning;
 #ifdef CONFIG_PREEMPT
 #define INIT_TRACE_RECURSION		.trace_recursion = 0,
 #endif
 #else /* CONFIG_TRACING */
 static inline void  disable_trace_on_warning(void) { }
 #endif /* CONFIG_TRACING */
 #ifndef INIT_TRACE_RECURSION
 #define INIT_TRACE_RECURSION
 #endif
 #ifdef CONFIG_FTRACE_SYSCALLS
 unsigned long arch_syscall_addr(int nr);
 #endif /* CONFIG_FTRACE_SYSCALLS */
 #endif /* _LINUX_FTRACE_H */

kernel/module.c

Diff comments View file @ 2aafe1a

 /*
    Copyright (C) 2002 Richard Henderson
    Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
     You should have received a copy of the GNU General Public License
     along with this program; if not, write to the Free Software
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
 #include <linux/export.h>
 #include <linux/moduleloader.h>
 #include <linux/ftrace_event.h>
 #include <linux/init.h>
 #include <linux/kallsyms.h>
 #include <linux/file.h>
 #include <linux/fs.h>
 #include <linux/sysfs.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 #include <linux/elf.h>
 #include <linux/proc_fs.h>
 #include <linux/security.h>
 #include <linux/seq_file.h>
 #include <linux/syscalls.h>
 #include <linux/fcntl.h>
 #include <linux/rcupdate.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
 #include <linux/moduleparam.h>
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/vermagic.h>
 #include <linux/notifier.h>
 #include <linux/sched.h>
 #include <linux/stop_machine.h>
 #include <linux/device.h>
 #include <linux/string.h>
 #include <linux/mutex.h>
 #include <linux/rculist.h>
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
 #include <asm/mmu_context.h>
 #include <linux/license.h>
 #include <asm/sections.h>
 #include <linux/tracepoint.h>
 #include <linux/ftrace.h>
 #include <linux/async.h>
 #include <linux/percpu.h>
 #include <linux/kmemleak.h>
 #include <linux/jump_label.h>
 #include <linux/pfn.h>
 #include <linux/bsearch.h>
 #include <linux/fips.h>
 #include <uapi/linux/module.h>
 #include "module-internal.h"
 #define CREATE_TRACE_POINTS
 #include <trace/events/module.h>
 #ifndef ARCH_SHF_SMALL
 #define ARCH_SHF_SMALL 0
 #endif
 /*
  * Modules' sections will be aligned on page boundaries
  * to ensure complete separation of code and data, but
  * only when CONFIG_DEBUG_SET_MODULE_RONX=y
  */
 #ifdef CONFIG_DEBUG_SET_MODULE_RONX
 # define debug_align(X) ALIGN(X, PAGE_SIZE)
 #else
 # define debug_align(X) (X)
 #endif
 /*
  * Given BASE and SIZE this macro calculates the number of pages the
  * memory regions occupies
  */
 #define MOD_NUMBER_OF_PAGES(BASE, SIZE) (((SIZE) > 0) ?		\
 		(PFN_DOWN((unsigned long)(BASE) + (SIZE) - 1) -	\
 			 PFN_DOWN((unsigned long)BASE) + 1)	\
 		: (0UL))
 /* If this is set, the section belongs in the init part of the module */
 #define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))
 /*
  * Mutex protects:
  * 1) List of modules (also safely readable with preempt_disable),
  * 2) module_use links,
  * 3) module_addr_min/module_addr_max.
  * (delete uses stop_machine/add uses RCU list operations). */
 DEFINE_MUTEX(module_mutex);
 EXPORT_SYMBOL_GPL(module_mutex);
 static LIST_HEAD(modules);
 #ifdef CONFIG_KGDB_KDB
 struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
 #endif /* CONFIG_KGDB_KDB */
 #ifdef CONFIG_MODULE_SIG
 #ifdef CONFIG_MODULE_SIG_FORCE
 static bool sig_enforce = true;
 #else
 static bool sig_enforce = false;
 static int param_set_bool_enable_only(const char *val,
 				      const struct kernel_param *kp)
 {
 	int err;
 	bool test;
 	struct kernel_param dummy_kp = *kp;
 	dummy_kp.arg = &test;
 	err = param_set_bool(val, &dummy_kp);
 	if (err)
 		return err;
 	/* Don't let them unset it once it's set! */
 	if (!test && sig_enforce)
 		return -EROFS;
 	if (test)
 		sig_enforce = true;
 	return 0;
 }
 static const struct kernel_param_ops param_ops_bool_enable_only = {
 	.flags = KERNEL_PARAM_FL_NOARG,
 	.set = param_set_bool_enable_only,
 	.get = param_get_bool,
 };
 #define param_check_bool_enable_only param_check_bool
 module_param(sig_enforce, bool_enable_only, 0644);
 #endif /* !CONFIG_MODULE_SIG_FORCE */
 #endif /* CONFIG_MODULE_SIG */
 /* Block module loading/unloading? */
 int modules_disabled = 0;
 core_param(nomodule, modules_disabled, bint, 0);
 /* Waiting for a module to finish initializing? */
 static DECLARE_WAIT_QUEUE_HEAD(module_wq);
 static BLOCKING_NOTIFIER_HEAD(module_notify_list);
 /* Bounds of module allocation, for speeding __module_address.
  * Protected by module_mutex. */
 static unsigned long module_addr_min = -1UL, module_addr_max = 0;
 int register_module_notifier(struct notifier_block * nb)
 {
 	return blocking_notifier_chain_register(&module_notify_list, nb);
 }
 EXPORT_SYMBOL(register_module_notifier);
 int unregister_module_notifier(struct notifier_block * nb)
 {
 	return blocking_notifier_chain_unregister(&module_notify_list, nb);
 }
 EXPORT_SYMBOL(unregister_module_notifier);
 struct load_info {
 	Elf_Ehdr *hdr;
 	unsigned long len;
 	Elf_Shdr *sechdrs;
 	char *secstrings, *strtab;
 	unsigned long symoffs, stroffs;
 	struct _ddebug *debug;
 	unsigned int num_debug;
 	bool sig_ok;
 	struct {
 		unsigned int sym, str, mod, vers, info, pcpu;
 	} index;
 };
 /* We require a truly strong try_module_get(): 0 means failure due to
    ongoing or failed initialization etc. */
 static inline int strong_try_module_get(struct module *mod)
 {
 	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
 	if (mod && mod->state == MODULE_STATE_COMING)
 		return -EBUSY;
 	if (try_module_get(mod))
 		return 0;
 	else
 		return -ENOENT;
 }
 static inline void add_taint_module(struct module *mod, unsigned flag,
 				    enum lockdep_ok lockdep_ok)
 {
 	add_taint(flag, lockdep_ok);
 	mod->taints |= (1U << flag);
 }
 /*
  * A thread that wants to hold a reference to a module only while it
  * is running can call this to safely exit.  nfsd and lockd use this.
  */
 void __module_put_and_exit(struct module *mod, long code)
 {
 	module_put(mod);
 	do_exit(code);
 }
 EXPORT_SYMBOL(__module_put_and_exit);
 /* Find a module section: 0 means not found. */
 static unsigned int find_sec(const struct load_info *info, const char *name)
 {
 	unsigned int i;
 	for (i = 1; i < info->hdr->e_shnum; i++) {
 		Elf_Shdr *shdr = &info->sechdrs[i];
 		/* Alloc bit cleared means "ignore it." */
 		if ((shdr->sh_flags & SHF_ALLOC)
 		    && strcmp(info->secstrings + shdr->sh_name, name) == 0)
 			return i;
 	}
 	return 0;
 }
 /* Find a module section, or NULL. */
 static void *section_addr(const struct load_info *info, const char *name)
 {
 	/* Section 0 has sh_addr 0. */
 	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
 }
 /* Find a module section, or NULL.  Fill in number of "objects" in section. */
 static void *section_objs(const struct load_info *info,
 			  const char *name,
 			  size_t object_size,
 			  unsigned int *num)
 {
 	unsigned int sec = find_sec(info, name);
 	/* Section 0 has sh_addr 0 and sh_size 0. */
 	*num = info->sechdrs[sec].sh_size / object_size;
 	return (void *)info->sechdrs[sec].sh_addr;
 }
 /* Provided by the linker */
 extern const struct kernel_symbol __start___ksymtab[];
 extern const struct kernel_symbol __stop___ksymtab[];
 extern const struct kernel_symbol __start___ksymtab_gpl[];
 extern const struct kernel_symbol __stop___ksymtab_gpl[];
 extern const struct kernel_symbol __start___ksymtab_gpl_future[];
 extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
 extern const unsigned long __start___kcrctab[];
 extern const unsigned long __start___kcrctab_gpl[];
 extern const unsigned long __start___kcrctab_gpl_future[];
 #ifdef CONFIG_UNUSED_SYMBOLS
 extern const struct kernel_symbol __start___ksymtab_unused[];
 extern const struct kernel_symbol __stop___ksymtab_unused[];
 extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
 extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
 extern const unsigned long __start___kcrctab_unused[];
 extern const unsigned long __start___kcrctab_unused_gpl[];
 #endif
 #ifndef CONFIG_MODVERSIONS
 #define symversion(base, idx) NULL
 #else
 #define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
 #endif
 static bool each_symbol_in_section(const struct symsearch *arr,
 				   unsigned int arrsize,
 				   struct module *owner,
 				   bool (*fn)(const struct symsearch *syms,
 					      struct module *owner,
 					      void *data),
 				   void *data)
 {
 	unsigned int j;
 	for (j = 0; j < arrsize; j++) {
 		if (fn(&arr[j], owner, data))
 			return true;
 	}
 	return false;
 }
 /* Returns true as soon as fn returns true, otherwise false. */
 bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
 				    struct module *owner,
 				    void *data),
 			 void *data)
 {
 	struct module *mod;
 	static const struct symsearch arr[] = {
 		{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
 		  NOT_GPL_ONLY, false },
 		{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
 		  __start___kcrctab_gpl,
 		  GPL_ONLY, false },
 		{ __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
 		  __start___kcrctab_gpl_future,
 		  WILL_BE_GPL_ONLY, false },
 #ifdef CONFIG_UNUSED_SYMBOLS
 		{ __start___ksymtab_unused, __stop___ksymtab_unused,
 		  __start___kcrctab_unused,
 		  NOT_GPL_ONLY, true },
 		{ __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
 		  __start___kcrctab_unused_gpl,
 		  GPL_ONLY, true },
 #endif
 	};
 	if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
 		return true;
 	list_for_each_entry_rcu(mod, &modules, list) {
 		struct symsearch arr[] = {
 			{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
 			  NOT_GPL_ONLY, false },
 			{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
 			  mod->gpl_crcs,
 			  GPL_ONLY, false },
 			{ mod->gpl_future_syms,
 			  mod->gpl_future_syms + mod->num_gpl_future_syms,
 			  mod->gpl_future_crcs,
 			  WILL_BE_GPL_ONLY, false },
 #ifdef CONFIG_UNUSED_SYMBOLS
 			{ mod->unused_syms,
 			  mod->unused_syms + mod->num_unused_syms,
 			  mod->unused_crcs,
 			  NOT_GPL_ONLY, true },
 			{ mod->unused_gpl_syms,
 			  mod->unused_gpl_syms + mod->num_unused_gpl_syms,
 			  mod->unused_gpl_crcs,
 			  GPL_ONLY, true },
 #endif
 		};
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
 			return true;
 	}
 	return false;
 }
 EXPORT_SYMBOL_GPL(each_symbol_section);
 struct find_symbol_arg {
 	/* Input */
 	const char *name;
 	bool gplok;
 	bool warn;
 	/* Output */
 	struct module *owner;
 	const unsigned long *crc;
 	const struct kernel_symbol *sym;
 };
 static bool check_symbol(const struct symsearch *syms,
 				 struct module *owner,
 				 unsigned int symnum, void *data)
 {
 	struct find_symbol_arg *fsa = data;
 	if (!fsa->gplok) {
 		if (syms->licence == GPL_ONLY)
 			return false;
 		if (syms->licence == WILL_BE_GPL_ONLY && fsa->warn) {
 			pr_warn("Symbol %s is being used by a non-GPL module, "
 				"which will not be allowed in the future\n",
 				fsa->name);
 		}
 	}
 #ifdef CONFIG_UNUSED_SYMBOLS
 	if (syms->unused && fsa->warn) {
 		pr_warn("Symbol %s is marked as UNUSED, however this module is "
 			"using it.\n", fsa->name);
 		pr_warn("This symbol will go away in the future.\n");
 		pr_warn("Please evalute if this is the right api to use and if "
 			"it really is, submit a report the linux kernel "
 			"mailinglist together with submitting your code for "
 			"inclusion.\n");
 	}
 #endif
 	fsa->owner = owner;
 	fsa->crc = symversion(syms->crcs, symnum);
 	fsa->sym = &syms->start[symnum];
 	return true;
 }
 static int cmp_name(const void *va, const void *vb)
 {
 	const char *a;
 	const struct kernel_symbol *b;
 	a = va; b = vb;
 	return strcmp(a, b->name);
 }
 static bool find_symbol_in_section(const struct symsearch *syms,
 				   struct module *owner,
 				   void *data)
 {
 	struct find_symbol_arg *fsa = data;
 	struct kernel_symbol *sym;
 	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
 			sizeof(struct kernel_symbol), cmp_name);
 	if (sym != NULL && check_symbol(syms, owner, sym - syms->start, data))
 		return true;
 	return false;
 }
 /* Find a symbol and return it, along with, (optional) crc and
  * (optional) module which owns it.  Needs preempt disabled or module_mutex. */
 const struct kernel_symbol *find_symbol(const char *name,
 					struct module **owner,
 					const unsigned long **crc,
 					bool gplok,
 					bool warn)
 {
 	struct find_symbol_arg fsa;
 	fsa.name = name;
 	fsa.gplok = gplok;
 	fsa.warn = warn;
 	if (each_symbol_section(find_symbol_in_section, &fsa)) {
 		if (owner)
 			*owner = fsa.owner;
 		if (crc)
 			*crc = fsa.crc;
 		return fsa.sym;
 	}
 	pr_debug("Failed to find symbol %s\n", name);
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(find_symbol);
 /* Search for module by name: must hold module_mutex. */
 static struct module *find_module_all(const char *name, size_t len,
 				      bool even_unformed)
 {
 	struct module *mod;
 	list_for_each_entry(mod, &modules, list) {
 		if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
 			return mod;
 	}
 	return NULL;
 }
 struct module *find_module(const char *name)
 {
 	return find_module_all(name, strlen(name), false);
 }
 EXPORT_SYMBOL_GPL(find_module);
 #ifdef CONFIG_SMP
 static inline void __percpu *mod_percpu(struct module *mod)
 {
 	return mod->percpu;
 }
 static int percpu_modalloc(struct module *mod, struct load_info *info)
 {
 	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
 	unsigned long align = pcpusec->sh_addralign;
 	if (!pcpusec->sh_size)
 		return 0;
 	if (align > PAGE_SIZE) {
 		pr_warn("%s: per-cpu alignment %li > %li\n",
 			mod->name, align, PAGE_SIZE);
 		align = PAGE_SIZE;
 	}
 	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
 	if (!mod->percpu) {
 		pr_warn("%s: Could not allocate %lu bytes percpu data\n",
 			mod->name, (unsigned long)pcpusec->sh_size);
 		return -ENOMEM;
 	}
 	mod->percpu_size = pcpusec->sh_size;
 	return 0;
 }
 static void percpu_modfree(struct module *mod)
 {
 	free_percpu(mod->percpu);
 }
 static unsigned int find_pcpusec(struct load_info *info)
 {
 	return find_sec(info, ".data..percpu");
 }
 static void percpu_modcopy(struct module *mod,
 			   const void *from, unsigned long size)
 {
 	int cpu;
 	for_each_possible_cpu(cpu)
 		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
 }
 /**
  * is_module_percpu_address - test whether address is from module static percpu
  * @addr: address to test
  *
  * Test whether @addr belongs to module static percpu area.
  *
  * RETURNS:
  * %true if @addr is from module static percpu area
  */
 bool is_module_percpu_address(unsigned long addr)
 {
 	struct module *mod;
 	unsigned int cpu;
 	preempt_disable();
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (!mod->percpu_size)
 			continue;
 		for_each_possible_cpu(cpu) {
 			void *start = per_cpu_ptr(mod->percpu, cpu);
 			if ((void *)addr >= start &&
 			    (void *)addr < start + mod->percpu_size) {
 				preempt_enable();
 				return true;
 			}
 		}
 	}
 	preempt_enable();
 	return false;
 }
 #else /* ... !CONFIG_SMP */
 static inline void __percpu *mod_percpu(struct module *mod)
 {
 	return NULL;
 }
 static int percpu_modalloc(struct module *mod, struct load_info *info)
 {
 	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
 	if (info->sechdrs[info->index.pcpu].sh_size != 0)
 		return -ENOMEM;
 	return 0;
 }
 static inline void percpu_modfree(struct module *mod)
 {
 }
 static unsigned int find_pcpusec(struct load_info *info)
 {
 	return 0;
 }
 static inline void percpu_modcopy(struct module *mod,
 				  const void *from, unsigned long size)
 {
 	/* pcpusec should be 0, and size of that section should be 0. */
 	BUG_ON(size != 0);
 }
 bool is_module_percpu_address(unsigned long addr)
 {
 	return false;
 }
 #endif /* CONFIG_SMP */
 #define MODINFO_ATTR(field)	\
 static void setup_modinfo_##field(struct module *mod, const char *s)  \
 {                                                                     \
 	mod->field = kstrdup(s, GFP_KERNEL);                          \
 }                                                                     \
 static ssize_t show_modinfo_##field(struct module_attribute *mattr,   \
 			struct module_kobject *mk, char *buffer)      \
 {                                                                     \
 	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field);  \
 }                                                                     \
 static int modinfo_##field##_exists(struct module *mod)               \
 {                                                                     \
 	return mod->field != NULL;                                    \
 }                                                                     \
 static void free_modinfo_##field(struct module *mod)                  \
 {                                                                     \
 	kfree(mod->field);                                            \
 	mod->field = NULL;                                            \
 }                                                                     \
 static struct module_attribute modinfo_##field = {                    \
 	.attr = { .name = __stringify(field), .mode = 0444 },         \
 	.show = show_modinfo_##field,                                 \
 	.setup = setup_modinfo_##field,                               \
 	.test = modinfo_##field##_exists,                             \
 	.free = free_modinfo_##field,                                 \
 };
 MODINFO_ATTR(version);
 MODINFO_ATTR(srcversion);
 static char last_unloaded_module[MODULE_NAME_LEN+1];
 #ifdef CONFIG_MODULE_UNLOAD
 EXPORT_TRACEPOINT_SYMBOL(module_get);
 /* Init the unload section of the module. */
 static int module_unload_init(struct module *mod)
 {
 	mod->refptr = alloc_percpu(struct module_ref);
 	if (!mod->refptr)
 		return -ENOMEM;
 	INIT_LIST_HEAD(&mod->source_list);
 	INIT_LIST_HEAD(&mod->target_list);
 	/* Hold reference count during initialization. */
 	raw_cpu_write(mod->refptr->incs, 1);
 	return 0;
 }
 /* Does a already use b? */
 static int already_uses(struct module *a, struct module *b)
 {
 	struct module_use *use;
 	list_for_each_entry(use, &b->source_list, source_list) {
 		if (use->source == a) {
 			pr_debug("%s uses %s!\n", a->name, b->name);
 			return 1;
 		}
 	}
 	pr_debug("%s does not use %s!\n", a->name, b->name);
 	return 0;
 }
 /*
  * Module a uses b
  *  - we add 'a' as a "source", 'b' as a "target" of module use
  *  - the module_use is added to the list of 'b' sources (so
  *    'b' can walk the list to see who sourced them), and of 'a'
  *    targets (so 'a' can see what modules it targets).
  */
 static int add_module_usage(struct module *a, struct module *b)
 {
 	struct module_use *use;
 	pr_debug("Allocating new usage for %s.\n", a->name);
 	use = kmalloc(sizeof(*use), GFP_ATOMIC);
 	if (!use) {
 		pr_warn("%s: out of memory loading\n", a->name);
 		return -ENOMEM;
 	}
 	use->source = a;
 	use->target = b;
 	list_add(&use->source_list, &b->source_list);
 	list_add(&use->target_list, &a->target_list);
 	return 0;
 }
 /* Module a uses b: caller needs module_mutex() */
 int ref_module(struct module *a, struct module *b)
 {
 	int err;
 	if (b == NULL || already_uses(a, b))
 		return 0;
 	/* If module isn't available, we fail. */
 	err = strong_try_module_get(b);
 	if (err)
 		return err;
 	err = add_module_usage(a, b);
 	if (err) {
 		module_put(b);
 		return err;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(ref_module);
 /* Clear the unload stuff of the module. */
 static void module_unload_free(struct module *mod)
 {
 	struct module_use *use, *tmp;
 	mutex_lock(&module_mutex);
 	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
 		struct module *i = use->target;
 		pr_debug("%s unusing %s\n", mod->name, i->name);
 		module_put(i);
 		list_del(&use->source_list);
 		list_del(&use->target_list);
 		kfree(use);
 	}
 	mutex_unlock(&module_mutex);
 	free_percpu(mod->refptr);
 }
 #ifdef CONFIG_MODULE_FORCE_UNLOAD
 static inline int try_force_unload(unsigned int flags)
 {
 	int ret = (flags & O_TRUNC);
 	if (ret)
 		add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
 	return ret;
 }
 #else
 static inline int try_force_unload(unsigned int flags)
 {
 	return 0;
 }
 #endif /* CONFIG_MODULE_FORCE_UNLOAD */
 struct stopref
 {
 	struct module *mod;
 	int flags;
 	int *forced;
 };
 /* Whole machine is stopped with interrupts off when this runs. */
 static int __try_stop_module(void *_sref)
 {
 	struct stopref *sref = _sref;
 	/* If it's not unused, quit unless we're forcing. */
 	if (module_refcount(sref->mod) != 0) {
 		if (!(*sref->forced = try_force_unload(sref->flags)))
 			return -EWOULDBLOCK;
 	}
 	/* Mark it as dying. */
 	sref->mod->state = MODULE_STATE_GOING;
 	return 0;
 }
 static int try_stop_module(struct module *mod, int flags, int *forced)
 {
 	struct stopref sref = { mod, flags, forced };
 	return stop_machine(__try_stop_module, &sref, NULL);
 }
 unsigned long module_refcount(struct module *mod)
 {
 	unsigned long incs = 0, decs = 0;
 	int cpu;
 	for_each_possible_cpu(cpu)
 		decs += per_cpu_ptr(mod->refptr, cpu)->decs;
 	/*
 	 * ensure the incs are added up after the decs.
 	 * module_put ensures incs are visible before decs with smp_wmb.
 	 *
 	 * This 2-count scheme avoids the situation where the refcount
 	 * for CPU0 is read, then CPU0 increments the module refcount,
 	 * then CPU1 drops that refcount, then the refcount for CPU1 is
 	 * read. We would record a decrement but not its corresponding
 	 * increment so we would see a low count (disaster).
 	 *
 	 * Rare situation? But module_refcount can be preempted, and we
 	 * might be tallying up 4096+ CPUs. So it is not impossible.
 	 */
 	smp_rmb();
 	for_each_possible_cpu(cpu)
 		incs += per_cpu_ptr(mod->refptr, cpu)->incs;
 	return incs - decs;
 }
 EXPORT_SYMBOL(module_refcount);
 /* This exists whether we can unload or not */
 static void free_module(struct module *mod);
 SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
 		unsigned int, flags)
 {
 	struct module *mod;
 	char name[MODULE_NAME_LEN];
 	int ret, forced = 0;
 	if (!capable(CAP_SYS_MODULE) || modules_disabled)
 		return -EPERM;
 	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
 		return -EFAULT;
 	name[MODULE_NAME_LEN-1] = '\0';
 	if (!(flags & O_NONBLOCK))
 		pr_warn("waiting module removal not supported: please upgrade\n");
 	if (mutex_lock_interruptible(&module_mutex) != 0)
 		return -EINTR;
 	mod = find_module(name);
 	if (!mod) {
 		ret = -ENOENT;
 		goto out;
 	}
 	if (!list_empty(&mod->source_list)) {
 		/* Other modules depend on us: get rid of them first. */
 		ret = -EWOULDBLOCK;
 		goto out;
 	}
 	/* Doing init or already dying? */
 	if (mod->state != MODULE_STATE_LIVE) {
 		/* FIXME: if (force), slam module count damn the torpedoes */
 		pr_debug("%s already dying\n", mod->name);
 		ret = -EBUSY;
 		goto out;
 	}
 	/* If it has an init func, it must have an exit func to unload */
 	if (mod->init && !mod->exit) {
 		forced = try_force_unload(flags);
 		if (!forced) {
 			/* This module can't be removed */
 			ret = -EBUSY;
 			goto out;
 		}
 	}
 	/* Stop the machine so refcounts can't move and disable module. */
 	ret = try_stop_module(mod, flags, &forced);
 	if (ret != 0)
 		goto out;
 	mutex_unlock(&module_mutex);
 	/* Final destruction now no one is using it. */
 	if (mod->exit != NULL)
 		mod->exit();
 	blocking_notifier_call_chain(&module_notify_list,
 				     MODULE_STATE_GOING, mod);
 	async_synchronize_full();
 	/* Store the name of the last unloaded module for diagnostic purposes */
 	strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
 	free_module(mod);
 	return 0;
 out:
 	mutex_unlock(&module_mutex);
 	return ret;
 }
 static inline void print_unload_info(struct seq_file *m, struct module *mod)
 {
 	struct module_use *use;
 	int printed_something = 0;
 	seq_printf(m, " %lu ", module_refcount(mod));
 	/* Always include a trailing , so userspace can differentiate
            between this and the old multi-field proc format. */
 	list_for_each_entry(use, &mod->source_list, source_list) {
 		printed_something = 1;
 		seq_printf(m, "%s,", use->source->name);
 	}
 	if (mod->init != NULL && mod->exit == NULL) {
 		printed_something = 1;
 		seq_printf(m, "[permanent],");
 	}
 	if (!printed_something)
 		seq_printf(m, "-");
 }
 void __symbol_put(const char *symbol)
 {
 	struct module *owner;
 	preempt_disable();
 	if (!find_symbol(symbol, &owner, NULL, true, false))
 		BUG();
 	module_put(owner);
 	preempt_enable();
 }
 EXPORT_SYMBOL(__symbol_put);
 /* Note this assumes addr is a function, which it currently always is. */
 void symbol_put_addr(void *addr)
 {
 	struct module *modaddr;
 	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
 	if (core_kernel_text(a))
 		return;
 	/* module_text_address is safe here: we're supposed to have reference
 	 * to module from symbol_get, so it can't go away. */
 	modaddr = __module_text_address(a);
 	BUG_ON(!modaddr);
 	module_put(modaddr);
 }
 EXPORT_SYMBOL_GPL(symbol_put_addr);
 static ssize_t show_refcnt(struct module_attribute *mattr,
 			   struct module_kobject *mk, char *buffer)
 {
 	return sprintf(buffer, "%lu\n", module_refcount(mk->mod));
 }
 static struct module_attribute modinfo_refcnt =
 	__ATTR(refcnt, 0444, show_refcnt, NULL);
 void __module_get(struct module *module)
 {
 	if (module) {
 		preempt_disable();
 		__this_cpu_inc(module->refptr->incs);
 		trace_module_get(module, _RET_IP_);
 		preempt_enable();
 	}
 }
 EXPORT_SYMBOL(__module_get);
 bool try_module_get(struct module *module)
 {
 	bool ret = true;
 	if (module) {
 		preempt_disable();
 		if (likely(module_is_live(module))) {
 			__this_cpu_inc(module->refptr->incs);
 			trace_module_get(module, _RET_IP_);
 		} else
 			ret = false;
 		preempt_enable();
 	}
 	return ret;
 }
 EXPORT_SYMBOL(try_module_get);
 void module_put(struct module *module)
 {
 	if (module) {
 		preempt_disable();
 		smp_wmb(); /* see comment in module_refcount */
 		__this_cpu_inc(module->refptr->decs);
 		trace_module_put(module, _RET_IP_);
 		preempt_enable();
 	}
 }
 EXPORT_SYMBOL(module_put);
 #else /* !CONFIG_MODULE_UNLOAD */
 static inline void print_unload_info(struct seq_file *m, struct module *mod)
 {
 	/* We don't know the usage count, or what modules are using. */
 	seq_printf(m, " - -");
 }
 static inline void module_unload_free(struct module *mod)
 {
 }
 int ref_module(struct module *a, struct module *b)
 {
 	return strong_try_module_get(b);
 }
 EXPORT_SYMBOL_GPL(ref_module);
 static inline int module_unload_init(struct module *mod)
 {
 	return 0;
 }
 #endif /* CONFIG_MODULE_UNLOAD */
 static size_t module_flags_taint(struct module *mod, char *buf)
 {
 	size_t l = 0;
 	if (mod->taints & (1 << TAINT_PROPRIETARY_MODULE))
 		buf[l++] = 'P';
 	if (mod->taints & (1 << TAINT_OOT_MODULE))
 		buf[l++] = 'O';
 	if (mod->taints & (1 << TAINT_FORCED_MODULE))
 		buf[l++] = 'F';
 	if (mod->taints & (1 << TAINT_CRAP))
 		buf[l++] = 'C';
 	if (mod->taints & (1 << TAINT_UNSIGNED_MODULE))
 		buf[l++] = 'E';
 	/*
 	 * TAINT_FORCED_RMMOD: could be added.
 	 * TAINT_CPU_OUT_OF_SPEC, TAINT_MACHINE_CHECK, TAINT_BAD_PAGE don't
 	 * apply to modules.
 	 */
 	return l;
 }
 static ssize_t show_initstate(struct module_attribute *mattr,
 			      struct module_kobject *mk, char *buffer)
 {
 	const char *state = "unknown";
 	switch (mk->mod->state) {
 	case MODULE_STATE_LIVE:
 		state = "live";
 		break;
 	case MODULE_STATE_COMING:
 		state = "coming";
 		break;
 	case MODULE_STATE_GOING:
 		state = "going";
 		break;
 	default:
 		BUG();
 	}
 	return sprintf(buffer, "%s\n", state);
 }
 static struct module_attribute modinfo_initstate =
 	__ATTR(initstate, 0444, show_initstate, NULL);
 static ssize_t store_uevent(struct module_attribute *mattr,
 			    struct module_kobject *mk,
 			    const char *buffer, size_t count)
 {
 	enum kobject_action action;
 	if (kobject_action_type(buffer, count, &action) == 0)
 		kobject_uevent(&mk->kobj, action);
 	return count;
 }
 struct module_attribute module_uevent =
 	__ATTR(uevent, 0200, NULL, store_uevent);
 static ssize_t show_coresize(struct module_attribute *mattr,
 			     struct module_kobject *mk, char *buffer)
 {
 	return sprintf(buffer, "%u\n", mk->mod->core_size);
 }
 static struct module_attribute modinfo_coresize =
 	__ATTR(coresize, 0444, show_coresize, NULL);
 static ssize_t show_initsize(struct module_attribute *mattr,
 			     struct module_kobject *mk, char *buffer)
 {
 	return sprintf(buffer, "%u\n", mk->mod->init_size);
 }
 static struct module_attribute modinfo_initsize =
 	__ATTR(initsize, 0444, show_initsize, NULL);
 static ssize_t show_taint(struct module_attribute *mattr,
 			  struct module_kobject *mk, char *buffer)
 {
 	size_t l;
 	l = module_flags_taint(mk->mod, buffer);
 	buffer[l++] = '\n';
 	return l;
 }
 static struct module_attribute modinfo_taint =
 	__ATTR(taint, 0444, show_taint, NULL);
 static struct module_attribute *modinfo_attrs[] = {
 	&module_uevent,
 	&modinfo_version,
 	&modinfo_srcversion,
 	&modinfo_initstate,
 	&modinfo_coresize,
 	&modinfo_initsize,
 	&modinfo_taint,
 #ifdef CONFIG_MODULE_UNLOAD
 	&modinfo_refcnt,
 #endif
 	NULL,
 };
 static const char vermagic[] = VERMAGIC_STRING;
 static int try_to_force_load(struct module *mod, const char *reason)
 {
 #ifdef CONFIG_MODULE_FORCE_LOAD
 	if (!test_taint(TAINT_FORCED_MODULE))
 		pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
 	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
 	return 0;
 #else
 	return -ENOEXEC;
 #endif
 }
 #ifdef CONFIG_MODVERSIONS
 /* If the arch applies (non-zero) relocations to kernel kcrctab, unapply it. */
 static unsigned long maybe_relocated(unsigned long crc,
 				     const struct module *crc_owner)
 {
 #ifdef ARCH_RELOCATES_KCRCTAB
 	if (crc_owner == NULL)
 		return crc - (unsigned long)reloc_start;
 #endif
 	return crc;
 }
 static int check_version(Elf_Shdr *sechdrs,
 			 unsigned int versindex,
 			 const char *symname,
 			 struct module *mod,
 			 const unsigned long *crc,
 			 const struct module *crc_owner)
 {
 	unsigned int i, num_versions;
 	struct modversion_info *versions;
 	/* Exporting module didn't supply crcs?  OK, we're already tainted. */
 	if (!crc)
 		return 1;
 	/* No versions at all?  modprobe --force does this. */
 	if (versindex == 0)
 		return try_to_force_load(mod, symname) == 0;
 	versions = (void *) sechdrs[versindex].sh_addr;
 	num_versions = sechdrs[versindex].sh_size
 		/ sizeof(struct modversion_info);
 	for (i = 0; i < num_versions; i++) {
 		if (strcmp(versions[i].name, symname) != 0)
 			continue;
 		if (versions[i].crc == maybe_relocated(*crc, crc_owner))
 			return 1;
 		pr_debug("Found checksum %lX vs module %lX\n",
 		       maybe_relocated(*crc, crc_owner), versions[i].crc);
 		goto bad_version;
 	}
 	pr_warn("%s: no symbol version for %s\n", mod->name, symname);
 	return 0;
 bad_version:
 	printk("%s: disagrees about version of symbol %s\n",
 	       mod->name, symname);
 	return 0;
 }
 static inline int check_modstruct_version(Elf_Shdr *sechdrs,
 					  unsigned int versindex,
 					  struct module *mod)
 {
 	const unsigned long *crc;
 	/* Since this should be found in kernel (which can't be removed),
 	 * no locking is necessary. */
 	if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL,
 			 &crc, true, false))
 		BUG();
 	return check_version(sechdrs, versindex,
 			     VMLINUX_SYMBOL_STR(module_layout), mod, crc,
 			     NULL);
 }
 /* First part is kernel version, which we ignore if module has crcs. */
 static inline int same_magic(const char *amagic, const char *bmagic,
 			     bool has_crcs)
 {
 	if (has_crcs) {
 		amagic += strcspn(amagic, " ");
 		bmagic += strcspn(bmagic, " ");
 	}
 	return strcmp(amagic, bmagic) == 0;
 }
 #else
 static inline int check_version(Elf_Shdr *sechdrs,
 				unsigned int versindex,
 				const char *symname,
 				struct module *mod,
 				const unsigned long *crc,
 				const struct module *crc_owner)
 {
 	return 1;
 }
 static inline int check_modstruct_version(Elf_Shdr *sechdrs,
 					  unsigned int versindex,
 					  struct module *mod)
 {
 	return 1;
 }
 static inline int same_magic(const char *amagic, const char *bmagic,
 			     bool has_crcs)
 {
 	return strcmp(amagic, bmagic) == 0;
 }
 #endif /* CONFIG_MODVERSIONS */
 /* Resolve a symbol for this module.  I.e. if we find one, record usage. */
 static const struct kernel_symbol *resolve_symbol(struct module *mod,
 						  const struct load_info *info,
 						  const char *name,
 						  char ownername[])
 {
 	struct module *owner;
 	const struct kernel_symbol *sym;
 	const unsigned long *crc;
 	int err;
 	mutex_lock(&module_mutex);
 	sym = find_symbol(name, &owner, &crc,
 			  !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
 	if (!sym)
 		goto unlock;
 	if (!check_version(info->sechdrs, info->index.vers, name, mod, crc,
 			   owner)) {
 		sym = ERR_PTR(-EINVAL);
 		goto getname;
 	}
 	err = ref_module(mod, owner);
 	if (err) {
 		sym = ERR_PTR(err);
 		goto getname;
 	}
 getname:
 	/* We must make copy under the lock if we failed to get ref. */
 	strncpy(ownername, module_name(owner), MODULE_NAME_LEN);
 unlock:
 	mutex_unlock(&module_mutex);
 	return sym;
 }
 static const struct kernel_symbol *
 resolve_symbol_wait(struct module *mod,
 		    const struct load_info *info,
 		    const char *name)
 {
 	const struct kernel_symbol *ksym;
 	char owner[MODULE_NAME_LEN];
 	if (wait_event_interruptible_timeout(module_wq,
 			!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
 			|| PTR_ERR(ksym) != -EBUSY,
 					     30 * HZ) <= 0) {
 		pr_warn("%s: gave up waiting for init of module %s.\n",
 			mod->name, owner);
 	}
 	return ksym;
 }
 /*
  * /sys/module/foo/sections stuff
  * J. Corbet <corbet@lwn.net>
  */
 #ifdef CONFIG_SYSFS
 #ifdef CONFIG_KALLSYMS
 static inline bool sect_empty(const Elf_Shdr *sect)
 {
 	return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
 }
 struct module_sect_attr
 {
 	struct module_attribute mattr;
 	char *name;
 	unsigned long address;
 };
 struct module_sect_attrs
 {
 	struct attribute_group grp;
 	unsigned int nsections;
 	struct module_sect_attr attrs[0];
 };
 static ssize_t module_sect_show(struct module_attribute *mattr,
 				struct module_kobject *mk, char *buf)
 {
 	struct module_sect_attr *sattr =
 		container_of(mattr, struct module_sect_attr, mattr);
 	return sprintf(buf, "0x%pK\n", (void *)sattr->address);
 }
 static void free_sect_attrs(struct module_sect_attrs *sect_attrs)
 {
 	unsigned int section;
 	for (section = 0; section < sect_attrs->nsections; section++)
 		kfree(sect_attrs->attrs[section].name);
 	kfree(sect_attrs);
 }
 static void add_sect_attrs(struct module *mod, const struct load_info *info)
 {
 	unsigned int nloaded = 0, i, size[2];
 	struct module_sect_attrs *sect_attrs;
 	struct module_sect_attr *sattr;
 	struct attribute **gattr;
 	/* Count loaded sections and allocate structures */
 	for (i = 0; i < info->hdr->e_shnum; i++)
 		if (!sect_empty(&info->sechdrs[i]))
 			nloaded++;
 	size[0] = ALIGN(sizeof(*sect_attrs)
 			+ nloaded * sizeof(sect_attrs->attrs[0]),
 			sizeof(sect_attrs->grp.attrs[0]));
 	size[1] = (nloaded + 1) * sizeof(sect_attrs->grp.attrs[0]);
 	sect_attrs = kzalloc(size[0] + size[1], GFP_KERNEL);
 	if (sect_attrs == NULL)
 		return;
 	/* Setup section attributes. */
 	sect_attrs->grp.name = "sections";
 	sect_attrs->grp.attrs = (void *)sect_attrs + size[0];
 	sect_attrs->nsections = 0;
 	sattr = &sect_attrs->attrs[0];
 	gattr = &sect_attrs->grp.attrs[0];
 	for (i = 0; i < info->hdr->e_shnum; i++) {
 		Elf_Shdr *sec = &info->sechdrs[i];
 		if (sect_empty(sec))
 			continue;
 		sattr->address = sec->sh_addr;
 		sattr->name = kstrdup(info->secstrings + sec->sh_name,
 					GFP_KERNEL);
 		if (sattr->name == NULL)
 			goto out;
 		sect_attrs->nsections++;
 		sysfs_attr_init(&sattr->mattr.attr);
 		sattr->mattr.show = module_sect_show;
 		sattr->mattr.store = NULL;
 		sattr->mattr.attr.name = sattr->name;
 		sattr->mattr.attr.mode = S_IRUGO;
 		*(gattr++) = &(sattr++)->mattr.attr;
 	}
 	*gattr = NULL;
 	if (sysfs_create_group(&mod->mkobj.kobj, &sect_attrs->grp))
 		goto out;
 	mod->sect_attrs = sect_attrs;
 	return;
   out:
 	free_sect_attrs(sect_attrs);
 }
 static void remove_sect_attrs(struct module *mod)
 {
 	if (mod->sect_attrs) {
 		sysfs_remove_group(&mod->mkobj.kobj,
 				   &mod->sect_attrs->grp);
 		/* We are positive that no one is using any sect attrs
 		 * at this point.  Deallocate immediately. */
 		free_sect_attrs(mod->sect_attrs);
 		mod->sect_attrs = NULL;
 	}
 }
 /*
  * /sys/module/foo/notes/.section.name gives contents of SHT_NOTE sections.
  */
 struct module_notes_attrs {
 	struct kobject *dir;
 	unsigned int notes;
 	struct bin_attribute attrs[0];
 };
 static ssize_t module_notes_read(struct file *filp, struct kobject *kobj,
 				 struct bin_attribute *bin_attr,
 				 char *buf, loff_t pos, size_t count)
 {
 	/*
 	 * The caller checked the pos and count against our size.
 	 */
 	memcpy(buf, bin_attr->private + pos, count);
 	return count;
 }
 static void free_notes_attrs(struct module_notes_attrs *notes_attrs,
 			     unsigned int i)
 {
 	if (notes_attrs->dir) {
 		while (i-- > 0)
 			sysfs_remove_bin_file(notes_attrs->dir,
 					      &notes_attrs->attrs[i]);
 		kobject_put(notes_attrs->dir);
 	}
 	kfree(notes_attrs);
 }
 static void add_notes_attrs(struct module *mod, const struct load_info *info)
 {
 	unsigned int notes, loaded, i;
 	struct module_notes_attrs *notes_attrs;
 	struct bin_attribute *nattr;
 	/* failed to create section attributes, so can't create notes */
 	if (!mod->sect_attrs)
 		return;
 	/* Count notes sections and allocate structures.  */
 	notes = 0;
 	for (i = 0; i < info->hdr->e_shnum; i++)
 		if (!sect_empty(&info->sechdrs[i]) &&
 		    (info->sechdrs[i].sh_type == SHT_NOTE))
 			++notes;
 	if (notes == 0)
 		return;
 	notes_attrs = kzalloc(sizeof(*notes_attrs)
 			      + notes * sizeof(notes_attrs->attrs[0]),
 			      GFP_KERNEL);
 	if (notes_attrs == NULL)
 		return;
 	notes_attrs->notes = notes;
 	nattr = &notes_attrs->attrs[0];
 	for (loaded = i = 0; i < info->hdr->e_shnum; ++i) {
 		if (sect_empty(&info->sechdrs[i]))
 			continue;
 		if (info->sechdrs[i].sh_type == SHT_NOTE) {
 			sysfs_bin_attr_init(nattr);
 			nattr->attr.name = mod->sect_attrs->attrs[loaded].name;
 			nattr->attr.mode = S_IRUGO;
 			nattr->size = info->sechdrs[i].sh_size;
 			nattr->private = (void *) info->sechdrs[i].sh_addr;
 			nattr->read = module_notes_read;
 			++nattr;
 		}
 		++loaded;
 	}
 	notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj);
 	if (!notes_attrs->dir)
 		goto out;
 	for (i = 0; i < notes; ++i)
 		if (sysfs_create_bin_file(notes_attrs->dir,
 					  &notes_attrs->attrs[i]))
 			goto out;
 	mod->notes_attrs = notes_attrs;
 	return;
   out:
 	free_notes_attrs(notes_attrs, i);
 }
 static void remove_notes_attrs(struct module *mod)
 {
 	if (mod->notes_attrs)
 		free_notes_attrs(mod->notes_attrs, mod->notes_attrs->notes);
 }
 #else
 static inline void add_sect_attrs(struct module *mod,
 				  const struct load_info *info)
 {
 }
 static inline void remove_sect_attrs(struct module *mod)
 {
 }
 static inline void add_notes_attrs(struct module *mod,
 				   const struct load_info *info)
 {
 }
 static inline void remove_notes_attrs(struct module *mod)
 {
 }
 #endif /* CONFIG_KALLSYMS */
 static void add_usage_links(struct module *mod)
 {
 #ifdef CONFIG_MODULE_UNLOAD
 	struct module_use *use;
 	int nowarn;
 	mutex_lock(&module_mutex);
 	list_for_each_entry(use, &mod->target_list, target_list) {
 		nowarn = sysfs_create_link(use->target->holders_dir,
 					   &mod->mkobj.kobj, mod->name);
 	}
 	mutex_unlock(&module_mutex);
 #endif
 }
 static void del_usage_links(struct module *mod)
 {
 #ifdef CONFIG_MODULE_UNLOAD
 	struct module_use *use;
 	mutex_lock(&module_mutex);
 	list_for_each_entry(use, &mod->target_list, target_list)
 		sysfs_remove_link(use->target->holders_dir, mod->name);
 	mutex_unlock(&module_mutex);
 #endif
 }
 static int module_add_modinfo_attrs(struct module *mod)
 {
 	struct module_attribute *attr;
 	struct module_attribute *temp_attr;
 	int error = 0;
 	int i;
 	mod->modinfo_attrs = kzalloc((sizeof(struct module_attribute) *
 					(ARRAY_SIZE(modinfo_attrs) + 1)),
 					GFP_KERNEL);
 	if (!mod->modinfo_attrs)
 		return -ENOMEM;
 	temp_attr = mod->modinfo_attrs;
 	for (i = 0; (attr = modinfo_attrs[i]) && !error; i++) {
 		if (!attr->test ||
 		    (attr->test && attr->test(mod))) {
 			memcpy(temp_attr, attr, sizeof(*temp_attr));
 			sysfs_attr_init(&temp_attr->attr);
 			error = sysfs_create_file(&mod->mkobj.kobj,&temp_attr->attr);
 			++temp_attr;
 		}
 	}
 	return error;
 }
 static void module_remove_modinfo_attrs(struct module *mod)
 {
 	struct module_attribute *attr;
 	int i;
 	for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) {
 		/* pick a field to test for end of list */
 		if (!attr->attr.name)
 			break;
 		sysfs_remove_file(&mod->mkobj.kobj,&attr->attr);
 		if (attr->free)
 			attr->free(mod);
 	}
 	kfree(mod->modinfo_attrs);
 }
 static void mod_kobject_put(struct module *mod)
 {
 	DECLARE_COMPLETION_ONSTACK(c);
 	mod->mkobj.kobj_completion = &c;
 	kobject_put(&mod->mkobj.kobj);
 	wait_for_completion(&c);
 }
 static int mod_sysfs_init(struct module *mod)
 {
 	int err;
 	struct kobject *kobj;
 	if (!module_sysfs_initialized) {
 		pr_err("%s: module sysfs not initialized\n", mod->name);
 		err = -EINVAL;
 		goto out;
 	}
 	kobj = kset_find_obj(module_kset, mod->name);
 	if (kobj) {
 		pr_err("%s: module is already loaded\n", mod->name);
 		kobject_put(kobj);
 		err = -EINVAL;
 		goto out;
 	}
 	mod->mkobj.mod = mod;
 	memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj));
 	mod->mkobj.kobj.kset = module_kset;
 	err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL,
 				   "%s", mod->name);
 	if (err)
 		mod_kobject_put(mod);
 	/* delay uevent until full sysfs population */
 out:
 	return err;
 }
 static int mod_sysfs_setup(struct module *mod,
 			   const struct load_info *info,
 			   struct kernel_param *kparam,
 			   unsigned int num_params)
 {
 	int err;
 	err = mod_sysfs_init(mod);
 	if (err)
 		goto out;
 	mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
 	if (!mod->holders_dir) {
 		err = -ENOMEM;
 		goto out_unreg;
 	}
 	err = module_param_sysfs_setup(mod, kparam, num_params);
 	if (err)
 		goto out_unreg_holders;
 	err = module_add_modinfo_attrs(mod);
 	if (err)
 		goto out_unreg_param;
 	add_usage_links(mod);
 	add_sect_attrs(mod, info);
 	add_notes_attrs(mod, info);
 	kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
 	return 0;
 out_unreg_param:
 	module_param_sysfs_remove(mod);
 out_unreg_holders:
 	kobject_put(mod->holders_dir);
 out_unreg:
 	mod_kobject_put(mod);
 out:
 	return err;
 }
 static void mod_sysfs_fini(struct module *mod)
 {
 	remove_notes_attrs(mod);
 	remove_sect_attrs(mod);
 	mod_kobject_put(mod);
 }
 #else /* !CONFIG_SYSFS */
 static int mod_sysfs_setup(struct module *mod,
 			   const struct load_info *info,
 			   struct kernel_param *kparam,
 			   unsigned int num_params)
 {
 	return 0;
 }
 static void mod_sysfs_fini(struct module *mod)
 {
 }
 static void module_remove_modinfo_attrs(struct module *mod)
 {
 }
 static void del_usage_links(struct module *mod)
 {
 }
 #endif /* CONFIG_SYSFS */
 static void mod_sysfs_teardown(struct module *mod)
 {
 	del_usage_links(mod);
 	module_remove_modinfo_attrs(mod);
 	module_param_sysfs_remove(mod);
 	kobject_put(mod->mkobj.drivers_dir);
 	kobject_put(mod->holders_dir);
 	mod_sysfs_fini(mod);
 }
 /*
  * unlink the module with the whole machine is stopped with interrupts off
  * - this defends against kallsyms not taking locks
  */
 static int __unlink_module(void *_mod)
 {
 	struct module *mod = _mod;
 	list_del(&mod->list);
 	module_bug_cleanup(mod);
 	return 0;
 }
 #ifdef CONFIG_DEBUG_SET_MODULE_RONX
 /*
  * LKM RO/NX protection: protect module's text/ro-data
  * from modification and any data from execution.
  */
 void set_page_attributes(void *start, void *end, int (*set)(unsigned long start, int num_pages))
 {
 	unsigned long begin_pfn = PFN_DOWN((unsigned long)start);
 	unsigned long end_pfn = PFN_DOWN((unsigned long)end);
 	if (end_pfn > begin_pfn)
 		set(begin_pfn << PAGE_SHIFT, end_pfn - begin_pfn);
 }
 static void set_section_ro_nx(void *base,
 			unsigned long text_size,
 			unsigned long ro_size,
 			unsigned long total_size)
 {
 	/* begin and end PFNs of the current subsection */
 	unsigned long begin_pfn;
 	unsigned long end_pfn;
 	/*
 	 * Set RO for module text and RO-data:
 	 * - Always protect first page.
 	 * - Do not protect last partial page.
 	 */
 	if (ro_size > 0)
 		set_page_attributes(base, base + ro_size, set_memory_ro);
 	/*
 	 * Set NX permissions for module data:
 	 * - Do not protect first partial page.
 	 * - Always protect last page.
 	 */
 	if (total_size > text_size) {
 		begin_pfn = PFN_UP((unsigned long)base + text_size);
 		end_pfn = PFN_UP((unsigned long)base + total_size);
 		if (end_pfn > begin_pfn)
 			set_memory_nx(begin_pfn << PAGE_SHIFT, end_pfn - begin_pfn);
 	}
 }
 static void unset_module_core_ro_nx(struct module *mod)
 {
 	set_page_attributes(mod->module_core + mod->core_text_size,
 		mod->module_core + mod->core_size,
 		set_memory_x);
 	set_page_attributes(mod->module_core,
 		mod->module_core + mod->core_ro_size,
 		set_memory_rw);
 }
 static void unset_module_init_ro_nx(struct module *mod)
 {
 	set_page_attributes(mod->module_init + mod->init_text_size,
 		mod->module_init + mod->init_size,
 		set_memory_x);
 	set_page_attributes(mod->module_init,
 		mod->module_init + mod->init_ro_size,
 		set_memory_rw);
 }
 /* Iterate through all modules and set each module's text as RW */
 void set_all_modules_text_rw(void)
 {
 	struct module *mod;
 	mutex_lock(&module_mutex);
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if ((mod->module_core) && (mod->core_text_size)) {
 			set_page_attributes(mod->module_core,
 						mod->module_core + mod->core_text_size,
 						set_memory_rw);
 		}
 		if ((mod->module_init) && (mod->init_text_size)) {
 			set_page_attributes(mod->module_init,
 						mod->module_init + mod->init_text_size,
 						set_memory_rw);
 		}
 	}
 	mutex_unlock(&module_mutex);
 }
 /* Iterate through all modules and set each module's text as RO */
 void set_all_modules_text_ro(void)
 {
 	struct module *mod;
 	mutex_lock(&module_mutex);
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if ((mod->module_core) && (mod->core_text_size)) {
 			set_page_attributes(mod->module_core,
 						mod->module_core + mod->core_text_size,
 						set_memory_ro);
 		}
 		if ((mod->module_init) && (mod->init_text_size)) {
 			set_page_attributes(mod->module_init,
 						mod->module_init + mod->init_text_size,
 						set_memory_ro);
 		}
 	}
 	mutex_unlock(&module_mutex);
 }
 #else
 static inline void set_section_ro_nx(void *base, unsigned long text_size, unsigned long ro_size, unsigned long total_size) { }
 static void unset_module_core_ro_nx(struct module *mod) { }
 static void unset_module_init_ro_nx(struct module *mod) { }
 #endif
 void __weak module_free(struct module *mod, void *module_region)
 {
 	vfree(module_region);
 }
 void __weak module_arch_cleanup(struct module *mod)
 {
 }
 /* Free a module, remove from lists, etc. */
 static void free_module(struct module *mod)
 {
 	trace_module_free(mod);
 	mod_sysfs_teardown(mod);
 	/* We leave it in list to prevent duplicate loads, but make sure
 	 * that noone uses it while it's being deconstructed. */
 	mod->state = MODULE_STATE_UNFORMED;
 	/* Remove dynamic debug info */
 	ddebug_remove_module(mod->name);
 	/* Arch-specific cleanup. */
 	module_arch_cleanup(mod);
 	/* Module unload stuff */
 	module_unload_free(mod);
 	/* Free any allocated parameters. */
 	destroy_params(mod->kp, mod->num_kp);
 	/* Now we can delete it from the lists */
 	mutex_lock(&module_mutex);
 	stop_machine(__unlink_module, mod, NULL);
 	mutex_unlock(&module_mutex);
 	/* This may be NULL, but that's OK */
 	unset_module_init_ro_nx(mod);
 	module_free(mod, mod->module_init);
 	kfree(mod->args);
 	percpu_modfree(mod);
 	/* Free lock-classes: */
 	lockdep_free_key_range(mod->module_core, mod->core_size);
 	/* Finally, free the core (containing the module structure) */
 	unset_module_core_ro_nx(mod);
 	module_free(mod, mod->module_core);
 #ifdef CONFIG_MPU
 	update_protections(current->mm);
 #endif
 }
 void *__symbol_get(const char *symbol)
 {
 	struct module *owner;
 	const struct kernel_symbol *sym;
 	preempt_disable();
 	sym = find_symbol(symbol, &owner, NULL, true, true);
 	if (sym && strong_try_module_get(owner))
 		sym = NULL;
 	preempt_enable();
 	return sym ? (void *)sym->value : NULL;
 }
 EXPORT_SYMBOL_GPL(__symbol_get);
 /*
  * Ensure that an exported symbol [global namespace] does not already exist
  * in the kernel or in some other module's exported symbol table.
  *
  * You must hold the module_mutex.
  */
 static int verify_export_symbols(struct module *mod)
 {
 	unsigned int i;
 	struct module *owner;
 	const struct kernel_symbol *s;
 	struct {
 		const struct kernel_symbol *sym;
 		unsigned int num;
 	} arr[] = {
 		{ mod->syms, mod->num_syms },
 		{ mod->gpl_syms, mod->num_gpl_syms },
 		{ mod->gpl_future_syms, mod->num_gpl_future_syms },
 #ifdef CONFIG_UNUSED_SYMBOLS
 		{ mod->unused_syms, mod->num_unused_syms },
 		{ mod->unused_gpl_syms, mod->num_unused_gpl_syms },
 #endif
 	};
 	for (i = 0; i < ARRAY_SIZE(arr); i++) {
 		for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
 			if (find_symbol(s->name, &owner, NULL, true, false)) {
 				pr_err("%s: exports duplicate symbol %s"
 				       " (owned by %s)\n",
 				       mod->name, s->name, module_name(owner));
 				return -ENOEXEC;
 			}
 		}
 	}
 	return 0;
 }
 /* Change all symbols so that st_value encodes the pointer directly. */
 static int simplify_symbols(struct module *mod, const struct load_info *info)
 {
 	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
 	Elf_Sym *sym = (void *)symsec->sh_addr;
 	unsigned long secbase;
 	unsigned int i;
 	int ret = 0;
 	const struct kernel_symbol *ksym;
 	for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
 		const char *name = info->strtab + sym[i].st_name;
 		switch (sym[i].st_shndx) {
 		case SHN_COMMON:
 			/* Ignore common symbols */
 			if (!strncmp(name, "__gnu_lto", 9))
 				break;
 			/* We compiled with -fno-common.  These are not
 			   supposed to happen.  */
 			pr_debug("Common symbol: %s\n", name);
 			printk("%s: please compile with -fno-common\n",
 			       mod->name);
 			ret = -ENOEXEC;
 			break;
 		case SHN_ABS:
 			/* Don't need to do anything */
 			pr_debug("Absolute symbol: 0x%08lx\n",
 			       (long)sym[i].st_value);
 			break;
 		case SHN_UNDEF:
 			ksym = resolve_symbol_wait(mod, info, name);
 			/* Ok if resolved.  */
 			if (ksym && !IS_ERR(ksym)) {
 				sym[i].st_value = ksym->value;
 				break;
 			}
 			/* Ok if weak.  */
 			if (!ksym && ELF_ST_BIND(sym[i].st_info) == STB_WEAK)
 				break;
 			pr_warn("%s: Unknown symbol %s (err %li)\n",
 				mod->name, name, PTR_ERR(ksym));
 			ret = PTR_ERR(ksym) ?: -ENOENT;
 			break;
 		default:
 			/* Divert to percpu allocation if a percpu var. */
 			if (sym[i].st_shndx == info->index.pcpu)
 				secbase = (unsigned long)mod_percpu(mod);
 			else
 				secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
 			sym[i].st_value += secbase;
 			break;
 		}
 	}
 	return ret;
 }
 static int apply_relocations(struct module *mod, const struct load_info *info)
 {
 	unsigned int i;
 	int err = 0;
 	/* Now do relocations. */
 	for (i = 1; i < info->hdr->e_shnum; i++) {
 		unsigned int infosec = info->sechdrs[i].sh_info;
 		/* Not a valid relocation section? */
 		if (infosec >= info->hdr->e_shnum)
 			continue;
 		/* Don't bother with non-allocated sections */
 		if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
 			continue;
 		if (info->sechdrs[i].sh_type == SHT_REL)
 			err = apply_relocate(info->sechdrs, info->strtab,
 					     info->index.sym, i, mod);
 		else if (info->sechdrs[i].sh_type == SHT_RELA)
 			err = apply_relocate_add(info->sechdrs, info->strtab,
 						 info->index.sym, i, mod);
 		if (err < 0)
 			break;
 	}
 	return err;
 }
 /* Additional bytes needed by arch in front of individual sections */
 unsigned int __weak arch_mod_section_prepend(struct module *mod,
 					     unsigned int section)
 {
 	/* default implementation just returns zero */
 	return 0;
 }
 /* Update size with this section: return offset. */
 static long get_offset(struct module *mod, unsigned int *size,
 		       Elf_Shdr *sechdr, unsigned int section)
 {
 	long ret;
 	*size += arch_mod_section_prepend(mod, section);
 	ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
 	*size = ret + sechdr->sh_size;
 	return ret;
 }
 /* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
    might -- code, read-only data, read-write data, small data.  Tally
    sizes, and place the offsets into sh_entsize fields: high bit means it
    belongs in init. */
 static void layout_sections(struct module *mod, struct load_info *info)
 {
 	static unsigned long const masks[][2] = {
 		/* NOTE: all executable code must be the first section
 		 * in this array; otherwise modify the text_size
 		 * finder in the two loops below */
 		{ SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL },
 		{ SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL },
 		{ SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
 		{ ARCH_SHF_SMALL | SHF_ALLOC, 0 }
 	};
 	unsigned int m, i;
 	for (i = 0; i < info->hdr->e_shnum; i++)
 		info->sechdrs[i].sh_entsize = ~0UL;
 	pr_debug("Core section allocation order:\n");
 	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
 		for (i = 0; i < info->hdr->e_shnum; ++i) {
 			Elf_Shdr *s = &info->sechdrs[i];
 			const char *sname = info->secstrings + s->sh_name;
 			if ((s->sh_flags & masks[m][0]) != masks[m][0]
 			    || (s->sh_flags & masks[m][1])
 			    || s->sh_entsize != ~0UL
 			    || strstarts(sname, ".init"))
 				continue;
 			s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
 			pr_debug("\t%s\n", sname);
 		}
 		switch (m) {
 		case 0: /* executable */
 			mod->core_size = debug_align(mod->core_size);
 			mod->core_text_size = mod->core_size;
 			break;
 		case 1: /* RO: text and ro-data */
 			mod->core_size = debug_align(mod->core_size);
 			mod->core_ro_size = mod->core_size;
 			break;
 		case 3: /* whole core */
 			mod->core_size = debug_align(mod->core_size);
 			break;
 		}
 	}
 	pr_debug("Init section allocation order:\n");
 	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
 		for (i = 0; i < info->hdr->e_shnum; ++i) {
 			Elf_Shdr *s = &info->sechdrs[i];
 			const char *sname = info->secstrings + s->sh_name;
 			if ((s->sh_flags & masks[m][0]) != masks[m][0]
 			    || (s->sh_flags & masks[m][1])
 			    || s->sh_entsize != ~0UL
 			    || !strstarts(sname, ".init"))
 				continue;
 			s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
 					 | INIT_OFFSET_MASK);
 			pr_debug("\t%s\n", sname);
 		}
 		switch (m) {
 		case 0: /* executable */
 			mod->init_size = debug_align(mod->init_size);
 			mod->init_text_size = mod->init_size;
 			break;
 		case 1: /* RO: text and ro-data */
 			mod->init_size = debug_align(mod->init_size);
 			mod->init_ro_size = mod->init_size;
 			break;
 		case 3: /* whole init */
 			mod->init_size = debug_align(mod->init_size);
 			break;
 		}
 	}
 }
 static void set_license(struct module *mod, const char *license)
 {
 	if (!license)
 		license = "unspecified";
 	if (!license_is_gpl_compatible(license)) {
 		if (!test_taint(TAINT_PROPRIETARY_MODULE))
 			pr_warn("%s: module license '%s' taints kernel.\n",
 				mod->name, license);
 		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
 				 LOCKDEP_NOW_UNRELIABLE);
 	}
 }
 /* Parse tag=value strings from .modinfo section */
 static char *next_string(char *string, unsigned long *secsize)
 {
 	/* Skip non-zero chars */
 	while (string[0]) {
 		string++;
 		if ((*secsize)-- <= 1)
 			return NULL;
 	}
 	/* Skip any zero padding. */
 	while (!string[0]) {
 		string++;
 		if ((*secsize)-- <= 1)
 			return NULL;
 	}
 	return string;
 }
 static char *get_modinfo(struct load_info *info, const char *tag)
 {
 	char *p;
 	unsigned int taglen = strlen(tag);
 	Elf_Shdr *infosec = &info->sechdrs[info->index.info];
 	unsigned long size = infosec->sh_size;
 	for (p = (char *)infosec->sh_addr; p; p = next_string(p, &size)) {
 		if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
 			return p + taglen + 1;
 	}
 	return NULL;
 }
 static void setup_modinfo(struct module *mod, struct load_info *info)
 {
 	struct module_attribute *attr;
 	int i;
 	for (i = 0; (attr = modinfo_attrs[i]); i++) {
 		if (attr->setup)
 			attr->setup(mod, get_modinfo(info, attr->attr.name));
 	}
 }
 static void free_modinfo(struct module *mod)
 {
 	struct module_attribute *attr;
 	int i;
 	for (i = 0; (attr = modinfo_attrs[i]); i++) {
 		if (attr->free)
 			attr->free(mod);
 	}
 }
 #ifdef CONFIG_KALLSYMS
 /* lookup symbol in given range of kernel_symbols */
 static const struct kernel_symbol *lookup_symbol(const char *name,
 	const struct kernel_symbol *start,
 	const struct kernel_symbol *stop)
 {
 	return bsearch(name, start, stop - start,
 			sizeof(struct kernel_symbol), cmp_name);
 }
 static int is_exported(const char *name, unsigned long value,
 		       const struct module *mod)
 {
 	const struct kernel_symbol *ks;
 	if (!mod)
 		ks = lookup_symbol(name, __start___ksymtab, __stop___ksymtab);
 	else
 		ks = lookup_symbol(name, mod->syms, mod->syms + mod->num_syms);
 	return ks != NULL && ks->value == value;
 }
 /* As per nm */
 static char elf_type(const Elf_Sym *sym, const struct load_info *info)
 {
 	const Elf_Shdr *sechdrs = info->sechdrs;
 	if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
 		if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
 			return 'v';
 		else
 			return 'w';
 	}
 	if (sym->st_shndx == SHN_UNDEF)
 		return 'U';
 	if (sym->st_shndx == SHN_ABS)
 		return 'a';
 	if (sym->st_shndx >= SHN_LORESERVE)
 		return '?';
 	if (sechdrs[sym->st_shndx].sh_flags & SHF_EXECINSTR)
 		return 't';
 	if (sechdrs[sym->st_shndx].sh_flags & SHF_ALLOC
 	    && sechdrs[sym->st_shndx].sh_type != SHT_NOBITS) {
 		if (!(sechdrs[sym->st_shndx].sh_flags & SHF_WRITE))
 			return 'r';
 		else if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
 			return 'g';
 		else
 			return 'd';
 	}
 	if (sechdrs[sym->st_shndx].sh_type == SHT_NOBITS) {
 		if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
 			return 's';
 		else
 			return 'b';
 	}
 	if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
 		      ".debug")) {
 		return 'n';
 	}
 	return '?';
 }
 static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
                            unsigned int shnum)
 {
 	const Elf_Shdr *sec;
 	if (src->st_shndx == SHN_UNDEF
 	    || src->st_shndx >= shnum
 	    || !src->st_name)
 		return false;
 	sec = sechdrs + src->st_shndx;
 	if (!(sec->sh_flags & SHF_ALLOC)
 #ifndef CONFIG_KALLSYMS_ALL
 	    || !(sec->sh_flags & SHF_EXECINSTR)
 #endif
 	    || (sec->sh_entsize & INIT_OFFSET_MASK))
 		return false;
 	return true;
 }
 /*
  * We only allocate and copy the strings needed by the parts of symtab
  * we keep.  This is simple, but has the effect of making multiple
  * copies of duplicates.  We could be more sophisticated, see
  * linux-kernel thread starting with
  * <73defb5e4bca04a6431392cc341112b1@localhost>.
  */
 static void layout_symtab(struct module *mod, struct load_info *info)
 {
 	Elf_Shdr *symsect = info->sechdrs + info->index.sym;
 	Elf_Shdr *strsect = info->sechdrs + info->index.str;
 	const Elf_Sym *src;
 	unsigned int i, nsrc, ndst, strtab_size = 0;
 	/* Put symbol section at end of init part of module. */
 	symsect->sh_flags |= SHF_ALLOC;
 	symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect,
 					 info->index.sym) | INIT_OFFSET_MASK;
 	pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
 	src = (void *)info->hdr + symsect->sh_offset;
 	nsrc = symsect->sh_size / sizeof(*src);
 	/* Compute total space required for the core symbols' strtab. */
 	for (ndst = i = 0; i < nsrc; i++) {
 		if (i == 0 ||
 		    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum)) {
 			strtab_size += strlen(&info->strtab[src[i].st_name])+1;
 			ndst++;
 		}
 	}
 	/* Append room for core symbols at end of core part. */
 	info->symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1);
 	info->stroffs = mod->core_size = info->symoffs + ndst * sizeof(Elf_Sym);
 	mod->core_size += strtab_size;
 	/* Put string table section at end of init part of module. */
 	strsect->sh_flags |= SHF_ALLOC;
 	strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect,
 					 info->index.str) | INIT_OFFSET_MASK;
 	pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
 }
 static void add_kallsyms(struct module *mod, const struct load_info *info)
 {
 	unsigned int i, ndst;
 	const Elf_Sym *src;
 	Elf_Sym *dst;
 	char *s;
 	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
 	mod->symtab = (void *)symsec->sh_addr;
 	mod->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
 	/* Make sure we get permanent strtab: don't use info->strtab. */
 	mod->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
 	/* Set types up while we still have access to sections. */
 	for (i = 0; i < mod->num_symtab; i++)
 		mod->symtab[i].st_info = elf_type(&mod->symtab[i], info);
 	mod->core_symtab = dst = mod->module_core + info->symoffs;
 	mod->core_strtab = s = mod->module_core + info->stroffs;
 	src = mod->symtab;
 	for (ndst = i = 0; i < mod->num_symtab; i++) {
 		if (i == 0 ||
 		    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum)) {
 			dst[ndst] = src[i];
 			dst[ndst++].st_name = s - mod->core_strtab;
 			s += strlcpy(s, &mod->strtab[src[i].st_name],
 				     KSYM_NAME_LEN) + 1;
 		}
 	}
 	mod->core_num_syms = ndst;
 }
 #else
 static inline void layout_symtab(struct module *mod, struct load_info *info)
 {
 }
 static void add_kallsyms(struct module *mod, const struct load_info *info)
 {
 }
 #endif /* CONFIG_KALLSYMS */
 static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num)
 {
 	if (!debug)
 		return;
 #ifdef CONFIG_DYNAMIC_DEBUG
 	if (ddebug_add_module(debug, num, debug->modname))
 		pr_err("dynamic debug error adding module: %s\n",
 			debug->modname);
 #endif
 }
 static void dynamic_debug_remove(struct _ddebug *debug)
 {
 	if (debug)
 		ddebug_remove_module(debug->modname);
 }
 void * __weak module_alloc(unsigned long size)
 {
 	return vmalloc_exec(size);
 }
 static void *module_alloc_update_bounds(unsigned long size)
 {
 	void *ret = module_alloc(size);
 	if (ret) {
 		mutex_lock(&module_mutex);
 		/* Update module bounds. */
 		if ((unsigned long)ret < module_addr_min)
 			module_addr_min = (unsigned long)ret;
 		if ((unsigned long)ret + size > module_addr_max)
 			module_addr_max = (unsigned long)ret + size;
 		mutex_unlock(&module_mutex);
 	}
 	return ret;
 }
 #ifdef CONFIG_DEBUG_KMEMLEAK
 static void kmemleak_load_module(const struct module *mod,
 				 const struct load_info *info)
 {
 	unsigned int i;
 	/* only scan the sections containing data */
 	kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
 	for (i = 1; i < info->hdr->e_shnum; i++) {
 		/* Scan all writable sections that's not executable */
 		if (!(info->sechdrs[i].sh_flags & SHF_ALLOC) ||
 		    !(info->sechdrs[i].sh_flags & SHF_WRITE) ||
 		    (info->sechdrs[i].sh_flags & SHF_EXECINSTR))
 			continue;
 		kmemleak_scan_area((void *)info->sechdrs[i].sh_addr,
 				   info->sechdrs[i].sh_size, GFP_KERNEL);
 	}
 }
 #else
 static inline void kmemleak_load_module(const struct module *mod,
 					const struct load_info *info)
 {
 }
 #endif
 #ifdef CONFIG_MODULE_SIG
 static int module_sig_check(struct load_info *info)
 {
 	int err = -ENOKEY;
 	const unsigned long markerlen = sizeof(MODULE_SIG_STRING) - 1;
 	const void *mod = info->hdr;
 	if (info->len > markerlen &&
 	    memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) {
 		/* We truncate the module to discard the signature */
 		info->len -= markerlen;
 		err = mod_verify_sig(mod, &info->len);
 	}
 	if (!err) {
 		info->sig_ok = true;
 		return 0;
 	}
 	/* Not having a signature is only an error if we're strict. */
 	if (err < 0 && fips_enabled)
 		panic("Module verification failed with error %d in FIPS mode\n",
 		      err);
 	if (err == -ENOKEY && !sig_enforce)
 		err = 0;
 	return err;
 }
 #else /* !CONFIG_MODULE_SIG */
 static int module_sig_check(struct load_info *info)
 {
 	return 0;
 }
 #endif /* !CONFIG_MODULE_SIG */
 /* Sanity checks against invalid binaries, wrong arch, weird elf version. */
 static int elf_header_check(struct load_info *info)
 {
 	if (info->len < sizeof(*(info->hdr)))
 		return -ENOEXEC;
 	if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0
 	    || info->hdr->e_type != ET_REL
 	    || !elf_check_arch(info->hdr)
 	    || info->hdr->e_shentsize != sizeof(Elf_Shdr))
 		return -ENOEXEC;
 	if (info->hdr->e_shoff >= info->len
 	    || (info->hdr->e_shnum * sizeof(Elf_Shdr) >
 		info->len - info->hdr->e_shoff))
 		return -ENOEXEC;
 	return 0;
 }
 /* Sets info->hdr and info->len. */
 static int copy_module_from_user(const void __user *umod, unsigned long len,
 				  struct load_info *info)
 {
 	int err;
 	info->len = len;
 	if (info->len < sizeof(*(info->hdr)))
 		return -ENOEXEC;
 	err = security_kernel_module_from_file(NULL);
 	if (err)
 		return err;
 	/* Suck in entire file: we'll want most of it. */
 	info->hdr = vmalloc(info->len);
 	if (!info->hdr)
 		return -ENOMEM;
 	if (copy_from_user(info->hdr, umod, info->len) != 0) {
 		vfree(info->hdr);
 		return -EFAULT;
 	}
 	return 0;
 }
 /* Sets info->hdr and info->len. */
 static int copy_module_from_fd(int fd, struct load_info *info)
 {
 	struct fd f = fdget(fd);
 	int err;
 	struct kstat stat;
 	loff_t pos;
 	ssize_t bytes = 0;
 	if (!f.file)
 		return -ENOEXEC;
 	err = security_kernel_module_from_file(f.file);
 	if (err)
 		goto out;
 	err = vfs_getattr(&f.file->f_path, &stat);
 	if (err)
 		goto out;
 	if (stat.size > INT_MAX) {
 		err = -EFBIG;
 		goto out;
 	}
 	/* Don't hand 0 to vmalloc, it whines. */
 	if (stat.size == 0) {
 		err = -EINVAL;
 		goto out;
 	}
 	info->hdr = vmalloc(stat.size);
 	if (!info->hdr) {
 		err = -ENOMEM;
 		goto out;
 	}
 	pos = 0;
 	while (pos < stat.size) {
 		bytes = kernel_read(f.file, pos, (char *)(info->hdr) + pos,
 				    stat.size - pos);
 		if (bytes < 0) {
 			vfree(info->hdr);
 			err = bytes;
 			goto out;
 		}
 		if (bytes == 0)
 			break;
 		pos += bytes;
 	}
 	info->len = pos;
 out:
 	fdput(f);
 	return err;
 }
 static void free_copy(struct load_info *info)
 {
 	vfree(info->hdr);
 }
 static int rewrite_section_headers(struct load_info *info, int flags)
 {
 	unsigned int i;
 	/* This should always be true, but let's be sure. */
 	info->sechdrs[0].sh_addr = 0;
 	for (i = 1; i < info->hdr->e_shnum; i++) {
 		Elf_Shdr *shdr = &info->sechdrs[i];
 		if (shdr->sh_type != SHT_NOBITS
 		    && info->len < shdr->sh_offset + shdr->sh_size) {
 			pr_err("Module len %lu truncated\n", info->len);
 			return -ENOEXEC;
 		}
 		/* Mark all sections sh_addr with their address in the
 		   temporary image. */
 		shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
 #ifndef CONFIG_MODULE_UNLOAD
 		/* Don't load .exit sections */
 		if (strstarts(info->secstrings+shdr->sh_name, ".exit"))
 			shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
 #endif
 	}
 	/* Track but don't keep modinfo and version sections. */
 	if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
 		info->index.vers = 0; /* Pretend no __versions section! */
 	else
 		info->index.vers = find_sec(info, "__versions");
 	info->index.info = find_sec(info, ".modinfo");
 	info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
 	info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
 	return 0;
 }
 /*
  * Set up our basic convenience variables (pointers to section headers,
  * search for module section index etc), and do some basic section
  * verification.
  *
  * Return the temporary module pointer (we'll replace it with the final
  * one when we move the module sections around).
  */
 static struct module *setup_load_info(struct load_info *info, int flags)
 {
 	unsigned int i;
 	int err;
 	struct module *mod;
 	/* Set up the convenience variables */
 	info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
 	info->secstrings = (void *)info->hdr
 		+ info->sechdrs[info->hdr->e_shstrndx].sh_offset;
 	err = rewrite_section_headers(info, flags);
 	if (err)
 		return ERR_PTR(err);
 	/* Find internal symbols and strings. */
 	for (i = 1; i < info->hdr->e_shnum; i++) {
 		if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
 			info->index.sym = i;
 			info->index.str = info->sechdrs[i].sh_link;
 			info->strtab = (char *)info->hdr
 				+ info->sechdrs[info->index.str].sh_offset;
 			break;
 		}
 	}
 	info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
 	if (!info->index.mod) {
 		pr_warn("No module found in object\n");
 		return ERR_PTR(-ENOEXEC);
 	}
 	/* This is temporary: point mod into copy of data. */
 	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
 	if (info->index.sym == 0) {
 		pr_warn("%s: module has no symbols (stripped?)\n", mod->name);
 		return ERR_PTR(-ENOEXEC);
 	}
 	info->index.pcpu = find_pcpusec(info);
 	/* Check module struct version now, before we try to use module. */
 	if (!check_modstruct_version(info->sechdrs, info->index.vers, mod))
 		return ERR_PTR(-ENOEXEC);
 	return mod;
 }
 static int check_modinfo(struct module *mod, struct load_info *info, int flags)
 {
 	const char *modmagic = get_modinfo(info, "vermagic");
 	int err;
 	if (flags & MODULE_INIT_IGNORE_VERMAGIC)
 		modmagic = NULL;
 	/* This is allowed: modprobe --force will invalidate it. */
 	if (!modmagic) {
 		err = try_to_force_load(mod, "bad vermagic");
 		if (err)
 			return err;
 	} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
 		pr_err("%s: version magic '%s' should be '%s'\n",
 		       mod->name, modmagic, vermagic);
 		return -ENOEXEC;
 	}
 	if (!get_modinfo(info, "intree"))
 		add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
 	if (get_modinfo(info, "staging")) {
 		add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
 		pr_warn("%s: module is from the staging directory, the quality "
 			"is unknown, you have been warned.\n", mod->name);
 	}
 	/* Set up license info based on the info section */
 	set_license(mod, get_modinfo(info, "license"));
 	return 0;
 }
 static int find_module_sections(struct module *mod, struct load_info *info)
 {
 	mod->kp = section_objs(info, "__param",
 			       sizeof(*mod->kp), &mod->num_kp);
 	mod->syms = section_objs(info, "__ksymtab",
 				 sizeof(*mod->syms), &mod->num_syms);
 	mod->crcs = section_addr(info, "__kcrctab");
 	mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
 				     sizeof(*mod->gpl_syms),
 				     &mod->num_gpl_syms);
 	mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
 	mod->gpl_future_syms = section_objs(info,
 					    "__ksymtab_gpl_future",
 					    sizeof(*mod->gpl_future_syms),
 					    &mod->num_gpl_future_syms);
 	mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
 #ifdef CONFIG_UNUSED_SYMBOLS
 	mod->unused_syms = section_objs(info, "__ksymtab_unused",
 					sizeof(*mod->unused_syms),
 					&mod->num_unused_syms);
 	mod->unused_crcs = section_addr(info, "__kcrctab_unused");
 	mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
 					    sizeof(*mod->unused_gpl_syms),
 					    &mod->num_unused_gpl_syms);
 	mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
 #endif
 #ifdef CONFIG_CONSTRUCTORS
 	mod->ctors = section_objs(info, ".ctors",
 				  sizeof(*mod->ctors), &mod->num_ctors);
 	if (!mod->ctors)
 		mod->ctors = section_objs(info, ".init_array",
 				sizeof(*mod->ctors), &mod->num_ctors);
 	else if (find_sec(info, ".init_array")) {
 		/*
 		 * This shouldn't happen with same compiler and binutils
 		 * building all parts of the module.
 		 */
 		printk(KERN_WARNING "%s: has both .ctors and .init_array.\n",
 		       mod->name);
 		return -EINVAL;
 	}
 #endif
 #ifdef CONFIG_TRACEPOINTS
 	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
 					     sizeof(*mod->tracepoints_ptrs),
 					     &mod->num_tracepoints);
 #endif
 #ifdef HAVE_JUMP_LABEL
 	mod->jump_entries = section_objs(info, "__jump_table",
 					sizeof(*mod->jump_entries),
 					&mod->num_jump_entries);
 #endif
 #ifdef CONFIG_EVENT_TRACING
 	mod->trace_events = section_objs(info, "_ftrace_events",
 					 sizeof(*mod->trace_events),
 					 &mod->num_trace_events);
 #endif
 #ifdef CONFIG_TRACING
 	mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
 					 sizeof(*mod->trace_bprintk_fmt_start),
 					 &mod->num_trace_bprintk_fmt);
 #endif
 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
 	/* sechdrs[0].sh_size is always zero */
 	mod->ftrace_callsites = section_objs(info, "__mcount_loc",
 					     sizeof(*mod->ftrace_callsites),
 					     &mod->num_ftrace_callsites);
 #endif
 	mod->extable = section_objs(info, "__ex_table",
 				    sizeof(*mod->extable), &mod->num_exentries);
 	if (section_addr(info, "__obsparm"))
 		pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
 	info->debug = section_objs(info, "__verbose",
 				   sizeof(*info->debug), &info->num_debug);
 	return 0;
 }
 static int move_module(struct module *mod, struct load_info *info)
 {
 	int i;
 	void *ptr;
 	/* Do the allocs. */
 	ptr = module_alloc_update_bounds(mod->core_size);
 	/*
 	 * The pointer to this block is stored in the module structure
 	 * which is inside the block. Just mark it as not being a
 	 * leak.
 	 */
 	kmemleak_not_leak(ptr);
 	if (!ptr)
 		return -ENOMEM;
 	memset(ptr, 0, mod->core_size);
 	mod->module_core = ptr;
 	if (mod->init_size) {
 		ptr = module_alloc_update_bounds(mod->init_size);
 		/*
 		 * The pointer to this block is stored in the module structure
 		 * which is inside the block. This block doesn't need to be
 		 * scanned as it contains data and code that will be freed
 		 * after the module is initialized.
 		 */
 		kmemleak_ignore(ptr);
 		if (!ptr) {
 			module_free(mod, mod->module_core);
 			return -ENOMEM;
 		}
 		memset(ptr, 0, mod->init_size);
 		mod->module_init = ptr;
 	} else
 		mod->module_init = NULL;
 	/* Transfer each section which specifies SHF_ALLOC */
 	pr_debug("final section addresses:\n");
 	for (i = 0; i < info->hdr->e_shnum; i++) {
 		void *dest;
 		Elf_Shdr *shdr = &info->sechdrs[i];
 		if (!(shdr->sh_flags & SHF_ALLOC))
 			continue;
 		if (shdr->sh_entsize & INIT_OFFSET_MASK)
 			dest = mod->module_init
 				+ (shdr->sh_entsize & ~INIT_OFFSET_MASK);
 		else
 			dest = mod->module_core + shdr->sh_entsize;
 		if (shdr->sh_type != SHT_NOBITS)
 			memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
 		/* Update sh_addr to point to copy in image. */
 		shdr->sh_addr = (unsigned long)dest;
 		pr_debug("\t0x%lx %s\n",
 			 (long)shdr->sh_addr, info->secstrings + shdr->sh_name);
 	}
 	return 0;
 }
 static int check_module_license_and_versions(struct module *mod)
 {
 	/*
 	 * ndiswrapper is under GPL by itself, but loads proprietary modules.
 	 * Don't use add_taint_module(), as it would prevent ndiswrapper from
 	 * using GPL-only symbols it needs.
 	 */
 	if (strcmp(mod->name, "ndiswrapper") == 0)
 		add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
 	/* driverloader was caught wrongly pretending to be under GPL */
 	if (strcmp(mod->name, "driverloader") == 0)
 		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
 				 LOCKDEP_NOW_UNRELIABLE);
 	/* lve claims to be GPL but upstream won't provide source */
 	if (strcmp(mod->name, "lve") == 0)
 		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
 				 LOCKDEP_NOW_UNRELIABLE);
 #ifdef CONFIG_MODVERSIONS
 	if ((mod->num_syms && !mod->crcs)
 	    || (mod->num_gpl_syms && !mod->gpl_crcs)
 	    || (mod->num_gpl_future_syms && !mod->gpl_future_crcs)
 #ifdef CONFIG_UNUSED_SYMBOLS
 	    || (mod->num_unused_syms && !mod->unused_crcs)
 	    || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
 #endif
 		) {
 		return try_to_force_load(mod,
 					 "no versions for exported symbols");
 	}
 #endif
 	return 0;
 }
 static void flush_module_icache(const struct module *mod)
 {
 	mm_segment_t old_fs;
 	/* flush the icache in correct context */
 	old_fs = get_fs();
 	set_fs(KERNEL_DS);
 	/*
 	 * Flush the instruction cache, since we've played with text.
 	 * Do it before processing of module parameters, so the module
 	 * can provide parameter accessor functions of its own.
 	 */
 	if (mod->module_init)
 		flush_icache_range((unsigned long)mod->module_init,
 				   (unsigned long)mod->module_init
 				   + mod->init_size);
 	flush_icache_range((unsigned long)mod->module_core,
 			   (unsigned long)mod->module_core + mod->core_size);
 	set_fs(old_fs);
 }
 int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
 				     Elf_Shdr *sechdrs,
 				     char *secstrings,
 				     struct module *mod)
 {
 	return 0;
 }
 static struct module *layout_and_allocate(struct load_info *info, int flags)
 {
 	/* Module within temporary copy. */
 	struct module *mod;
 	int err;
 	mod = setup_load_info(info, flags);
 	if (IS_ERR(mod))
 		return mod;
 	err = check_modinfo(mod, info, flags);
 	if (err)
 		return ERR_PTR(err);
 	/* Allow arches to frob section contents and sizes.  */
 	err = module_frob_arch_sections(info->hdr, info->sechdrs,
 					info->secstrings, mod);
 	if (err < 0)
 		return ERR_PTR(err);
 	/* We will do a special allocation for per-cpu sections later. */
 	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
 	/* Determine total sizes, and put offsets in sh_entsize.  For now
 	   this is done generically; there doesn't appear to be any
 	   special cases for the architectures. */
 	layout_sections(mod, info);
 	layout_symtab(mod, info);
 	/* Allocate and move to the final place */
 	err = move_module(mod, info);
 	if (err)
 		return ERR_PTR(err);
 	/* Module has been copied to its final place now: return it. */
 	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
 	kmemleak_load_module(mod, info);
 	return mod;
 }
 /* mod is no longer valid after this! */
 static void module_deallocate(struct module *mod, struct load_info *info)
 {
 	percpu_modfree(mod);
 	module_free(mod, mod->module_init);
 	module_free(mod, mod->module_core);
 }
 int __weak module_finalize(const Elf_Ehdr *hdr,
 			   const Elf_Shdr *sechdrs,
 			   struct module *me)
 {
 	return 0;
 }
 static int post_relocation(struct module *mod, const struct load_info *info)
 {
 	/* Sort exception table now relocations are done. */
 	sort_extable(mod->extable, mod->extable + mod->num_exentries);
 	/* Copy relocated percpu area over. */
 	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
 		       info->sechdrs[info->index.pcpu].sh_size);
 	/* Setup kallsyms-specific fields. */
 	add_kallsyms(mod, info);
 	/* Arch-specific module finalizing. */
 	return module_finalize(info->hdr, info->sechdrs, mod);
 }
 /* Is this module of this name done loading?  No locks held. */
 static bool finished_loading(const char *name)
 {
 	struct module *mod;
 	bool ret;
 	mutex_lock(&module_mutex);
 	mod = find_module_all(name, strlen(name), true);
 	ret = !mod || mod->state == MODULE_STATE_LIVE
 		|| mod->state == MODULE_STATE_GOING;
 	mutex_unlock(&module_mutex);
 	return ret;
 }
 /* Call module constructors. */
 static void do_mod_ctors(struct module *mod)
 {
 #ifdef CONFIG_CONSTRUCTORS
 	unsigned long i;
 	for (i = 0; i < mod->num_ctors; i++)
 		mod->ctors[i]();
 #endif
 }
 /* This is where the real work happens */
 static int do_init_module(struct module *mod)
 {
 	int ret = 0;
 	/*
 	 * We want to find out whether @mod uses async during init.  Clear
 	 * PF_USED_ASYNC.  async_schedule*() will set it.
 	 */
 	current->flags &= ~PF_USED_ASYNC;
 	blocking_notifier_call_chain(&module_notify_list,
 			MODULE_STATE_COMING, mod);
 	/* Set RO and NX regions for core */
 	set_section_ro_nx(mod->module_core,
 				mod->core_text_size,
 				mod->core_ro_size,
 				mod->core_size);
 	/* Set RO and NX regions for init */
 	set_section_ro_nx(mod->module_init,
 				mod->init_text_size,
 				mod->init_ro_size,
 				mod->init_size);
 	do_mod_ctors(mod);
 	/* Start the module */
 	if (mod->init != NULL)
 		ret = do_one_initcall(mod->init);
 	if (ret < 0) {
 		/* Init routine failed: abort.  Try to protect us from
                    buggy refcounters. */
 		mod->state = MODULE_STATE_GOING;
 		synchronize_sched();
 		module_put(mod);
 		blocking_notifier_call_chain(&module_notify_list,
 					     MODULE_STATE_GOING, mod);
 		free_module(mod);
 		wake_up_all(&module_wq);
 		return ret;
 	}
 	if (ret > 0) {
 		pr_warn("%s: '%s'->init suspiciously returned %d, it should "
 			"follow 0/-E convention\n"
 			"%s: loading module anyway...\n",
 			__func__, mod->name, ret, __func__);
 		dump_stack();
 	}
 	/* Now it's a first class citizen! */
 	mod->state = MODULE_STATE_LIVE;
 	blocking_notifier_call_chain(&module_notify_list,
 				     MODULE_STATE_LIVE, mod);
 	/*
 	 * We need to finish all async code before the module init sequence
 	 * is done.  This has potential to deadlock.  For example, a newly
 	 * detected block device can trigger request_module() of the
 	 * default iosched from async probing task.  Once userland helper
 	 * reaches here, async_synchronize_full() will wait on the async
 	 * task waiting on request_module() and deadlock.
 	 *
 	 * This deadlock is avoided by perfomring async_synchronize_full()
 	 * iff module init queued any async jobs.  This isn't a full
 	 * solution as it will deadlock the same if module loading from
 	 * async jobs nests more than once; however, due to the various
 	 * constraints, this hack seems to be the best option for now.
 	 * Please refer to the following thread for details.
 	 *
 	 * http://thread.gmane.org/gmane.linux.kernel/1420814
 	 */
 	if (current->flags & PF_USED_ASYNC)
 		async_synchronize_full();
 	mutex_lock(&module_mutex);
 	/* Drop initial reference. */
 	module_put(mod);
 	trim_init_extable(mod);
 #ifdef CONFIG_KALLSYMS
 	mod->num_symtab = mod->core_num_syms;
 	mod->symtab = mod->core_symtab;
 	mod->strtab = mod->core_strtab;
 #endif
 	unset_module_init_ro_nx(mod);
 	module_free(mod, mod->module_init);
 	mod->module_init = NULL;
 	mod->init_size = 0;
 	mod->init_ro_size = 0;
 	mod->init_text_size = 0;
 	mutex_unlock(&module_mutex);
 	wake_up_all(&module_wq);
 	return 0;
 }
 static int may_init_module(void)
 {
 	if (!capable(CAP_SYS_MODULE) || modules_disabled)
 		return -EPERM;
 	return 0;
 }
 /*
  * We try to place it in the list now to make sure it's unique before
  * we dedicate too many resources.  In particular, temporary percpu
  * memory exhaustion.
  */
 static int add_unformed_module(struct module *mod)
 {
 	int err;
 	struct module *old;
 	mod->state = MODULE_STATE_UNFORMED;
 again:
 	mutex_lock(&module_mutex);
 	old = find_module_all(mod->name, strlen(mod->name), true);
 	if (old != NULL) {
 		if (old->state == MODULE_STATE_COMING
 		    || old->state == MODULE_STATE_UNFORMED) {
 			/* Wait in case it fails to load. */
 			mutex_unlock(&module_mutex);
 			err = wait_event_interruptible(module_wq,
 					       finished_loading(mod->name));
 			if (err)
 				goto out_unlocked;
 			goto again;
 		}
 		err = -EEXIST;
 		goto out;
 	}
 	list_add_rcu(&mod->list, &modules);
 	err = 0;
 out:
 	mutex_unlock(&module_mutex);
 out_unlocked:
 	return err;
 }
 static int complete_formation(struct module *mod, struct load_info *info)
 {
 	int err;
 	mutex_lock(&module_mutex);
 	/* Find duplicate symbols (must be called under lock). */
 	err = verify_export_symbols(mod);
 	if (err < 0)
 		goto out;
 	/* This relies on module_mutex for list integrity. */
 	module_bug_finalize(info->hdr, info->sechdrs, mod);
 	/* Mark state as coming so strong_try_module_get() ignores us,
 	 * but kallsyms etc. can see us. */
 	mod->state = MODULE_STATE_COMING;
 out:
 	mutex_unlock(&module_mutex);
 	return err;
 }
 static int unknown_module_param_cb(char *param, char *val, const char *modname)
 {
 	/* Check for magic 'dyndbg' arg */
 	int ret = ddebug_dyndbg_module_param_cb(param, val, modname);
 	if (ret != 0)
 		pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
 	return 0;
 }
 /* Allocate and load the module: note that size of section 0 is always
    zero, and we rely on this for optional sections. */
 static int load_module(struct load_info *info, const char __user *uargs,
 		       int flags)
 {
 	struct module *mod;
 	long err;
 	err = module_sig_check(info);
 	if (err)
 		goto free_copy;
 	err = elf_header_check(info);
 	if (err)
 		goto free_copy;
 	/* Figure out module layout, and allocate all the memory. */
 	mod = layout_and_allocate(info, flags);
 	if (IS_ERR(mod)) {
 		err = PTR_ERR(mod);
 		goto free_copy;
 	}
 	/* Reserve our place in the list. */
 	err = add_unformed_module(mod);
 	if (err)
 		goto free_module;
 #ifdef CONFIG_MODULE_SIG
 	mod->sig_ok = info->sig_ok;
 	if (!mod->sig_ok) {
 		pr_notice_once("%s: module verification failed: signature "
 			       "and/or  required key missing - tainting "
 			       "kernel\n", mod->name);
 		add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
 	}
 #endif
 	/* To avoid stressing percpu allocator, do this once we're unique. */
 	err = percpu_modalloc(mod, info);
 	if (err)
 		goto unlink_mod;
 	/* Now module is in final location, initialize linked lists, etc. */
 	err = module_unload_init(mod);
 	if (err)
 		goto unlink_mod;
 	/* Now we've got everything in the final locations, we can
 	 * find optional sections. */
 	err = find_module_sections(mod, info);
 	if (err)
 		goto free_unload;
 	err = check_module_license_and_versions(mod);
 	if (err)
 		goto free_unload;
 	/* Set up MODINFO_ATTR fields */
 	setup_modinfo(mod, info);
 	/* Fix up syms, so that st_value is a pointer to location. */
 	err = simplify_symbols(mod, info);
 	if (err < 0)
 		goto free_modinfo;
 	err = apply_relocations(mod, info);
 	if (err < 0)
 		goto free_modinfo;
 	err = post_relocation(mod, info);
 	if (err < 0)
 		goto free_modinfo;
 	flush_module_icache(mod);
 	/* Now copy in args */
 	mod->args = strndup_user(uargs, ~0UL >> 1);
 	if (IS_ERR(mod->args)) {
 		err = PTR_ERR(mod->args);
 		goto free_arch_cleanup;
 	}
 	dynamic_debug_setup(info->debug, info->num_debug);
+	/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
+	ftrace_module_init(mod);
 	/* Finally it's fully formed, ready to start executing. */
 	err = complete_formation(mod, info);
 	if (err)
 		goto ddebug_cleanup;
 	/* Module is ready to execute: parsing args may do that. */
 	err = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
 			 -32768, 32767, unknown_module_param_cb);
 	if (err < 0)
 		goto bug_cleanup;
 	/* Link in to syfs. */
 	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
 	if (err < 0)
 		goto bug_cleanup;
 	/* Get rid of temporary copy. */
 	free_copy(info);
 	/* Done! */
 	trace_module_load(mod);
 	return do_init_module(mod);
  bug_cleanup:
 	/* module_bug_cleanup needs module_mutex protection */
 	mutex_lock(&module_mutex);
 	module_bug_cleanup(mod);
 	mutex_unlock(&module_mutex);
  ddebug_cleanup:
 	dynamic_debug_remove(info->debug);
 	synchronize_sched();
 	kfree(mod->args);
  free_arch_cleanup:
 	module_arch_cleanup(mod);
  free_modinfo:
 	free_modinfo(mod);
  free_unload:
 	module_unload_free(mod);
  unlink_mod:
 	mutex_lock(&module_mutex);
 	/* Unlink carefully: kallsyms could be walking list. */
 	list_del_rcu(&mod->list);
 	wake_up_all(&module_wq);
 	mutex_unlock(&module_mutex);
  free_module:
 	module_deallocate(mod, info);
  free_copy:
 	free_copy(info);
 	return err;
 }
 SYSCALL_DEFINE3(init_module, void __user *, umod,
 		unsigned long, len, const char __user *, uargs)
 {
 	int err;
 	struct load_info info = { };
 	err = may_init_module();
 	if (err)
 		return err;
 	pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
 	       umod, len, uargs);
 	err = copy_module_from_user(umod, len, &info);
 	if (err)
 		return err;
 	return load_module(&info, uargs, 0);
 }
 SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
 {
 	int err;
 	struct load_info info = { };
 	err = may_init_module();
 	if (err)
 		return err;
 	pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
 	if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
 		      |MODULE_INIT_IGNORE_VERMAGIC))
 		return -EINVAL;
 	err = copy_module_from_fd(fd, &info);
 	if (err)
 		return err;
 	return load_module(&info, uargs, flags);
 }
 static inline int within(unsigned long addr, void *start, unsigned long size)
 {
 	return ((void *)addr >= start && (void *)addr < start + size);
 }
 #ifdef CONFIG_KALLSYMS
 /*
  * This ignores the intensely annoying "mapping symbols" found
  * in ARM ELF files: $a, $t and $d.
  */
 static inline int is_arm_mapping_symbol(const char *str)
 {
 	return str[0] == '$' && strchr("atd", str[1])
 	       && (str[2] == '\0' || str[2] == '.');
 }
 static const char *get_ksymbol(struct module *mod,
 			       unsigned long addr,
 			       unsigned long *size,
 			       unsigned long *offset)
 {
 	unsigned int i, best = 0;
 	unsigned long nextval;
 	/* At worse, next value is at end of module */
 	if (within_module_init(addr, mod))
 		nextval = (unsigned long)mod->module_init+mod->init_text_size;
 	else
 		nextval = (unsigned long)mod->module_core+mod->core_text_size;
 	/* Scan for closest preceding symbol, and next symbol. (ELF
 	   starts real symbols at 1). */
 	for (i = 1; i < mod->num_symtab; i++) {
 		if (mod->symtab[i].st_shndx == SHN_UNDEF)
 			continue;
 		/* We ignore unnamed symbols: they're uninformative
 		 * and inserted at a whim. */
 		if (mod->symtab[i].st_value <= addr
 		    && mod->symtab[i].st_value > mod->symtab[best].st_value
 		    && *(mod->strtab + mod->symtab[i].st_name) != '\0'
 		    && !is_arm_mapping_symbol(mod->strtab + mod->symtab[i].st_name))
 			best = i;
 		if (mod->symtab[i].st_value > addr
 		    && mod->symtab[i].st_value < nextval
 		    && *(mod->strtab + mod->symtab[i].st_name) != '\0'
 		    && !is_arm_mapping_symbol(mod->strtab + mod->symtab[i].st_name))
 			nextval = mod->symtab[i].st_value;
 	}
 	if (!best)
 		return NULL;
 	if (size)
 		*size = nextval - mod->symtab[best].st_value;
 	if (offset)
 		*offset = addr - mod->symtab[best].st_value;
 	return mod->strtab + mod->symtab[best].st_name;
 }
 /* For kallsyms to ask for address resolution.  NULL means not found.  Careful
  * not to lock to avoid deadlock on oopses, simply disable preemption. */
 const char *module_address_lookup(unsigned long addr,
 			    unsigned long *size,
 			    unsigned long *offset,
 			    char **modname,
 			    char *namebuf)
 {
 	struct module *mod;
 	const char *ret = NULL;
 	preempt_disable();
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (within_module_init(addr, mod) ||
 		    within_module_core(addr, mod)) {
 			if (modname)
 				*modname = mod->name;
 			ret = get_ksymbol(mod, addr, size, offset);
 			break;
 		}
 	}
 	/* Make a copy in here where it's safe */
 	if (ret) {
 		strncpy(namebuf, ret, KSYM_NAME_LEN - 1);
 		ret = namebuf;
 	}
 	preempt_enable();
 	return ret;
 }
 int lookup_module_symbol_name(unsigned long addr, char *symname)
 {
 	struct module *mod;
 	preempt_disable();
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (within_module_init(addr, mod) ||
 		    within_module_core(addr, mod)) {
 			const char *sym;
 			sym = get_ksymbol(mod, addr, NULL, NULL);
 			if (!sym)
 				goto out;
 			strlcpy(symname, sym, KSYM_NAME_LEN);
 			preempt_enable();
 			return 0;
 		}
 	}
 out:
 	preempt_enable();
 	return -ERANGE;
 }
 int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size,
 			unsigned long *offset, char *modname, char *name)
 {
 	struct module *mod;
 	preempt_disable();
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (within_module_init(addr, mod) ||
 		    within_module_core(addr, mod)) {
 			const char *sym;
 			sym = get_ksymbol(mod, addr, size, offset);
 			if (!sym)
 				goto out;
 			if (modname)
 				strlcpy(modname, mod->name, MODULE_NAME_LEN);
 			if (name)
 				strlcpy(name, sym, KSYM_NAME_LEN);
 			preempt_enable();
 			return 0;
 		}
 	}
 out:
 	preempt_enable();
 	return -ERANGE;
 }
 int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
 			char *name, char *module_name, int *exported)
 {
 	struct module *mod;
 	preempt_disable();
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (symnum < mod->num_symtab) {
 			*value = mod->symtab[symnum].st_value;
 			*type = mod->symtab[symnum].st_info;
 			strlcpy(name, mod->strtab + mod->symtab[symnum].st_name,
 				KSYM_NAME_LEN);
 			strlcpy(module_name, mod->name, MODULE_NAME_LEN);
 			*exported = is_exported(name, *value, mod);
 			preempt_enable();
 			return 0;
 		}
 		symnum -= mod->num_symtab;
 	}
 	preempt_enable();
 	return -ERANGE;
 }
 static unsigned long mod_find_symname(struct module *mod, const char *name)
 {
 	unsigned int i;
 	for (i = 0; i < mod->num_symtab; i++)
 		if (strcmp(name, mod->strtab+mod->symtab[i].st_name) == 0 &&
 		    mod->symtab[i].st_info != 'U')
 			return mod->symtab[i].st_value;
 	return 0;
 }
 /* Look for this name: can be of form module:name. */
 unsigned long module_kallsyms_lookup_name(const char *name)
 {
 	struct module *mod;
 	char *colon;
 	unsigned long ret = 0;
 	/* Don't lock: we're in enough trouble already. */
 	preempt_disable();
 	if ((colon = strchr(name, ':')) != NULL) {
 		if ((mod = find_module_all(name, colon - name, false)) != NULL)
 			ret = mod_find_symname(mod, colon+1);
 	} else {
 		list_for_each_entry_rcu(mod, &modules, list) {
 			if (mod->state == MODULE_STATE_UNFORMED)
 				continue;
 			if ((ret = mod_find_symname(mod, name)) != 0)
 				break;
 		}
 	}
 	preempt_enable();
 	return ret;
 }
 int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
 					     struct module *, unsigned long),
 				   void *data)
 {
 	struct module *mod;
 	unsigned int i;
 	int ret;
 	list_for_each_entry(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		for (i = 0; i < mod->num_symtab; i++) {
 			ret = fn(data, mod->strtab + mod->symtab[i].st_name,
 				 mod, mod->symtab[i].st_value);
 			if (ret != 0)
 				return ret;
 		}
 	}
 	return 0;
 }
 #endif /* CONFIG_KALLSYMS */
 static char *module_flags(struct module *mod, char *buf)
 {
 	int bx = 0;
 	BUG_ON(mod->state == MODULE_STATE_UNFORMED);
 	if (mod->taints ||
 	    mod->state == MODULE_STATE_GOING ||
 	    mod->state == MODULE_STATE_COMING) {
 		buf[bx++] = '(';
 		bx += module_flags_taint(mod, buf + bx);
 		/* Show a - for module-is-being-unloaded */
 		if (mod->state == MODULE_STATE_GOING)
 			buf[bx++] = '-';
 		/* Show a + for module-is-being-loaded */
 		if (mod->state == MODULE_STATE_COMING)
 			buf[bx++] = '+';
 		buf[bx++] = ')';
 	}
 	buf[bx] = '\0';
 	return buf;
 }
 #ifdef CONFIG_PROC_FS
 /* Called by the /proc file system to return a list of modules. */
 static void *m_start(struct seq_file *m, loff_t *pos)
 {
 	mutex_lock(&module_mutex);
 	return seq_list_start(&modules, *pos);
 }
 static void *m_next(struct seq_file *m, void *p, loff_t *pos)
 {
 	return seq_list_next(p, &modules, pos);
 }
 static void m_stop(struct seq_file *m, void *p)
 {
 	mutex_unlock(&module_mutex);
 }
 static int m_show(struct seq_file *m, void *p)
 {
 	struct module *mod = list_entry(p, struct module, list);
 	char buf[8];
 	/* We always ignore unformed modules. */
 	if (mod->state == MODULE_STATE_UNFORMED)
 		return 0;
 	seq_printf(m, "%s %u",
 		   mod->name, mod->init_size + mod->core_size);
 	print_unload_info(m, mod);
 	/* Informative for users. */
 	seq_printf(m, " %s",
 		   mod->state == MODULE_STATE_GOING ? "Unloading":
 		   mod->state == MODULE_STATE_COMING ? "Loading":
 		   "Live");
 	/* Used by oprofile and other similar tools. */
 	seq_printf(m, " 0x%pK", mod->module_core);
 	/* Taints info */
 	if (mod->taints)
 		seq_printf(m, " %s", module_flags(mod, buf));
 	seq_printf(m, "\n");
 	return 0;
 }
 /* Format: modulename size refcount deps address
    Where refcount is a number or -, and deps is a comma-separated list
    of depends or -.
 */
 static const struct seq_operations modules_op = {
 	.start	= m_start,
 	.next	= m_next,
 	.stop	= m_stop,
 	.show	= m_show
 };
 static int modules_open(struct inode *inode, struct file *file)
 {
 	return seq_open(file, &modules_op);
 }
 static const struct file_operations proc_modules_operations = {
 	.open		= modules_open,
 	.read		= seq_read,
 	.llseek		= seq_lseek,
 	.release	= seq_release,
 };
 static int __init proc_modules_init(void)
 {
 	proc_create("modules", 0, NULL, &proc_modules_operations);
 	return 0;
 }
 module_init(proc_modules_init);
 #endif
 /* Given an address, look for it in the module exception tables. */
 const struct exception_table_entry *search_module_extables(unsigned long addr)
 {
 	const struct exception_table_entry *e = NULL;
 	struct module *mod;
 	preempt_disable();
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (mod->num_exentries == 0)
 			continue;
 		e = search_extable(mod->extable,
 				   mod->extable + mod->num_exentries - 1,
 				   addr);
 		if (e)
 			break;
 	}
 	preempt_enable();
 	/* Now, if we found one, we are running inside it now, hence
 	   we cannot unload the module, hence no refcnt needed. */
 	return e;
 }
 /*
  * is_module_address - is this address inside a module?
  * @addr: the address to check.
  *
  * See is_module_text_address() if you simply want to see if the address
  * is code (not data).
  */
 bool is_module_address(unsigned long addr)
 {
 	bool ret;
 	preempt_disable();
 	ret = __module_address(addr) != NULL;
 	preempt_enable();
 	return ret;
 }
 /*
  * __module_address - get the module which contains an address.
  * @addr: the address.
  *
  * Must be called with preempt disabled or module mutex held so that
  * module doesn't get freed during this.
  */
 struct module *__module_address(unsigned long addr)
 {
 	struct module *mod;
 	if (addr < module_addr_min || addr > module_addr_max)
 		return NULL;
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		if (within_module_core(addr, mod)
 		    || within_module_init(addr, mod))
 			return mod;
 	}
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(__module_address);
 /*
  * is_module_text_address - is this address inside module code?
  * @addr: the address to check.
  *
  * See is_module_address() if you simply want to see if the address is
  * anywhere in a module.  See kernel_text_address() for testing if an
  * address corresponds to kernel or module code.
  */
 bool is_module_text_address(unsigned long addr)
 {
 	bool ret;
 	preempt_disable();
 	ret = __module_text_address(addr) != NULL;
 	preempt_enable();
 	return ret;
 }
 /*
  * __module_text_address - get the module whose code contains an address.
  * @addr: the address.
  *
  * Must be called with preempt disabled or module mutex held so that
  * module doesn't get freed during this.
  */
 struct module *__module_text_address(unsigned long addr)
 {
 	struct module *mod = __module_address(addr);
 	if (mod) {
 		/* Make sure it's within the text section. */
 		if (!within(addr, mod->module_init, mod->init_text_size)
 		    && !within(addr, mod->module_core, mod->core_text_size))
 			mod = NULL;
 	}
 	return mod;
 }
 EXPORT_SYMBOL_GPL(__module_text_address);
 /* Don't grab lock, we're oopsing. */
 void print_modules(void)
 {
 	struct module *mod;
 	char buf[8];
 	printk(KERN_DEFAULT "Modules linked in:");
 	/* Most callers should already have preempt disabled, but make sure */
 	preempt_disable();
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
 			continue;
 		pr_cont(" %s%s", mod->name, module_flags(mod, buf));
 	}
 	preempt_enable();
 	if (last_unloaded_module[0])
 		pr_cont(" [last unloaded: %s]", last_unloaded_module);
 	pr_cont("\n");
 }
 #ifdef CONFIG_MODVERSIONS
 /* Generate the signature for all relevant module structures here.
  * If these change, we don't want to try to parse the module. */
 void module_layout(struct module *mod,
 		   struct modversion_info *ver,
 		   struct kernel_param *kp,
 		   struct kernel_symbol *ks,
 		   struct tracepoint * const *tp)
 {
 }
 EXPORT_SYMBOL(module_layout);
 #endif

kernel/trace/ftrace.c

Diff comments View file @ 2aafe1a

1	/*	1	/*
2	* Infrastructure for profiling code inserted by 'gcc -pg'.	2	* Infrastructure for profiling code inserted by 'gcc -pg'.
3	*	3	*
4	* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>	4	* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5	* Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>	5	* Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6	*	6	*
7	* Originally ported from the -rt patch by:	7	* Originally ported from the -rt patch by:
8	* Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>	8	* Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9	*	9	*
10	* Based on code in the latency_tracer, that is:	10	* Based on code in the latency_tracer, that is:
11	*	11	*
12	* Copyright (C) 2004-2006 Ingo Molnar	12	* Copyright (C) 2004-2006 Ingo Molnar
13	* Copyright (C) 2004 Nadia Yvette Chambers	13	* Copyright (C) 2004 Nadia Yvette Chambers
14	*/	14	*/
15		15
16	#include <linux/stop_machine.h>	16	#include <linux/stop_machine.h>
17	#include <linux/clocksource.h>	17	#include <linux/clocksource.h>
18	#include <linux/kallsyms.h>	18	#include <linux/kallsyms.h>
19	#include <linux/seq_file.h>	19	#include <linux/seq_file.h>
20	#include <linux/suspend.h>	20	#include <linux/suspend.h>
21	#include <linux/debugfs.h>	21	#include <linux/debugfs.h>
22	#include <linux/hardirq.h>	22	#include <linux/hardirq.h>
23	#include <linux/kthread.h>	23	#include <linux/kthread.h>
24	#include <linux/uaccess.h>	24	#include <linux/uaccess.h>
25	#include <linux/bsearch.h>	25	#include <linux/bsearch.h>
26	#include <linux/module.h>	26	#include <linux/module.h>
27	#include <linux/ftrace.h>	27	#include <linux/ftrace.h>
28	#include <linux/sysctl.h>	28	#include <linux/sysctl.h>
29	#include <linux/slab.h>	29	#include <linux/slab.h>
30	#include <linux/ctype.h>	30	#include <linux/ctype.h>
31	#include <linux/sort.h>	31	#include <linux/sort.h>
32	#include <linux/list.h>	32	#include <linux/list.h>
33	#include <linux/hash.h>	33	#include <linux/hash.h>
34	#include <linux/rcupdate.h>	34	#include <linux/rcupdate.h>
35		35
36	#include <trace/events/sched.h>	36	#include <trace/events/sched.h>
37		37
38	#include <asm/setup.h>	38	#include <asm/setup.h>
39		39
40	#include "trace_output.h"	40	#include "trace_output.h"
41	#include "trace_stat.h"	41	#include "trace_stat.h"
42		42
43	#define FTRACE_WARN_ON(cond) \	43	#define FTRACE_WARN_ON(cond) \
44	({ \	44	({ \
45	int ___r = cond; \	45	int ___r = cond; \
46	if (WARN_ON(___r)) \	46	if (WARN_ON(___r)) \
47	ftrace_kill(); \	47	ftrace_kill(); \
48	___r; \	48	___r; \
49	})	49	})
50		50
51	#define FTRACE_WARN_ON_ONCE(cond) \	51	#define FTRACE_WARN_ON_ONCE(cond) \
52	({ \	52	({ \
53	int ___r = cond; \	53	int ___r = cond; \
54	if (WARN_ON_ONCE(___r)) \	54	if (WARN_ON_ONCE(___r)) \
55	ftrace_kill(); \	55	ftrace_kill(); \
56	___r; \	56	___r; \
57	})	57	})
58		58
59	/* hash bits for specific function selection */	59	/* hash bits for specific function selection */
60	#define FTRACE_HASH_BITS 7	60	#define FTRACE_HASH_BITS 7
61	#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)	61	#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
62	#define FTRACE_HASH_DEFAULT_BITS 10	62	#define FTRACE_HASH_DEFAULT_BITS 10
63	#define FTRACE_HASH_MAX_BITS 12	63	#define FTRACE_HASH_MAX_BITS 12
64		64
65	#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL \| FTRACE_OPS_FL_CONTROL)	65	#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL \| FTRACE_OPS_FL_CONTROL)
66		66
67	#ifdef CONFIG_DYNAMIC_FTRACE	67	#ifdef CONFIG_DYNAMIC_FTRACE
68	#define INIT_REGEX_LOCK(opsname) \	68	#define INIT_REGEX_LOCK(opsname) \
69	.regex_lock = __MUTEX_INITIALIZER(opsname.regex_lock),	69	.regex_lock = __MUTEX_INITIALIZER(opsname.regex_lock),
70	#else	70	#else
71	#define INIT_REGEX_LOCK(opsname)	71	#define INIT_REGEX_LOCK(opsname)
72	#endif	72	#endif
73		73
74	static struct ftrace_ops ftrace_list_end __read_mostly = {	74	static struct ftrace_ops ftrace_list_end __read_mostly = {
75	.func = ftrace_stub,	75	.func = ftrace_stub,
76	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_STUB,	76	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_STUB,
77	};	77	};
78		78
79	/* ftrace_enabled is a method to turn ftrace on or off */	79	/* ftrace_enabled is a method to turn ftrace on or off */
80	int ftrace_enabled __read_mostly;	80	int ftrace_enabled __read_mostly;
81	static int last_ftrace_enabled;	81	static int last_ftrace_enabled;
82		82
83	/* Quick disabling of function tracer. */	83	/* Quick disabling of function tracer. */
84	int function_trace_stop __read_mostly;	84	int function_trace_stop __read_mostly;
85		85
86	/* Current function tracing op */	86	/* Current function tracing op */
87	struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;	87	struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
88	/* What to set function_trace_op to */	88	/* What to set function_trace_op to */
89	static struct ftrace_ops *set_function_trace_op;	89	static struct ftrace_ops *set_function_trace_op;
90		90
91	/* List for set_ftrace_pid's pids. */	91	/* List for set_ftrace_pid's pids. */
92	LIST_HEAD(ftrace_pids);	92	LIST_HEAD(ftrace_pids);
93	struct ftrace_pid {	93	struct ftrace_pid {
94	struct list_head list;	94	struct list_head list;
95	struct pid *pid;	95	struct pid *pid;
96	};	96	};
97		97
98	/*	98	/*
99	* ftrace_disabled is set when an anomaly is discovered.	99	* ftrace_disabled is set when an anomaly is discovered.
100	* ftrace_disabled is much stronger than ftrace_enabled.	100	* ftrace_disabled is much stronger than ftrace_enabled.
101	*/	101	*/
102	static int ftrace_disabled __read_mostly;	102	static int ftrace_disabled __read_mostly;
103		103
104	static DEFINE_MUTEX(ftrace_lock);	104	static DEFINE_MUTEX(ftrace_lock);
105		105
106	static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;	106	static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
107	static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;	107	static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
108	static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;	108	static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
109	ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;	109	ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
110	ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;	110	ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
111	static struct ftrace_ops global_ops;	111	static struct ftrace_ops global_ops;
112	static struct ftrace_ops control_ops;	112	static struct ftrace_ops control_ops;
113		113
114	#if ARCH_SUPPORTS_FTRACE_OPS	114	#if ARCH_SUPPORTS_FTRACE_OPS
115	static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,	115	static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
116	struct ftrace_ops op, struct pt_regs regs);	116	struct ftrace_ops op, struct pt_regs regs);
117	#else	117	#else
118	/* See comment below, where ftrace_ops_list_func is defined */	118	/* See comment below, where ftrace_ops_list_func is defined */
119	static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);	119	static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
120	#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)	120	#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
121	#endif	121	#endif
122		122
123	/*	123	/*
124	* Traverse the ftrace_global_list, invoking all entries. The reason that we	124	* Traverse the ftrace_global_list, invoking all entries. The reason that we
125	* can use rcu_dereference_raw_notrace() is that elements removed from this list	125	* can use rcu_dereference_raw_notrace() is that elements removed from this list
126	* are simply leaked, so there is no need to interact with a grace-period	126	* are simply leaked, so there is no need to interact with a grace-period
127	* mechanism. The rcu_dereference_raw_notrace() calls are needed to handle	127	* mechanism. The rcu_dereference_raw_notrace() calls are needed to handle
128	* concurrent insertions into the ftrace_global_list.	128	* concurrent insertions into the ftrace_global_list.
129	*	129	*
130	* Silly Alpha and silly pointer-speculation compiler optimizations!	130	* Silly Alpha and silly pointer-speculation compiler optimizations!
131	*/	131	*/
132	#define do_for_each_ftrace_op(op, list) \	132	#define do_for_each_ftrace_op(op, list) \
133	op = rcu_dereference_raw_notrace(list); \	133	op = rcu_dereference_raw_notrace(list); \
134	do	134	do
135		135
136	/*	136	/*
137	* Optimized for just a single item in the list (as that is the normal case).	137	* Optimized for just a single item in the list (as that is the normal case).
138	*/	138	*/
139	#define while_for_each_ftrace_op(op) \	139	#define while_for_each_ftrace_op(op) \
140	while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \	140	while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \
141	unlikely((op) != &ftrace_list_end))	141	unlikely((op) != &ftrace_list_end))
142		142
143	static inline void ftrace_ops_init(struct ftrace_ops *ops)	143	static inline void ftrace_ops_init(struct ftrace_ops *ops)
144	{	144	{
145	#ifdef CONFIG_DYNAMIC_FTRACE	145	#ifdef CONFIG_DYNAMIC_FTRACE
146	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {	146	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
147	mutex_init(&ops->regex_lock);	147	mutex_init(&ops->regex_lock);
148	ops->flags \|= FTRACE_OPS_FL_INITIALIZED;	148	ops->flags \|= FTRACE_OPS_FL_INITIALIZED;
149	}	149	}
150	#endif	150	#endif
151	}	151	}
152		152
153	/**	153	/**
154	* ftrace_nr_registered_ops - return number of ops registered	154	* ftrace_nr_registered_ops - return number of ops registered
155	*	155	*
156	* Returns the number of ftrace_ops registered and tracing functions	156	* Returns the number of ftrace_ops registered and tracing functions
157	*/	157	*/
158	int ftrace_nr_registered_ops(void)	158	int ftrace_nr_registered_ops(void)
159	{	159	{
160	struct ftrace_ops *ops;	160	struct ftrace_ops *ops;
161	int cnt = 0;	161	int cnt = 0;
162		162
163	mutex_lock(&ftrace_lock);	163	mutex_lock(&ftrace_lock);
164		164
165	for (ops = ftrace_ops_list;	165	for (ops = ftrace_ops_list;
166	ops != &ftrace_list_end; ops = ops->next)	166	ops != &ftrace_list_end; ops = ops->next)
167	cnt++;	167	cnt++;
168		168
169	mutex_unlock(&ftrace_lock);	169	mutex_unlock(&ftrace_lock);
170		170
171	return cnt;	171	return cnt;
172	}	172	}
173		173
174	static void	174	static void
175	ftrace_global_list_func(unsigned long ip, unsigned long parent_ip,	175	ftrace_global_list_func(unsigned long ip, unsigned long parent_ip,
176	struct ftrace_ops op, struct pt_regs regs)	176	struct ftrace_ops op, struct pt_regs regs)
177	{	177	{
178	int bit;	178	int bit;
179		179
180	bit = trace_test_and_set_recursion(TRACE_GLOBAL_START, TRACE_GLOBAL_MAX);	180	bit = trace_test_and_set_recursion(TRACE_GLOBAL_START, TRACE_GLOBAL_MAX);
181	if (bit < 0)	181	if (bit < 0)
182	return;	182	return;
183		183
184	do_for_each_ftrace_op(op, ftrace_global_list) {	184	do_for_each_ftrace_op(op, ftrace_global_list) {
185	op->func(ip, parent_ip, op, regs);	185	op->func(ip, parent_ip, op, regs);
186	} while_for_each_ftrace_op(op);	186	} while_for_each_ftrace_op(op);
187		187
188	trace_clear_recursion(bit);	188	trace_clear_recursion(bit);
189	}	189	}
190		190
191	static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,	191	static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
192	struct ftrace_ops op, struct pt_regs regs)	192	struct ftrace_ops op, struct pt_regs regs)
193	{	193	{
194	if (!test_tsk_trace_trace(current))	194	if (!test_tsk_trace_trace(current))
195	return;	195	return;
196		196
197	ftrace_pid_function(ip, parent_ip, op, regs);	197	ftrace_pid_function(ip, parent_ip, op, regs);
198	}	198	}
199		199
200	static void set_ftrace_pid_function(ftrace_func_t func)	200	static void set_ftrace_pid_function(ftrace_func_t func)
201	{	201	{
202	/* do not set ftrace_pid_function to itself! */	202	/* do not set ftrace_pid_function to itself! */
203	if (func != ftrace_pid_func)	203	if (func != ftrace_pid_func)
204	ftrace_pid_function = func;	204	ftrace_pid_function = func;
205	}	205	}
206		206
207	/**	207	/**
208	* clear_ftrace_function - reset the ftrace function	208	* clear_ftrace_function - reset the ftrace function
209	*	209	*
210	* This NULLs the ftrace function and in essence stops	210	* This NULLs the ftrace function and in essence stops
211	* tracing. There may be lag	211	* tracing. There may be lag
212	*/	212	*/
213	void clear_ftrace_function(void)	213	void clear_ftrace_function(void)
214	{	214	{
215	ftrace_trace_function = ftrace_stub;	215	ftrace_trace_function = ftrace_stub;
216	ftrace_pid_function = ftrace_stub;	216	ftrace_pid_function = ftrace_stub;
217	}	217	}
218		218
219	static void control_ops_disable_all(struct ftrace_ops *ops)	219	static void control_ops_disable_all(struct ftrace_ops *ops)
220	{	220	{
221	int cpu;	221	int cpu;
222		222
223	for_each_possible_cpu(cpu)	223	for_each_possible_cpu(cpu)
224	*per_cpu_ptr(ops->disabled, cpu) = 1;	224	*per_cpu_ptr(ops->disabled, cpu) = 1;
225	}	225	}
226		226
227	static int control_ops_alloc(struct ftrace_ops *ops)	227	static int control_ops_alloc(struct ftrace_ops *ops)
228	{	228	{
229	int __percpu *disabled;	229	int __percpu *disabled;
230		230
231	disabled = alloc_percpu(int);	231	disabled = alloc_percpu(int);
232	if (!disabled)	232	if (!disabled)
233	return -ENOMEM;	233	return -ENOMEM;
234		234
235	ops->disabled = disabled;	235	ops->disabled = disabled;
236	control_ops_disable_all(ops);	236	control_ops_disable_all(ops);
237	return 0;	237	return 0;
238	}	238	}
239		239
240	static void update_global_ops(void)	240	static void update_global_ops(void)
241	{	241	{
242	ftrace_func_t func = ftrace_global_list_func;	242	ftrace_func_t func = ftrace_global_list_func;
243	void *private = NULL;	243	void *private = NULL;
244		244
245	/* The list has its own recursion protection. */	245	/* The list has its own recursion protection. */
246	global_ops.flags \|= FTRACE_OPS_FL_RECURSION_SAFE;	246	global_ops.flags \|= FTRACE_OPS_FL_RECURSION_SAFE;
247		247
248	/*	248	/*
249	* If there's only one function registered, then call that	249	* If there's only one function registered, then call that
250	* function directly. Otherwise, we need to iterate over the	250	* function directly. Otherwise, we need to iterate over the
251	* registered callers.	251	* registered callers.
252	*/	252	*/
253	if (ftrace_global_list == &ftrace_list_end \|\|	253	if (ftrace_global_list == &ftrace_list_end \|\|
254	ftrace_global_list->next == &ftrace_list_end) {	254	ftrace_global_list->next == &ftrace_list_end) {
255	func = ftrace_global_list->func;	255	func = ftrace_global_list->func;
256	private = ftrace_global_list->private;	256	private = ftrace_global_list->private;
257	/*	257	/*
258	* As we are calling the function directly.	258	* As we are calling the function directly.
259	* If it does not have recursion protection,	259	* If it does not have recursion protection,
260	* the function_trace_op needs to be updated	260	* the function_trace_op needs to be updated
261	* accordingly.	261	* accordingly.
262	*/	262	*/
263	if (!(ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE))	263	if (!(ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE))
264	global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE;	264	global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE;
265	}	265	}
266		266
267	/* If we filter on pids, update to use the pid function */	267	/* If we filter on pids, update to use the pid function */
268	if (!list_empty(&ftrace_pids)) {	268	if (!list_empty(&ftrace_pids)) {
269	set_ftrace_pid_function(func);	269	set_ftrace_pid_function(func);
270	func = ftrace_pid_func;	270	func = ftrace_pid_func;
271	}	271	}
272		272
273	global_ops.func = func;	273	global_ops.func = func;
274	global_ops.private = private;	274	global_ops.private = private;
275	}	275	}
276		276
277	static void ftrace_sync(struct work_struct *work)	277	static void ftrace_sync(struct work_struct *work)
278	{	278	{
279	/*	279	/*
280	* This function is just a stub to implement a hard force	280	* This function is just a stub to implement a hard force
281	* of synchronize_sched(). This requires synchronizing	281	* of synchronize_sched(). This requires synchronizing
282	* tasks even in userspace and idle.	282	* tasks even in userspace and idle.
283	*	283	*
284	* Yes, function tracing is rude.	284	* Yes, function tracing is rude.
285	*/	285	*/
286	}	286	}
287		287
288	static void ftrace_sync_ipi(void *data)	288	static void ftrace_sync_ipi(void *data)
289	{	289	{
290	/* Probably not needed, but do it anyway */	290	/* Probably not needed, but do it anyway */
291	smp_rmb();	291	smp_rmb();
292	}	292	}
293		293
294	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	294	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
295	static void update_function_graph_func(void);	295	static void update_function_graph_func(void);
296	#else	296	#else
297	static inline void update_function_graph_func(void) { }	297	static inline void update_function_graph_func(void) { }
298	#endif	298	#endif
299		299
300	static void update_ftrace_function(void)	300	static void update_ftrace_function(void)
301	{	301	{
302	ftrace_func_t func;	302	ftrace_func_t func;
303		303
304	update_global_ops();	304	update_global_ops();
305		305
306	/*	306	/*
307	* If we are at the end of the list and this ops is	307	* If we are at the end of the list and this ops is
308	* recursion safe and not dynamic and the arch supports passing ops,	308	* recursion safe and not dynamic and the arch supports passing ops,
309	* then have the mcount trampoline call the function directly.	309	* then have the mcount trampoline call the function directly.
310	*/	310	*/
311	if (ftrace_ops_list == &ftrace_list_end \|\|	311	if (ftrace_ops_list == &ftrace_list_end \|\|
312	(ftrace_ops_list->next == &ftrace_list_end &&	312	(ftrace_ops_list->next == &ftrace_list_end &&
313	!(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) &&	313	!(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) &&
314	(ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) &&	314	(ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) &&
315	!FTRACE_FORCE_LIST_FUNC)) {	315	!FTRACE_FORCE_LIST_FUNC)) {
316	/* Set the ftrace_ops that the arch callback uses */	316	/* Set the ftrace_ops that the arch callback uses */
317	if (ftrace_ops_list == &global_ops)	317	if (ftrace_ops_list == &global_ops)
318	set_function_trace_op = ftrace_global_list;	318	set_function_trace_op = ftrace_global_list;
319	else	319	else
320	set_function_trace_op = ftrace_ops_list;	320	set_function_trace_op = ftrace_ops_list;
321	func = ftrace_ops_list->func;	321	func = ftrace_ops_list->func;
322	} else {	322	} else {
323	/* Just use the default ftrace_ops */	323	/* Just use the default ftrace_ops */
324	set_function_trace_op = &ftrace_list_end;	324	set_function_trace_op = &ftrace_list_end;
325	func = ftrace_ops_list_func;	325	func = ftrace_ops_list_func;
326	}	326	}
327		327
328	/* If there's no change, then do nothing more here */	328	/* If there's no change, then do nothing more here */
329	if (ftrace_trace_function == func)	329	if (ftrace_trace_function == func)
330	return;	330	return;
331		331
332	update_function_graph_func();	332	update_function_graph_func();
333		333
334	/*	334	/*
335	* If we are using the list function, it doesn't care	335	* If we are using the list function, it doesn't care
336	* about the function_trace_ops.	336	* about the function_trace_ops.
337	*/	337	*/
338	if (func == ftrace_ops_list_func) {	338	if (func == ftrace_ops_list_func) {
339	ftrace_trace_function = func;	339	ftrace_trace_function = func;
340	/*	340	/*
341	* Don't even bother setting function_trace_ops,	341	* Don't even bother setting function_trace_ops,
342	* it would be racy to do so anyway.	342	* it would be racy to do so anyway.
343	*/	343	*/
344	return;	344	return;
345	}	345	}
346		346
347	#ifndef CONFIG_DYNAMIC_FTRACE	347	#ifndef CONFIG_DYNAMIC_FTRACE
348	/*	348	/*
349	* For static tracing, we need to be a bit more careful.	349	* For static tracing, we need to be a bit more careful.
350	* The function change takes affect immediately. Thus,	350	* The function change takes affect immediately. Thus,
351	* we need to coorditate the setting of the function_trace_ops	351	* we need to coorditate the setting of the function_trace_ops
352	* with the setting of the ftrace_trace_function.	352	* with the setting of the ftrace_trace_function.
353	*	353	*
354	* Set the function to the list ops, which will call the	354	* Set the function to the list ops, which will call the
355	* function we want, albeit indirectly, but it handles the	355	* function we want, albeit indirectly, but it handles the
356	* ftrace_ops and doesn't depend on function_trace_op.	356	* ftrace_ops and doesn't depend on function_trace_op.
357	*/	357	*/
358	ftrace_trace_function = ftrace_ops_list_func;	358	ftrace_trace_function = ftrace_ops_list_func;
359	/*	359	/*
360	* Make sure all CPUs see this. Yes this is slow, but static	360	* Make sure all CPUs see this. Yes this is slow, but static
361	* tracing is slow and nasty to have enabled.	361	* tracing is slow and nasty to have enabled.
362	*/	362	*/
363	schedule_on_each_cpu(ftrace_sync);	363	schedule_on_each_cpu(ftrace_sync);
364	/* Now all cpus are using the list ops. */	364	/* Now all cpus are using the list ops. */
365	function_trace_op = set_function_trace_op;	365	function_trace_op = set_function_trace_op;
366	/* Make sure the function_trace_op is visible on all CPUs */	366	/* Make sure the function_trace_op is visible on all CPUs */
367	smp_wmb();	367	smp_wmb();
368	/* Nasty way to force a rmb on all cpus */	368	/* Nasty way to force a rmb on all cpus */
369	smp_call_function(ftrace_sync_ipi, NULL, 1);	369	smp_call_function(ftrace_sync_ipi, NULL, 1);
370	/* OK, we are all set to update the ftrace_trace_function now! */	370	/* OK, we are all set to update the ftrace_trace_function now! */
371	#endif /* !CONFIG_DYNAMIC_FTRACE */	371	#endif /* !CONFIG_DYNAMIC_FTRACE */
372		372
373	ftrace_trace_function = func;	373	ftrace_trace_function = func;
374	}	374	}
375		375
376	static void add_ftrace_ops(struct ftrace_ops *list, struct ftrace_ops ops)	376	static void add_ftrace_ops(struct ftrace_ops *list, struct ftrace_ops ops)
377	{	377	{
378	ops->next = *list;	378	ops->next = *list;
379	/*	379	/*
380	* We are entering ops into the list but another	380	* We are entering ops into the list but another
381	* CPU might be walking that list. We need to make sure	381	* CPU might be walking that list. We need to make sure
382	* the ops->next pointer is valid before another CPU sees	382	* the ops->next pointer is valid before another CPU sees
383	* the ops pointer included into the list.	383	* the ops pointer included into the list.
384	*/	384	*/
385	rcu_assign_pointer(*list, ops);	385	rcu_assign_pointer(*list, ops);
386	}	386	}
387		387
388	static int remove_ftrace_ops(struct ftrace_ops *list, struct ftrace_ops ops)	388	static int remove_ftrace_ops(struct ftrace_ops *list, struct ftrace_ops ops)
389	{	389	{
390	struct ftrace_ops **p;	390	struct ftrace_ops **p;
391		391
392	/*	392	/*
393	* If we are removing the last function, then simply point	393	* If we are removing the last function, then simply point
394	* to the ftrace_stub.	394	* to the ftrace_stub.
395	*/	395	*/
396	if (*list == ops && ops->next == &ftrace_list_end) {	396	if (*list == ops && ops->next == &ftrace_list_end) {
397	*list = &ftrace_list_end;	397	*list = &ftrace_list_end;
398	return 0;	398	return 0;
399	}	399	}
400		400
401	for (p = list; p != &ftrace_list_end; p = &(p)->next)	401	for (p = list; p != &ftrace_list_end; p = &(p)->next)
402	if (*p == ops)	402	if (*p == ops)
403	break;	403	break;
404		404
405	if (*p != ops)	405	if (*p != ops)
406	return -1;	406	return -1;
407		407
408	p = (p)->next;	408	p = (p)->next;
409	return 0;	409	return 0;
410	}	410	}
411		411
412	static void add_ftrace_list_ops(struct ftrace_ops **list,	412	static void add_ftrace_list_ops(struct ftrace_ops **list,
413	struct ftrace_ops *main_ops,	413	struct ftrace_ops *main_ops,
414	struct ftrace_ops *ops)	414	struct ftrace_ops *ops)
415	{	415	{
416	int first = *list == &ftrace_list_end;	416	int first = *list == &ftrace_list_end;
417	add_ftrace_ops(list, ops);	417	add_ftrace_ops(list, ops);
418	if (first)	418	if (first)
419	add_ftrace_ops(&ftrace_ops_list, main_ops);	419	add_ftrace_ops(&ftrace_ops_list, main_ops);
420	}	420	}
421		421
422	static int remove_ftrace_list_ops(struct ftrace_ops **list,	422	static int remove_ftrace_list_ops(struct ftrace_ops **list,
423	struct ftrace_ops *main_ops,	423	struct ftrace_ops *main_ops,
424	struct ftrace_ops *ops)	424	struct ftrace_ops *ops)
425	{	425	{
426	int ret = remove_ftrace_ops(list, ops);	426	int ret = remove_ftrace_ops(list, ops);
427	if (!ret && *list == &ftrace_list_end)	427	if (!ret && *list == &ftrace_list_end)
428	ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);	428	ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
429	return ret;	429	return ret;
430	}	430	}
431		431
432	static int __register_ftrace_function(struct ftrace_ops *ops)	432	static int __register_ftrace_function(struct ftrace_ops *ops)
433	{	433	{
434	if (ops->flags & FTRACE_OPS_FL_DELETED)	434	if (ops->flags & FTRACE_OPS_FL_DELETED)
435	return -EINVAL;	435	return -EINVAL;
436		436
437	if (FTRACE_WARN_ON(ops == &global_ops))	437	if (FTRACE_WARN_ON(ops == &global_ops))
438	return -EINVAL;	438	return -EINVAL;
439		439
440	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))	440	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
441	return -EBUSY;	441	return -EBUSY;
442		442
443	/* We don't support both control and global flags set. */	443	/* We don't support both control and global flags set. */
444	if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)	444	if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
445	return -EINVAL;	445	return -EINVAL;
446		446
447	#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS	447	#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
448	/*	448	/*
449	* If the ftrace_ops specifies SAVE_REGS, then it only can be used	449	* If the ftrace_ops specifies SAVE_REGS, then it only can be used
450	* if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.	450	* if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
451	* Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.	451	* Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
452	*/	452	*/
453	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&	453	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
454	!(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))	454	!(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
455	return -EINVAL;	455	return -EINVAL;
456		456
457	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)	457	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
458	ops->flags \|= FTRACE_OPS_FL_SAVE_REGS;	458	ops->flags \|= FTRACE_OPS_FL_SAVE_REGS;
459	#endif	459	#endif
460		460
461	if (!core_kernel_data((unsigned long)ops))	461	if (!core_kernel_data((unsigned long)ops))
462	ops->flags \|= FTRACE_OPS_FL_DYNAMIC;	462	ops->flags \|= FTRACE_OPS_FL_DYNAMIC;
463		463
464	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {	464	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
465	add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);	465	add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);
466	ops->flags \|= FTRACE_OPS_FL_ENABLED;	466	ops->flags \|= FTRACE_OPS_FL_ENABLED;
467	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {	467	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
468	if (control_ops_alloc(ops))	468	if (control_ops_alloc(ops))
469	return -ENOMEM;	469	return -ENOMEM;
470	add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);	470	add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
471	} else	471	} else
472	add_ftrace_ops(&ftrace_ops_list, ops);	472	add_ftrace_ops(&ftrace_ops_list, ops);
473		473
474	if (ftrace_enabled)	474	if (ftrace_enabled)
475	update_ftrace_function();	475	update_ftrace_function();
476		476
477	return 0;	477	return 0;
478	}	478	}
479		479
480	static int __unregister_ftrace_function(struct ftrace_ops *ops)	480	static int __unregister_ftrace_function(struct ftrace_ops *ops)
481	{	481	{
482	int ret;	482	int ret;
483		483
484	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))	484	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
485	return -EBUSY;	485	return -EBUSY;
486		486
487	if (FTRACE_WARN_ON(ops == &global_ops))	487	if (FTRACE_WARN_ON(ops == &global_ops))
488	return -EINVAL;	488	return -EINVAL;
489		489
490	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {	490	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
491	ret = remove_ftrace_list_ops(&ftrace_global_list,	491	ret = remove_ftrace_list_ops(&ftrace_global_list,
492	&global_ops, ops);	492	&global_ops, ops);
493	if (!ret)	493	if (!ret)
494	ops->flags &= ~FTRACE_OPS_FL_ENABLED;	494	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
495	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {	495	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
496	ret = remove_ftrace_list_ops(&ftrace_control_list,	496	ret = remove_ftrace_list_ops(&ftrace_control_list,
497	&control_ops, ops);	497	&control_ops, ops);
498	} else	498	} else
499	ret = remove_ftrace_ops(&ftrace_ops_list, ops);	499	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
500		500
501	if (ret < 0)	501	if (ret < 0)
502	return ret;	502	return ret;
503		503
504	if (ftrace_enabled)	504	if (ftrace_enabled)
505	update_ftrace_function();	505	update_ftrace_function();
506		506
507	return 0;	507	return 0;
508	}	508	}
509		509
510	static void ftrace_update_pid_func(void)	510	static void ftrace_update_pid_func(void)
511	{	511	{
512	/* Only do something if we are tracing something */	512	/* Only do something if we are tracing something */
513	if (ftrace_trace_function == ftrace_stub)	513	if (ftrace_trace_function == ftrace_stub)
514	return;	514	return;
515		515
516	update_ftrace_function();	516	update_ftrace_function();
517	}	517	}
518		518
519	#ifdef CONFIG_FUNCTION_PROFILER	519	#ifdef CONFIG_FUNCTION_PROFILER
520	struct ftrace_profile {	520	struct ftrace_profile {
521	struct hlist_node node;	521	struct hlist_node node;
522	unsigned long ip;	522	unsigned long ip;
523	unsigned long counter;	523	unsigned long counter;
524	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	524	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
525	unsigned long long time;	525	unsigned long long time;
526	unsigned long long time_squared;	526	unsigned long long time_squared;
527	#endif	527	#endif
528	};	528	};
529		529
530	struct ftrace_profile_page {	530	struct ftrace_profile_page {
531	struct ftrace_profile_page *next;	531	struct ftrace_profile_page *next;
532	unsigned long index;	532	unsigned long index;
533	struct ftrace_profile records[];	533	struct ftrace_profile records[];
534	};	534	};
535		535
536	struct ftrace_profile_stat {	536	struct ftrace_profile_stat {
537	atomic_t disabled;	537	atomic_t disabled;
538	struct hlist_head *hash;	538	struct hlist_head *hash;
539	struct ftrace_profile_page *pages;	539	struct ftrace_profile_page *pages;
540	struct ftrace_profile_page *start;	540	struct ftrace_profile_page *start;
541	struct tracer_stat stat;	541	struct tracer_stat stat;
542	};	542	};
543		543
544	#define PROFILE_RECORDS_SIZE \	544	#define PROFILE_RECORDS_SIZE \
545	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))	545	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
546		546
547	#define PROFILES_PER_PAGE \	547	#define PROFILES_PER_PAGE \
548	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))	548	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
549		549
550	static int ftrace_profile_enabled __read_mostly;	550	static int ftrace_profile_enabled __read_mostly;
551		551
552	/* ftrace_profile_lock - synchronize the enable and disable of the profiler */	552	/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
553	static DEFINE_MUTEX(ftrace_profile_lock);	553	static DEFINE_MUTEX(ftrace_profile_lock);
554		554
555	static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);	555	static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
556		556
557	#define FTRACE_PROFILE_HASH_BITS 10	557	#define FTRACE_PROFILE_HASH_BITS 10
558	#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)	558	#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
559		559
560	static void *	560	static void *
561	function_stat_next(void *v, int idx)	561	function_stat_next(void *v, int idx)
562	{	562	{
563	struct ftrace_profile *rec = v;	563	struct ftrace_profile *rec = v;
564	struct ftrace_profile_page *pg;	564	struct ftrace_profile_page *pg;
565		565
566	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);	566	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
567		567
568	again:	568	again:
569	if (idx != 0)	569	if (idx != 0)
570	rec++;	570	rec++;
571		571
572	if ((void )rec >= (void )&pg->records[pg->index]) {	572	if ((void )rec >= (void )&pg->records[pg->index]) {
573	pg = pg->next;	573	pg = pg->next;
574	if (!pg)	574	if (!pg)
575	return NULL;	575	return NULL;
576	rec = &pg->records[0];	576	rec = &pg->records[0];
577	if (!rec->counter)	577	if (!rec->counter)
578	goto again;	578	goto again;
579	}	579	}
580		580
581	return rec;	581	return rec;
582	}	582	}
583		583
584	static void function_stat_start(struct tracer_stat trace)	584	static void function_stat_start(struct tracer_stat trace)
585	{	585	{
586	struct ftrace_profile_stat *stat =	586	struct ftrace_profile_stat *stat =
587	container_of(trace, struct ftrace_profile_stat, stat);	587	container_of(trace, struct ftrace_profile_stat, stat);
588		588
589	if (!stat \|\| !stat->start)	589	if (!stat \|\| !stat->start)
590	return NULL;	590	return NULL;
591		591
592	return function_stat_next(&stat->start->records[0], 0);	592	return function_stat_next(&stat->start->records[0], 0);
593	}	593	}
594		594
595	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	595	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
596	/* function graph compares on total time */	596	/* function graph compares on total time */
597	static int function_stat_cmp(void p1, void p2)	597	static int function_stat_cmp(void p1, void p2)
598	{	598	{
599	struct ftrace_profile *a = p1;	599	struct ftrace_profile *a = p1;
600	struct ftrace_profile *b = p2;	600	struct ftrace_profile *b = p2;
601		601
602	if (a->time < b->time)	602	if (a->time < b->time)
603	return -1;	603	return -1;
604	if (a->time > b->time)	604	if (a->time > b->time)
605	return 1;	605	return 1;
606	else	606	else
607	return 0;	607	return 0;
608	}	608	}
609	#else	609	#else
610	/* not function graph compares against hits */	610	/* not function graph compares against hits */
611	static int function_stat_cmp(void p1, void p2)	611	static int function_stat_cmp(void p1, void p2)
612	{	612	{
613	struct ftrace_profile *a = p1;	613	struct ftrace_profile *a = p1;
614	struct ftrace_profile *b = p2;	614	struct ftrace_profile *b = p2;
615		615
616	if (a->counter < b->counter)	616	if (a->counter < b->counter)
617	return -1;	617	return -1;
618	if (a->counter > b->counter)	618	if (a->counter > b->counter)
619	return 1;	619	return 1;
620	else	620	else
621	return 0;	621	return 0;
622	}	622	}
623	#endif	623	#endif
624		624
625	static int function_stat_headers(struct seq_file *m)	625	static int function_stat_headers(struct seq_file *m)
626	{	626	{
627	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	627	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
628	seq_printf(m, " Function "	628	seq_printf(m, " Function "
629	"Hit Time Avg s^2\n"	629	"Hit Time Avg s^2\n"
630	" -------- "	630	" -------- "
631	"--- ---- --- ---\n");	631	"--- ---- --- ---\n");
632	#else	632	#else
633	seq_printf(m, " Function Hit\n"	633	seq_printf(m, " Function Hit\n"
634	" -------- ---\n");	634	" -------- ---\n");
635	#endif	635	#endif
636	return 0;	636	return 0;
637	}	637	}
638		638
639	static int function_stat_show(struct seq_file m, void v)	639	static int function_stat_show(struct seq_file m, void v)
640	{	640	{
641	struct ftrace_profile *rec = v;	641	struct ftrace_profile *rec = v;
642	char str[KSYM_SYMBOL_LEN];	642	char str[KSYM_SYMBOL_LEN];
643	int ret = 0;	643	int ret = 0;
644	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	644	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
645	static struct trace_seq s;	645	static struct trace_seq s;
646	unsigned long long avg;	646	unsigned long long avg;
647	unsigned long long stddev;	647	unsigned long long stddev;
648	#endif	648	#endif
649	mutex_lock(&ftrace_profile_lock);	649	mutex_lock(&ftrace_profile_lock);
650		650
651	/* we raced with function_profile_reset() */	651	/* we raced with function_profile_reset() */
652	if (unlikely(rec->counter == 0)) {	652	if (unlikely(rec->counter == 0)) {
653	ret = -EBUSY;	653	ret = -EBUSY;
654	goto out;	654	goto out;
655	}	655	}
656		656
657	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);	657	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
658	seq_printf(m, " %-30.30s %10lu", str, rec->counter);	658	seq_printf(m, " %-30.30s %10lu", str, rec->counter);
659		659
660	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	660	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
661	seq_printf(m, " ");	661	seq_printf(m, " ");
662	avg = rec->time;	662	avg = rec->time;
663	do_div(avg, rec->counter);	663	do_div(avg, rec->counter);
664		664
665	/* Sample standard deviation (s^2) */	665	/* Sample standard deviation (s^2) */
666	if (rec->counter <= 1)	666	if (rec->counter <= 1)
667	stddev = 0;	667	stddev = 0;
668	else {	668	else {
669	/*	669	/*
670	* Apply Welford's method:	670	* Apply Welford's method:
671	* s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)	671	* s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
672	*/	672	*/
673	stddev = rec->counter * rec->time_squared -	673	stddev = rec->counter * rec->time_squared -
674	rec->time * rec->time;	674	rec->time * rec->time;
675		675
676	/*	676	/*
677	* Divide only 1000 for ns^2 -> us^2 conversion.	677	* Divide only 1000 for ns^2 -> us^2 conversion.
678	* trace_print_graph_duration will divide 1000 again.	678	* trace_print_graph_duration will divide 1000 again.
679	*/	679	*/
680	do_div(stddev, rec->counter * (rec->counter - 1) * 1000);	680	do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
681	}	681	}
682		682
683	trace_seq_init(&s);	683	trace_seq_init(&s);
684	trace_print_graph_duration(rec->time, &s);	684	trace_print_graph_duration(rec->time, &s);
685	trace_seq_puts(&s, " ");	685	trace_seq_puts(&s, " ");
686	trace_print_graph_duration(avg, &s);	686	trace_print_graph_duration(avg, &s);
687	trace_seq_puts(&s, " ");	687	trace_seq_puts(&s, " ");
688	trace_print_graph_duration(stddev, &s);	688	trace_print_graph_duration(stddev, &s);
689	trace_print_seq(m, &s);	689	trace_print_seq(m, &s);
690	#endif	690	#endif
691	seq_putc(m, '\n');	691	seq_putc(m, '\n');
692	out:	692	out:
693	mutex_unlock(&ftrace_profile_lock);	693	mutex_unlock(&ftrace_profile_lock);
694		694
695	return ret;	695	return ret;
696	}	696	}
697		697
698	static void ftrace_profile_reset(struct ftrace_profile_stat *stat)	698	static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
699	{	699	{
700	struct ftrace_profile_page *pg;	700	struct ftrace_profile_page *pg;
701		701
702	pg = stat->pages = stat->start;	702	pg = stat->pages = stat->start;
703		703
704	while (pg) {	704	while (pg) {
705	memset(pg->records, 0, PROFILE_RECORDS_SIZE);	705	memset(pg->records, 0, PROFILE_RECORDS_SIZE);
706	pg->index = 0;	706	pg->index = 0;
707	pg = pg->next;	707	pg = pg->next;
708	}	708	}
709		709
710	memset(stat->hash, 0,	710	memset(stat->hash, 0,
711	FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));	711	FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
712	}	712	}
713		713
714	int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)	714	int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
715	{	715	{
716	struct ftrace_profile_page *pg;	716	struct ftrace_profile_page *pg;
717	int functions;	717	int functions;
718	int pages;	718	int pages;
719	int i;	719	int i;
720		720
721	/* If we already allocated, do nothing */	721	/* If we already allocated, do nothing */
722	if (stat->pages)	722	if (stat->pages)
723	return 0;	723	return 0;
724		724
725	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);	725	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
726	if (!stat->pages)	726	if (!stat->pages)
727	return -ENOMEM;	727	return -ENOMEM;
728		728
729	#ifdef CONFIG_DYNAMIC_FTRACE	729	#ifdef CONFIG_DYNAMIC_FTRACE
730	functions = ftrace_update_tot_cnt;	730	functions = ftrace_update_tot_cnt;
731	#else	731	#else
732	/*	732	/*
733	* We do not know the number of functions that exist because	733	* We do not know the number of functions that exist because
734	* dynamic tracing is what counts them. With past experience	734	* dynamic tracing is what counts them. With past experience
735	* we have around 20K functions. That should be more than enough.	735	* we have around 20K functions. That should be more than enough.
736	* It is highly unlikely we will execute every function in	736	* It is highly unlikely we will execute every function in
737	* the kernel.	737	* the kernel.
738	*/	738	*/
739	functions = 20000;	739	functions = 20000;
740	#endif	740	#endif
741		741
742	pg = stat->start = stat->pages;	742	pg = stat->start = stat->pages;
743		743
744	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);	744	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
745		745
746	for (i = 1; i < pages; i++) {	746	for (i = 1; i < pages; i++) {
747	pg->next = (void *)get_zeroed_page(GFP_KERNEL);	747	pg->next = (void *)get_zeroed_page(GFP_KERNEL);
748	if (!pg->next)	748	if (!pg->next)
749	goto out_free;	749	goto out_free;
750	pg = pg->next;	750	pg = pg->next;
751	}	751	}
752		752
753	return 0;	753	return 0;
754		754
755	out_free:	755	out_free:
756	pg = stat->start;	756	pg = stat->start;
757	while (pg) {	757	while (pg) {
758	unsigned long tmp = (unsigned long)pg;	758	unsigned long tmp = (unsigned long)pg;
759		759
760	pg = pg->next;	760	pg = pg->next;
761	free_page(tmp);	761	free_page(tmp);
762	}	762	}
763		763
764	stat->pages = NULL;	764	stat->pages = NULL;
765	stat->start = NULL;	765	stat->start = NULL;
766		766
767	return -ENOMEM;	767	return -ENOMEM;
768	}	768	}
769		769
770	static int ftrace_profile_init_cpu(int cpu)	770	static int ftrace_profile_init_cpu(int cpu)
771	{	771	{
772	struct ftrace_profile_stat *stat;	772	struct ftrace_profile_stat *stat;
773	int size;	773	int size;
774		774
775	stat = &per_cpu(ftrace_profile_stats, cpu);	775	stat = &per_cpu(ftrace_profile_stats, cpu);
776		776
777	if (stat->hash) {	777	if (stat->hash) {
778	/* If the profile is already created, simply reset it */	778	/* If the profile is already created, simply reset it */
779	ftrace_profile_reset(stat);	779	ftrace_profile_reset(stat);
780	return 0;	780	return 0;
781	}	781	}
782		782
783	/*	783	/*
784	* We are profiling all functions, but usually only a few thousand	784	* We are profiling all functions, but usually only a few thousand
785	* functions are hit. We'll make a hash of 1024 items.	785	* functions are hit. We'll make a hash of 1024 items.
786	*/	786	*/
787	size = FTRACE_PROFILE_HASH_SIZE;	787	size = FTRACE_PROFILE_HASH_SIZE;
788		788
789	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);	789	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
790		790
791	if (!stat->hash)	791	if (!stat->hash)
792	return -ENOMEM;	792	return -ENOMEM;
793		793
794	/* Preallocate the function profiling pages */	794	/* Preallocate the function profiling pages */
795	if (ftrace_profile_pages_init(stat) < 0) {	795	if (ftrace_profile_pages_init(stat) < 0) {
796	kfree(stat->hash);	796	kfree(stat->hash);
797	stat->hash = NULL;	797	stat->hash = NULL;
798	return -ENOMEM;	798	return -ENOMEM;
799	}	799	}
800		800
801	return 0;	801	return 0;
802	}	802	}
803		803
804	static int ftrace_profile_init(void)	804	static int ftrace_profile_init(void)
805	{	805	{
806	int cpu;	806	int cpu;
807	int ret = 0;	807	int ret = 0;
808		808
809	for_each_possible_cpu(cpu) {	809	for_each_possible_cpu(cpu) {
810	ret = ftrace_profile_init_cpu(cpu);	810	ret = ftrace_profile_init_cpu(cpu);
811	if (ret)	811	if (ret)
812	break;	812	break;
813	}	813	}
814		814
815	return ret;	815	return ret;
816	}	816	}
817		817
818	/* interrupts must be disabled */	818	/* interrupts must be disabled */
819	static struct ftrace_profile *	819	static struct ftrace_profile *
820	ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)	820	ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
821	{	821	{
822	struct ftrace_profile *rec;	822	struct ftrace_profile *rec;
823	struct hlist_head *hhd;	823	struct hlist_head *hhd;
824	unsigned long key;	824	unsigned long key;
825		825
826	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);	826	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
827	hhd = &stat->hash[key];	827	hhd = &stat->hash[key];
828		828
829	if (hlist_empty(hhd))	829	if (hlist_empty(hhd))
830	return NULL;	830	return NULL;
831		831
832	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {	832	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
833	if (rec->ip == ip)	833	if (rec->ip == ip)
834	return rec;	834	return rec;
835	}	835	}
836		836
837	return NULL;	837	return NULL;
838	}	838	}
839		839
840	static void ftrace_add_profile(struct ftrace_profile_stat *stat,	840	static void ftrace_add_profile(struct ftrace_profile_stat *stat,
841	struct ftrace_profile *rec)	841	struct ftrace_profile *rec)
842	{	842	{
843	unsigned long key;	843	unsigned long key;
844		844
845	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);	845	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
846	hlist_add_head_rcu(&rec->node, &stat->hash[key]);	846	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
847	}	847	}
848		848
849	/*	849	/*
850	* The memory is already allocated, this simply finds a new record to use.	850	* The memory is already allocated, this simply finds a new record to use.
851	*/	851	*/
852	static struct ftrace_profile *	852	static struct ftrace_profile *
853	ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)	853	ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
854	{	854	{
855	struct ftrace_profile *rec = NULL;	855	struct ftrace_profile *rec = NULL;
856		856
857	/* prevent recursion (from NMIs) */	857	/* prevent recursion (from NMIs) */
858	if (atomic_inc_return(&stat->disabled) != 1)	858	if (atomic_inc_return(&stat->disabled) != 1)
859	goto out;	859	goto out;
860		860
861	/*	861	/*
862	* Try to find the function again since an NMI	862	* Try to find the function again since an NMI
863	* could have added it	863	* could have added it
864	*/	864	*/
865	rec = ftrace_find_profiled_func(stat, ip);	865	rec = ftrace_find_profiled_func(stat, ip);
866	if (rec)	866	if (rec)
867	goto out;	867	goto out;
868		868
869	if (stat->pages->index == PROFILES_PER_PAGE) {	869	if (stat->pages->index == PROFILES_PER_PAGE) {
870	if (!stat->pages->next)	870	if (!stat->pages->next)
871	goto out;	871	goto out;
872	stat->pages = stat->pages->next;	872	stat->pages = stat->pages->next;
873	}	873	}
874		874
875	rec = &stat->pages->records[stat->pages->index++];	875	rec = &stat->pages->records[stat->pages->index++];
876	rec->ip = ip;	876	rec->ip = ip;
877	ftrace_add_profile(stat, rec);	877	ftrace_add_profile(stat, rec);
878		878
879	out:	879	out:
880	atomic_dec(&stat->disabled);	880	atomic_dec(&stat->disabled);
881		881
882	return rec;	882	return rec;
883	}	883	}
884		884
885	static void	885	static void
886	function_profile_call(unsigned long ip, unsigned long parent_ip,	886	function_profile_call(unsigned long ip, unsigned long parent_ip,
887	struct ftrace_ops ops, struct pt_regs regs)	887	struct ftrace_ops ops, struct pt_regs regs)
888	{	888	{
889	struct ftrace_profile_stat *stat;	889	struct ftrace_profile_stat *stat;
890	struct ftrace_profile *rec;	890	struct ftrace_profile *rec;
891	unsigned long flags;	891	unsigned long flags;
892		892
893	if (!ftrace_profile_enabled)	893	if (!ftrace_profile_enabled)
894	return;	894	return;
895		895
896	local_irq_save(flags);	896	local_irq_save(flags);
897		897
898	stat = &__get_cpu_var(ftrace_profile_stats);	898	stat = &__get_cpu_var(ftrace_profile_stats);
899	if (!stat->hash \|\| !ftrace_profile_enabled)	899	if (!stat->hash \|\| !ftrace_profile_enabled)
900	goto out;	900	goto out;
901		901
902	rec = ftrace_find_profiled_func(stat, ip);	902	rec = ftrace_find_profiled_func(stat, ip);
903	if (!rec) {	903	if (!rec) {
904	rec = ftrace_profile_alloc(stat, ip);	904	rec = ftrace_profile_alloc(stat, ip);
905	if (!rec)	905	if (!rec)
906	goto out;	906	goto out;
907	}	907	}
908		908
909	rec->counter++;	909	rec->counter++;
910	out:	910	out:
911	local_irq_restore(flags);	911	local_irq_restore(flags);
912	}	912	}
913		913
914	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	914	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
915	static int profile_graph_entry(struct ftrace_graph_ent *trace)	915	static int profile_graph_entry(struct ftrace_graph_ent *trace)
916	{	916	{
917	function_profile_call(trace->func, 0, NULL, NULL);	917	function_profile_call(trace->func, 0, NULL, NULL);
918	return 1;	918	return 1;
919	}	919	}
920		920
921	static void profile_graph_return(struct ftrace_graph_ret *trace)	921	static void profile_graph_return(struct ftrace_graph_ret *trace)
922	{	922	{
923	struct ftrace_profile_stat *stat;	923	struct ftrace_profile_stat *stat;
924	unsigned long long calltime;	924	unsigned long long calltime;
925	struct ftrace_profile *rec;	925	struct ftrace_profile *rec;
926	unsigned long flags;	926	unsigned long flags;
927		927
928	local_irq_save(flags);	928	local_irq_save(flags);
929	stat = &__get_cpu_var(ftrace_profile_stats);	929	stat = &__get_cpu_var(ftrace_profile_stats);
930	if (!stat->hash \|\| !ftrace_profile_enabled)	930	if (!stat->hash \|\| !ftrace_profile_enabled)
931	goto out;	931	goto out;
932		932
933	/* If the calltime was zero'd ignore it */	933	/* If the calltime was zero'd ignore it */
934	if (!trace->calltime)	934	if (!trace->calltime)
935	goto out;	935	goto out;
936		936
937	calltime = trace->rettime - trace->calltime;	937	calltime = trace->rettime - trace->calltime;
938		938
939	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {	939	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
940	int index;	940	int index;
941		941
942	index = trace->depth;	942	index = trace->depth;
943		943
944	/* Append this call time to the parent time to subtract */	944	/* Append this call time to the parent time to subtract */
945	if (index)	945	if (index)
946	current->ret_stack[index - 1].subtime += calltime;	946	current->ret_stack[index - 1].subtime += calltime;
947		947
948	if (current->ret_stack[index].subtime < calltime)	948	if (current->ret_stack[index].subtime < calltime)
949	calltime -= current->ret_stack[index].subtime;	949	calltime -= current->ret_stack[index].subtime;
950	else	950	else
951	calltime = 0;	951	calltime = 0;
952	}	952	}
953		953
954	rec = ftrace_find_profiled_func(stat, trace->func);	954	rec = ftrace_find_profiled_func(stat, trace->func);
955	if (rec) {	955	if (rec) {
956	rec->time += calltime;	956	rec->time += calltime;
957	rec->time_squared += calltime * calltime;	957	rec->time_squared += calltime * calltime;
958	}	958	}
959		959
960	out:	960	out:
961	local_irq_restore(flags);	961	local_irq_restore(flags);
962	}	962	}
963		963
964	static int register_ftrace_profiler(void)	964	static int register_ftrace_profiler(void)
965	{	965	{
966	return register_ftrace_graph(&profile_graph_return,	966	return register_ftrace_graph(&profile_graph_return,
967	&profile_graph_entry);	967	&profile_graph_entry);
968	}	968	}
969		969
970	static void unregister_ftrace_profiler(void)	970	static void unregister_ftrace_profiler(void)
971	{	971	{
972	unregister_ftrace_graph();	972	unregister_ftrace_graph();
973	}	973	}
974	#else	974	#else
975	static struct ftrace_ops ftrace_profile_ops __read_mostly = {	975	static struct ftrace_ops ftrace_profile_ops __read_mostly = {
976	.func = function_profile_call,	976	.func = function_profile_call,
977	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,	977	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,
978	INIT_REGEX_LOCK(ftrace_profile_ops)	978	INIT_REGEX_LOCK(ftrace_profile_ops)
979	};	979	};
980		980
981	static int register_ftrace_profiler(void)	981	static int register_ftrace_profiler(void)
982	{	982	{
983	return register_ftrace_function(&ftrace_profile_ops);	983	return register_ftrace_function(&ftrace_profile_ops);
984	}	984	}
985		985
986	static void unregister_ftrace_profiler(void)	986	static void unregister_ftrace_profiler(void)
987	{	987	{
988	unregister_ftrace_function(&ftrace_profile_ops);	988	unregister_ftrace_function(&ftrace_profile_ops);
989	}	989	}
990	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */	990	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
991		991
992	static ssize_t	992	static ssize_t
993	ftrace_profile_write(struct file filp, const char __user ubuf,	993	ftrace_profile_write(struct file filp, const char __user ubuf,
994	size_t cnt, loff_t *ppos)	994	size_t cnt, loff_t *ppos)
995	{	995	{
996	unsigned long val;	996	unsigned long val;
997	int ret;	997	int ret;
998		998
999	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);	999	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1000	if (ret)	1000	if (ret)
1001	return ret;	1001	return ret;
1002		1002
1003	val = !!val;	1003	val = !!val;
1004		1004
1005	mutex_lock(&ftrace_profile_lock);	1005	mutex_lock(&ftrace_profile_lock);
1006	if (ftrace_profile_enabled ^ val) {	1006	if (ftrace_profile_enabled ^ val) {
1007	if (val) {	1007	if (val) {
1008	ret = ftrace_profile_init();	1008	ret = ftrace_profile_init();
1009	if (ret < 0) {	1009	if (ret < 0) {
1010	cnt = ret;	1010	cnt = ret;
1011	goto out;	1011	goto out;
1012	}	1012	}
1013		1013
1014	ret = register_ftrace_profiler();	1014	ret = register_ftrace_profiler();
1015	if (ret < 0) {	1015	if (ret < 0) {
1016	cnt = ret;	1016	cnt = ret;
1017	goto out;	1017	goto out;
1018	}	1018	}
1019	ftrace_profile_enabled = 1;	1019	ftrace_profile_enabled = 1;
1020	} else {	1020	} else {
1021	ftrace_profile_enabled = 0;	1021	ftrace_profile_enabled = 0;
1022	/*	1022	/*
1023	* unregister_ftrace_profiler calls stop_machine	1023	* unregister_ftrace_profiler calls stop_machine
1024	* so this acts like an synchronize_sched.	1024	* so this acts like an synchronize_sched.
1025	*/	1025	*/
1026	unregister_ftrace_profiler();	1026	unregister_ftrace_profiler();
1027	}	1027	}
1028	}	1028	}
1029	out:	1029	out:
1030	mutex_unlock(&ftrace_profile_lock);	1030	mutex_unlock(&ftrace_profile_lock);
1031		1031
1032	*ppos += cnt;	1032	*ppos += cnt;
1033		1033
1034	return cnt;	1034	return cnt;
1035	}	1035	}
1036		1036
1037	static ssize_t	1037	static ssize_t
1038	ftrace_profile_read(struct file filp, char __user ubuf,	1038	ftrace_profile_read(struct file filp, char __user ubuf,
1039	size_t cnt, loff_t *ppos)	1039	size_t cnt, loff_t *ppos)
1040	{	1040	{
1041	char buf[64]; /* big enough to hold a number */	1041	char buf[64]; /* big enough to hold a number */
1042	int r;	1042	int r;
1043		1043
1044	r = sprintf(buf, "%u\n", ftrace_profile_enabled);	1044	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
1045	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);	1045	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1046	}	1046	}
1047		1047
1048	static const struct file_operations ftrace_profile_fops = {	1048	static const struct file_operations ftrace_profile_fops = {
1049	.open = tracing_open_generic,	1049	.open = tracing_open_generic,
1050	.read = ftrace_profile_read,	1050	.read = ftrace_profile_read,
1051	.write = ftrace_profile_write,	1051	.write = ftrace_profile_write,
1052	.llseek = default_llseek,	1052	.llseek = default_llseek,
1053	};	1053	};
1054		1054
1055	/* used to initialize the real stat files */	1055	/* used to initialize the real stat files */
1056	static struct tracer_stat function_stats __initdata = {	1056	static struct tracer_stat function_stats __initdata = {
1057	.name = "functions",	1057	.name = "functions",
1058	.stat_start = function_stat_start,	1058	.stat_start = function_stat_start,
1059	.stat_next = function_stat_next,	1059	.stat_next = function_stat_next,
1060	.stat_cmp = function_stat_cmp,	1060	.stat_cmp = function_stat_cmp,
1061	.stat_headers = function_stat_headers,	1061	.stat_headers = function_stat_headers,
1062	.stat_show = function_stat_show	1062	.stat_show = function_stat_show
1063	};	1063	};
1064		1064
1065	static __init void ftrace_profile_debugfs(struct dentry *d_tracer)	1065	static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
1066	{	1066	{
1067	struct ftrace_profile_stat *stat;	1067	struct ftrace_profile_stat *stat;
1068	struct dentry *entry;	1068	struct dentry *entry;
1069	char *name;	1069	char *name;
1070	int ret;	1070	int ret;
1071	int cpu;	1071	int cpu;
1072		1072
1073	for_each_possible_cpu(cpu) {	1073	for_each_possible_cpu(cpu) {
1074	stat = &per_cpu(ftrace_profile_stats, cpu);	1074	stat = &per_cpu(ftrace_profile_stats, cpu);
1075		1075
1076	/* allocate enough for function name + cpu number */	1076	/* allocate enough for function name + cpu number */
1077	name = kmalloc(32, GFP_KERNEL);	1077	name = kmalloc(32, GFP_KERNEL);
1078	if (!name) {	1078	if (!name) {
1079	/*	1079	/*
1080	* The files created are permanent, if something happens	1080	* The files created are permanent, if something happens
1081	* we still do not free memory.	1081	* we still do not free memory.
1082	*/	1082	*/
1083	WARN(1,	1083	WARN(1,
1084	"Could not allocate stat file for cpu %d\n",	1084	"Could not allocate stat file for cpu %d\n",
1085	cpu);	1085	cpu);
1086	return;	1086	return;
1087	}	1087	}
1088	stat->stat = function_stats;	1088	stat->stat = function_stats;
1089	snprintf(name, 32, "function%d", cpu);	1089	snprintf(name, 32, "function%d", cpu);
1090	stat->stat.name = name;	1090	stat->stat.name = name;
1091	ret = register_stat_tracer(&stat->stat);	1091	ret = register_stat_tracer(&stat->stat);
1092	if (ret) {	1092	if (ret) {
1093	WARN(1,	1093	WARN(1,
1094	"Could not register function stat for cpu %d\n",	1094	"Could not register function stat for cpu %d\n",
1095	cpu);	1095	cpu);
1096	kfree(name);	1096	kfree(name);
1097	return;	1097	return;
1098	}	1098	}
1099	}	1099	}
1100		1100
1101	entry = debugfs_create_file("function_profile_enabled", 0644,	1101	entry = debugfs_create_file("function_profile_enabled", 0644,
1102	d_tracer, NULL, &ftrace_profile_fops);	1102	d_tracer, NULL, &ftrace_profile_fops);
1103	if (!entry)	1103	if (!entry)
1104	pr_warning("Could not create debugfs "	1104	pr_warning("Could not create debugfs "
1105	"'function_profile_enabled' entry\n");	1105	"'function_profile_enabled' entry\n");
1106	}	1106	}
1107		1107
1108	#else /* CONFIG_FUNCTION_PROFILER */	1108	#else /* CONFIG_FUNCTION_PROFILER */
1109	static __init void ftrace_profile_debugfs(struct dentry *d_tracer)	1109	static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
1110	{	1110	{
1111	}	1111	}
1112	#endif /* CONFIG_FUNCTION_PROFILER */	1112	#endif /* CONFIG_FUNCTION_PROFILER */
1113		1113
1114	static struct pid * const ftrace_swapper_pid = &init_struct_pid;	1114	static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1115		1115
1116	#ifdef CONFIG_DYNAMIC_FTRACE	1116	#ifdef CONFIG_DYNAMIC_FTRACE
1117		1117
1118	#ifndef CONFIG_FTRACE_MCOUNT_RECORD	1118	#ifndef CONFIG_FTRACE_MCOUNT_RECORD
1119	# error Dynamic ftrace depends on MCOUNT_RECORD	1119	# error Dynamic ftrace depends on MCOUNT_RECORD
1120	#endif	1120	#endif
1121		1121
1122	static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;	1122	static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
1123		1123
1124	struct ftrace_func_probe {	1124	struct ftrace_func_probe {
1125	struct hlist_node node;	1125	struct hlist_node node;
1126	struct ftrace_probe_ops *ops;	1126	struct ftrace_probe_ops *ops;
1127	unsigned long flags;	1127	unsigned long flags;
1128	unsigned long ip;	1128	unsigned long ip;
1129	void *data;	1129	void *data;
1130	struct list_head free_list;	1130	struct list_head free_list;
1131	};	1131	};
1132		1132
1133	struct ftrace_func_entry {	1133	struct ftrace_func_entry {
1134	struct hlist_node hlist;	1134	struct hlist_node hlist;
1135	unsigned long ip;	1135	unsigned long ip;
1136	};	1136	};
1137		1137
1138	struct ftrace_hash {	1138	struct ftrace_hash {
1139	unsigned long size_bits;	1139	unsigned long size_bits;
1140	struct hlist_head *buckets;	1140	struct hlist_head *buckets;
1141	unsigned long count;	1141	unsigned long count;
1142	struct rcu_head rcu;	1142	struct rcu_head rcu;
1143	};	1143	};
1144		1144
1145	/*	1145	/*
1146	* We make these constant because no one should touch them,	1146	* We make these constant because no one should touch them,
1147	* but they are used as the default "empty hash", to avoid allocating	1147	* but they are used as the default "empty hash", to avoid allocating
1148	* it all the time. These are in a read only section such that if	1148	* it all the time. These are in a read only section such that if
1149	* anyone does try to modify it, it will cause an exception.	1149	* anyone does try to modify it, it will cause an exception.
1150	*/	1150	*/
1151	static const struct hlist_head empty_buckets[1];	1151	static const struct hlist_head empty_buckets[1];
1152	static const struct ftrace_hash empty_hash = {	1152	static const struct ftrace_hash empty_hash = {
1153	.buckets = (struct hlist_head *)empty_buckets,	1153	.buckets = (struct hlist_head *)empty_buckets,
1154	};	1154	};
1155	#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)	1155	#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
1156		1156
1157	static struct ftrace_ops global_ops = {	1157	static struct ftrace_ops global_ops = {
1158	.func = ftrace_stub,	1158	.func = ftrace_stub,
1159	.notrace_hash = EMPTY_HASH,	1159	.notrace_hash = EMPTY_HASH,
1160	.filter_hash = EMPTY_HASH,	1160	.filter_hash = EMPTY_HASH,
1161	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,	1161	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,
1162	INIT_REGEX_LOCK(global_ops)	1162	INIT_REGEX_LOCK(global_ops)
1163	};	1163	};
1164		1164
1165	struct ftrace_page {	1165	struct ftrace_page {
1166	struct ftrace_page *next;	1166	struct ftrace_page *next;
1167	struct dyn_ftrace *records;	1167	struct dyn_ftrace *records;
1168	int index;	1168	int index;
1169	int size;	1169	int size;
1170	};	1170	};
1171		1171
1172	#define ENTRY_SIZE sizeof(struct dyn_ftrace)	1172	#define ENTRY_SIZE sizeof(struct dyn_ftrace)
1173	#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)	1173	#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1174		1174
1175	/* estimate from running different kernels */	1175	/* estimate from running different kernels */
1176	#define NR_TO_INIT 10000	1176	#define NR_TO_INIT 10000
1177		1177
1178	static struct ftrace_page *ftrace_pages_start;	1178	static struct ftrace_page *ftrace_pages_start;
1179	static struct ftrace_page *ftrace_pages;	1179	static struct ftrace_page *ftrace_pages;
1180		1180
1181	static bool ftrace_hash_empty(struct ftrace_hash *hash)	1181	static bool ftrace_hash_empty(struct ftrace_hash *hash)
1182	{	1182	{
1183	return !hash \|\| !hash->count;	1183	return !hash \|\| !hash->count;
1184	}	1184	}
1185		1185
1186	static struct ftrace_func_entry *	1186	static struct ftrace_func_entry *
1187	ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)	1187	ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1188	{	1188	{
1189	unsigned long key;	1189	unsigned long key;
1190	struct ftrace_func_entry *entry;	1190	struct ftrace_func_entry *entry;
1191	struct hlist_head *hhd;	1191	struct hlist_head *hhd;
1192		1192
1193	if (ftrace_hash_empty(hash))	1193	if (ftrace_hash_empty(hash))
1194	return NULL;	1194	return NULL;
1195		1195
1196	if (hash->size_bits > 0)	1196	if (hash->size_bits > 0)
1197	key = hash_long(ip, hash->size_bits);	1197	key = hash_long(ip, hash->size_bits);
1198	else	1198	else
1199	key = 0;	1199	key = 0;
1200		1200
1201	hhd = &hash->buckets[key];	1201	hhd = &hash->buckets[key];
1202		1202
1203	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {	1203	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1204	if (entry->ip == ip)	1204	if (entry->ip == ip)
1205	return entry;	1205	return entry;
1206	}	1206	}
1207	return NULL;	1207	return NULL;
1208	}	1208	}
1209		1209
1210	static void __add_hash_entry(struct ftrace_hash *hash,	1210	static void __add_hash_entry(struct ftrace_hash *hash,
1211	struct ftrace_func_entry *entry)	1211	struct ftrace_func_entry *entry)
1212	{	1212	{
1213	struct hlist_head *hhd;	1213	struct hlist_head *hhd;
1214	unsigned long key;	1214	unsigned long key;
1215		1215
1216	if (hash->size_bits)	1216	if (hash->size_bits)
1217	key = hash_long(entry->ip, hash->size_bits);	1217	key = hash_long(entry->ip, hash->size_bits);
1218	else	1218	else
1219	key = 0;	1219	key = 0;
1220		1220
1221	hhd = &hash->buckets[key];	1221	hhd = &hash->buckets[key];
1222	hlist_add_head(&entry->hlist, hhd);	1222	hlist_add_head(&entry->hlist, hhd);
1223	hash->count++;	1223	hash->count++;
1224	}	1224	}
1225		1225
1226	static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)	1226	static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1227	{	1227	{
1228	struct ftrace_func_entry *entry;	1228	struct ftrace_func_entry *entry;
1229		1229
1230	entry = kmalloc(sizeof(*entry), GFP_KERNEL);	1230	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1231	if (!entry)	1231	if (!entry)
1232	return -ENOMEM;	1232	return -ENOMEM;
1233		1233
1234	entry->ip = ip;	1234	entry->ip = ip;
1235	__add_hash_entry(hash, entry);	1235	__add_hash_entry(hash, entry);
1236		1236
1237	return 0;	1237	return 0;
1238	}	1238	}
1239		1239
1240	static void	1240	static void
1241	free_hash_entry(struct ftrace_hash *hash,	1241	free_hash_entry(struct ftrace_hash *hash,
1242	struct ftrace_func_entry *entry)	1242	struct ftrace_func_entry *entry)
1243	{	1243	{
1244	hlist_del(&entry->hlist);	1244	hlist_del(&entry->hlist);
1245	kfree(entry);	1245	kfree(entry);
1246	hash->count--;	1246	hash->count--;
1247	}	1247	}
1248		1248
1249	static void	1249	static void
1250	remove_hash_entry(struct ftrace_hash *hash,	1250	remove_hash_entry(struct ftrace_hash *hash,
1251	struct ftrace_func_entry *entry)	1251	struct ftrace_func_entry *entry)
1252	{	1252	{
1253	hlist_del(&entry->hlist);	1253	hlist_del(&entry->hlist);
1254	hash->count--;	1254	hash->count--;
1255	}	1255	}
1256		1256
1257	static void ftrace_hash_clear(struct ftrace_hash *hash)	1257	static void ftrace_hash_clear(struct ftrace_hash *hash)
1258	{	1258	{
1259	struct hlist_head *hhd;	1259	struct hlist_head *hhd;
1260	struct hlist_node *tn;	1260	struct hlist_node *tn;
1261	struct ftrace_func_entry *entry;	1261	struct ftrace_func_entry *entry;
1262	int size = 1 << hash->size_bits;	1262	int size = 1 << hash->size_bits;
1263	int i;	1263	int i;
1264		1264
1265	if (!hash->count)	1265	if (!hash->count)
1266	return;	1266	return;
1267		1267
1268	for (i = 0; i < size; i++) {	1268	for (i = 0; i < size; i++) {
1269	hhd = &hash->buckets[i];	1269	hhd = &hash->buckets[i];
1270	hlist_for_each_entry_safe(entry, tn, hhd, hlist)	1270	hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1271	free_hash_entry(hash, entry);	1271	free_hash_entry(hash, entry);
1272	}	1272	}
1273	FTRACE_WARN_ON(hash->count);	1273	FTRACE_WARN_ON(hash->count);
1274	}	1274	}
1275		1275
1276	static void free_ftrace_hash(struct ftrace_hash *hash)	1276	static void free_ftrace_hash(struct ftrace_hash *hash)
1277	{	1277	{
1278	if (!hash \|\| hash == EMPTY_HASH)	1278	if (!hash \|\| hash == EMPTY_HASH)
1279	return;	1279	return;
1280	ftrace_hash_clear(hash);	1280	ftrace_hash_clear(hash);
1281	kfree(hash->buckets);	1281	kfree(hash->buckets);
1282	kfree(hash);	1282	kfree(hash);
1283	}	1283	}
1284		1284
1285	static void __free_ftrace_hash_rcu(struct rcu_head *rcu)	1285	static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1286	{	1286	{
1287	struct ftrace_hash *hash;	1287	struct ftrace_hash *hash;
1288		1288
1289	hash = container_of(rcu, struct ftrace_hash, rcu);	1289	hash = container_of(rcu, struct ftrace_hash, rcu);
1290	free_ftrace_hash(hash);	1290	free_ftrace_hash(hash);
1291	}	1291	}
1292		1292
1293	static void free_ftrace_hash_rcu(struct ftrace_hash *hash)	1293	static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1294	{	1294	{
1295	if (!hash \|\| hash == EMPTY_HASH)	1295	if (!hash \|\| hash == EMPTY_HASH)
1296	return;	1296	return;
1297	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);	1297	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1298	}	1298	}
1299		1299
1300	void ftrace_free_filter(struct ftrace_ops *ops)	1300	void ftrace_free_filter(struct ftrace_ops *ops)
1301	{	1301	{
1302	ftrace_ops_init(ops);	1302	ftrace_ops_init(ops);
1303	free_ftrace_hash(ops->filter_hash);	1303	free_ftrace_hash(ops->filter_hash);
1304	free_ftrace_hash(ops->notrace_hash);	1304	free_ftrace_hash(ops->notrace_hash);
1305	}	1305	}
1306		1306
1307	static struct ftrace_hash *alloc_ftrace_hash(int size_bits)	1307	static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1308	{	1308	{
1309	struct ftrace_hash *hash;	1309	struct ftrace_hash *hash;
1310	int size;	1310	int size;
1311		1311
1312	hash = kzalloc(sizeof(*hash), GFP_KERNEL);	1312	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1313	if (!hash)	1313	if (!hash)
1314	return NULL;	1314	return NULL;
1315		1315
1316	size = 1 << size_bits;	1316	size = 1 << size_bits;
1317	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);	1317	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1318		1318
1319	if (!hash->buckets) {	1319	if (!hash->buckets) {
1320	kfree(hash);	1320	kfree(hash);
1321	return NULL;	1321	return NULL;
1322	}	1322	}
1323		1323
1324	hash->size_bits = size_bits;	1324	hash->size_bits = size_bits;
1325		1325
1326	return hash;	1326	return hash;
1327	}	1327	}
1328		1328
1329	static struct ftrace_hash *	1329	static struct ftrace_hash *
1330	alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)	1330	alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1331	{	1331	{
1332	struct ftrace_func_entry *entry;	1332	struct ftrace_func_entry *entry;
1333	struct ftrace_hash *new_hash;	1333	struct ftrace_hash *new_hash;
1334	int size;	1334	int size;
1335	int ret;	1335	int ret;
1336	int i;	1336	int i;
1337		1337
1338	new_hash = alloc_ftrace_hash(size_bits);	1338	new_hash = alloc_ftrace_hash(size_bits);
1339	if (!new_hash)	1339	if (!new_hash)
1340	return NULL;	1340	return NULL;
1341		1341
1342	/* Empty hash? */	1342	/* Empty hash? */
1343	if (ftrace_hash_empty(hash))	1343	if (ftrace_hash_empty(hash))
1344	return new_hash;	1344	return new_hash;
1345		1345
1346	size = 1 << hash->size_bits;	1346	size = 1 << hash->size_bits;
1347	for (i = 0; i < size; i++) {	1347	for (i = 0; i < size; i++) {
1348	hlist_for_each_entry(entry, &hash->buckets[i], hlist) {	1348	hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1349	ret = add_hash_entry(new_hash, entry->ip);	1349	ret = add_hash_entry(new_hash, entry->ip);
1350	if (ret < 0)	1350	if (ret < 0)
1351	goto free_hash;	1351	goto free_hash;
1352	}	1352	}
1353	}	1353	}
1354		1354
1355	FTRACE_WARN_ON(new_hash->count != hash->count);	1355	FTRACE_WARN_ON(new_hash->count != hash->count);
1356		1356
1357	return new_hash;	1357	return new_hash;
1358		1358
1359	free_hash:	1359	free_hash:
1360	free_ftrace_hash(new_hash);	1360	free_ftrace_hash(new_hash);
1361	return NULL;	1361	return NULL;
1362	}	1362	}
1363		1363
1364	static void	1364	static void
1365	ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);	1365	ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1366	static void	1366	static void
1367	ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);	1367	ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1368		1368
1369	static int	1369	static int
1370	ftrace_hash_move(struct ftrace_ops *ops, int enable,	1370	ftrace_hash_move(struct ftrace_ops *ops, int enable,
1371	struct ftrace_hash *dst, struct ftrace_hash src)	1371	struct ftrace_hash *dst, struct ftrace_hash src)
1372	{	1372	{
1373	struct ftrace_func_entry *entry;	1373	struct ftrace_func_entry *entry;
1374	struct hlist_node *tn;	1374	struct hlist_node *tn;
1375	struct hlist_head *hhd;	1375	struct hlist_head *hhd;
1376	struct ftrace_hash *old_hash;	1376	struct ftrace_hash *old_hash;
1377	struct ftrace_hash *new_hash;	1377	struct ftrace_hash *new_hash;
1378	int size = src->count;	1378	int size = src->count;
1379	int bits = 0;	1379	int bits = 0;
1380	int ret;	1380	int ret;
1381	int i;	1381	int i;
1382		1382
1383	/*	1383	/*
1384	* Remove the current set, update the hash and add	1384	* Remove the current set, update the hash and add
1385	* them back.	1385	* them back.
1386	*/	1386	*/
1387	ftrace_hash_rec_disable(ops, enable);	1387	ftrace_hash_rec_disable(ops, enable);
1388		1388
1389	/*	1389	/*
1390	* If the new source is empty, just free dst and assign it	1390	* If the new source is empty, just free dst and assign it
1391	* the empty_hash.	1391	* the empty_hash.
1392	*/	1392	*/
1393	if (!src->count) {	1393	if (!src->count) {
1394	free_ftrace_hash_rcu(*dst);	1394	free_ftrace_hash_rcu(*dst);
1395	rcu_assign_pointer(*dst, EMPTY_HASH);	1395	rcu_assign_pointer(*dst, EMPTY_HASH);
1396	/* still need to update the function records */	1396	/* still need to update the function records */
1397	ret = 0;	1397	ret = 0;
1398	goto out;	1398	goto out;
1399	}	1399	}
1400		1400
1401	/*	1401	/*
1402	* Make the hash size about 1/2 the # found	1402	* Make the hash size about 1/2 the # found
1403	*/	1403	*/
1404	for (size /= 2; size; size >>= 1)	1404	for (size /= 2; size; size >>= 1)
1405	bits++;	1405	bits++;
1406		1406
1407	/* Don't allocate too much */	1407	/* Don't allocate too much */
1408	if (bits > FTRACE_HASH_MAX_BITS)	1408	if (bits > FTRACE_HASH_MAX_BITS)
1409	bits = FTRACE_HASH_MAX_BITS;	1409	bits = FTRACE_HASH_MAX_BITS;
1410		1410
1411	ret = -ENOMEM;	1411	ret = -ENOMEM;
1412	new_hash = alloc_ftrace_hash(bits);	1412	new_hash = alloc_ftrace_hash(bits);
1413	if (!new_hash)	1413	if (!new_hash)
1414	goto out;	1414	goto out;
1415		1415
1416	size = 1 << src->size_bits;	1416	size = 1 << src->size_bits;
1417	for (i = 0; i < size; i++) {	1417	for (i = 0; i < size; i++) {
1418	hhd = &src->buckets[i];	1418	hhd = &src->buckets[i];
1419	hlist_for_each_entry_safe(entry, tn, hhd, hlist) {	1419	hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1420	remove_hash_entry(src, entry);	1420	remove_hash_entry(src, entry);
1421	__add_hash_entry(new_hash, entry);	1421	__add_hash_entry(new_hash, entry);
1422	}	1422	}
1423	}	1423	}
1424		1424
1425	old_hash = *dst;	1425	old_hash = *dst;
1426	rcu_assign_pointer(*dst, new_hash);	1426	rcu_assign_pointer(*dst, new_hash);
1427	free_ftrace_hash_rcu(old_hash);	1427	free_ftrace_hash_rcu(old_hash);
1428		1428
1429	ret = 0;	1429	ret = 0;
1430	out:	1430	out:
1431	/*	1431	/*
1432	* Enable regardless of ret:	1432	* Enable regardless of ret:
1433	* On success, we enable the new hash.	1433	* On success, we enable the new hash.
1434	* On failure, we re-enable the original hash.	1434	* On failure, we re-enable the original hash.
1435	*/	1435	*/
1436	ftrace_hash_rec_enable(ops, enable);	1436	ftrace_hash_rec_enable(ops, enable);
1437		1437
1438	return ret;	1438	return ret;
1439	}	1439	}
1440		1440
1441	/*	1441	/*
1442	* Test the hashes for this ops to see if we want to call	1442	* Test the hashes for this ops to see if we want to call
1443	* the ops->func or not.	1443	* the ops->func or not.
1444	*	1444	*
1445	* It's a match if the ip is in the ops->filter_hash or	1445	* It's a match if the ip is in the ops->filter_hash or
1446	* the filter_hash does not exist or is empty,	1446	* the filter_hash does not exist or is empty,
1447	* AND	1447	* AND
1448	* the ip is not in the ops->notrace_hash.	1448	* the ip is not in the ops->notrace_hash.
1449	*	1449	*
1450	* This needs to be called with preemption disabled as	1450	* This needs to be called with preemption disabled as
1451	* the hashes are freed with call_rcu_sched().	1451	* the hashes are freed with call_rcu_sched().
1452	*/	1452	*/
1453	static int	1453	static int
1454	ftrace_ops_test(struct ftrace_ops ops, unsigned long ip, void regs)	1454	ftrace_ops_test(struct ftrace_ops ops, unsigned long ip, void regs)
1455	{	1455	{
1456	struct ftrace_hash *filter_hash;	1456	struct ftrace_hash *filter_hash;
1457	struct ftrace_hash *notrace_hash;	1457	struct ftrace_hash *notrace_hash;
1458	int ret;	1458	int ret;
1459		1459
1460	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS	1460	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1461	/*	1461	/*
1462	* There's a small race when adding ops that the ftrace handler	1462	* There's a small race when adding ops that the ftrace handler
1463	* that wants regs, may be called without them. We can not	1463	* that wants regs, may be called without them. We can not
1464	* allow that handler to be called if regs is NULL.	1464	* allow that handler to be called if regs is NULL.
1465	*/	1465	*/
1466	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))	1466	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1467	return 0;	1467	return 0;
1468	#endif	1468	#endif
1469		1469
1470	filter_hash = rcu_dereference_raw_notrace(ops->filter_hash);	1470	filter_hash = rcu_dereference_raw_notrace(ops->filter_hash);
1471	notrace_hash = rcu_dereference_raw_notrace(ops->notrace_hash);	1471	notrace_hash = rcu_dereference_raw_notrace(ops->notrace_hash);
1472		1472
1473	if ((ftrace_hash_empty(filter_hash) \|\|	1473	if ((ftrace_hash_empty(filter_hash) \|\|
1474	ftrace_lookup_ip(filter_hash, ip)) &&	1474	ftrace_lookup_ip(filter_hash, ip)) &&
1475	(ftrace_hash_empty(notrace_hash) \|\|	1475	(ftrace_hash_empty(notrace_hash) \|\|
1476	!ftrace_lookup_ip(notrace_hash, ip)))	1476	!ftrace_lookup_ip(notrace_hash, ip)))
1477	ret = 1;	1477	ret = 1;
1478	else	1478	else
1479	ret = 0;	1479	ret = 0;
1480		1480
1481	return ret;	1481	return ret;
1482	}	1482	}
1483		1483
1484	/*	1484	/*
1485	* This is a double for. Do not use 'break' to break out of the loop,	1485	* This is a double for. Do not use 'break' to break out of the loop,
1486	* you must use a goto.	1486	* you must use a goto.
1487	*/	1487	*/
1488	#define do_for_each_ftrace_rec(pg, rec) \	1488	#define do_for_each_ftrace_rec(pg, rec) \
1489	for (pg = ftrace_pages_start; pg; pg = pg->next) { \	1489	for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1490	int _____i; \	1490	int _____i; \
1491	for (_____i = 0; _____i < pg->index; _____i++) { \	1491	for (_____i = 0; _____i < pg->index; _____i++) { \
1492	rec = &pg->records[_____i];	1492	rec = &pg->records[_____i];
1493		1493
1494	#define while_for_each_ftrace_rec() \	1494	#define while_for_each_ftrace_rec() \
1495	} \	1495	} \
1496	}	1496	}
1497		1497
1498		1498
1499	static int ftrace_cmp_recs(const void a, const void b)	1499	static int ftrace_cmp_recs(const void a, const void b)
1500	{	1500	{
1501	const struct dyn_ftrace *key = a;	1501	const struct dyn_ftrace *key = a;
1502	const struct dyn_ftrace *rec = b;	1502	const struct dyn_ftrace *rec = b;
1503		1503
1504	if (key->flags < rec->ip)	1504	if (key->flags < rec->ip)
1505	return -1;	1505	return -1;
1506	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)	1506	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1507	return 1;	1507	return 1;
1508	return 0;	1508	return 0;
1509	}	1509	}
1510		1510
1511	static unsigned long ftrace_location_range(unsigned long start, unsigned long end)	1511	static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1512	{	1512	{
1513	struct ftrace_page *pg;	1513	struct ftrace_page *pg;
1514	struct dyn_ftrace *rec;	1514	struct dyn_ftrace *rec;
1515	struct dyn_ftrace key;	1515	struct dyn_ftrace key;
1516		1516
1517	key.ip = start;	1517	key.ip = start;
1518	key.flags = end; /* overload flags, as it is unsigned long */	1518	key.flags = end; /* overload flags, as it is unsigned long */
1519		1519
1520	for (pg = ftrace_pages_start; pg; pg = pg->next) {	1520	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1521	if (end < pg->records[0].ip \|\|	1521	if (end < pg->records[0].ip \|\|
1522	start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))	1522	start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1523	continue;	1523	continue;
1524	rec = bsearch(&key, pg->records, pg->index,	1524	rec = bsearch(&key, pg->records, pg->index,
1525	sizeof(struct dyn_ftrace),	1525	sizeof(struct dyn_ftrace),
1526	ftrace_cmp_recs);	1526	ftrace_cmp_recs);
1527	if (rec)	1527	if (rec)
1528	return rec->ip;	1528	return rec->ip;
1529	}	1529	}
1530		1530
1531	return 0;	1531	return 0;
1532	}	1532	}
1533		1533
1534	/**	1534	/**
1535	* ftrace_location - return true if the ip giving is a traced location	1535	* ftrace_location - return true if the ip giving is a traced location
1536	* @ip: the instruction pointer to check	1536	* @ip: the instruction pointer to check
1537	*	1537	*
1538	* Returns rec->ip if @ip given is a pointer to a ftrace location.	1538	* Returns rec->ip if @ip given is a pointer to a ftrace location.
1539	* That is, the instruction that is either a NOP or call to	1539	* That is, the instruction that is either a NOP or call to
1540	* the function tracer. It checks the ftrace internal tables to	1540	* the function tracer. It checks the ftrace internal tables to
1541	* determine if the address belongs or not.	1541	* determine if the address belongs or not.
1542	*/	1542	*/
1543	unsigned long ftrace_location(unsigned long ip)	1543	unsigned long ftrace_location(unsigned long ip)
1544	{	1544	{
1545	return ftrace_location_range(ip, ip);	1545	return ftrace_location_range(ip, ip);
1546	}	1546	}
1547		1547
1548	/**	1548	/**
1549	* ftrace_text_reserved - return true if range contains an ftrace location	1549	* ftrace_text_reserved - return true if range contains an ftrace location
1550	* @start: start of range to search	1550	* @start: start of range to search
1551	* @end: end of range to search (inclusive). @end points to the last byte to check.	1551	* @end: end of range to search (inclusive). @end points to the last byte to check.
1552	*	1552	*
1553	* Returns 1 if @start and @end contains a ftrace location.	1553	* Returns 1 if @start and @end contains a ftrace location.
1554	* That is, the instruction that is either a NOP or call to	1554	* That is, the instruction that is either a NOP or call to
1555	* the function tracer. It checks the ftrace internal tables to	1555	* the function tracer. It checks the ftrace internal tables to
1556	* determine if the address belongs or not.	1556	* determine if the address belongs or not.
1557	*/	1557	*/
1558	int ftrace_text_reserved(const void start, const void end)	1558	int ftrace_text_reserved(const void start, const void end)
1559	{	1559	{
1560	unsigned long ret;	1560	unsigned long ret;
1561		1561
1562	ret = ftrace_location_range((unsigned long)start,	1562	ret = ftrace_location_range((unsigned long)start,
1563	(unsigned long)end);	1563	(unsigned long)end);
1564		1564
1565	return (int)!!ret;	1565	return (int)!!ret;
1566	}	1566	}
1567		1567
1568	static void __ftrace_hash_rec_update(struct ftrace_ops *ops,	1568	static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1569	int filter_hash,	1569	int filter_hash,
1570	bool inc)	1570	bool inc)
1571	{	1571	{
1572	struct ftrace_hash *hash;	1572	struct ftrace_hash *hash;
1573	struct ftrace_hash *other_hash;	1573	struct ftrace_hash *other_hash;
1574	struct ftrace_page *pg;	1574	struct ftrace_page *pg;
1575	struct dyn_ftrace *rec;	1575	struct dyn_ftrace *rec;
1576	int count = 0;	1576	int count = 0;
1577	int all = 0;	1577	int all = 0;
1578		1578
1579	/* Only update if the ops has been registered */	1579	/* Only update if the ops has been registered */
1580	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))	1580	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1581	return;	1581	return;
1582		1582
1583	/*	1583	/*
1584	* In the filter_hash case:	1584	* In the filter_hash case:
1585	* If the count is zero, we update all records.	1585	* If the count is zero, we update all records.
1586	* Otherwise we just update the items in the hash.	1586	* Otherwise we just update the items in the hash.
1587	*	1587	*
1588	* In the notrace_hash case:	1588	* In the notrace_hash case:
1589	* We enable the update in the hash.	1589	* We enable the update in the hash.
1590	* As disabling notrace means enabling the tracing,	1590	* As disabling notrace means enabling the tracing,
1591	* and enabling notrace means disabling, the inc variable	1591	* and enabling notrace means disabling, the inc variable
1592	* gets inversed.	1592	* gets inversed.
1593	*/	1593	*/
1594	if (filter_hash) {	1594	if (filter_hash) {
1595	hash = ops->filter_hash;	1595	hash = ops->filter_hash;
1596	other_hash = ops->notrace_hash;	1596	other_hash = ops->notrace_hash;
1597	if (ftrace_hash_empty(hash))	1597	if (ftrace_hash_empty(hash))
1598	all = 1;	1598	all = 1;
1599	} else {	1599	} else {
1600	inc = !inc;	1600	inc = !inc;
1601	hash = ops->notrace_hash;	1601	hash = ops->notrace_hash;
1602	other_hash = ops->filter_hash;	1602	other_hash = ops->filter_hash;
1603	/*	1603	/*
1604	* If the notrace hash has no items,	1604	* If the notrace hash has no items,
1605	* then there's nothing to do.	1605	* then there's nothing to do.
1606	*/	1606	*/
1607	if (ftrace_hash_empty(hash))	1607	if (ftrace_hash_empty(hash))
1608	return;	1608	return;
1609	}	1609	}
1610		1610
1611	do_for_each_ftrace_rec(pg, rec) {	1611	do_for_each_ftrace_rec(pg, rec) {
1612	int in_other_hash = 0;	1612	int in_other_hash = 0;
1613	int in_hash = 0;	1613	int in_hash = 0;
1614	int match = 0;	1614	int match = 0;
1615		1615
1616	if (all) {	1616	if (all) {
1617	/*	1617	/*
1618	* Only the filter_hash affects all records.	1618	* Only the filter_hash affects all records.
1619	* Update if the record is not in the notrace hash.	1619	* Update if the record is not in the notrace hash.
1620	*/	1620	*/
1621	if (!other_hash \|\| !ftrace_lookup_ip(other_hash, rec->ip))	1621	if (!other_hash \|\| !ftrace_lookup_ip(other_hash, rec->ip))
1622	match = 1;	1622	match = 1;
1623	} else {	1623	} else {
1624	in_hash = !!ftrace_lookup_ip(hash, rec->ip);	1624	in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1625	in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);	1625	in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1626		1626
1627	/*	1627	/*
1628	*	1628	*
1629	*/	1629	*/
1630	if (filter_hash && in_hash && !in_other_hash)	1630	if (filter_hash && in_hash && !in_other_hash)
1631	match = 1;	1631	match = 1;
1632	else if (!filter_hash && in_hash &&	1632	else if (!filter_hash && in_hash &&
1633	(in_other_hash \|\| ftrace_hash_empty(other_hash)))	1633	(in_other_hash \|\| ftrace_hash_empty(other_hash)))
1634	match = 1;	1634	match = 1;
1635	}	1635	}
1636	if (!match)	1636	if (!match)
1637	continue;	1637	continue;
1638		1638
1639	if (inc) {	1639	if (inc) {
1640	rec->flags++;	1640	rec->flags++;
1641	if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))	1641	if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1642	return;	1642	return;
1643	/*	1643	/*
1644	* If any ops wants regs saved for this function	1644	* If any ops wants regs saved for this function
1645	* then all ops will get saved regs.	1645	* then all ops will get saved regs.
1646	*/	1646	*/
1647	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)	1647	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1648	rec->flags \|= FTRACE_FL_REGS;	1648	rec->flags \|= FTRACE_FL_REGS;
1649	} else {	1649	} else {
1650	if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))	1650	if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1651	return;	1651	return;
1652	rec->flags--;	1652	rec->flags--;
1653	}	1653	}
1654	count++;	1654	count++;
1655	/* Shortcut, if we handled all records, we are done. */	1655	/* Shortcut, if we handled all records, we are done. */
1656	if (!all && count == hash->count)	1656	if (!all && count == hash->count)
1657	return;	1657	return;
1658	} while_for_each_ftrace_rec();	1658	} while_for_each_ftrace_rec();
1659	}	1659	}
1660		1660
1661	static void ftrace_hash_rec_disable(struct ftrace_ops *ops,	1661	static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1662	int filter_hash)	1662	int filter_hash)
1663	{	1663	{
1664	__ftrace_hash_rec_update(ops, filter_hash, 0);	1664	__ftrace_hash_rec_update(ops, filter_hash, 0);
1665	}	1665	}
1666		1666
1667	static void ftrace_hash_rec_enable(struct ftrace_ops *ops,	1667	static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1668	int filter_hash)	1668	int filter_hash)
1669	{	1669	{
1670	__ftrace_hash_rec_update(ops, filter_hash, 1);	1670	__ftrace_hash_rec_update(ops, filter_hash, 1);
1671	}	1671	}
1672		1672
1673	static void print_ip_ins(const char fmt, unsigned char p)	1673	static void print_ip_ins(const char fmt, unsigned char p)
1674	{	1674	{
1675	int i;	1675	int i;
1676		1676
1677	printk(KERN_CONT "%s", fmt);	1677	printk(KERN_CONT "%s", fmt);
1678		1678
1679	for (i = 0; i < MCOUNT_INSN_SIZE; i++)	1679	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1680	printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);	1680	printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1681	}	1681	}
1682		1682
1683	/**	1683	/**
1684	* ftrace_bug - report and shutdown function tracer	1684	* ftrace_bug - report and shutdown function tracer
1685	* @failed: The failed type (EFAULT, EINVAL, EPERM)	1685	* @failed: The failed type (EFAULT, EINVAL, EPERM)
1686	* @ip: The address that failed	1686	* @ip: The address that failed
1687	*	1687	*
1688	* The arch code that enables or disables the function tracing	1688	* The arch code that enables or disables the function tracing
1689	* can call ftrace_bug() when it has detected a problem in	1689	* can call ftrace_bug() when it has detected a problem in
1690	* modifying the code. @failed should be one of either:	1690	* modifying the code. @failed should be one of either:
1691	* EFAULT - if the problem happens on reading the @ip address	1691	* EFAULT - if the problem happens on reading the @ip address
1692	* EINVAL - if what is read at @ip is not what was expected	1692	* EINVAL - if what is read at @ip is not what was expected
1693	* EPERM - if the problem happens on writting to the @ip address	1693	* EPERM - if the problem happens on writting to the @ip address
1694	*/	1694	*/
1695	void ftrace_bug(int failed, unsigned long ip)	1695	void ftrace_bug(int failed, unsigned long ip)
1696	{	1696	{
1697	switch (failed) {	1697	switch (failed) {
1698	case -EFAULT:	1698	case -EFAULT:
1699	FTRACE_WARN_ON_ONCE(1);	1699	FTRACE_WARN_ON_ONCE(1);
1700	pr_info("ftrace faulted on modifying ");	1700	pr_info("ftrace faulted on modifying ");
1701	print_ip_sym(ip);	1701	print_ip_sym(ip);
1702	break;	1702	break;
1703	case -EINVAL:	1703	case -EINVAL:
1704	FTRACE_WARN_ON_ONCE(1);	1704	FTRACE_WARN_ON_ONCE(1);
1705	pr_info("ftrace failed to modify ");	1705	pr_info("ftrace failed to modify ");
1706	print_ip_sym(ip);	1706	print_ip_sym(ip);
1707	print_ip_ins(" actual: ", (unsigned char *)ip);	1707	print_ip_ins(" actual: ", (unsigned char *)ip);
1708	printk(KERN_CONT "\n");	1708	printk(KERN_CONT "\n");
1709	break;	1709	break;
1710	case -EPERM:	1710	case -EPERM:
1711	FTRACE_WARN_ON_ONCE(1);	1711	FTRACE_WARN_ON_ONCE(1);
1712	pr_info("ftrace faulted on writing ");	1712	pr_info("ftrace faulted on writing ");
1713	print_ip_sym(ip);	1713	print_ip_sym(ip);
1714	break;	1714	break;
1715	default:	1715	default:
1716	FTRACE_WARN_ON_ONCE(1);	1716	FTRACE_WARN_ON_ONCE(1);
1717	pr_info("ftrace faulted on unknown error ");	1717	pr_info("ftrace faulted on unknown error ");
1718	print_ip_sym(ip);	1718	print_ip_sym(ip);
1719	}	1719	}
1720	}	1720	}
1721		1721
1722	static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)	1722	static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
1723	{	1723	{
1724	unsigned long flag = 0UL;	1724	unsigned long flag = 0UL;
1725		1725
1726	/*	1726	/*
1727	* If we are updating calls:	1727	* If we are updating calls:
1728	*	1728	*
1729	* If the record has a ref count, then we need to enable it	1729	* If the record has a ref count, then we need to enable it
1730	* because someone is using it.	1730	* because someone is using it.
1731	*	1731	*
1732	* Otherwise we make sure its disabled.	1732	* Otherwise we make sure its disabled.
1733	*	1733	*
1734	* If we are disabling calls, then disable all records that	1734	* If we are disabling calls, then disable all records that
1735	* are enabled.	1735	* are enabled.
1736	*/	1736	*/
1737	if (enable && (rec->flags & ~FTRACE_FL_MASK))	1737	if (enable && (rec->flags & ~FTRACE_FL_MASK))
1738	flag = FTRACE_FL_ENABLED;	1738	flag = FTRACE_FL_ENABLED;
1739		1739
1740	/*	1740	/*
1741	* If enabling and the REGS flag does not match the REGS_EN, then	1741	* If enabling and the REGS flag does not match the REGS_EN, then
1742	* do not ignore this record. Set flags to fail the compare against	1742	* do not ignore this record. Set flags to fail the compare against
1743	* ENABLED.	1743	* ENABLED.
1744	*/	1744	*/
1745	if (flag &&	1745	if (flag &&
1746	(!(rec->flags & FTRACE_FL_REGS) != !(rec->flags & FTRACE_FL_REGS_EN)))	1746	(!(rec->flags & FTRACE_FL_REGS) != !(rec->flags & FTRACE_FL_REGS_EN)))
1747	flag \|= FTRACE_FL_REGS;	1747	flag \|= FTRACE_FL_REGS;
1748		1748
1749	/* If the state of this record hasn't changed, then do nothing */	1749	/* If the state of this record hasn't changed, then do nothing */
1750	if ((rec->flags & FTRACE_FL_ENABLED) == flag)	1750	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1751	return FTRACE_UPDATE_IGNORE;	1751	return FTRACE_UPDATE_IGNORE;
1752		1752
1753	if (flag) {	1753	if (flag) {
1754	/* Save off if rec is being enabled (for return value) */	1754	/* Save off if rec is being enabled (for return value) */
1755	flag ^= rec->flags & FTRACE_FL_ENABLED;	1755	flag ^= rec->flags & FTRACE_FL_ENABLED;
1756		1756
1757	if (update) {	1757	if (update) {
1758	rec->flags \|= FTRACE_FL_ENABLED;	1758	rec->flags \|= FTRACE_FL_ENABLED;
1759	if (flag & FTRACE_FL_REGS) {	1759	if (flag & FTRACE_FL_REGS) {
1760	if (rec->flags & FTRACE_FL_REGS)	1760	if (rec->flags & FTRACE_FL_REGS)
1761	rec->flags \|= FTRACE_FL_REGS_EN;	1761	rec->flags \|= FTRACE_FL_REGS_EN;
1762	else	1762	else
1763	rec->flags &= ~FTRACE_FL_REGS_EN;	1763	rec->flags &= ~FTRACE_FL_REGS_EN;
1764	}	1764	}
1765	}	1765	}
1766		1766
1767	/*	1767	/*
1768	* If this record is being updated from a nop, then	1768	* If this record is being updated from a nop, then
1769	* return UPDATE_MAKE_CALL.	1769	* return UPDATE_MAKE_CALL.
1770	* Otherwise, if the EN flag is set, then return	1770	* Otherwise, if the EN flag is set, then return
1771	* UPDATE_MODIFY_CALL_REGS to tell the caller to convert	1771	* UPDATE_MODIFY_CALL_REGS to tell the caller to convert
1772	* from the non-save regs, to a save regs function.	1772	* from the non-save regs, to a save regs function.
1773	* Otherwise,	1773	* Otherwise,
1774	* return UPDATE_MODIFY_CALL to tell the caller to convert	1774	* return UPDATE_MODIFY_CALL to tell the caller to convert
1775	* from the save regs, to a non-save regs function.	1775	* from the save regs, to a non-save regs function.
1776	*/	1776	*/
1777	if (flag & FTRACE_FL_ENABLED)	1777	if (flag & FTRACE_FL_ENABLED)
1778	return FTRACE_UPDATE_MAKE_CALL;	1778	return FTRACE_UPDATE_MAKE_CALL;
1779	else if (rec->flags & FTRACE_FL_REGS_EN)	1779	else if (rec->flags & FTRACE_FL_REGS_EN)
1780	return FTRACE_UPDATE_MODIFY_CALL_REGS;	1780	return FTRACE_UPDATE_MODIFY_CALL_REGS;
1781	else	1781	else
1782	return FTRACE_UPDATE_MODIFY_CALL;	1782	return FTRACE_UPDATE_MODIFY_CALL;
1783	}	1783	}
1784		1784
1785	if (update) {	1785	if (update) {
1786	/* If there's no more users, clear all flags */	1786	/* If there's no more users, clear all flags */
1787	if (!(rec->flags & ~FTRACE_FL_MASK))	1787	if (!(rec->flags & ~FTRACE_FL_MASK))
1788	rec->flags = 0;	1788	rec->flags = 0;
1789	else	1789	else
1790	/* Just disable the record (keep REGS state) */	1790	/* Just disable the record (keep REGS state) */
1791	rec->flags &= ~FTRACE_FL_ENABLED;	1791	rec->flags &= ~FTRACE_FL_ENABLED;
1792	}	1792	}
1793		1793
1794	return FTRACE_UPDATE_MAKE_NOP;	1794	return FTRACE_UPDATE_MAKE_NOP;
1795	}	1795	}
1796		1796
1797	/**	1797	/**
1798	* ftrace_update_record, set a record that now is tracing or not	1798	* ftrace_update_record, set a record that now is tracing or not
1799	* @rec: the record to update	1799	* @rec: the record to update
1800	* @enable: set to 1 if the record is tracing, zero to force disable	1800	* @enable: set to 1 if the record is tracing, zero to force disable
1801	*	1801	*
1802	* The records that represent all functions that can be traced need	1802	* The records that represent all functions that can be traced need
1803	* to be updated when tracing has been enabled.	1803	* to be updated when tracing has been enabled.
1804	*/	1804	*/
1805	int ftrace_update_record(struct dyn_ftrace *rec, int enable)	1805	int ftrace_update_record(struct dyn_ftrace *rec, int enable)
1806	{	1806	{
1807	return ftrace_check_record(rec, enable, 1);	1807	return ftrace_check_record(rec, enable, 1);
1808	}	1808	}
1809		1809
1810	/**	1810	/**
1811	* ftrace_test_record, check if the record has been enabled or not	1811	* ftrace_test_record, check if the record has been enabled or not
1812	* @rec: the record to test	1812	* @rec: the record to test
1813	* @enable: set to 1 to check if enabled, 0 if it is disabled	1813	* @enable: set to 1 to check if enabled, 0 if it is disabled
1814	*	1814	*
1815	* The arch code may need to test if a record is already set to	1815	* The arch code may need to test if a record is already set to
1816	* tracing to determine how to modify the function code that it	1816	* tracing to determine how to modify the function code that it
1817	* represents.	1817	* represents.
1818	*/	1818	*/
1819	int ftrace_test_record(struct dyn_ftrace *rec, int enable)	1819	int ftrace_test_record(struct dyn_ftrace *rec, int enable)
1820	{	1820	{
1821	return ftrace_check_record(rec, enable, 0);	1821	return ftrace_check_record(rec, enable, 0);
1822	}	1822	}
1823		1823
1824	static int	1824	static int
1825	__ftrace_replace_code(struct dyn_ftrace *rec, int enable)	1825	__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1826	{	1826	{
1827	unsigned long ftrace_old_addr;	1827	unsigned long ftrace_old_addr;
1828	unsigned long ftrace_addr;	1828	unsigned long ftrace_addr;
1829	int ret;	1829	int ret;
1830		1830
1831	ret = ftrace_update_record(rec, enable);	1831	ret = ftrace_update_record(rec, enable);
1832		1832
1833	if (rec->flags & FTRACE_FL_REGS)	1833	if (rec->flags & FTRACE_FL_REGS)
1834	ftrace_addr = (unsigned long)FTRACE_REGS_ADDR;	1834	ftrace_addr = (unsigned long)FTRACE_REGS_ADDR;
1835	else	1835	else
1836	ftrace_addr = (unsigned long)FTRACE_ADDR;	1836	ftrace_addr = (unsigned long)FTRACE_ADDR;
1837		1837
1838	switch (ret) {	1838	switch (ret) {
1839	case FTRACE_UPDATE_IGNORE:	1839	case FTRACE_UPDATE_IGNORE:
1840	return 0;	1840	return 0;
1841		1841
1842	case FTRACE_UPDATE_MAKE_CALL:	1842	case FTRACE_UPDATE_MAKE_CALL:
1843	return ftrace_make_call(rec, ftrace_addr);	1843	return ftrace_make_call(rec, ftrace_addr);
1844		1844
1845	case FTRACE_UPDATE_MAKE_NOP:	1845	case FTRACE_UPDATE_MAKE_NOP:
1846	return ftrace_make_nop(NULL, rec, ftrace_addr);	1846	return ftrace_make_nop(NULL, rec, ftrace_addr);
1847		1847
1848	case FTRACE_UPDATE_MODIFY_CALL_REGS:	1848	case FTRACE_UPDATE_MODIFY_CALL_REGS:
1849	case FTRACE_UPDATE_MODIFY_CALL:	1849	case FTRACE_UPDATE_MODIFY_CALL:
1850	if (rec->flags & FTRACE_FL_REGS)	1850	if (rec->flags & FTRACE_FL_REGS)
1851	ftrace_old_addr = (unsigned long)FTRACE_ADDR;	1851	ftrace_old_addr = (unsigned long)FTRACE_ADDR;
1852	else	1852	else
1853	ftrace_old_addr = (unsigned long)FTRACE_REGS_ADDR;	1853	ftrace_old_addr = (unsigned long)FTRACE_REGS_ADDR;
1854		1854
1855	return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);	1855	return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
1856	}	1856	}
1857		1857
1858	return -1; /* unknow ftrace bug */	1858	return -1; /* unknow ftrace bug */
1859	}	1859	}
1860		1860
1861	void __weak ftrace_replace_code(int enable)	1861	void __weak ftrace_replace_code(int enable)
1862	{	1862	{
1863	struct dyn_ftrace *rec;	1863	struct dyn_ftrace *rec;
1864	struct ftrace_page *pg;	1864	struct ftrace_page *pg;
1865	int failed;	1865	int failed;
1866		1866
1867	if (unlikely(ftrace_disabled))	1867	if (unlikely(ftrace_disabled))
1868	return;	1868	return;
1869		1869
1870	do_for_each_ftrace_rec(pg, rec) {	1870	do_for_each_ftrace_rec(pg, rec) {
1871	failed = __ftrace_replace_code(rec, enable);	1871	failed = __ftrace_replace_code(rec, enable);
1872	if (failed) {	1872	if (failed) {
1873	ftrace_bug(failed, rec->ip);	1873	ftrace_bug(failed, rec->ip);
1874	/* Stop processing */	1874	/* Stop processing */
1875	return;	1875	return;
1876	}	1876	}
1877	} while_for_each_ftrace_rec();	1877	} while_for_each_ftrace_rec();
1878	}	1878	}
1879		1879
1880	struct ftrace_rec_iter {	1880	struct ftrace_rec_iter {
1881	struct ftrace_page *pg;	1881	struct ftrace_page *pg;
1882	int index;	1882	int index;
1883	};	1883	};
1884		1884
1885	/**	1885	/**
1886	* ftrace_rec_iter_start, start up iterating over traced functions	1886	* ftrace_rec_iter_start, start up iterating over traced functions
1887	*	1887	*
1888	* Returns an iterator handle that is used to iterate over all	1888	* Returns an iterator handle that is used to iterate over all
1889	* the records that represent address locations where functions	1889	* the records that represent address locations where functions
1890	* are traced.	1890	* are traced.
1891	*	1891	*
1892	* May return NULL if no records are available.	1892	* May return NULL if no records are available.
1893	*/	1893	*/
1894	struct ftrace_rec_iter *ftrace_rec_iter_start(void)	1894	struct ftrace_rec_iter *ftrace_rec_iter_start(void)
1895	{	1895	{
1896	/*	1896	/*
1897	* We only use a single iterator.	1897	* We only use a single iterator.
1898	* Protected by the ftrace_lock mutex.	1898	* Protected by the ftrace_lock mutex.
1899	*/	1899	*/
1900	static struct ftrace_rec_iter ftrace_rec_iter;	1900	static struct ftrace_rec_iter ftrace_rec_iter;
1901	struct ftrace_rec_iter *iter = &ftrace_rec_iter;	1901	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
1902		1902
1903	iter->pg = ftrace_pages_start;	1903	iter->pg = ftrace_pages_start;
1904	iter->index = 0;	1904	iter->index = 0;
1905		1905
1906	/* Could have empty pages */	1906	/* Could have empty pages */
1907	while (iter->pg && !iter->pg->index)	1907	while (iter->pg && !iter->pg->index)
1908	iter->pg = iter->pg->next;	1908	iter->pg = iter->pg->next;
1909		1909
1910	if (!iter->pg)	1910	if (!iter->pg)
1911	return NULL;	1911	return NULL;
1912		1912
1913	return iter;	1913	return iter;
1914	}	1914	}
1915		1915
1916	/**	1916	/**
1917	* ftrace_rec_iter_next, get the next record to process.	1917	* ftrace_rec_iter_next, get the next record to process.
1918	* @iter: The handle to the iterator.	1918	* @iter: The handle to the iterator.
1919	*	1919	*
1920	* Returns the next iterator after the given iterator @iter.	1920	* Returns the next iterator after the given iterator @iter.
1921	*/	1921	*/
1922	struct ftrace_rec_iter ftrace_rec_iter_next(struct ftrace_rec_iter iter)	1922	struct ftrace_rec_iter ftrace_rec_iter_next(struct ftrace_rec_iter iter)
1923	{	1923	{
1924	iter->index++;	1924	iter->index++;
1925		1925
1926	if (iter->index >= iter->pg->index) {	1926	if (iter->index >= iter->pg->index) {
1927	iter->pg = iter->pg->next;	1927	iter->pg = iter->pg->next;
1928	iter->index = 0;	1928	iter->index = 0;
1929		1929
1930	/* Could have empty pages */	1930	/* Could have empty pages */
1931	while (iter->pg && !iter->pg->index)	1931	while (iter->pg && !iter->pg->index)
1932	iter->pg = iter->pg->next;	1932	iter->pg = iter->pg->next;
1933	}	1933	}
1934		1934
1935	if (!iter->pg)	1935	if (!iter->pg)
1936	return NULL;	1936	return NULL;
1937		1937
1938	return iter;	1938	return iter;
1939	}	1939	}
1940		1940
1941	/**	1941	/**
1942	* ftrace_rec_iter_record, get the record at the iterator location	1942	* ftrace_rec_iter_record, get the record at the iterator location
1943	* @iter: The current iterator location	1943	* @iter: The current iterator location
1944	*	1944	*
1945	* Returns the record that the current @iter is at.	1945	* Returns the record that the current @iter is at.
1946	*/	1946	*/
1947	struct dyn_ftrace ftrace_rec_iter_record(struct ftrace_rec_iter iter)	1947	struct dyn_ftrace ftrace_rec_iter_record(struct ftrace_rec_iter iter)
1948	{	1948	{
1949	return &iter->pg->records[iter->index];	1949	return &iter->pg->records[iter->index];
1950	}	1950	}
1951		1951
1952	static int	1952	static int
1953	ftrace_code_disable(struct module mod, struct dyn_ftrace rec)	1953	ftrace_code_disable(struct module mod, struct dyn_ftrace rec)
1954	{	1954	{
1955	unsigned long ip;	1955	unsigned long ip;
1956	int ret;	1956	int ret;
1957		1957
1958	ip = rec->ip;	1958	ip = rec->ip;
1959		1959
1960	if (unlikely(ftrace_disabled))	1960	if (unlikely(ftrace_disabled))
1961	return 0;	1961	return 0;
1962		1962
1963	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);	1963	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1964	if (ret) {	1964	if (ret) {
1965	ftrace_bug(ret, ip);	1965	ftrace_bug(ret, ip);
1966	return 0;	1966	return 0;
1967	}	1967	}
1968	return 1;	1968	return 1;
1969	}	1969	}
1970		1970
1971	/*	1971	/*
1972	* archs can override this function if they must do something	1972	* archs can override this function if they must do something
1973	* before the modifying code is performed.	1973	* before the modifying code is performed.
1974	*/	1974	*/
1975	int __weak ftrace_arch_code_modify_prepare(void)	1975	int __weak ftrace_arch_code_modify_prepare(void)
1976	{	1976	{
1977	return 0;	1977	return 0;
1978	}	1978	}
1979		1979
1980	/*	1980	/*
1981	* archs can override this function if they must do something	1981	* archs can override this function if they must do something
1982	* after the modifying code is performed.	1982	* after the modifying code is performed.
1983	*/	1983	*/
1984	int __weak ftrace_arch_code_modify_post_process(void)	1984	int __weak ftrace_arch_code_modify_post_process(void)
1985	{	1985	{
1986	return 0;	1986	return 0;
1987	}	1987	}
1988		1988
1989	void ftrace_modify_all_code(int command)	1989	void ftrace_modify_all_code(int command)
1990	{	1990	{
1991	int update = command & FTRACE_UPDATE_TRACE_FUNC;	1991	int update = command & FTRACE_UPDATE_TRACE_FUNC;
1992	int err = 0;	1992	int err = 0;
1993		1993
1994	/*	1994	/*
1995	* If the ftrace_caller calls a ftrace_ops func directly,	1995	* If the ftrace_caller calls a ftrace_ops func directly,
1996	* we need to make sure that it only traces functions it	1996	* we need to make sure that it only traces functions it
1997	* expects to trace. When doing the switch of functions,	1997	* expects to trace. When doing the switch of functions,
1998	* we need to update to the ftrace_ops_list_func first	1998	* we need to update to the ftrace_ops_list_func first
1999	* before the transition between old and new calls are set,	1999	* before the transition between old and new calls are set,
2000	* as the ftrace_ops_list_func will check the ops hashes	2000	* as the ftrace_ops_list_func will check the ops hashes
2001	* to make sure the ops are having the right functions	2001	* to make sure the ops are having the right functions
2002	* traced.	2002	* traced.
2003	*/	2003	*/
2004	if (update) {	2004	if (update) {
2005	err = ftrace_update_ftrace_func(ftrace_ops_list_func);	2005	err = ftrace_update_ftrace_func(ftrace_ops_list_func);
2006	if (FTRACE_WARN_ON(err))	2006	if (FTRACE_WARN_ON(err))
2007	return;	2007	return;
2008	}	2008	}
2009		2009
2010	if (command & FTRACE_UPDATE_CALLS)	2010	if (command & FTRACE_UPDATE_CALLS)
2011	ftrace_replace_code(1);	2011	ftrace_replace_code(1);
2012	else if (command & FTRACE_DISABLE_CALLS)	2012	else if (command & FTRACE_DISABLE_CALLS)
2013	ftrace_replace_code(0);	2013	ftrace_replace_code(0);
2014		2014
2015	if (update && ftrace_trace_function != ftrace_ops_list_func) {	2015	if (update && ftrace_trace_function != ftrace_ops_list_func) {
2016	function_trace_op = set_function_trace_op;	2016	function_trace_op = set_function_trace_op;
2017	smp_wmb();	2017	smp_wmb();
2018	/* If irqs are disabled, we are in stop machine */	2018	/* If irqs are disabled, we are in stop machine */
2019	if (!irqs_disabled())	2019	if (!irqs_disabled())
2020	smp_call_function(ftrace_sync_ipi, NULL, 1);	2020	smp_call_function(ftrace_sync_ipi, NULL, 1);
2021	err = ftrace_update_ftrace_func(ftrace_trace_function);	2021	err = ftrace_update_ftrace_func(ftrace_trace_function);
2022	if (FTRACE_WARN_ON(err))	2022	if (FTRACE_WARN_ON(err))
2023	return;	2023	return;
2024	}	2024	}
2025		2025
2026	if (command & FTRACE_START_FUNC_RET)	2026	if (command & FTRACE_START_FUNC_RET)
2027	err = ftrace_enable_ftrace_graph_caller();	2027	err = ftrace_enable_ftrace_graph_caller();
2028	else if (command & FTRACE_STOP_FUNC_RET)	2028	else if (command & FTRACE_STOP_FUNC_RET)
2029	err = ftrace_disable_ftrace_graph_caller();	2029	err = ftrace_disable_ftrace_graph_caller();
2030	FTRACE_WARN_ON(err);	2030	FTRACE_WARN_ON(err);
2031	}	2031	}
2032		2032
2033	static int __ftrace_modify_code(void *data)	2033	static int __ftrace_modify_code(void *data)
2034	{	2034	{
2035	int *command = data;	2035	int *command = data;
2036		2036
2037	ftrace_modify_all_code(*command);	2037	ftrace_modify_all_code(*command);
2038		2038
2039	return 0;	2039	return 0;
2040	}	2040	}
2041		2041
2042	/**	2042	/**
2043	* ftrace_run_stop_machine, go back to the stop machine method	2043	* ftrace_run_stop_machine, go back to the stop machine method
2044	* @command: The command to tell ftrace what to do	2044	* @command: The command to tell ftrace what to do
2045	*	2045	*
2046	* If an arch needs to fall back to the stop machine method, the	2046	* If an arch needs to fall back to the stop machine method, the
2047	* it can call this function.	2047	* it can call this function.
2048	*/	2048	*/
2049	void ftrace_run_stop_machine(int command)	2049	void ftrace_run_stop_machine(int command)
2050	{	2050	{
2051	stop_machine(__ftrace_modify_code, &command, NULL);	2051	stop_machine(__ftrace_modify_code, &command, NULL);
2052	}	2052	}
2053		2053
2054	/**	2054	/**
2055	* arch_ftrace_update_code, modify the code to trace or not trace	2055	* arch_ftrace_update_code, modify the code to trace or not trace
2056	* @command: The command that needs to be done	2056	* @command: The command that needs to be done
2057	*	2057	*
2058	* Archs can override this function if it does not need to	2058	* Archs can override this function if it does not need to
2059	* run stop_machine() to modify code.	2059	* run stop_machine() to modify code.
2060	*/	2060	*/
2061	void __weak arch_ftrace_update_code(int command)	2061	void __weak arch_ftrace_update_code(int command)
2062	{	2062	{
2063	ftrace_run_stop_machine(command);	2063	ftrace_run_stop_machine(command);
2064	}	2064	}
2065		2065
2066	static void ftrace_run_update_code(int command)	2066	static void ftrace_run_update_code(int command)
2067	{	2067	{
2068	int ret;	2068	int ret;
2069		2069
2070	ret = ftrace_arch_code_modify_prepare();	2070	ret = ftrace_arch_code_modify_prepare();
2071	FTRACE_WARN_ON(ret);	2071	FTRACE_WARN_ON(ret);
2072	if (ret)	2072	if (ret)
2073	return;	2073	return;
2074	/*	2074	/*
2075	* Do not call function tracer while we update the code.	2075	* Do not call function tracer while we update the code.
2076	* We are in stop machine.	2076	* We are in stop machine.
2077	*/	2077	*/
2078	function_trace_stop++;	2078	function_trace_stop++;
2079		2079
2080	/*	2080	/*
2081	* By default we use stop_machine() to modify the code.	2081	* By default we use stop_machine() to modify the code.
2082	* But archs can do what ever they want as long as it	2082	* But archs can do what ever they want as long as it
2083	* is safe. The stop_machine() is the safest, but also	2083	* is safe. The stop_machine() is the safest, but also
2084	* produces the most overhead.	2084	* produces the most overhead.
2085	*/	2085	*/
2086	arch_ftrace_update_code(command);	2086	arch_ftrace_update_code(command);
2087		2087
2088	function_trace_stop--;	2088	function_trace_stop--;
2089		2089
2090	ret = ftrace_arch_code_modify_post_process();	2090	ret = ftrace_arch_code_modify_post_process();
2091	FTRACE_WARN_ON(ret);	2091	FTRACE_WARN_ON(ret);
2092	}	2092	}
2093		2093
2094	static ftrace_func_t saved_ftrace_func;	2094	static ftrace_func_t saved_ftrace_func;
2095	static int ftrace_start_up;	2095	static int ftrace_start_up;
2096	static int global_start_up;	2096	static int global_start_up;
2097		2097
2098	static void control_ops_free(struct ftrace_ops *ops)	2098	static void control_ops_free(struct ftrace_ops *ops)
2099	{	2099	{
2100	free_percpu(ops->disabled);	2100	free_percpu(ops->disabled);
2101	}	2101	}
2102		2102
2103	static void ftrace_startup_enable(int command)	2103	static void ftrace_startup_enable(int command)
2104	{	2104	{
2105	if (saved_ftrace_func != ftrace_trace_function) {	2105	if (saved_ftrace_func != ftrace_trace_function) {
2106	saved_ftrace_func = ftrace_trace_function;	2106	saved_ftrace_func = ftrace_trace_function;
2107	command \|= FTRACE_UPDATE_TRACE_FUNC;	2107	command \|= FTRACE_UPDATE_TRACE_FUNC;
2108	}	2108	}
2109		2109
2110	if (!command \|\| !ftrace_enabled)	2110	if (!command \|\| !ftrace_enabled)
2111	return;	2111	return;
2112		2112
2113	ftrace_run_update_code(command);	2113	ftrace_run_update_code(command);
2114	}	2114	}
2115		2115
2116	static int ftrace_startup(struct ftrace_ops *ops, int command)	2116	static int ftrace_startup(struct ftrace_ops *ops, int command)
2117	{	2117	{
2118	bool hash_enable = true;	2118	bool hash_enable = true;
2119	int ret;	2119	int ret;
2120		2120
2121	if (unlikely(ftrace_disabled))	2121	if (unlikely(ftrace_disabled))
2122	return -ENODEV;	2122	return -ENODEV;
2123		2123
2124	ret = __register_ftrace_function(ops);	2124	ret = __register_ftrace_function(ops);
2125	if (ret)	2125	if (ret)
2126	return ret;	2126	return ret;
2127		2127
2128	ftrace_start_up++;	2128	ftrace_start_up++;
2129	command \|= FTRACE_UPDATE_CALLS;	2129	command \|= FTRACE_UPDATE_CALLS;
2130		2130
2131	/* ops marked global share the filter hashes */	2131	/* ops marked global share the filter hashes */
2132	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {	2132	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
2133	ops = &global_ops;	2133	ops = &global_ops;
2134	/* Don't update hash if global is already set */	2134	/* Don't update hash if global is already set */
2135	if (global_start_up)	2135	if (global_start_up)
2136	hash_enable = false;	2136	hash_enable = false;
2137	global_start_up++;	2137	global_start_up++;
2138	}	2138	}
2139		2139
2140	ops->flags \|= FTRACE_OPS_FL_ENABLED;	2140	ops->flags \|= FTRACE_OPS_FL_ENABLED;
2141	if (hash_enable)	2141	if (hash_enable)
2142	ftrace_hash_rec_enable(ops, 1);	2142	ftrace_hash_rec_enable(ops, 1);
2143		2143
2144	ftrace_startup_enable(command);	2144	ftrace_startup_enable(command);
2145		2145
2146	return 0;	2146	return 0;
2147	}	2147	}
2148		2148
2149	static int ftrace_shutdown(struct ftrace_ops *ops, int command)	2149	static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2150	{	2150	{
2151	bool hash_disable = true;	2151	bool hash_disable = true;
2152	int ret;	2152	int ret;
2153		2153
2154	if (unlikely(ftrace_disabled))	2154	if (unlikely(ftrace_disabled))
2155	return -ENODEV;	2155	return -ENODEV;
2156		2156
2157	ret = __unregister_ftrace_function(ops);	2157	ret = __unregister_ftrace_function(ops);
2158	if (ret)	2158	if (ret)
2159	return ret;	2159	return ret;
2160		2160
2161	ftrace_start_up--;	2161	ftrace_start_up--;
2162	/*	2162	/*
2163	* Just warn in case of unbalance, no need to kill ftrace, it's not	2163	* Just warn in case of unbalance, no need to kill ftrace, it's not
2164	* critical but the ftrace_call callers may be never nopped again after	2164	* critical but the ftrace_call callers may be never nopped again after
2165	* further ftrace uses.	2165	* further ftrace uses.
2166	*/	2166	*/
2167	WARN_ON_ONCE(ftrace_start_up < 0);	2167	WARN_ON_ONCE(ftrace_start_up < 0);
2168		2168
2169	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {	2169	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
2170	ops = &global_ops;	2170	ops = &global_ops;
2171	global_start_up--;	2171	global_start_up--;
2172	WARN_ON_ONCE(global_start_up < 0);	2172	WARN_ON_ONCE(global_start_up < 0);
2173	/* Don't update hash if global still has users */	2173	/* Don't update hash if global still has users */
2174	if (global_start_up) {	2174	if (global_start_up) {
2175	WARN_ON_ONCE(!ftrace_start_up);	2175	WARN_ON_ONCE(!ftrace_start_up);
2176	hash_disable = false;	2176	hash_disable = false;
2177	}	2177	}
2178	}	2178	}
2179		2179
2180	if (hash_disable)	2180	if (hash_disable)
2181	ftrace_hash_rec_disable(ops, 1);	2181	ftrace_hash_rec_disable(ops, 1);
2182		2182
2183	if (ops != &global_ops \|\| !global_start_up)	2183	if (ops != &global_ops \|\| !global_start_up)
2184	ops->flags &= ~FTRACE_OPS_FL_ENABLED;	2184	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2185		2185
2186	command \|= FTRACE_UPDATE_CALLS;	2186	command \|= FTRACE_UPDATE_CALLS;
2187		2187
2188	if (saved_ftrace_func != ftrace_trace_function) {	2188	if (saved_ftrace_func != ftrace_trace_function) {
2189	saved_ftrace_func = ftrace_trace_function;	2189	saved_ftrace_func = ftrace_trace_function;
2190	command \|= FTRACE_UPDATE_TRACE_FUNC;	2190	command \|= FTRACE_UPDATE_TRACE_FUNC;
2191	}	2191	}
2192		2192
2193	if (!command \|\| !ftrace_enabled) {	2193	if (!command \|\| !ftrace_enabled) {
2194	/*	2194	/*
2195	* If these are control ops, they still need their	2195	* If these are control ops, they still need their
2196	* per_cpu field freed. Since, function tracing is	2196	* per_cpu field freed. Since, function tracing is
2197	* not currently active, we can just free them	2197	* not currently active, we can just free them
2198	* without synchronizing all CPUs.	2198	* without synchronizing all CPUs.
2199	*/	2199	*/
2200	if (ops->flags & FTRACE_OPS_FL_CONTROL)	2200	if (ops->flags & FTRACE_OPS_FL_CONTROL)
2201	control_ops_free(ops);	2201	control_ops_free(ops);
2202	return 0;	2202	return 0;
2203	}	2203	}
2204		2204
2205	ftrace_run_update_code(command);	2205	ftrace_run_update_code(command);
2206		2206
2207	/*	2207	/*
2208	* Dynamic ops may be freed, we must make sure that all	2208	* Dynamic ops may be freed, we must make sure that all
2209	* callers are done before leaving this function.	2209	* callers are done before leaving this function.
2210	* The same goes for freeing the per_cpu data of the control	2210	* The same goes for freeing the per_cpu data of the control
2211	* ops.	2211	* ops.
2212	*	2212	*
2213	* Again, normal synchronize_sched() is not good enough.	2213	* Again, normal synchronize_sched() is not good enough.
2214	* We need to do a hard force of sched synchronization.	2214	* We need to do a hard force of sched synchronization.
2215	* This is because we use preempt_disable() to do RCU, but	2215	* This is because we use preempt_disable() to do RCU, but
2216	* the function tracers can be called where RCU is not watching	2216	* the function tracers can be called where RCU is not watching
2217	* (like before user_exit()). We can not rely on the RCU	2217	* (like before user_exit()). We can not rely on the RCU
2218	* infrastructure to do the synchronization, thus we must do it	2218	* infrastructure to do the synchronization, thus we must do it
2219	* ourselves.	2219	* ourselves.
2220	*/	2220	*/
2221	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC \| FTRACE_OPS_FL_CONTROL)) {	2221	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC \| FTRACE_OPS_FL_CONTROL)) {
2222	schedule_on_each_cpu(ftrace_sync);	2222	schedule_on_each_cpu(ftrace_sync);
2223		2223
2224	if (ops->flags & FTRACE_OPS_FL_CONTROL)	2224	if (ops->flags & FTRACE_OPS_FL_CONTROL)
2225	control_ops_free(ops);	2225	control_ops_free(ops);
2226	}	2226	}
2227		2227
2228	return 0;	2228	return 0;
2229	}	2229	}
2230		2230
2231	static void ftrace_startup_sysctl(void)	2231	static void ftrace_startup_sysctl(void)
2232	{	2232	{
2233	if (unlikely(ftrace_disabled))	2233	if (unlikely(ftrace_disabled))
2234	return;	2234	return;
2235		2235
2236	/* Force update next time */	2236	/* Force update next time */
2237	saved_ftrace_func = NULL;	2237	saved_ftrace_func = NULL;
2238	/* ftrace_start_up is true if we want ftrace running */	2238	/* ftrace_start_up is true if we want ftrace running */
2239	if (ftrace_start_up)	2239	if (ftrace_start_up)
2240	ftrace_run_update_code(FTRACE_UPDATE_CALLS);	2240	ftrace_run_update_code(FTRACE_UPDATE_CALLS);
2241	}	2241	}
2242		2242
2243	static void ftrace_shutdown_sysctl(void)	2243	static void ftrace_shutdown_sysctl(void)
2244	{	2244	{
2245	if (unlikely(ftrace_disabled))	2245	if (unlikely(ftrace_disabled))
2246	return;	2246	return;
2247		2247
2248	/* ftrace_start_up is true if ftrace is running */	2248	/* ftrace_start_up is true if ftrace is running */
2249	if (ftrace_start_up)	2249	if (ftrace_start_up)
2250	ftrace_run_update_code(FTRACE_DISABLE_CALLS);	2250	ftrace_run_update_code(FTRACE_DISABLE_CALLS);
2251	}	2251	}
2252		2252
2253	static cycle_t ftrace_update_time;	2253	static cycle_t ftrace_update_time;
2254	unsigned long ftrace_update_tot_cnt;	2254	unsigned long ftrace_update_tot_cnt;
2255		2255
2256	static inline int ops_traces_mod(struct ftrace_ops *ops)	2256	static inline int ops_traces_mod(struct ftrace_ops *ops)
2257	{	2257	{
2258	/*	2258	/*
2259	* Filter_hash being empty will default to trace module.	2259	* Filter_hash being empty will default to trace module.
2260	* But notrace hash requires a test of individual module functions.	2260	* But notrace hash requires a test of individual module functions.
2261	*/	2261	*/
2262	return ftrace_hash_empty(ops->filter_hash) &&	2262	return ftrace_hash_empty(ops->filter_hash) &&
2263	ftrace_hash_empty(ops->notrace_hash);	2263	ftrace_hash_empty(ops->notrace_hash);
2264	}	2264	}
2265		2265
2266	/*	2266	/*
2267	* Check if the current ops references the record.	2267	* Check if the current ops references the record.
2268	*	2268	*
2269	* If the ops traces all functions, then it was already accounted for.	2269	* If the ops traces all functions, then it was already accounted for.
2270	* If the ops does not trace the current record function, skip it.	2270	* If the ops does not trace the current record function, skip it.
2271	* If the ops ignores the function via notrace filter, skip it.	2271	* If the ops ignores the function via notrace filter, skip it.
2272	*/	2272	*/
2273	static inline bool	2273	static inline bool
2274	ops_references_rec(struct ftrace_ops ops, struct dyn_ftrace rec)	2274	ops_references_rec(struct ftrace_ops ops, struct dyn_ftrace rec)
2275	{	2275	{
2276	/* If ops isn't enabled, ignore it */	2276	/* If ops isn't enabled, ignore it */
2277	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))	2277	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2278	return 0;	2278	return 0;
2279		2279
2280	/* If ops traces all mods, we already accounted for it */	2280	/* If ops traces all mods, we already accounted for it */
2281	if (ops_traces_mod(ops))	2281	if (ops_traces_mod(ops))
2282	return 0;	2282	return 0;
2283		2283
2284	/* The function must be in the filter */	2284	/* The function must be in the filter */
2285	if (!ftrace_hash_empty(ops->filter_hash) &&	2285	if (!ftrace_hash_empty(ops->filter_hash) &&
2286	!ftrace_lookup_ip(ops->filter_hash, rec->ip))	2286	!ftrace_lookup_ip(ops->filter_hash, rec->ip))
2287	return 0;	2287	return 0;
2288		2288
2289	/* If in notrace hash, we ignore it too */	2289	/* If in notrace hash, we ignore it too */
2290	if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))	2290	if (ftrace_lookup_ip(ops->notrace_hash, rec->ip))
2291	return 0;	2291	return 0;
2292		2292
2293	return 1;	2293	return 1;
2294	}	2294	}
2295		2295
2296	static int referenced_filters(struct dyn_ftrace *rec)	2296	static int referenced_filters(struct dyn_ftrace *rec)
2297	{	2297	{
2298	struct ftrace_ops *ops;	2298	struct ftrace_ops *ops;
2299	int cnt = 0;	2299	int cnt = 0;
2300		2300
2301	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {	2301	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
2302	if (ops_references_rec(ops, rec))	2302	if (ops_references_rec(ops, rec))
2303	cnt++;	2303	cnt++;
2304	}	2304	}
2305		2305
2306	return cnt;	2306	return cnt;
2307	}	2307	}
2308		2308
2309	static int ftrace_update_code(struct module mod, struct ftrace_page new_pgs)	2309	static int ftrace_update_code(struct module mod, struct ftrace_page new_pgs)
2310	{	2310	{
2311	struct ftrace_page *pg;	2311	struct ftrace_page *pg;
2312	struct dyn_ftrace *p;	2312	struct dyn_ftrace *p;
2313	cycle_t start, stop;	2313	cycle_t start, stop;
2314	unsigned long update_cnt = 0;	2314	unsigned long update_cnt = 0;
2315	unsigned long ref = 0;	2315	unsigned long ref = 0;
2316	bool test = false;	2316	bool test = false;
2317	int i;	2317	int i;
2318		2318
2319	/*	2319	/*
2320	* When adding a module, we need to check if tracers are	2320	* When adding a module, we need to check if tracers are
2321	* currently enabled and if they are set to trace all functions.	2321	* currently enabled and if they are set to trace all functions.
2322	* If they are, we need to enable the module functions as well	2322	* If they are, we need to enable the module functions as well
2323	* as update the reference counts for those function records.	2323	* as update the reference counts for those function records.
2324	*/	2324	*/
2325	if (mod) {	2325	if (mod) {
2326	struct ftrace_ops *ops;	2326	struct ftrace_ops *ops;
2327		2327
2328	for (ops = ftrace_ops_list;	2328	for (ops = ftrace_ops_list;
2329	ops != &ftrace_list_end; ops = ops->next) {	2329	ops != &ftrace_list_end; ops = ops->next) {
2330	if (ops->flags & FTRACE_OPS_FL_ENABLED) {	2330	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
2331	if (ops_traces_mod(ops))	2331	if (ops_traces_mod(ops))
2332	ref++;	2332	ref++;
2333	else	2333	else
2334	test = true;	2334	test = true;
2335	}	2335	}
2336	}	2336	}
2337	}	2337	}
2338		2338
2339	start = ftrace_now(raw_smp_processor_id());	2339	start = ftrace_now(raw_smp_processor_id());
2340		2340
2341	for (pg = new_pgs; pg; pg = pg->next) {	2341	for (pg = new_pgs; pg; pg = pg->next) {
2342		2342
2343	for (i = 0; i < pg->index; i++) {	2343	for (i = 0; i < pg->index; i++) {
2344	int cnt = ref;	2344	int cnt = ref;
2345		2345
2346	/* If something went wrong, bail without enabling anything */	2346	/* If something went wrong, bail without enabling anything */
2347	if (unlikely(ftrace_disabled))	2347	if (unlikely(ftrace_disabled))
2348	return -1;	2348	return -1;
2349		2349
2350	p = &pg->records[i];	2350	p = &pg->records[i];
2351	if (test)	2351	if (test)
2352	cnt += referenced_filters(p);	2352	cnt += referenced_filters(p);
2353	p->flags = cnt;	2353	p->flags = cnt;
2354		2354
2355	/*	2355	/*
2356	* Do the initial record conversion from mcount jump	2356	* Do the initial record conversion from mcount jump
2357	* to the NOP instructions.	2357	* to the NOP instructions.
2358	*/	2358	*/
2359	if (!ftrace_code_disable(mod, p))	2359	if (!ftrace_code_disable(mod, p))
2360	break;	2360	break;
2361		2361
2362	update_cnt++;	2362	update_cnt++;
2363		2363
2364	/*	2364	/*
2365	* If the tracing is enabled, go ahead and enable the record.	2365	* If the tracing is enabled, go ahead and enable the record.
2366	*	2366	*
2367	* The reason not to enable the record immediatelly is the	2367	* The reason not to enable the record immediatelly is the
2368	* inherent check of ftrace_make_nop/ftrace_make_call for	2368	* inherent check of ftrace_make_nop/ftrace_make_call for
2369	* correct previous instructions. Making first the NOP	2369	* correct previous instructions. Making first the NOP
2370	* conversion puts the module to the correct state, thus	2370	* conversion puts the module to the correct state, thus
2371	* passing the ftrace_make_call check.	2371	* passing the ftrace_make_call check.
2372	*/	2372	*/
2373	if (ftrace_start_up && cnt) {	2373	if (ftrace_start_up && cnt) {
2374	int failed = __ftrace_replace_code(p, 1);	2374	int failed = __ftrace_replace_code(p, 1);
2375	if (failed)	2375	if (failed)
2376	ftrace_bug(failed, p->ip);	2376	ftrace_bug(failed, p->ip);
2377	}	2377	}
2378	}	2378	}
2379	}	2379	}
2380		2380
2381	stop = ftrace_now(raw_smp_processor_id());	2381	stop = ftrace_now(raw_smp_processor_id());
2382	ftrace_update_time = stop - start;	2382	ftrace_update_time = stop - start;
2383	ftrace_update_tot_cnt += update_cnt;	2383	ftrace_update_tot_cnt += update_cnt;
2384		2384
2385	return 0;	2385	return 0;
2386	}	2386	}
2387		2387
2388	static int ftrace_allocate_records(struct ftrace_page *pg, int count)	2388	static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2389	{	2389	{
2390	int order;	2390	int order;
2391	int cnt;	2391	int cnt;
2392		2392
2393	if (WARN_ON(!count))	2393	if (WARN_ON(!count))
2394	return -EINVAL;	2394	return -EINVAL;
2395		2395
2396	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));	2396	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2397		2397
2398	/*	2398	/*
2399	* We want to fill as much as possible. No more than a page	2399	* We want to fill as much as possible. No more than a page
2400	* may be empty.	2400	* may be empty.
2401	*/	2401	*/
2402	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)	2402	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2403	order--;	2403	order--;
2404		2404
2405	again:	2405	again:
2406	pg->records = (void *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO, order);	2406	pg->records = (void *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO, order);
2407		2407
2408	if (!pg->records) {	2408	if (!pg->records) {
2409	/* if we can't allocate this size, try something smaller */	2409	/* if we can't allocate this size, try something smaller */
2410	if (!order)	2410	if (!order)
2411	return -ENOMEM;	2411	return -ENOMEM;
2412	order >>= 1;	2412	order >>= 1;
2413	goto again;	2413	goto again;
2414	}	2414	}
2415		2415
2416	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;	2416	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2417	pg->size = cnt;	2417	pg->size = cnt;
2418		2418
2419	if (cnt > count)	2419	if (cnt > count)
2420	cnt = count;	2420	cnt = count;
2421		2421
2422	return cnt;	2422	return cnt;
2423	}	2423	}
2424		2424
2425	static struct ftrace_page *	2425	static struct ftrace_page *
2426	ftrace_allocate_pages(unsigned long num_to_init)	2426	ftrace_allocate_pages(unsigned long num_to_init)
2427	{	2427	{
2428	struct ftrace_page *start_pg;	2428	struct ftrace_page *start_pg;
2429	struct ftrace_page *pg;	2429	struct ftrace_page *pg;
2430	int order;	2430	int order;
2431	int cnt;	2431	int cnt;
2432		2432
2433	if (!num_to_init)	2433	if (!num_to_init)
2434	return 0;	2434	return 0;
2435		2435
2436	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);	2436	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
2437	if (!pg)	2437	if (!pg)
2438	return NULL;	2438	return NULL;
2439		2439
2440	/*	2440	/*
2441	* Try to allocate as much as possible in one continues	2441	* Try to allocate as much as possible in one continues
2442	* location that fills in all of the space. We want to	2442	* location that fills in all of the space. We want to
2443	* waste as little space as possible.	2443	* waste as little space as possible.
2444	*/	2444	*/
2445	for (;;) {	2445	for (;;) {
2446	cnt = ftrace_allocate_records(pg, num_to_init);	2446	cnt = ftrace_allocate_records(pg, num_to_init);
2447	if (cnt < 0)	2447	if (cnt < 0)
2448	goto free_pages;	2448	goto free_pages;
2449		2449
2450	num_to_init -= cnt;	2450	num_to_init -= cnt;
2451	if (!num_to_init)	2451	if (!num_to_init)
2452	break;	2452	break;
2453		2453
2454	pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);	2454	pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
2455	if (!pg->next)	2455	if (!pg->next)
2456	goto free_pages;	2456	goto free_pages;
2457		2457
2458	pg = pg->next;	2458	pg = pg->next;
2459	}	2459	}
2460		2460
2461	return start_pg;	2461	return start_pg;
2462		2462
2463	free_pages:	2463	free_pages:
2464	while (start_pg) {	2464	while (start_pg) {
2465	order = get_count_order(pg->size / ENTRIES_PER_PAGE);	2465	order = get_count_order(pg->size / ENTRIES_PER_PAGE);
2466	free_pages((unsigned long)pg->records, order);	2466	free_pages((unsigned long)pg->records, order);
2467	start_pg = pg->next;	2467	start_pg = pg->next;
2468	kfree(pg);	2468	kfree(pg);
2469	pg = start_pg;	2469	pg = start_pg;
2470	}	2470	}
2471	pr_info("ftrace: FAILED to allocate memory for functions\n");	2471	pr_info("ftrace: FAILED to allocate memory for functions\n");
2472	return NULL;	2472	return NULL;
2473	}	2473	}
2474		2474
2475	#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */	2475	#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
2476		2476
2477	struct ftrace_iterator {	2477	struct ftrace_iterator {
2478	loff_t pos;	2478	loff_t pos;
2479	loff_t func_pos;	2479	loff_t func_pos;
2480	struct ftrace_page *pg;	2480	struct ftrace_page *pg;
2481	struct dyn_ftrace *func;	2481	struct dyn_ftrace *func;
2482	struct ftrace_func_probe *probe;	2482	struct ftrace_func_probe *probe;
2483	struct trace_parser parser;	2483	struct trace_parser parser;
2484	struct ftrace_hash *hash;	2484	struct ftrace_hash *hash;
2485	struct ftrace_ops *ops;	2485	struct ftrace_ops *ops;
2486	int hidx;	2486	int hidx;
2487	int idx;	2487	int idx;
2488	unsigned flags;	2488	unsigned flags;
2489	};	2489	};
2490		2490
2491	static void *	2491	static void *
2492	t_hash_next(struct seq_file m, loff_t pos)	2492	t_hash_next(struct seq_file m, loff_t pos)
2493	{	2493	{
2494	struct ftrace_iterator *iter = m->private;	2494	struct ftrace_iterator *iter = m->private;
2495	struct hlist_node *hnd = NULL;	2495	struct hlist_node *hnd = NULL;
2496	struct hlist_head *hhd;	2496	struct hlist_head *hhd;
2497		2497
2498	(*pos)++;	2498	(*pos)++;
2499	iter->pos = *pos;	2499	iter->pos = *pos;
2500		2500
2501	if (iter->probe)	2501	if (iter->probe)
2502	hnd = &iter->probe->node;	2502	hnd = &iter->probe->node;
2503	retry:	2503	retry:
2504	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)	2504	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
2505	return NULL;	2505	return NULL;
2506		2506
2507	hhd = &ftrace_func_hash[iter->hidx];	2507	hhd = &ftrace_func_hash[iter->hidx];
2508		2508
2509	if (hlist_empty(hhd)) {	2509	if (hlist_empty(hhd)) {
2510	iter->hidx++;	2510	iter->hidx++;
2511	hnd = NULL;	2511	hnd = NULL;
2512	goto retry;	2512	goto retry;
2513	}	2513	}
2514		2514
2515	if (!hnd)	2515	if (!hnd)
2516	hnd = hhd->first;	2516	hnd = hhd->first;
2517	else {	2517	else {
2518	hnd = hnd->next;	2518	hnd = hnd->next;
2519	if (!hnd) {	2519	if (!hnd) {
2520	iter->hidx++;	2520	iter->hidx++;
2521	goto retry;	2521	goto retry;
2522	}	2522	}
2523	}	2523	}
2524		2524
2525	if (WARN_ON_ONCE(!hnd))	2525	if (WARN_ON_ONCE(!hnd))
2526	return NULL;	2526	return NULL;
2527		2527
2528	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);	2528	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
2529		2529
2530	return iter;	2530	return iter;
2531	}	2531	}
2532		2532
2533	static void t_hash_start(struct seq_file m, loff_t *pos)	2533	static void t_hash_start(struct seq_file m, loff_t *pos)
2534	{	2534	{
2535	struct ftrace_iterator *iter = m->private;	2535	struct ftrace_iterator *iter = m->private;
2536	void *p = NULL;	2536	void *p = NULL;
2537	loff_t l;	2537	loff_t l;
2538		2538
2539	if (!(iter->flags & FTRACE_ITER_DO_HASH))	2539	if (!(iter->flags & FTRACE_ITER_DO_HASH))
2540	return NULL;	2540	return NULL;
2541		2541
2542	if (iter->func_pos > *pos)	2542	if (iter->func_pos > *pos)
2543	return NULL;	2543	return NULL;
2544		2544
2545	iter->hidx = 0;	2545	iter->hidx = 0;
2546	for (l = 0; l <= (*pos - iter->func_pos); ) {	2546	for (l = 0; l <= (*pos - iter->func_pos); ) {
2547	p = t_hash_next(m, &l);	2547	p = t_hash_next(m, &l);
2548	if (!p)	2548	if (!p)
2549	break;	2549	break;
2550	}	2550	}
2551	if (!p)	2551	if (!p)
2552	return NULL;	2552	return NULL;
2553		2553
2554	/* Only set this if we have an item */	2554	/* Only set this if we have an item */
2555	iter->flags \|= FTRACE_ITER_HASH;	2555	iter->flags \|= FTRACE_ITER_HASH;
2556		2556
2557	return iter;	2557	return iter;
2558	}	2558	}
2559		2559
2560	static int	2560	static int
2561	t_hash_show(struct seq_file m, struct ftrace_iterator iter)	2561	t_hash_show(struct seq_file m, struct ftrace_iterator iter)
2562	{	2562	{
2563	struct ftrace_func_probe *rec;	2563	struct ftrace_func_probe *rec;
2564		2564
2565	rec = iter->probe;	2565	rec = iter->probe;
2566	if (WARN_ON_ONCE(!rec))	2566	if (WARN_ON_ONCE(!rec))
2567	return -EIO;	2567	return -EIO;
2568		2568
2569	if (rec->ops->print)	2569	if (rec->ops->print)
2570	return rec->ops->print(m, rec->ip, rec->ops, rec->data);	2570	return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2571		2571
2572	seq_printf(m, "%ps:%ps", (void )rec->ip, (void )rec->ops->func);	2572	seq_printf(m, "%ps:%ps", (void )rec->ip, (void )rec->ops->func);
2573		2573
2574	if (rec->data)	2574	if (rec->data)
2575	seq_printf(m, ":%p", rec->data);	2575	seq_printf(m, ":%p", rec->data);
2576	seq_putc(m, '\n');	2576	seq_putc(m, '\n');
2577		2577
2578	return 0;	2578	return 0;
2579	}	2579	}
2580		2580
2581	static void *	2581	static void *
2582	t_next(struct seq_file m, void v, loff_t *pos)	2582	t_next(struct seq_file m, void v, loff_t *pos)
2583	{	2583	{
2584	struct ftrace_iterator *iter = m->private;	2584	struct ftrace_iterator *iter = m->private;
2585	struct ftrace_ops *ops = iter->ops;	2585	struct ftrace_ops *ops = iter->ops;
2586	struct dyn_ftrace *rec = NULL;	2586	struct dyn_ftrace *rec = NULL;
2587		2587
2588	if (unlikely(ftrace_disabled))	2588	if (unlikely(ftrace_disabled))
2589	return NULL;	2589	return NULL;
2590		2590
2591	if (iter->flags & FTRACE_ITER_HASH)	2591	if (iter->flags & FTRACE_ITER_HASH)
2592	return t_hash_next(m, pos);	2592	return t_hash_next(m, pos);
2593		2593
2594	(*pos)++;	2594	(*pos)++;
2595	iter->pos = iter->func_pos = *pos;	2595	iter->pos = iter->func_pos = *pos;
2596		2596
2597	if (iter->flags & FTRACE_ITER_PRINTALL)	2597	if (iter->flags & FTRACE_ITER_PRINTALL)
2598	return t_hash_start(m, pos);	2598	return t_hash_start(m, pos);
2599		2599
2600	retry:	2600	retry:
2601	if (iter->idx >= iter->pg->index) {	2601	if (iter->idx >= iter->pg->index) {
2602	if (iter->pg->next) {	2602	if (iter->pg->next) {
2603	iter->pg = iter->pg->next;	2603	iter->pg = iter->pg->next;
2604	iter->idx = 0;	2604	iter->idx = 0;
2605	goto retry;	2605	goto retry;
2606	}	2606	}
2607	} else {	2607	} else {
2608	rec = &iter->pg->records[iter->idx++];	2608	rec = &iter->pg->records[iter->idx++];
2609	if (((iter->flags & FTRACE_ITER_FILTER) &&	2609	if (((iter->flags & FTRACE_ITER_FILTER) &&
2610	!(ftrace_lookup_ip(ops->filter_hash, rec->ip))) \|\|	2610	!(ftrace_lookup_ip(ops->filter_hash, rec->ip))) \|\|
2611		2611
2612	((iter->flags & FTRACE_ITER_NOTRACE) &&	2612	((iter->flags & FTRACE_ITER_NOTRACE) &&
2613	!ftrace_lookup_ip(ops->notrace_hash, rec->ip)) \|\|	2613	!ftrace_lookup_ip(ops->notrace_hash, rec->ip)) \|\|
2614		2614
2615	((iter->flags & FTRACE_ITER_ENABLED) &&	2615	((iter->flags & FTRACE_ITER_ENABLED) &&
2616	!(rec->flags & FTRACE_FL_ENABLED))) {	2616	!(rec->flags & FTRACE_FL_ENABLED))) {
2617		2617
2618	rec = NULL;	2618	rec = NULL;
2619	goto retry;	2619	goto retry;
2620	}	2620	}
2621	}	2621	}
2622		2622
2623	if (!rec)	2623	if (!rec)
2624	return t_hash_start(m, pos);	2624	return t_hash_start(m, pos);
2625		2625
2626	iter->func = rec;	2626	iter->func = rec;
2627		2627
2628	return iter;	2628	return iter;
2629	}	2629	}
2630		2630
2631	static void reset_iter_read(struct ftrace_iterator *iter)	2631	static void reset_iter_read(struct ftrace_iterator *iter)
2632	{	2632	{
2633	iter->pos = 0;	2633	iter->pos = 0;
2634	iter->func_pos = 0;	2634	iter->func_pos = 0;
2635	iter->flags &= ~(FTRACE_ITER_PRINTALL \| FTRACE_ITER_HASH);	2635	iter->flags &= ~(FTRACE_ITER_PRINTALL \| FTRACE_ITER_HASH);
2636	}	2636	}
2637		2637
2638	static void t_start(struct seq_file m, loff_t *pos)	2638	static void t_start(struct seq_file m, loff_t *pos)
2639	{	2639	{
2640	struct ftrace_iterator *iter = m->private;	2640	struct ftrace_iterator *iter = m->private;
2641	struct ftrace_ops *ops = iter->ops;	2641	struct ftrace_ops *ops = iter->ops;
2642	void *p = NULL;	2642	void *p = NULL;
2643	loff_t l;	2643	loff_t l;
2644		2644
2645	mutex_lock(&ftrace_lock);	2645	mutex_lock(&ftrace_lock);
2646		2646
2647	if (unlikely(ftrace_disabled))	2647	if (unlikely(ftrace_disabled))
2648	return NULL;	2648	return NULL;
2649		2649
2650	/*	2650	/*
2651	* If an lseek was done, then reset and start from beginning.	2651	* If an lseek was done, then reset and start from beginning.
2652	*/	2652	*/
2653	if (*pos < iter->pos)	2653	if (*pos < iter->pos)
2654	reset_iter_read(iter);	2654	reset_iter_read(iter);
2655		2655
2656	/*	2656	/*
2657	* For set_ftrace_filter reading, if we have the filter	2657	* For set_ftrace_filter reading, if we have the filter
2658	* off, we can short cut and just print out that all	2658	* off, we can short cut and just print out that all
2659	* functions are enabled.	2659	* functions are enabled.
2660	*/	2660	*/
2661	if (iter->flags & FTRACE_ITER_FILTER &&	2661	if (iter->flags & FTRACE_ITER_FILTER &&
2662	ftrace_hash_empty(ops->filter_hash)) {	2662	ftrace_hash_empty(ops->filter_hash)) {
2663	if (*pos > 0)	2663	if (*pos > 0)
2664	return t_hash_start(m, pos);	2664	return t_hash_start(m, pos);
2665	iter->flags \|= FTRACE_ITER_PRINTALL;	2665	iter->flags \|= FTRACE_ITER_PRINTALL;
2666	/* reset in case of seek/pread */	2666	/* reset in case of seek/pread */
2667	iter->flags &= ~FTRACE_ITER_HASH;	2667	iter->flags &= ~FTRACE_ITER_HASH;
2668	return iter;	2668	return iter;
2669	}	2669	}
2670		2670
2671	if (iter->flags & FTRACE_ITER_HASH)	2671	if (iter->flags & FTRACE_ITER_HASH)
2672	return t_hash_start(m, pos);	2672	return t_hash_start(m, pos);
2673		2673
2674	/*	2674	/*
2675	* Unfortunately, we need to restart at ftrace_pages_start	2675	* Unfortunately, we need to restart at ftrace_pages_start
2676	* every time we let go of the ftrace_mutex. This is because	2676	* every time we let go of the ftrace_mutex. This is because
2677	* those pointers can change without the lock.	2677	* those pointers can change without the lock.
2678	*/	2678	*/
2679	iter->pg = ftrace_pages_start;	2679	iter->pg = ftrace_pages_start;
2680	iter->idx = 0;	2680	iter->idx = 0;
2681	for (l = 0; l <= *pos; ) {	2681	for (l = 0; l <= *pos; ) {
2682	p = t_next(m, p, &l);	2682	p = t_next(m, p, &l);
2683	if (!p)	2683	if (!p)
2684	break;	2684	break;
2685	}	2685	}
2686		2686
2687	if (!p)	2687	if (!p)
2688	return t_hash_start(m, pos);	2688	return t_hash_start(m, pos);
2689		2689
2690	return iter;	2690	return iter;
2691	}	2691	}
2692		2692
2693	static void t_stop(struct seq_file m, void p)	2693	static void t_stop(struct seq_file m, void p)
2694	{	2694	{
2695	mutex_unlock(&ftrace_lock);	2695	mutex_unlock(&ftrace_lock);
2696	}	2696	}
2697		2697
2698	static int t_show(struct seq_file m, void v)	2698	static int t_show(struct seq_file m, void v)
2699	{	2699	{
2700	struct ftrace_iterator *iter = m->private;	2700	struct ftrace_iterator *iter = m->private;
2701	struct dyn_ftrace *rec;	2701	struct dyn_ftrace *rec;
2702		2702
2703	if (iter->flags & FTRACE_ITER_HASH)	2703	if (iter->flags & FTRACE_ITER_HASH)
2704	return t_hash_show(m, iter);	2704	return t_hash_show(m, iter);
2705		2705
2706	if (iter->flags & FTRACE_ITER_PRINTALL) {	2706	if (iter->flags & FTRACE_ITER_PRINTALL) {
2707	seq_printf(m, "#### all functions enabled ####\n");	2707	seq_printf(m, "#### all functions enabled ####\n");
2708	return 0;	2708	return 0;
2709	}	2709	}
2710		2710
2711	rec = iter->func;	2711	rec = iter->func;
2712		2712
2713	if (!rec)	2713	if (!rec)
2714	return 0;	2714	return 0;
2715		2715
2716	seq_printf(m, "%ps", (void *)rec->ip);	2716	seq_printf(m, "%ps", (void *)rec->ip);
2717	if (iter->flags & FTRACE_ITER_ENABLED)	2717	if (iter->flags & FTRACE_ITER_ENABLED)
2718	seq_printf(m, " (%ld)%s",	2718	seq_printf(m, " (%ld)%s",
2719	rec->flags & ~FTRACE_FL_MASK,	2719	rec->flags & ~FTRACE_FL_MASK,
2720	rec->flags & FTRACE_FL_REGS ? " R" : "");	2720	rec->flags & FTRACE_FL_REGS ? " R" : "");
2721	seq_printf(m, "\n");	2721	seq_printf(m, "\n");
2722		2722
2723	return 0;	2723	return 0;
2724	}	2724	}
2725		2725
2726	static const struct seq_operations show_ftrace_seq_ops = {	2726	static const struct seq_operations show_ftrace_seq_ops = {
2727	.start = t_start,	2727	.start = t_start,
2728	.next = t_next,	2728	.next = t_next,
2729	.stop = t_stop,	2729	.stop = t_stop,
2730	.show = t_show,	2730	.show = t_show,
2731	};	2731	};
2732		2732
2733	static int	2733	static int
2734	ftrace_avail_open(struct inode inode, struct file file)	2734	ftrace_avail_open(struct inode inode, struct file file)
2735	{	2735	{
2736	struct ftrace_iterator *iter;	2736	struct ftrace_iterator *iter;
2737		2737
2738	if (unlikely(ftrace_disabled))	2738	if (unlikely(ftrace_disabled))
2739	return -ENODEV;	2739	return -ENODEV;
2740		2740
2741	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));	2741	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2742	if (iter) {	2742	if (iter) {
2743	iter->pg = ftrace_pages_start;	2743	iter->pg = ftrace_pages_start;
2744	iter->ops = &global_ops;	2744	iter->ops = &global_ops;
2745	}	2745	}
2746		2746
2747	return iter ? 0 : -ENOMEM;	2747	return iter ? 0 : -ENOMEM;
2748	}	2748	}
2749		2749
2750	static int	2750	static int
2751	ftrace_enabled_open(struct inode inode, struct file file)	2751	ftrace_enabled_open(struct inode inode, struct file file)
2752	{	2752	{
2753	struct ftrace_iterator *iter;	2753	struct ftrace_iterator *iter;
2754		2754
2755	if (unlikely(ftrace_disabled))	2755	if (unlikely(ftrace_disabled))
2756	return -ENODEV;	2756	return -ENODEV;
2757		2757
2758	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));	2758	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2759	if (iter) {	2759	if (iter) {
2760	iter->pg = ftrace_pages_start;	2760	iter->pg = ftrace_pages_start;
2761	iter->flags = FTRACE_ITER_ENABLED;	2761	iter->flags = FTRACE_ITER_ENABLED;
2762	iter->ops = &global_ops;	2762	iter->ops = &global_ops;
2763	}	2763	}
2764		2764
2765	return iter ? 0 : -ENOMEM;	2765	return iter ? 0 : -ENOMEM;
2766	}	2766	}
2767		2767
2768	static void ftrace_filter_reset(struct ftrace_hash *hash)	2768	static void ftrace_filter_reset(struct ftrace_hash *hash)
2769	{	2769	{
2770	mutex_lock(&ftrace_lock);	2770	mutex_lock(&ftrace_lock);
2771	ftrace_hash_clear(hash);	2771	ftrace_hash_clear(hash);
2772	mutex_unlock(&ftrace_lock);	2772	mutex_unlock(&ftrace_lock);
2773	}	2773	}
2774		2774
2775	/**	2775	/**
2776	* ftrace_regex_open - initialize function tracer filter files	2776	* ftrace_regex_open - initialize function tracer filter files
2777	* @ops: The ftrace_ops that hold the hash filters	2777	* @ops: The ftrace_ops that hold the hash filters
2778	* @flag: The type of filter to process	2778	* @flag: The type of filter to process
2779	* @inode: The inode, usually passed in to your open routine	2779	* @inode: The inode, usually passed in to your open routine
2780	* @file: The file, usually passed in to your open routine	2780	* @file: The file, usually passed in to your open routine
2781	*	2781	*
2782	* ftrace_regex_open() initializes the filter files for the	2782	* ftrace_regex_open() initializes the filter files for the
2783	* @ops. Depending on @flag it may process the filter hash or	2783	* @ops. Depending on @flag it may process the filter hash or
2784	* the notrace hash of @ops. With this called from the open	2784	* the notrace hash of @ops. With this called from the open
2785	* routine, you can use ftrace_filter_write() for the write	2785	* routine, you can use ftrace_filter_write() for the write
2786	* routine if @flag has FTRACE_ITER_FILTER set, or	2786	* routine if @flag has FTRACE_ITER_FILTER set, or
2787	* ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.	2787	* ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
2788	* tracing_lseek() should be used as the lseek routine, and	2788	* tracing_lseek() should be used as the lseek routine, and
2789	* release must call ftrace_regex_release().	2789	* release must call ftrace_regex_release().
2790	*/	2790	*/
2791	int	2791	int
2792	ftrace_regex_open(struct ftrace_ops *ops, int flag,	2792	ftrace_regex_open(struct ftrace_ops *ops, int flag,
2793	struct inode inode, struct file file)	2793	struct inode inode, struct file file)
2794	{	2794	{
2795	struct ftrace_iterator *iter;	2795	struct ftrace_iterator *iter;
2796	struct ftrace_hash *hash;	2796	struct ftrace_hash *hash;
2797	int ret = 0;	2797	int ret = 0;
2798		2798
2799	ftrace_ops_init(ops);	2799	ftrace_ops_init(ops);
2800		2800
2801	if (unlikely(ftrace_disabled))	2801	if (unlikely(ftrace_disabled))
2802	return -ENODEV;	2802	return -ENODEV;
2803		2803
2804	iter = kzalloc(sizeof(*iter), GFP_KERNEL);	2804	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2805	if (!iter)	2805	if (!iter)
2806	return -ENOMEM;	2806	return -ENOMEM;
2807		2807
2808	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {	2808	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2809	kfree(iter);	2809	kfree(iter);
2810	return -ENOMEM;	2810	return -ENOMEM;
2811	}	2811	}
2812		2812
2813	iter->ops = ops;	2813	iter->ops = ops;
2814	iter->flags = flag;	2814	iter->flags = flag;
2815		2815
2816	mutex_lock(&ops->regex_lock);	2816	mutex_lock(&ops->regex_lock);
2817		2817
2818	if (flag & FTRACE_ITER_NOTRACE)	2818	if (flag & FTRACE_ITER_NOTRACE)
2819	hash = ops->notrace_hash;	2819	hash = ops->notrace_hash;
2820	else	2820	else
2821	hash = ops->filter_hash;	2821	hash = ops->filter_hash;
2822		2822
2823	if (file->f_mode & FMODE_WRITE) {	2823	if (file->f_mode & FMODE_WRITE) {
2824	iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);	2824	iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2825	if (!iter->hash) {	2825	if (!iter->hash) {
2826	trace_parser_put(&iter->parser);	2826	trace_parser_put(&iter->parser);
2827	kfree(iter);	2827	kfree(iter);
2828	ret = -ENOMEM;	2828	ret = -ENOMEM;
2829	goto out_unlock;	2829	goto out_unlock;
2830	}	2830	}
2831	}	2831	}
2832		2832
2833	if ((file->f_mode & FMODE_WRITE) &&	2833	if ((file->f_mode & FMODE_WRITE) &&
2834	(file->f_flags & O_TRUNC))	2834	(file->f_flags & O_TRUNC))
2835	ftrace_filter_reset(iter->hash);	2835	ftrace_filter_reset(iter->hash);
2836		2836
2837	if (file->f_mode & FMODE_READ) {	2837	if (file->f_mode & FMODE_READ) {
2838	iter->pg = ftrace_pages_start;	2838	iter->pg = ftrace_pages_start;
2839		2839
2840	ret = seq_open(file, &show_ftrace_seq_ops);	2840	ret = seq_open(file, &show_ftrace_seq_ops);
2841	if (!ret) {	2841	if (!ret) {
2842	struct seq_file *m = file->private_data;	2842	struct seq_file *m = file->private_data;
2843	m->private = iter;	2843	m->private = iter;
2844	} else {	2844	} else {
2845	/* Failed */	2845	/* Failed */
2846	free_ftrace_hash(iter->hash);	2846	free_ftrace_hash(iter->hash);
2847	trace_parser_put(&iter->parser);	2847	trace_parser_put(&iter->parser);
2848	kfree(iter);	2848	kfree(iter);
2849	}	2849	}
2850	} else	2850	} else
2851	file->private_data = iter;	2851	file->private_data = iter;
2852		2852
2853	out_unlock:	2853	out_unlock:
2854	mutex_unlock(&ops->regex_lock);	2854	mutex_unlock(&ops->regex_lock);
2855		2855
2856	return ret;	2856	return ret;
2857	}	2857	}
2858		2858
2859	static int	2859	static int
2860	ftrace_filter_open(struct inode inode, struct file file)	2860	ftrace_filter_open(struct inode inode, struct file file)
2861	{	2861	{
2862	struct ftrace_ops *ops = inode->i_private;	2862	struct ftrace_ops *ops = inode->i_private;
2863		2863
2864	return ftrace_regex_open(ops,	2864	return ftrace_regex_open(ops,
2865	FTRACE_ITER_FILTER \| FTRACE_ITER_DO_HASH,	2865	FTRACE_ITER_FILTER \| FTRACE_ITER_DO_HASH,
2866	inode, file);	2866	inode, file);
2867	}	2867	}
2868		2868
2869	static int	2869	static int
2870	ftrace_notrace_open(struct inode inode, struct file file)	2870	ftrace_notrace_open(struct inode inode, struct file file)
2871	{	2871	{
2872	struct ftrace_ops *ops = inode->i_private;	2872	struct ftrace_ops *ops = inode->i_private;
2873		2873
2874	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,	2874	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
2875	inode, file);	2875	inode, file);
2876	}	2876	}
2877		2877
2878	static int ftrace_match(char str, char regex, int len, int type)	2878	static int ftrace_match(char str, char regex, int len, int type)
2879	{	2879	{
2880	int matched = 0;	2880	int matched = 0;
2881	int slen;	2881	int slen;
2882		2882
2883	switch (type) {	2883	switch (type) {
2884	case MATCH_FULL:	2884	case MATCH_FULL:
2885	if (strcmp(str, regex) == 0)	2885	if (strcmp(str, regex) == 0)
2886	matched = 1;	2886	matched = 1;
2887	break;	2887	break;
2888	case MATCH_FRONT_ONLY:	2888	case MATCH_FRONT_ONLY:
2889	if (strncmp(str, regex, len) == 0)	2889	if (strncmp(str, regex, len) == 0)
2890	matched = 1;	2890	matched = 1;
2891	break;	2891	break;
2892	case MATCH_MIDDLE_ONLY:	2892	case MATCH_MIDDLE_ONLY:
2893	if (strstr(str, regex))	2893	if (strstr(str, regex))
2894	matched = 1;	2894	matched = 1;
2895	break;	2895	break;
2896	case MATCH_END_ONLY:	2896	case MATCH_END_ONLY:
2897	slen = strlen(str);	2897	slen = strlen(str);
2898	if (slen >= len && memcmp(str + slen - len, regex, len) == 0)	2898	if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2899	matched = 1;	2899	matched = 1;
2900	break;	2900	break;
2901	}	2901	}
2902		2902
2903	return matched;	2903	return matched;
2904	}	2904	}
2905		2905
2906	static int	2906	static int
2907	enter_record(struct ftrace_hash hash, struct dyn_ftrace rec, int not)	2907	enter_record(struct ftrace_hash hash, struct dyn_ftrace rec, int not)
2908	{	2908	{
2909	struct ftrace_func_entry *entry;	2909	struct ftrace_func_entry *entry;
2910	int ret = 0;	2910	int ret = 0;
2911		2911
2912	entry = ftrace_lookup_ip(hash, rec->ip);	2912	entry = ftrace_lookup_ip(hash, rec->ip);
2913	if (not) {	2913	if (not) {
2914	/* Do nothing if it doesn't exist */	2914	/* Do nothing if it doesn't exist */
2915	if (!entry)	2915	if (!entry)
2916	return 0;	2916	return 0;
2917		2917
2918	free_hash_entry(hash, entry);	2918	free_hash_entry(hash, entry);
2919	} else {	2919	} else {
2920	/* Do nothing if it exists */	2920	/* Do nothing if it exists */
2921	if (entry)	2921	if (entry)
2922	return 0;	2922	return 0;
2923		2923
2924	ret = add_hash_entry(hash, rec->ip);	2924	ret = add_hash_entry(hash, rec->ip);
2925	}	2925	}
2926	return ret;	2926	return ret;
2927	}	2927	}
2928		2928
2929	static int	2929	static int
2930	ftrace_match_record(struct dyn_ftrace rec, char mod,	2930	ftrace_match_record(struct dyn_ftrace rec, char mod,
2931	char *regex, int len, int type)	2931	char *regex, int len, int type)
2932	{	2932	{
2933	char str[KSYM_SYMBOL_LEN];	2933	char str[KSYM_SYMBOL_LEN];
2934	char *modname;	2934	char *modname;
2935		2935
2936	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);	2936	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2937		2937
2938	if (mod) {	2938	if (mod) {
2939	/* module lookup requires matching the module */	2939	/* module lookup requires matching the module */
2940	if (!modname \|\| strcmp(modname, mod))	2940	if (!modname \|\| strcmp(modname, mod))
2941	return 0;	2941	return 0;
2942		2942
2943	/* blank search means to match all funcs in the mod */	2943	/* blank search means to match all funcs in the mod */
2944	if (!len)	2944	if (!len)
2945	return 1;	2945	return 1;
2946	}	2946	}
2947		2947
2948	return ftrace_match(str, regex, len, type);	2948	return ftrace_match(str, regex, len, type);
2949	}	2949	}
2950		2950
2951	static int	2951	static int
2952	match_records(struct ftrace_hash hash, char buff,	2952	match_records(struct ftrace_hash hash, char buff,
2953	int len, char *mod, int not)	2953	int len, char *mod, int not)
2954	{	2954	{
2955	unsigned search_len = 0;	2955	unsigned search_len = 0;
2956	struct ftrace_page *pg;	2956	struct ftrace_page *pg;
2957	struct dyn_ftrace *rec;	2957	struct dyn_ftrace *rec;
2958	int type = MATCH_FULL;	2958	int type = MATCH_FULL;
2959	char *search = buff;	2959	char *search = buff;
2960	int found = 0;	2960	int found = 0;
2961	int ret;	2961	int ret;
2962		2962
2963	if (len) {	2963	if (len) {
2964	type = filter_parse_regex(buff, len, &search, &not);	2964	type = filter_parse_regex(buff, len, &search, &not);
2965	search_len = strlen(search);	2965	search_len = strlen(search);
2966	}	2966	}
2967		2967
2968	mutex_lock(&ftrace_lock);	2968	mutex_lock(&ftrace_lock);
2969		2969
2970	if (unlikely(ftrace_disabled))	2970	if (unlikely(ftrace_disabled))
2971	goto out_unlock;	2971	goto out_unlock;
2972		2972
2973	do_for_each_ftrace_rec(pg, rec) {	2973	do_for_each_ftrace_rec(pg, rec) {
2974	if (ftrace_match_record(rec, mod, search, search_len, type)) {	2974	if (ftrace_match_record(rec, mod, search, search_len, type)) {
2975	ret = enter_record(hash, rec, not);	2975	ret = enter_record(hash, rec, not);
2976	if (ret < 0) {	2976	if (ret < 0) {
2977	found = ret;	2977	found = ret;
2978	goto out_unlock;	2978	goto out_unlock;
2979	}	2979	}
2980	found = 1;	2980	found = 1;
2981	}	2981	}
2982	} while_for_each_ftrace_rec();	2982	} while_for_each_ftrace_rec();
2983	out_unlock:	2983	out_unlock:
2984	mutex_unlock(&ftrace_lock);	2984	mutex_unlock(&ftrace_lock);
2985		2985
2986	return found;	2986	return found;
2987	}	2987	}
2988		2988
2989	static int	2989	static int
2990	ftrace_match_records(struct ftrace_hash hash, char buff, int len)	2990	ftrace_match_records(struct ftrace_hash hash, char buff, int len)
2991	{	2991	{
2992	return match_records(hash, buff, len, NULL, 0);	2992	return match_records(hash, buff, len, NULL, 0);
2993	}	2993	}
2994		2994
2995	static int	2995	static int
2996	ftrace_match_module_records(struct ftrace_hash hash, char buff, char *mod)	2996	ftrace_match_module_records(struct ftrace_hash hash, char buff, char *mod)
2997	{	2997	{
2998	int not = 0;	2998	int not = 0;
2999		2999
3000	/* blank or '' mean the same /	3000	/* blank or '' mean the same /
3001	if (strcmp(buff, "*") == 0)	3001	if (strcmp(buff, "*") == 0)
3002	buff[0] = 0;	3002	buff[0] = 0;
3003		3003
3004	/* handle the case of 'dont filter this module' */	3004	/* handle the case of 'dont filter this module' */
3005	if (strcmp(buff, "!") == 0 \|\| strcmp(buff, "!*") == 0) {	3005	if (strcmp(buff, "!") == 0 \|\| strcmp(buff, "!*") == 0) {
3006	buff[0] = 0;	3006	buff[0] = 0;
3007	not = 1;	3007	not = 1;
3008	}	3008	}
3009		3009
3010	return match_records(hash, buff, strlen(buff), mod, not);	3010	return match_records(hash, buff, strlen(buff), mod, not);
3011	}	3011	}
3012		3012
3013	/*	3013	/*
3014	* We register the module command as a template to show others how	3014	* We register the module command as a template to show others how
3015	* to register the a command as well.	3015	* to register the a command as well.
3016	*/	3016	*/
3017		3017
3018	static int	3018	static int
3019	ftrace_mod_callback(struct ftrace_hash *hash,	3019	ftrace_mod_callback(struct ftrace_hash *hash,
3020	char func, char cmd, char *param, int enable)	3020	char func, char cmd, char *param, int enable)
3021	{	3021	{
3022	char *mod;	3022	char *mod;
3023	int ret = -EINVAL;	3023	int ret = -EINVAL;
3024		3024
3025	/*	3025	/*
3026	* cmd == 'mod' because we only registered this func	3026	* cmd == 'mod' because we only registered this func
3027	* for the 'mod' ftrace_func_command.	3027	* for the 'mod' ftrace_func_command.
3028	* But if you register one func with multiple commands,	3028	* But if you register one func with multiple commands,
3029	* you can tell which command was used by the cmd	3029	* you can tell which command was used by the cmd
3030	* parameter.	3030	* parameter.
3031	*/	3031	*/
3032		3032
3033	/* we must have a module name */	3033	/* we must have a module name */
3034	if (!param)	3034	if (!param)
3035	return ret;	3035	return ret;
3036		3036
3037	mod = strsep(&param, ":");	3037	mod = strsep(&param, ":");
3038	if (!strlen(mod))	3038	if (!strlen(mod))
3039	return ret;	3039	return ret;
3040		3040
3041	ret = ftrace_match_module_records(hash, func, mod);	3041	ret = ftrace_match_module_records(hash, func, mod);
3042	if (!ret)	3042	if (!ret)
3043	ret = -EINVAL;	3043	ret = -EINVAL;
3044	if (ret < 0)	3044	if (ret < 0)
3045	return ret;	3045	return ret;
3046		3046
3047	return 0;	3047	return 0;
3048	}	3048	}
3049		3049
3050	static struct ftrace_func_command ftrace_mod_cmd = {	3050	static struct ftrace_func_command ftrace_mod_cmd = {
3051	.name = "mod",	3051	.name = "mod",
3052	.func = ftrace_mod_callback,	3052	.func = ftrace_mod_callback,
3053	};	3053	};
3054		3054
3055	static int __init ftrace_mod_cmd_init(void)	3055	static int __init ftrace_mod_cmd_init(void)
3056	{	3056	{
3057	return register_ftrace_command(&ftrace_mod_cmd);	3057	return register_ftrace_command(&ftrace_mod_cmd);
3058	}	3058	}
3059	core_initcall(ftrace_mod_cmd_init);	3059	core_initcall(ftrace_mod_cmd_init);
3060		3060
3061	static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,	3061	static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
3062	struct ftrace_ops op, struct pt_regs pt_regs)	3062	struct ftrace_ops op, struct pt_regs pt_regs)
3063	{	3063	{
3064	struct ftrace_func_probe *entry;	3064	struct ftrace_func_probe *entry;
3065	struct hlist_head *hhd;	3065	struct hlist_head *hhd;
3066	unsigned long key;	3066	unsigned long key;
3067		3067
3068	key = hash_long(ip, FTRACE_HASH_BITS);	3068	key = hash_long(ip, FTRACE_HASH_BITS);
3069		3069
3070	hhd = &ftrace_func_hash[key];	3070	hhd = &ftrace_func_hash[key];
3071		3071
3072	if (hlist_empty(hhd))	3072	if (hlist_empty(hhd))
3073	return;	3073	return;
3074		3074
3075	/*	3075	/*
3076	* Disable preemption for these calls to prevent a RCU grace	3076	* Disable preemption for these calls to prevent a RCU grace
3077	* period. This syncs the hash iteration and freeing of items	3077	* period. This syncs the hash iteration and freeing of items
3078	* on the hash. rcu_read_lock is too dangerous here.	3078	* on the hash. rcu_read_lock is too dangerous here.
3079	*/	3079	*/
3080	preempt_disable_notrace();	3080	preempt_disable_notrace();
3081	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {	3081	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
3082	if (entry->ip == ip)	3082	if (entry->ip == ip)
3083	entry->ops->func(ip, parent_ip, &entry->data);	3083	entry->ops->func(ip, parent_ip, &entry->data);
3084	}	3084	}
3085	preempt_enable_notrace();	3085	preempt_enable_notrace();
3086	}	3086	}
3087		3087
3088	static struct ftrace_ops trace_probe_ops __read_mostly =	3088	static struct ftrace_ops trace_probe_ops __read_mostly =
3089	{	3089	{
3090	.func = function_trace_probe_call,	3090	.func = function_trace_probe_call,
3091	.flags = FTRACE_OPS_FL_INITIALIZED,	3091	.flags = FTRACE_OPS_FL_INITIALIZED,
3092	INIT_REGEX_LOCK(trace_probe_ops)	3092	INIT_REGEX_LOCK(trace_probe_ops)
3093	};	3093	};
3094		3094
3095	static int ftrace_probe_registered;	3095	static int ftrace_probe_registered;
3096		3096
3097	static void __enable_ftrace_function_probe(void)	3097	static void __enable_ftrace_function_probe(void)
3098	{	3098	{
3099	int ret;	3099	int ret;
3100	int i;	3100	int i;
3101		3101
3102	if (ftrace_probe_registered) {	3102	if (ftrace_probe_registered) {
3103	/* still need to update the function call sites */	3103	/* still need to update the function call sites */
3104	if (ftrace_enabled)	3104	if (ftrace_enabled)
3105	ftrace_run_update_code(FTRACE_UPDATE_CALLS);	3105	ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3106	return;	3106	return;
3107	}	3107	}
3108		3108
3109	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {	3109	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3110	struct hlist_head *hhd = &ftrace_func_hash[i];	3110	struct hlist_head *hhd = &ftrace_func_hash[i];
3111	if (hhd->first)	3111	if (hhd->first)
3112	break;	3112	break;
3113	}	3113	}
3114	/* Nothing registered? */	3114	/* Nothing registered? */
3115	if (i == FTRACE_FUNC_HASHSIZE)	3115	if (i == FTRACE_FUNC_HASHSIZE)
3116	return;	3116	return;
3117		3117
3118	ret = ftrace_startup(&trace_probe_ops, 0);	3118	ret = ftrace_startup(&trace_probe_ops, 0);
3119		3119
3120	ftrace_probe_registered = 1;	3120	ftrace_probe_registered = 1;
3121	}	3121	}
3122		3122
3123	static void __disable_ftrace_function_probe(void)	3123	static void __disable_ftrace_function_probe(void)
3124	{	3124	{
3125	int i;	3125	int i;
3126		3126
3127	if (!ftrace_probe_registered)	3127	if (!ftrace_probe_registered)
3128	return;	3128	return;
3129		3129
3130	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {	3130	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3131	struct hlist_head *hhd = &ftrace_func_hash[i];	3131	struct hlist_head *hhd = &ftrace_func_hash[i];
3132	if (hhd->first)	3132	if (hhd->first)
3133	return;	3133	return;
3134	}	3134	}
3135		3135
3136	/* no more funcs left */	3136	/* no more funcs left */
3137	ftrace_shutdown(&trace_probe_ops, 0);	3137	ftrace_shutdown(&trace_probe_ops, 0);
3138		3138
3139	ftrace_probe_registered = 0;	3139	ftrace_probe_registered = 0;
3140	}	3140	}
3141		3141
3142		3142
3143	static void ftrace_free_entry(struct ftrace_func_probe *entry)	3143	static void ftrace_free_entry(struct ftrace_func_probe *entry)
3144	{	3144	{
3145	if (entry->ops->free)	3145	if (entry->ops->free)
3146	entry->ops->free(entry->ops, entry->ip, &entry->data);	3146	entry->ops->free(entry->ops, entry->ip, &entry->data);
3147	kfree(entry);	3147	kfree(entry);
3148	}	3148	}
3149		3149
3150	int	3150	int
3151	register_ftrace_function_probe(char glob, struct ftrace_probe_ops ops,	3151	register_ftrace_function_probe(char glob, struct ftrace_probe_ops ops,
3152	void *data)	3152	void *data)
3153	{	3153	{
3154	struct ftrace_func_probe *entry;	3154	struct ftrace_func_probe *entry;
3155	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;	3155	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
3156	struct ftrace_hash *hash;	3156	struct ftrace_hash *hash;
3157	struct ftrace_page *pg;	3157	struct ftrace_page *pg;
3158	struct dyn_ftrace *rec;	3158	struct dyn_ftrace *rec;
3159	int type, len, not;	3159	int type, len, not;
3160	unsigned long key;	3160	unsigned long key;
3161	int count = 0;	3161	int count = 0;
3162	char *search;	3162	char *search;
3163	int ret;	3163	int ret;
3164		3164
3165	type = filter_parse_regex(glob, strlen(glob), &search, &not);	3165	type = filter_parse_regex(glob, strlen(glob), &search, &not);
3166	len = strlen(search);	3166	len = strlen(search);
3167		3167
3168	/* we do not support '!' for function probes */	3168	/* we do not support '!' for function probes */
3169	if (WARN_ON(not))	3169	if (WARN_ON(not))
3170	return -EINVAL;	3170	return -EINVAL;
3171		3171
3172	mutex_lock(&trace_probe_ops.regex_lock);	3172	mutex_lock(&trace_probe_ops.regex_lock);
3173		3173
3174	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);	3174	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3175	if (!hash) {	3175	if (!hash) {
3176	count = -ENOMEM;	3176	count = -ENOMEM;
3177	goto out;	3177	goto out;
3178	}	3178	}
3179		3179
3180	if (unlikely(ftrace_disabled)) {	3180	if (unlikely(ftrace_disabled)) {
3181	count = -ENODEV;	3181	count = -ENODEV;
3182	goto out;	3182	goto out;
3183	}	3183	}
3184		3184
3185	mutex_lock(&ftrace_lock);	3185	mutex_lock(&ftrace_lock);
3186		3186
3187	do_for_each_ftrace_rec(pg, rec) {	3187	do_for_each_ftrace_rec(pg, rec) {
3188		3188
3189	if (!ftrace_match_record(rec, NULL, search, len, type))	3189	if (!ftrace_match_record(rec, NULL, search, len, type))
3190	continue;	3190	continue;
3191		3191
3192	entry = kmalloc(sizeof(*entry), GFP_KERNEL);	3192	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
3193	if (!entry) {	3193	if (!entry) {
3194	/* If we did not process any, then return error */	3194	/* If we did not process any, then return error */
3195	if (!count)	3195	if (!count)
3196	count = -ENOMEM;	3196	count = -ENOMEM;
3197	goto out_unlock;	3197	goto out_unlock;
3198	}	3198	}
3199		3199
3200	count++;	3200	count++;
3201		3201
3202	entry->data = data;	3202	entry->data = data;
3203		3203
3204	/*	3204	/*
3205	* The caller might want to do something special	3205	* The caller might want to do something special
3206	* for each function we find. We call the callback	3206	* for each function we find. We call the callback
3207	* to give the caller an opportunity to do so.	3207	* to give the caller an opportunity to do so.
3208	*/	3208	*/
3209	if (ops->init) {	3209	if (ops->init) {
3210	if (ops->init(ops, rec->ip, &entry->data) < 0) {	3210	if (ops->init(ops, rec->ip, &entry->data) < 0) {
3211	/* caller does not like this func */	3211	/* caller does not like this func */
3212	kfree(entry);	3212	kfree(entry);
3213	continue;	3213	continue;
3214	}	3214	}
3215	}	3215	}
3216		3216
3217	ret = enter_record(hash, rec, 0);	3217	ret = enter_record(hash, rec, 0);
3218	if (ret < 0) {	3218	if (ret < 0) {
3219	kfree(entry);	3219	kfree(entry);
3220	count = ret;	3220	count = ret;
3221	goto out_unlock;	3221	goto out_unlock;
3222	}	3222	}
3223		3223
3224	entry->ops = ops;	3224	entry->ops = ops;
3225	entry->ip = rec->ip;	3225	entry->ip = rec->ip;
3226		3226
3227	key = hash_long(entry->ip, FTRACE_HASH_BITS);	3227	key = hash_long(entry->ip, FTRACE_HASH_BITS);
3228	hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);	3228	hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
3229		3229
3230	} while_for_each_ftrace_rec();	3230	} while_for_each_ftrace_rec();
3231		3231
3232	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);	3232	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3233	if (ret < 0)	3233	if (ret < 0)
3234	count = ret;	3234	count = ret;
3235		3235
3236	__enable_ftrace_function_probe();	3236	__enable_ftrace_function_probe();
3237		3237
3238	out_unlock:	3238	out_unlock:
3239	mutex_unlock(&ftrace_lock);	3239	mutex_unlock(&ftrace_lock);
3240	out:	3240	out:
3241	mutex_unlock(&trace_probe_ops.regex_lock);	3241	mutex_unlock(&trace_probe_ops.regex_lock);
3242	free_ftrace_hash(hash);	3242	free_ftrace_hash(hash);
3243		3243
3244	return count;	3244	return count;
3245	}	3245	}
3246		3246
3247	enum {	3247	enum {
3248	PROBE_TEST_FUNC = 1,	3248	PROBE_TEST_FUNC = 1,
3249	PROBE_TEST_DATA = 2	3249	PROBE_TEST_DATA = 2
3250	};	3250	};
3251		3251
3252	static void	3252	static void
3253	__unregister_ftrace_function_probe(char glob, struct ftrace_probe_ops ops,	3253	__unregister_ftrace_function_probe(char glob, struct ftrace_probe_ops ops,
3254	void *data, int flags)	3254	void *data, int flags)
3255	{	3255	{
3256	struct ftrace_func_entry *rec_entry;	3256	struct ftrace_func_entry *rec_entry;
3257	struct ftrace_func_probe *entry;	3257	struct ftrace_func_probe *entry;
3258	struct ftrace_func_probe *p;	3258	struct ftrace_func_probe *p;
3259	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;	3259	struct ftrace_hash **orig_hash = &trace_probe_ops.filter_hash;
3260	struct list_head free_list;	3260	struct list_head free_list;
3261	struct ftrace_hash *hash;	3261	struct ftrace_hash *hash;
3262	struct hlist_node *tmp;	3262	struct hlist_node *tmp;
3263	char str[KSYM_SYMBOL_LEN];	3263	char str[KSYM_SYMBOL_LEN];
3264	int type = MATCH_FULL;	3264	int type = MATCH_FULL;
3265	int i, len = 0;	3265	int i, len = 0;
3266	char *search;	3266	char *search;
3267		3267
3268	if (glob && (strcmp(glob, "*") == 0 \|\| !strlen(glob)))	3268	if (glob && (strcmp(glob, "*") == 0 \|\| !strlen(glob)))
3269	glob = NULL;	3269	glob = NULL;
3270	else if (glob) {	3270	else if (glob) {
3271	int not;	3271	int not;
3272		3272
3273	type = filter_parse_regex(glob, strlen(glob), &search, &not);	3273	type = filter_parse_regex(glob, strlen(glob), &search, &not);
3274	len = strlen(search);	3274	len = strlen(search);
3275		3275
3276	/* we do not support '!' for function probes */	3276	/* we do not support '!' for function probes */
3277	if (WARN_ON(not))	3277	if (WARN_ON(not))
3278	return;	3278	return;
3279	}	3279	}
3280		3280
3281	mutex_lock(&trace_probe_ops.regex_lock);	3281	mutex_lock(&trace_probe_ops.regex_lock);
3282		3282
3283	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);	3283	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3284	if (!hash)	3284	if (!hash)
3285	/* Hmm, should report this somehow */	3285	/* Hmm, should report this somehow */
3286	goto out_unlock;	3286	goto out_unlock;
3287		3287
3288	INIT_LIST_HEAD(&free_list);	3288	INIT_LIST_HEAD(&free_list);
3289		3289
3290	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {	3290	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3291	struct hlist_head *hhd = &ftrace_func_hash[i];	3291	struct hlist_head *hhd = &ftrace_func_hash[i];
3292		3292
3293	hlist_for_each_entry_safe(entry, tmp, hhd, node) {	3293	hlist_for_each_entry_safe(entry, tmp, hhd, node) {
3294		3294
3295	/* break up if statements for readability */	3295	/* break up if statements for readability */
3296	if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)	3296	if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3297	continue;	3297	continue;
3298		3298
3299	if ((flags & PROBE_TEST_DATA) && entry->data != data)	3299	if ((flags & PROBE_TEST_DATA) && entry->data != data)
3300	continue;	3300	continue;
3301		3301
3302	/* do this last, since it is the most expensive */	3302	/* do this last, since it is the most expensive */
3303	if (glob) {	3303	if (glob) {
3304	kallsyms_lookup(entry->ip, NULL, NULL,	3304	kallsyms_lookup(entry->ip, NULL, NULL,
3305	NULL, str);	3305	NULL, str);
3306	if (!ftrace_match(str, glob, len, type))	3306	if (!ftrace_match(str, glob, len, type))
3307	continue;	3307	continue;
3308	}	3308	}
3309		3309
3310	rec_entry = ftrace_lookup_ip(hash, entry->ip);	3310	rec_entry = ftrace_lookup_ip(hash, entry->ip);
3311	/* It is possible more than one entry had this ip */	3311	/* It is possible more than one entry had this ip */
3312	if (rec_entry)	3312	if (rec_entry)
3313	free_hash_entry(hash, rec_entry);	3313	free_hash_entry(hash, rec_entry);
3314		3314
3315	hlist_del_rcu(&entry->node);	3315	hlist_del_rcu(&entry->node);
3316	list_add(&entry->free_list, &free_list);	3316	list_add(&entry->free_list, &free_list);
3317	}	3317	}
3318	}	3318	}
3319	mutex_lock(&ftrace_lock);	3319	mutex_lock(&ftrace_lock);
3320	__disable_ftrace_function_probe();	3320	__disable_ftrace_function_probe();
3321	/*	3321	/*
3322	* Remove after the disable is called. Otherwise, if the last	3322	* Remove after the disable is called. Otherwise, if the last
3323	* probe is removed, a null hash means all enabled.	3323	* probe is removed, a null hash means all enabled.
3324	*/	3324	*/
3325	ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);	3325	ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3326	synchronize_sched();	3326	synchronize_sched();
3327	list_for_each_entry_safe(entry, p, &free_list, free_list) {	3327	list_for_each_entry_safe(entry, p, &free_list, free_list) {
3328	list_del(&entry->free_list);	3328	list_del(&entry->free_list);
3329	ftrace_free_entry(entry);	3329	ftrace_free_entry(entry);
3330	}	3330	}
3331	mutex_unlock(&ftrace_lock);	3331	mutex_unlock(&ftrace_lock);
3332		3332
3333	out_unlock:	3333	out_unlock:
3334	mutex_unlock(&trace_probe_ops.regex_lock);	3334	mutex_unlock(&trace_probe_ops.regex_lock);
3335	free_ftrace_hash(hash);	3335	free_ftrace_hash(hash);
3336	}	3336	}
3337		3337
3338	void	3338	void
3339	unregister_ftrace_function_probe(char glob, struct ftrace_probe_ops ops,	3339	unregister_ftrace_function_probe(char glob, struct ftrace_probe_ops ops,
3340	void *data)	3340	void *data)
3341	{	3341	{
3342	__unregister_ftrace_function_probe(glob, ops, data,	3342	__unregister_ftrace_function_probe(glob, ops, data,
3343	PROBE_TEST_FUNC \| PROBE_TEST_DATA);	3343	PROBE_TEST_FUNC \| PROBE_TEST_DATA);
3344	}	3344	}
3345		3345
3346	void	3346	void
3347	unregister_ftrace_function_probe_func(char glob, struct ftrace_probe_ops ops)	3347	unregister_ftrace_function_probe_func(char glob, struct ftrace_probe_ops ops)
3348	{	3348	{
3349	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);	3349	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
3350	}	3350	}
3351		3351
3352	void unregister_ftrace_function_probe_all(char *glob)	3352	void unregister_ftrace_function_probe_all(char *glob)
3353	{	3353	{
3354	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);	3354	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
3355	}	3355	}
3356		3356
3357	static LIST_HEAD(ftrace_commands);	3357	static LIST_HEAD(ftrace_commands);
3358	static DEFINE_MUTEX(ftrace_cmd_mutex);	3358	static DEFINE_MUTEX(ftrace_cmd_mutex);
3359		3359
3360	/*	3360	/*
3361	* Currently we only register ftrace commands from __init, so mark this	3361	* Currently we only register ftrace commands from __init, so mark this
3362	* __init too.	3362	* __init too.
3363	*/	3363	*/
3364	__init int register_ftrace_command(struct ftrace_func_command *cmd)	3364	__init int register_ftrace_command(struct ftrace_func_command *cmd)
3365	{	3365	{
3366	struct ftrace_func_command *p;	3366	struct ftrace_func_command *p;
3367	int ret = 0;	3367	int ret = 0;
3368		3368
3369	mutex_lock(&ftrace_cmd_mutex);	3369	mutex_lock(&ftrace_cmd_mutex);
3370	list_for_each_entry(p, &ftrace_commands, list) {	3370	list_for_each_entry(p, &ftrace_commands, list) {
3371	if (strcmp(cmd->name, p->name) == 0) {	3371	if (strcmp(cmd->name, p->name) == 0) {
3372	ret = -EBUSY;	3372	ret = -EBUSY;
3373	goto out_unlock;	3373	goto out_unlock;
3374	}	3374	}
3375	}	3375	}
3376	list_add(&cmd->list, &ftrace_commands);	3376	list_add(&cmd->list, &ftrace_commands);
3377	out_unlock:	3377	out_unlock:
3378	mutex_unlock(&ftrace_cmd_mutex);	3378	mutex_unlock(&ftrace_cmd_mutex);
3379		3379
3380	return ret;	3380	return ret;
3381	}	3381	}
3382		3382
3383	/*	3383	/*
3384	* Currently we only unregister ftrace commands from __init, so mark	3384	* Currently we only unregister ftrace commands from __init, so mark
3385	* this __init too.	3385	* this __init too.
3386	*/	3386	*/
3387	__init int unregister_ftrace_command(struct ftrace_func_command *cmd)	3387	__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
3388	{	3388	{
3389	struct ftrace_func_command p, n;	3389	struct ftrace_func_command p, n;
3390	int ret = -ENODEV;	3390	int ret = -ENODEV;
3391		3391
3392	mutex_lock(&ftrace_cmd_mutex);	3392	mutex_lock(&ftrace_cmd_mutex);
3393	list_for_each_entry_safe(p, n, &ftrace_commands, list) {	3393	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
3394	if (strcmp(cmd->name, p->name) == 0) {	3394	if (strcmp(cmd->name, p->name) == 0) {
3395	ret = 0;	3395	ret = 0;
3396	list_del_init(&p->list);	3396	list_del_init(&p->list);
3397	goto out_unlock;	3397	goto out_unlock;
3398	}	3398	}
3399	}	3399	}
3400	out_unlock:	3400	out_unlock:
3401	mutex_unlock(&ftrace_cmd_mutex);	3401	mutex_unlock(&ftrace_cmd_mutex);
3402		3402
3403	return ret;	3403	return ret;
3404	}	3404	}
3405		3405
3406	static int ftrace_process_regex(struct ftrace_hash *hash,	3406	static int ftrace_process_regex(struct ftrace_hash *hash,
3407	char *buff, int len, int enable)	3407	char *buff, int len, int enable)
3408	{	3408	{
3409	char func, command, *next = buff;	3409	char func, command, *next = buff;
3410	struct ftrace_func_command *p;	3410	struct ftrace_func_command *p;
3411	int ret = -EINVAL;	3411	int ret = -EINVAL;
3412		3412
3413	func = strsep(&next, ":");	3413	func = strsep(&next, ":");
3414		3414
3415	if (!next) {	3415	if (!next) {
3416	ret = ftrace_match_records(hash, func, len);	3416	ret = ftrace_match_records(hash, func, len);
3417	if (!ret)	3417	if (!ret)
3418	ret = -EINVAL;	3418	ret = -EINVAL;
3419	if (ret < 0)	3419	if (ret < 0)
3420	return ret;	3420	return ret;
3421	return 0;	3421	return 0;
3422	}	3422	}
3423		3423
3424	/* command found */	3424	/* command found */
3425		3425
3426	command = strsep(&next, ":");	3426	command = strsep(&next, ":");
3427		3427
3428	mutex_lock(&ftrace_cmd_mutex);	3428	mutex_lock(&ftrace_cmd_mutex);
3429	list_for_each_entry(p, &ftrace_commands, list) {	3429	list_for_each_entry(p, &ftrace_commands, list) {
3430	if (strcmp(p->name, command) == 0) {	3430	if (strcmp(p->name, command) == 0) {
3431	ret = p->func(hash, func, command, next, enable);	3431	ret = p->func(hash, func, command, next, enable);
3432	goto out_unlock;	3432	goto out_unlock;
3433	}	3433	}
3434	}	3434	}
3435	out_unlock:	3435	out_unlock:
3436	mutex_unlock(&ftrace_cmd_mutex);	3436	mutex_unlock(&ftrace_cmd_mutex);
3437		3437
3438	return ret;	3438	return ret;
3439	}	3439	}
3440		3440
3441	static ssize_t	3441	static ssize_t
3442	ftrace_regex_write(struct file file, const char __user ubuf,	3442	ftrace_regex_write(struct file file, const char __user ubuf,
3443	size_t cnt, loff_t *ppos, int enable)	3443	size_t cnt, loff_t *ppos, int enable)
3444	{	3444	{
3445	struct ftrace_iterator *iter;	3445	struct ftrace_iterator *iter;
3446	struct trace_parser *parser;	3446	struct trace_parser *parser;
3447	ssize_t ret, read;	3447	ssize_t ret, read;
3448		3448
3449	if (!cnt)	3449	if (!cnt)
3450	return 0;	3450	return 0;
3451		3451
3452	if (file->f_mode & FMODE_READ) {	3452	if (file->f_mode & FMODE_READ) {
3453	struct seq_file *m = file->private_data;	3453	struct seq_file *m = file->private_data;
3454	iter = m->private;	3454	iter = m->private;
3455	} else	3455	} else
3456	iter = file->private_data;	3456	iter = file->private_data;
3457		3457
3458	if (unlikely(ftrace_disabled))	3458	if (unlikely(ftrace_disabled))
3459	return -ENODEV;	3459	return -ENODEV;
3460		3460
3461	/* iter->hash is a local copy, so we don't need regex_lock */	3461	/* iter->hash is a local copy, so we don't need regex_lock */
3462		3462
3463	parser = &iter->parser;	3463	parser = &iter->parser;
3464	read = trace_get_user(parser, ubuf, cnt, ppos);	3464	read = trace_get_user(parser, ubuf, cnt, ppos);
3465		3465
3466	if (read >= 0 && trace_parser_loaded(parser) &&	3466	if (read >= 0 && trace_parser_loaded(parser) &&
3467	!trace_parser_cont(parser)) {	3467	!trace_parser_cont(parser)) {
3468	ret = ftrace_process_regex(iter->hash, parser->buffer,	3468	ret = ftrace_process_regex(iter->hash, parser->buffer,
3469	parser->idx, enable);	3469	parser->idx, enable);
3470	trace_parser_clear(parser);	3470	trace_parser_clear(parser);
3471	if (ret < 0)	3471	if (ret < 0)
3472	goto out;	3472	goto out;
3473	}	3473	}
3474		3474
3475	ret = read;	3475	ret = read;
3476	out:	3476	out:
3477	return ret;	3477	return ret;
3478	}	3478	}
3479		3479
3480	ssize_t	3480	ssize_t
3481	ftrace_filter_write(struct file file, const char __user ubuf,	3481	ftrace_filter_write(struct file file, const char __user ubuf,
3482	size_t cnt, loff_t *ppos)	3482	size_t cnt, loff_t *ppos)
3483	{	3483	{
3484	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);	3484	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
3485	}	3485	}
3486		3486
3487	ssize_t	3487	ssize_t
3488	ftrace_notrace_write(struct file file, const char __user ubuf,	3488	ftrace_notrace_write(struct file file, const char __user ubuf,
3489	size_t cnt, loff_t *ppos)	3489	size_t cnt, loff_t *ppos)
3490	{	3490	{
3491	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);	3491	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
3492	}	3492	}
3493		3493
3494	static int	3494	static int
3495	ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)	3495	ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
3496	{	3496	{
3497	struct ftrace_func_entry *entry;	3497	struct ftrace_func_entry *entry;
3498		3498
3499	if (!ftrace_location(ip))	3499	if (!ftrace_location(ip))
3500	return -EINVAL;	3500	return -EINVAL;
3501		3501
3502	if (remove) {	3502	if (remove) {
3503	entry = ftrace_lookup_ip(hash, ip);	3503	entry = ftrace_lookup_ip(hash, ip);
3504	if (!entry)	3504	if (!entry)
3505	return -ENOENT;	3505	return -ENOENT;
3506	free_hash_entry(hash, entry);	3506	free_hash_entry(hash, entry);
3507	return 0;	3507	return 0;
3508	}	3508	}
3509		3509
3510	return add_hash_entry(hash, ip);	3510	return add_hash_entry(hash, ip);
3511	}	3511	}
3512		3512
3513	static void ftrace_ops_update_code(struct ftrace_ops *ops)	3513	static void ftrace_ops_update_code(struct ftrace_ops *ops)
3514	{	3514	{
3515	if (ops->flags & FTRACE_OPS_FL_ENABLED && ftrace_enabled)	3515	if (ops->flags & FTRACE_OPS_FL_ENABLED && ftrace_enabled)
3516	ftrace_run_update_code(FTRACE_UPDATE_CALLS);	3516	ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3517	}	3517	}
3518		3518
3519	static int	3519	static int
3520	ftrace_set_hash(struct ftrace_ops ops, unsigned char buf, int len,	3520	ftrace_set_hash(struct ftrace_ops ops, unsigned char buf, int len,
3521	unsigned long ip, int remove, int reset, int enable)	3521	unsigned long ip, int remove, int reset, int enable)
3522	{	3522	{
3523	struct ftrace_hash **orig_hash;	3523	struct ftrace_hash **orig_hash;
3524	struct ftrace_hash *hash;	3524	struct ftrace_hash *hash;
3525	int ret;	3525	int ret;
3526		3526
3527	/* All global ops uses the global ops filters */	3527	/* All global ops uses the global ops filters */
3528	if (ops->flags & FTRACE_OPS_FL_GLOBAL)	3528	if (ops->flags & FTRACE_OPS_FL_GLOBAL)
3529	ops = &global_ops;	3529	ops = &global_ops;
3530		3530
3531	if (unlikely(ftrace_disabled))	3531	if (unlikely(ftrace_disabled))
3532	return -ENODEV;	3532	return -ENODEV;
3533		3533
3534	mutex_lock(&ops->regex_lock);	3534	mutex_lock(&ops->regex_lock);
3535		3535
3536	if (enable)	3536	if (enable)
3537	orig_hash = &ops->filter_hash;	3537	orig_hash = &ops->filter_hash;
3538	else	3538	else
3539	orig_hash = &ops->notrace_hash;	3539	orig_hash = &ops->notrace_hash;
3540		3540
3541	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);	3541	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3542	if (!hash) {	3542	if (!hash) {
3543	ret = -ENOMEM;	3543	ret = -ENOMEM;
3544	goto out_regex_unlock;	3544	goto out_regex_unlock;
3545	}	3545	}
3546		3546
3547	if (reset)	3547	if (reset)
3548	ftrace_filter_reset(hash);	3548	ftrace_filter_reset(hash);
3549	if (buf && !ftrace_match_records(hash, buf, len)) {	3549	if (buf && !ftrace_match_records(hash, buf, len)) {
3550	ret = -EINVAL;	3550	ret = -EINVAL;
3551	goto out_regex_unlock;	3551	goto out_regex_unlock;
3552	}	3552	}
3553	if (ip) {	3553	if (ip) {
3554	ret = ftrace_match_addr(hash, ip, remove);	3554	ret = ftrace_match_addr(hash, ip, remove);
3555	if (ret < 0)	3555	if (ret < 0)
3556	goto out_regex_unlock;	3556	goto out_regex_unlock;
3557	}	3557	}
3558		3558
3559	mutex_lock(&ftrace_lock);	3559	mutex_lock(&ftrace_lock);
3560	ret = ftrace_hash_move(ops, enable, orig_hash, hash);	3560	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3561	if (!ret)	3561	if (!ret)
3562	ftrace_ops_update_code(ops);	3562	ftrace_ops_update_code(ops);
3563		3563
3564	mutex_unlock(&ftrace_lock);	3564	mutex_unlock(&ftrace_lock);
3565		3565
3566	out_regex_unlock:	3566	out_regex_unlock:
3567	mutex_unlock(&ops->regex_lock);	3567	mutex_unlock(&ops->regex_lock);
3568		3568
3569	free_ftrace_hash(hash);	3569	free_ftrace_hash(hash);
3570	return ret;	3570	return ret;
3571	}	3571	}
3572		3572
3573	static int	3573	static int
3574	ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,	3574	ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
3575	int reset, int enable)	3575	int reset, int enable)
3576	{	3576	{
3577	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);	3577	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
3578	}	3578	}
3579		3579
3580	/**	3580	/**
3581	* ftrace_set_filter_ip - set a function to filter on in ftrace by address	3581	* ftrace_set_filter_ip - set a function to filter on in ftrace by address
3582	* @ops - the ops to set the filter with	3582	* @ops - the ops to set the filter with
3583	* @ip - the address to add to or remove from the filter.	3583	* @ip - the address to add to or remove from the filter.
3584	* @remove - non zero to remove the ip from the filter	3584	* @remove - non zero to remove the ip from the filter
3585	* @reset - non zero to reset all filters before applying this filter.	3585	* @reset - non zero to reset all filters before applying this filter.
3586	*	3586	*
3587	* Filters denote which functions should be enabled when tracing is enabled	3587	* Filters denote which functions should be enabled when tracing is enabled
3588	* If @ip is NULL, it failes to update filter.	3588	* If @ip is NULL, it failes to update filter.
3589	*/	3589	*/
3590	int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,	3590	int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
3591	int remove, int reset)	3591	int remove, int reset)
3592	{	3592	{
3593	ftrace_ops_init(ops);	3593	ftrace_ops_init(ops);
3594	return ftrace_set_addr(ops, ip, remove, reset, 1);	3594	return ftrace_set_addr(ops, ip, remove, reset, 1);
3595	}	3595	}
3596	EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);	3596	EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
3597		3597
3598	static int	3598	static int
3599	ftrace_set_regex(struct ftrace_ops ops, unsigned char buf, int len,	3599	ftrace_set_regex(struct ftrace_ops ops, unsigned char buf, int len,
3600	int reset, int enable)	3600	int reset, int enable)
3601	{	3601	{
3602	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);	3602	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
3603	}	3603	}
3604		3604
3605	/**	3605	/**
3606	* ftrace_set_filter - set a function to filter on in ftrace	3606	* ftrace_set_filter - set a function to filter on in ftrace
3607	* @ops - the ops to set the filter with	3607	* @ops - the ops to set the filter with
3608	* @buf - the string that holds the function filter text.	3608	* @buf - the string that holds the function filter text.
3609	* @len - the length of the string.	3609	* @len - the length of the string.
3610	* @reset - non zero to reset all filters before applying this filter.	3610	* @reset - non zero to reset all filters before applying this filter.
3611	*	3611	*
3612	* Filters denote which functions should be enabled when tracing is enabled.	3612	* Filters denote which functions should be enabled when tracing is enabled.
3613	* If @buf is NULL and reset is set, all functions will be enabled for tracing.	3613	* If @buf is NULL and reset is set, all functions will be enabled for tracing.
3614	*/	3614	*/
3615	int ftrace_set_filter(struct ftrace_ops ops, unsigned char buf,	3615	int ftrace_set_filter(struct ftrace_ops ops, unsigned char buf,
3616	int len, int reset)	3616	int len, int reset)
3617	{	3617	{
3618	ftrace_ops_init(ops);	3618	ftrace_ops_init(ops);
3619	return ftrace_set_regex(ops, buf, len, reset, 1);	3619	return ftrace_set_regex(ops, buf, len, reset, 1);
3620	}	3620	}
3621	EXPORT_SYMBOL_GPL(ftrace_set_filter);	3621	EXPORT_SYMBOL_GPL(ftrace_set_filter);
3622		3622
3623	/**	3623	/**
3624	* ftrace_set_notrace - set a function to not trace in ftrace	3624	* ftrace_set_notrace - set a function to not trace in ftrace
3625	* @ops - the ops to set the notrace filter with	3625	* @ops - the ops to set the notrace filter with
3626	* @buf - the string that holds the function notrace text.	3626	* @buf - the string that holds the function notrace text.
3627	* @len - the length of the string.	3627	* @len - the length of the string.
3628	* @reset - non zero to reset all filters before applying this filter.	3628	* @reset - non zero to reset all filters before applying this filter.
3629	*	3629	*
3630	* Notrace Filters denote which functions should not be enabled when tracing	3630	* Notrace Filters denote which functions should not be enabled when tracing
3631	* is enabled. If @buf is NULL and reset is set, all functions will be enabled	3631	* is enabled. If @buf is NULL and reset is set, all functions will be enabled
3632	* for tracing.	3632	* for tracing.
3633	*/	3633	*/
3634	int ftrace_set_notrace(struct ftrace_ops ops, unsigned char buf,	3634	int ftrace_set_notrace(struct ftrace_ops ops, unsigned char buf,
3635	int len, int reset)	3635	int len, int reset)
3636	{	3636	{
3637	ftrace_ops_init(ops);	3637	ftrace_ops_init(ops);
3638	return ftrace_set_regex(ops, buf, len, reset, 0);	3638	return ftrace_set_regex(ops, buf, len, reset, 0);
3639	}	3639	}
3640	EXPORT_SYMBOL_GPL(ftrace_set_notrace);	3640	EXPORT_SYMBOL_GPL(ftrace_set_notrace);
3641	/**	3641	/**
3642	* ftrace_set_filter - set a function to filter on in ftrace	3642	* ftrace_set_filter - set a function to filter on in ftrace
3643	* @ops - the ops to set the filter with	3643	* @ops - the ops to set the filter with
3644	* @buf - the string that holds the function filter text.	3644	* @buf - the string that holds the function filter text.
3645	* @len - the length of the string.	3645	* @len - the length of the string.
3646	* @reset - non zero to reset all filters before applying this filter.	3646	* @reset - non zero to reset all filters before applying this filter.
3647	*	3647	*
3648	* Filters denote which functions should be enabled when tracing is enabled.	3648	* Filters denote which functions should be enabled when tracing is enabled.
3649	* If @buf is NULL and reset is set, all functions will be enabled for tracing.	3649	* If @buf is NULL and reset is set, all functions will be enabled for tracing.
3650	*/	3650	*/
3651	void ftrace_set_global_filter(unsigned char *buf, int len, int reset)	3651	void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3652	{	3652	{
3653	ftrace_set_regex(&global_ops, buf, len, reset, 1);	3653	ftrace_set_regex(&global_ops, buf, len, reset, 1);
3654	}	3654	}
3655	EXPORT_SYMBOL_GPL(ftrace_set_global_filter);	3655	EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3656		3656
3657	/**	3657	/**
3658	* ftrace_set_notrace - set a function to not trace in ftrace	3658	* ftrace_set_notrace - set a function to not trace in ftrace
3659	* @ops - the ops to set the notrace filter with	3659	* @ops - the ops to set the notrace filter with
3660	* @buf - the string that holds the function notrace text.	3660	* @buf - the string that holds the function notrace text.
3661	* @len - the length of the string.	3661	* @len - the length of the string.
3662	* @reset - non zero to reset all filters before applying this filter.	3662	* @reset - non zero to reset all filters before applying this filter.
3663	*	3663	*
3664	* Notrace Filters denote which functions should not be enabled when tracing	3664	* Notrace Filters denote which functions should not be enabled when tracing
3665	* is enabled. If @buf is NULL and reset is set, all functions will be enabled	3665	* is enabled. If @buf is NULL and reset is set, all functions will be enabled
3666	* for tracing.	3666	* for tracing.
3667	*/	3667	*/
3668	void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)	3668	void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3669	{	3669	{
3670	ftrace_set_regex(&global_ops, buf, len, reset, 0);	3670	ftrace_set_regex(&global_ops, buf, len, reset, 0);
3671	}	3671	}
3672	EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);	3672	EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3673		3673
3674	/*	3674	/*
3675	* command line interface to allow users to set filters on boot up.	3675	* command line interface to allow users to set filters on boot up.
3676	*/	3676	*/
3677	#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE	3677	#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
3678	static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;	3678	static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3679	static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;	3679	static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3680		3680
3681	/* Used by function selftest to not test if filter is set */	3681	/* Used by function selftest to not test if filter is set */
3682	bool ftrace_filter_param __initdata;	3682	bool ftrace_filter_param __initdata;
3683		3683
3684	static int __init set_ftrace_notrace(char *str)	3684	static int __init set_ftrace_notrace(char *str)
3685	{	3685	{
3686	ftrace_filter_param = true;	3686	ftrace_filter_param = true;
3687	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);	3687	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3688	return 1;	3688	return 1;
3689	}	3689	}
3690	__setup("ftrace_notrace=", set_ftrace_notrace);	3690	__setup("ftrace_notrace=", set_ftrace_notrace);
3691		3691
3692	static int __init set_ftrace_filter(char *str)	3692	static int __init set_ftrace_filter(char *str)
3693	{	3693	{
3694	ftrace_filter_param = true;	3694	ftrace_filter_param = true;
3695	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);	3695	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3696	return 1;	3696	return 1;
3697	}	3697	}
3698	__setup("ftrace_filter=", set_ftrace_filter);	3698	__setup("ftrace_filter=", set_ftrace_filter);
3699		3699
3700	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	3700	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3701	static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;	3701	static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3702	static int ftrace_set_func(unsigned long array, int idx, int size, char *buffer);	3702	static int ftrace_set_func(unsigned long array, int idx, int size, char *buffer);
3703		3703
3704	static int __init set_graph_function(char *str)	3704	static int __init set_graph_function(char *str)
3705	{	3705	{
3706	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);	3706	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3707	return 1;	3707	return 1;
3708	}	3708	}
3709	__setup("ftrace_graph_filter=", set_graph_function);	3709	__setup("ftrace_graph_filter=", set_graph_function);
3710		3710
3711	static void __init set_ftrace_early_graph(char *buf)	3711	static void __init set_ftrace_early_graph(char *buf)
3712	{	3712	{
3713	int ret;	3713	int ret;
3714	char *func;	3714	char *func;
3715		3715
3716	while (buf) {	3716	while (buf) {
3717	func = strsep(&buf, ",");	3717	func = strsep(&buf, ",");
3718	/* we allow only one expression at a time */	3718	/* we allow only one expression at a time */
3719	ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,	3719	ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3720	FTRACE_GRAPH_MAX_FUNCS, func);	3720	FTRACE_GRAPH_MAX_FUNCS, func);
3721	if (ret)	3721	if (ret)
3722	printk(KERN_DEBUG "ftrace: function %s not "	3722	printk(KERN_DEBUG "ftrace: function %s not "
3723	"traceable\n", func);	3723	"traceable\n", func);
3724	}	3724	}
3725	}	3725	}
3726	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */	3726	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3727		3727
3728	void __init	3728	void __init
3729	ftrace_set_early_filter(struct ftrace_ops ops, char buf, int enable)	3729	ftrace_set_early_filter(struct ftrace_ops ops, char buf, int enable)
3730	{	3730	{
3731	char *func;	3731	char *func;
3732		3732
3733	ftrace_ops_init(ops);	3733	ftrace_ops_init(ops);
3734		3734
3735	while (buf) {	3735	while (buf) {
3736	func = strsep(&buf, ",");	3736	func = strsep(&buf, ",");
3737	ftrace_set_regex(ops, func, strlen(func), 0, enable);	3737	ftrace_set_regex(ops, func, strlen(func), 0, enable);
3738	}	3738	}
3739	}	3739	}
3740		3740
3741	static void __init set_ftrace_early_filters(void)	3741	static void __init set_ftrace_early_filters(void)
3742	{	3742	{
3743	if (ftrace_filter_buf[0])	3743	if (ftrace_filter_buf[0])
3744	ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);	3744	ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
3745	if (ftrace_notrace_buf[0])	3745	if (ftrace_notrace_buf[0])
3746	ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);	3746	ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
3747	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	3747	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3748	if (ftrace_graph_buf[0])	3748	if (ftrace_graph_buf[0])
3749	set_ftrace_early_graph(ftrace_graph_buf);	3749	set_ftrace_early_graph(ftrace_graph_buf);
3750	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */	3750	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3751	}	3751	}
3752		3752
3753	int ftrace_regex_release(struct inode inode, struct file file)	3753	int ftrace_regex_release(struct inode inode, struct file file)
3754	{	3754	{
3755	struct seq_file m = (struct seq_file )file->private_data;	3755	struct seq_file m = (struct seq_file )file->private_data;
3756	struct ftrace_iterator *iter;	3756	struct ftrace_iterator *iter;
3757	struct ftrace_hash **orig_hash;	3757	struct ftrace_hash **orig_hash;
3758	struct trace_parser *parser;	3758	struct trace_parser *parser;
3759	int filter_hash;	3759	int filter_hash;
3760	int ret;	3760	int ret;
3761		3761
3762	if (file->f_mode & FMODE_READ) {	3762	if (file->f_mode & FMODE_READ) {
3763	iter = m->private;	3763	iter = m->private;
3764	seq_release(inode, file);	3764	seq_release(inode, file);
3765	} else	3765	} else
3766	iter = file->private_data;	3766	iter = file->private_data;
3767		3767
3768	parser = &iter->parser;	3768	parser = &iter->parser;
3769	if (trace_parser_loaded(parser)) {	3769	if (trace_parser_loaded(parser)) {
3770	parser->buffer[parser->idx] = 0;	3770	parser->buffer[parser->idx] = 0;
3771	ftrace_match_records(iter->hash, parser->buffer, parser->idx);	3771	ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3772	}	3772	}
3773		3773
3774	trace_parser_put(parser);	3774	trace_parser_put(parser);
3775		3775
3776	mutex_lock(&iter->ops->regex_lock);	3776	mutex_lock(&iter->ops->regex_lock);
3777		3777
3778	if (file->f_mode & FMODE_WRITE) {	3778	if (file->f_mode & FMODE_WRITE) {
3779	filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);	3779	filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3780		3780
3781	if (filter_hash)	3781	if (filter_hash)
3782	orig_hash = &iter->ops->filter_hash;	3782	orig_hash = &iter->ops->filter_hash;
3783	else	3783	else
3784	orig_hash = &iter->ops->notrace_hash;	3784	orig_hash = &iter->ops->notrace_hash;
3785		3785
3786	mutex_lock(&ftrace_lock);	3786	mutex_lock(&ftrace_lock);
3787	ret = ftrace_hash_move(iter->ops, filter_hash,	3787	ret = ftrace_hash_move(iter->ops, filter_hash,
3788	orig_hash, iter->hash);	3788	orig_hash, iter->hash);
3789	if (!ret)	3789	if (!ret)
3790	ftrace_ops_update_code(iter->ops);	3790	ftrace_ops_update_code(iter->ops);
3791		3791
3792	mutex_unlock(&ftrace_lock);	3792	mutex_unlock(&ftrace_lock);
3793	}	3793	}
3794		3794
3795	mutex_unlock(&iter->ops->regex_lock);	3795	mutex_unlock(&iter->ops->regex_lock);
3796	free_ftrace_hash(iter->hash);	3796	free_ftrace_hash(iter->hash);
3797	kfree(iter);	3797	kfree(iter);
3798		3798
3799	return 0;	3799	return 0;
3800	}	3800	}
3801		3801
3802	static const struct file_operations ftrace_avail_fops = {	3802	static const struct file_operations ftrace_avail_fops = {
3803	.open = ftrace_avail_open,	3803	.open = ftrace_avail_open,
3804	.read = seq_read,	3804	.read = seq_read,
3805	.llseek = seq_lseek,	3805	.llseek = seq_lseek,
3806	.release = seq_release_private,	3806	.release = seq_release_private,
3807	};	3807	};
3808		3808
3809	static const struct file_operations ftrace_enabled_fops = {	3809	static const struct file_operations ftrace_enabled_fops = {
3810	.open = ftrace_enabled_open,	3810	.open = ftrace_enabled_open,
3811	.read = seq_read,	3811	.read = seq_read,
3812	.llseek = seq_lseek,	3812	.llseek = seq_lseek,
3813	.release = seq_release_private,	3813	.release = seq_release_private,
3814	};	3814	};
3815		3815
3816	static const struct file_operations ftrace_filter_fops = {	3816	static const struct file_operations ftrace_filter_fops = {
3817	.open = ftrace_filter_open,	3817	.open = ftrace_filter_open,
3818	.read = seq_read,	3818	.read = seq_read,
3819	.write = ftrace_filter_write,	3819	.write = ftrace_filter_write,
3820	.llseek = tracing_lseek,	3820	.llseek = tracing_lseek,
3821	.release = ftrace_regex_release,	3821	.release = ftrace_regex_release,
3822	};	3822	};
3823		3823
3824	static const struct file_operations ftrace_notrace_fops = {	3824	static const struct file_operations ftrace_notrace_fops = {
3825	.open = ftrace_notrace_open,	3825	.open = ftrace_notrace_open,
3826	.read = seq_read,	3826	.read = seq_read,
3827	.write = ftrace_notrace_write,	3827	.write = ftrace_notrace_write,
3828	.llseek = tracing_lseek,	3828	.llseek = tracing_lseek,
3829	.release = ftrace_regex_release,	3829	.release = ftrace_regex_release,
3830	};	3830	};
3831		3831
3832	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	3832	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3833		3833
3834	static DEFINE_MUTEX(graph_lock);	3834	static DEFINE_MUTEX(graph_lock);
3835		3835
3836	int ftrace_graph_count;	3836	int ftrace_graph_count;
3837	int ftrace_graph_notrace_count;	3837	int ftrace_graph_notrace_count;
3838	unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;	3838	unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3839	unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;	3839	unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3840		3840
3841	struct ftrace_graph_data {	3841	struct ftrace_graph_data {
3842	unsigned long *table;	3842	unsigned long *table;
3843	size_t size;	3843	size_t size;
3844	int *count;	3844	int *count;
3845	const struct seq_operations *seq_ops;	3845	const struct seq_operations *seq_ops;
3846	};	3846	};
3847		3847
3848	static void *	3848	static void *
3849	__g_next(struct seq_file m, loff_t pos)	3849	__g_next(struct seq_file m, loff_t pos)
3850	{	3850	{
3851	struct ftrace_graph_data *fgd = m->private;	3851	struct ftrace_graph_data *fgd = m->private;
3852		3852
3853	if (pos >= fgd->count)	3853	if (pos >= fgd->count)
3854	return NULL;	3854	return NULL;
3855	return &fgd->table[*pos];	3855	return &fgd->table[*pos];
3856	}	3856	}
3857		3857
3858	static void *	3858	static void *
3859	g_next(struct seq_file m, void v, loff_t *pos)	3859	g_next(struct seq_file m, void v, loff_t *pos)
3860	{	3860	{
3861	(*pos)++;	3861	(*pos)++;
3862	return __g_next(m, pos);	3862	return __g_next(m, pos);
3863	}	3863	}
3864		3864
3865	static void g_start(struct seq_file m, loff_t *pos)	3865	static void g_start(struct seq_file m, loff_t *pos)
3866	{	3866	{
3867	struct ftrace_graph_data *fgd = m->private;	3867	struct ftrace_graph_data *fgd = m->private;
3868		3868
3869	mutex_lock(&graph_lock);	3869	mutex_lock(&graph_lock);
3870		3870
3871	/* Nothing, tell g_show to print all functions are enabled */	3871	/* Nothing, tell g_show to print all functions are enabled */
3872	if (!fgd->count && !pos)	3872	if (!fgd->count && !pos)
3873	return (void *)1;	3873	return (void *)1;
3874		3874
3875	return __g_next(m, pos);	3875	return __g_next(m, pos);
3876	}	3876	}
3877		3877
3878	static void g_stop(struct seq_file m, void p)	3878	static void g_stop(struct seq_file m, void p)
3879	{	3879	{
3880	mutex_unlock(&graph_lock);	3880	mutex_unlock(&graph_lock);
3881	}	3881	}
3882		3882
3883	static int g_show(struct seq_file m, void v)	3883	static int g_show(struct seq_file m, void v)
3884	{	3884	{
3885	unsigned long *ptr = v;	3885	unsigned long *ptr = v;
3886		3886
3887	if (!ptr)	3887	if (!ptr)
3888	return 0;	3888	return 0;
3889		3889
3890	if (ptr == (unsigned long *)1) {	3890	if (ptr == (unsigned long *)1) {
3891	seq_printf(m, "#### all functions enabled ####\n");	3891	seq_printf(m, "#### all functions enabled ####\n");
3892	return 0;	3892	return 0;
3893	}	3893	}
3894		3894
3895	seq_printf(m, "%ps\n", (void )ptr);	3895	seq_printf(m, "%ps\n", (void )ptr);
3896		3896
3897	return 0;	3897	return 0;
3898	}	3898	}
3899		3899
3900	static const struct seq_operations ftrace_graph_seq_ops = {	3900	static const struct seq_operations ftrace_graph_seq_ops = {
3901	.start = g_start,	3901	.start = g_start,
3902	.next = g_next,	3902	.next = g_next,
3903	.stop = g_stop,	3903	.stop = g_stop,
3904	.show = g_show,	3904	.show = g_show,
3905	};	3905	};
3906		3906
3907	static int	3907	static int
3908	__ftrace_graph_open(struct inode inode, struct file file,	3908	__ftrace_graph_open(struct inode inode, struct file file,
3909	struct ftrace_graph_data *fgd)	3909	struct ftrace_graph_data *fgd)
3910	{	3910	{
3911	int ret = 0;	3911	int ret = 0;
3912		3912
3913	mutex_lock(&graph_lock);	3913	mutex_lock(&graph_lock);
3914	if ((file->f_mode & FMODE_WRITE) &&	3914	if ((file->f_mode & FMODE_WRITE) &&
3915	(file->f_flags & O_TRUNC)) {	3915	(file->f_flags & O_TRUNC)) {
3916	*fgd->count = 0;	3916	*fgd->count = 0;
3917	memset(fgd->table, 0, fgd->size * sizeof(*fgd->table));	3917	memset(fgd->table, 0, fgd->size * sizeof(*fgd->table));
3918	}	3918	}
3919	mutex_unlock(&graph_lock);	3919	mutex_unlock(&graph_lock);
3920		3920
3921	if (file->f_mode & FMODE_READ) {	3921	if (file->f_mode & FMODE_READ) {
3922	ret = seq_open(file, fgd->seq_ops);	3922	ret = seq_open(file, fgd->seq_ops);
3923	if (!ret) {	3923	if (!ret) {
3924	struct seq_file *m = file->private_data;	3924	struct seq_file *m = file->private_data;
3925	m->private = fgd;	3925	m->private = fgd;
3926	}	3926	}
3927	} else	3927	} else
3928	file->private_data = fgd;	3928	file->private_data = fgd;
3929		3929
3930	return ret;	3930	return ret;
3931	}	3931	}
3932		3932
3933	static int	3933	static int
3934	ftrace_graph_open(struct inode inode, struct file file)	3934	ftrace_graph_open(struct inode inode, struct file file)
3935	{	3935	{
3936	struct ftrace_graph_data *fgd;	3936	struct ftrace_graph_data *fgd;
3937		3937
3938	if (unlikely(ftrace_disabled))	3938	if (unlikely(ftrace_disabled))
3939	return -ENODEV;	3939	return -ENODEV;
3940		3940
3941	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);	3941	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
3942	if (fgd == NULL)	3942	if (fgd == NULL)
3943	return -ENOMEM;	3943	return -ENOMEM;
3944		3944
3945	fgd->table = ftrace_graph_funcs;	3945	fgd->table = ftrace_graph_funcs;
3946	fgd->size = FTRACE_GRAPH_MAX_FUNCS;	3946	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
3947	fgd->count = &ftrace_graph_count;	3947	fgd->count = &ftrace_graph_count;
3948	fgd->seq_ops = &ftrace_graph_seq_ops;	3948	fgd->seq_ops = &ftrace_graph_seq_ops;
3949		3949
3950	return __ftrace_graph_open(inode, file, fgd);	3950	return __ftrace_graph_open(inode, file, fgd);
3951	}	3951	}
3952		3952
3953	static int	3953	static int
3954	ftrace_graph_notrace_open(struct inode inode, struct file file)	3954	ftrace_graph_notrace_open(struct inode inode, struct file file)
3955	{	3955	{
3956	struct ftrace_graph_data *fgd;	3956	struct ftrace_graph_data *fgd;
3957		3957
3958	if (unlikely(ftrace_disabled))	3958	if (unlikely(ftrace_disabled))
3959	return -ENODEV;	3959	return -ENODEV;
3960		3960
3961	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);	3961	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
3962	if (fgd == NULL)	3962	if (fgd == NULL)
3963	return -ENOMEM;	3963	return -ENOMEM;
3964		3964
3965	fgd->table = ftrace_graph_notrace_funcs;	3965	fgd->table = ftrace_graph_notrace_funcs;
3966	fgd->size = FTRACE_GRAPH_MAX_FUNCS;	3966	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
3967	fgd->count = &ftrace_graph_notrace_count;	3967	fgd->count = &ftrace_graph_notrace_count;
3968	fgd->seq_ops = &ftrace_graph_seq_ops;	3968	fgd->seq_ops = &ftrace_graph_seq_ops;
3969		3969
3970	return __ftrace_graph_open(inode, file, fgd);	3970	return __ftrace_graph_open(inode, file, fgd);
3971	}	3971	}
3972		3972
3973	static int	3973	static int
3974	ftrace_graph_release(struct inode inode, struct file file)	3974	ftrace_graph_release(struct inode inode, struct file file)
3975	{	3975	{
3976	if (file->f_mode & FMODE_READ) {	3976	if (file->f_mode & FMODE_READ) {
3977	struct seq_file *m = file->private_data;	3977	struct seq_file *m = file->private_data;
3978		3978
3979	kfree(m->private);	3979	kfree(m->private);
3980	seq_release(inode, file);	3980	seq_release(inode, file);
3981	} else {	3981	} else {
3982	kfree(file->private_data);	3982	kfree(file->private_data);
3983	}	3983	}
3984		3984
3985	return 0;	3985	return 0;
3986	}	3986	}
3987		3987
3988	static int	3988	static int
3989	ftrace_set_func(unsigned long array, int idx, int size, char *buffer)	3989	ftrace_set_func(unsigned long array, int idx, int size, char *buffer)
3990	{	3990	{
3991	struct dyn_ftrace *rec;	3991	struct dyn_ftrace *rec;
3992	struct ftrace_page *pg;	3992	struct ftrace_page *pg;
3993	int search_len;	3993	int search_len;
3994	int fail = 1;	3994	int fail = 1;
3995	int type, not;	3995	int type, not;
3996	char *search;	3996	char *search;
3997	bool exists;	3997	bool exists;
3998	int i;	3998	int i;
3999		3999
4000	/* decode regex */	4000	/* decode regex */
4001	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);	4001	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
4002	if (!not && *idx >= size)	4002	if (!not && *idx >= size)
4003	return -EBUSY;	4003	return -EBUSY;
4004		4004
4005	search_len = strlen(search);	4005	search_len = strlen(search);
4006		4006
4007	mutex_lock(&ftrace_lock);	4007	mutex_lock(&ftrace_lock);
4008		4008
4009	if (unlikely(ftrace_disabled)) {	4009	if (unlikely(ftrace_disabled)) {
4010	mutex_unlock(&ftrace_lock);	4010	mutex_unlock(&ftrace_lock);
4011	return -ENODEV;	4011	return -ENODEV;
4012	}	4012	}
4013		4013
4014	do_for_each_ftrace_rec(pg, rec) {	4014	do_for_each_ftrace_rec(pg, rec) {
4015		4015
4016	if (ftrace_match_record(rec, NULL, search, search_len, type)) {	4016	if (ftrace_match_record(rec, NULL, search, search_len, type)) {
4017	/* if it is in the array */	4017	/* if it is in the array */
4018	exists = false;	4018	exists = false;
4019	for (i = 0; i < *idx; i++) {	4019	for (i = 0; i < *idx; i++) {
4020	if (array[i] == rec->ip) {	4020	if (array[i] == rec->ip) {
4021	exists = true;	4021	exists = true;
4022	break;	4022	break;
4023	}	4023	}
4024	}	4024	}
4025		4025
4026	if (!not) {	4026	if (!not) {
4027	fail = 0;	4027	fail = 0;
4028	if (!exists) {	4028	if (!exists) {
4029	array[(*idx)++] = rec->ip;	4029	array[(*idx)++] = rec->ip;
4030	if (*idx >= size)	4030	if (*idx >= size)
4031	goto out;	4031	goto out;
4032	}	4032	}
4033	} else {	4033	} else {
4034	if (exists) {	4034	if (exists) {
4035	array[i] = array[--(*idx)];	4035	array[i] = array[--(*idx)];
4036	array[*idx] = 0;	4036	array[*idx] = 0;
4037	fail = 0;	4037	fail = 0;
4038	}	4038	}
4039	}	4039	}
4040	}	4040	}
4041	} while_for_each_ftrace_rec();	4041	} while_for_each_ftrace_rec();
4042	out:	4042	out:
4043	mutex_unlock(&ftrace_lock);	4043	mutex_unlock(&ftrace_lock);
4044		4044
4045	if (fail)	4045	if (fail)
4046	return -EINVAL;	4046	return -EINVAL;
4047		4047
4048	return 0;	4048	return 0;
4049	}	4049	}
4050		4050
4051	static ssize_t	4051	static ssize_t
4052	ftrace_graph_write(struct file file, const char __user ubuf,	4052	ftrace_graph_write(struct file file, const char __user ubuf,
4053	size_t cnt, loff_t *ppos)	4053	size_t cnt, loff_t *ppos)
4054	{	4054	{
4055	struct trace_parser parser;	4055	struct trace_parser parser;
4056	ssize_t read, ret = 0;	4056	ssize_t read, ret = 0;
4057	struct ftrace_graph_data *fgd = file->private_data;	4057	struct ftrace_graph_data *fgd = file->private_data;
4058		4058
4059	if (!cnt)	4059	if (!cnt)
4060	return 0;	4060	return 0;
4061		4061
4062	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX))	4062	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX))
4063	return -ENOMEM;	4063	return -ENOMEM;
4064		4064
4065	read = trace_get_user(&parser, ubuf, cnt, ppos);	4065	read = trace_get_user(&parser, ubuf, cnt, ppos);
4066		4066
4067	if (read >= 0 && trace_parser_loaded((&parser))) {	4067	if (read >= 0 && trace_parser_loaded((&parser))) {
4068	parser.buffer[parser.idx] = 0;	4068	parser.buffer[parser.idx] = 0;
4069		4069
4070	mutex_lock(&graph_lock);	4070	mutex_lock(&graph_lock);
4071		4071
4072	/* we allow only one expression at a time */	4072	/* we allow only one expression at a time */
4073	ret = ftrace_set_func(fgd->table, fgd->count, fgd->size,	4073	ret = ftrace_set_func(fgd->table, fgd->count, fgd->size,
4074	parser.buffer);	4074	parser.buffer);
4075		4075
4076	mutex_unlock(&graph_lock);	4076	mutex_unlock(&graph_lock);
4077	}	4077	}
4078		4078
4079	if (!ret)	4079	if (!ret)
4080	ret = read;	4080	ret = read;
4081		4081
4082	trace_parser_put(&parser);	4082	trace_parser_put(&parser);
4083		4083
4084	return ret;	4084	return ret;
4085	}	4085	}
4086		4086
4087	static const struct file_operations ftrace_graph_fops = {	4087	static const struct file_operations ftrace_graph_fops = {
4088	.open = ftrace_graph_open,	4088	.open = ftrace_graph_open,
4089	.read = seq_read,	4089	.read = seq_read,
4090	.write = ftrace_graph_write,	4090	.write = ftrace_graph_write,
4091	.llseek = tracing_lseek,	4091	.llseek = tracing_lseek,
4092	.release = ftrace_graph_release,	4092	.release = ftrace_graph_release,
4093	};	4093	};
4094		4094
4095	static const struct file_operations ftrace_graph_notrace_fops = {	4095	static const struct file_operations ftrace_graph_notrace_fops = {
4096	.open = ftrace_graph_notrace_open,	4096	.open = ftrace_graph_notrace_open,
4097	.read = seq_read,	4097	.read = seq_read,
4098	.write = ftrace_graph_write,	4098	.write = ftrace_graph_write,
4099	.llseek = tracing_lseek,	4099	.llseek = tracing_lseek,
4100	.release = ftrace_graph_release,	4100	.release = ftrace_graph_release,
4101	};	4101	};
4102	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */	4102	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4103		4103
4104	void ftrace_create_filter_files(struct ftrace_ops *ops,	4104	void ftrace_create_filter_files(struct ftrace_ops *ops,
4105	struct dentry *parent)	4105	struct dentry *parent)
4106	{	4106	{
4107		4107
4108	trace_create_file("set_ftrace_filter", 0644, parent,	4108	trace_create_file("set_ftrace_filter", 0644, parent,
4109	ops, &ftrace_filter_fops);	4109	ops, &ftrace_filter_fops);
4110		4110
4111	trace_create_file("set_ftrace_notrace", 0644, parent,	4111	trace_create_file("set_ftrace_notrace", 0644, parent,
4112	ops, &ftrace_notrace_fops);	4112	ops, &ftrace_notrace_fops);
4113	}	4113	}
4114		4114
4115	/*	4115	/*
4116	* The name "destroy_filter_files" is really a misnomer. Although	4116	* The name "destroy_filter_files" is really a misnomer. Although
4117	* in the future, it may actualy delete the files, but this is	4117	* in the future, it may actualy delete the files, but this is
4118	* really intended to make sure the ops passed in are disabled	4118	* really intended to make sure the ops passed in are disabled
4119	* and that when this function returns, the caller is free to	4119	* and that when this function returns, the caller is free to
4120	* free the ops.	4120	* free the ops.
4121	*	4121	*
4122	* The "destroy" name is only to match the "create" name that this	4122	* The "destroy" name is only to match the "create" name that this
4123	* should be paired with.	4123	* should be paired with.
4124	*/	4124	*/
4125	void ftrace_destroy_filter_files(struct ftrace_ops *ops)	4125	void ftrace_destroy_filter_files(struct ftrace_ops *ops)
4126	{	4126	{
4127	mutex_lock(&ftrace_lock);	4127	mutex_lock(&ftrace_lock);
4128	if (ops->flags & FTRACE_OPS_FL_ENABLED)	4128	if (ops->flags & FTRACE_OPS_FL_ENABLED)
4129	ftrace_shutdown(ops, 0);	4129	ftrace_shutdown(ops, 0);
4130	ops->flags \|= FTRACE_OPS_FL_DELETED;	4130	ops->flags \|= FTRACE_OPS_FL_DELETED;
4131	mutex_unlock(&ftrace_lock);	4131	mutex_unlock(&ftrace_lock);
4132	}	4132	}
4133		4133
4134	static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)	4134	static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
4135	{	4135	{
4136		4136
4137	trace_create_file("available_filter_functions", 0444,	4137	trace_create_file("available_filter_functions", 0444,
4138	d_tracer, NULL, &ftrace_avail_fops);	4138	d_tracer, NULL, &ftrace_avail_fops);
4139		4139
4140	trace_create_file("enabled_functions", 0444,	4140	trace_create_file("enabled_functions", 0444,
4141	d_tracer, NULL, &ftrace_enabled_fops);	4141	d_tracer, NULL, &ftrace_enabled_fops);
4142		4142
4143	ftrace_create_filter_files(&global_ops, d_tracer);	4143	ftrace_create_filter_files(&global_ops, d_tracer);
4144		4144
4145	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	4145	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4146	trace_create_file("set_graph_function", 0444, d_tracer,	4146	trace_create_file("set_graph_function", 0444, d_tracer,
4147	NULL,	4147	NULL,
4148	&ftrace_graph_fops);	4148	&ftrace_graph_fops);
4149	trace_create_file("set_graph_notrace", 0444, d_tracer,	4149	trace_create_file("set_graph_notrace", 0444, d_tracer,
4150	NULL,	4150	NULL,
4151	&ftrace_graph_notrace_fops);	4151	&ftrace_graph_notrace_fops);
4152	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */	4152	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4153		4153
4154	return 0;	4154	return 0;
4155	}	4155	}
4156		4156
4157	static int ftrace_cmp_ips(const void a, const void b)	4157	static int ftrace_cmp_ips(const void a, const void b)
4158	{	4158	{
4159	const unsigned long *ipa = a;	4159	const unsigned long *ipa = a;
4160	const unsigned long *ipb = b;	4160	const unsigned long *ipb = b;
4161		4161
4162	if (ipa > ipb)	4162	if (ipa > ipb)
4163	return 1;	4163	return 1;
4164	if (ipa < ipb)	4164	if (ipa < ipb)
4165	return -1;	4165	return -1;
4166	return 0;	4166	return 0;
4167	}	4167	}
4168		4168
4169	static void ftrace_swap_ips(void a, void b, int size)	4169	static void ftrace_swap_ips(void a, void b, int size)
4170	{	4170	{
4171	unsigned long *ipa = a;	4171	unsigned long *ipa = a;
4172	unsigned long *ipb = b;	4172	unsigned long *ipb = b;
4173	unsigned long t;	4173	unsigned long t;
4174		4174
4175	t = *ipa;	4175	t = *ipa;
4176	ipa = ipb;	4176	ipa = ipb;
4177	*ipb = t;	4177	*ipb = t;
4178	}	4178	}
4179		4179
4180	static int ftrace_process_locs(struct module *mod,	4180	static int ftrace_process_locs(struct module *mod,
4181	unsigned long *start,	4181	unsigned long *start,
4182	unsigned long *end)	4182	unsigned long *end)
4183	{	4183	{
4184	struct ftrace_page *start_pg;	4184	struct ftrace_page *start_pg;
4185	struct ftrace_page *pg;	4185	struct ftrace_page *pg;
4186	struct dyn_ftrace *rec;	4186	struct dyn_ftrace *rec;
4187	unsigned long count;	4187	unsigned long count;
4188	unsigned long *p;	4188	unsigned long *p;
4189	unsigned long addr;	4189	unsigned long addr;
4190	unsigned long flags = 0; /* Shut up gcc */	4190	unsigned long flags = 0; /* Shut up gcc */
4191	int ret = -ENOMEM;	4191	int ret = -ENOMEM;
4192		4192
4193	count = end - start;	4193	count = end - start;
4194		4194
4195	if (!count)	4195	if (!count)
4196	return 0;	4196	return 0;
4197		4197
4198	sort(start, count, sizeof(*start),	4198	sort(start, count, sizeof(*start),
4199	ftrace_cmp_ips, ftrace_swap_ips);	4199	ftrace_cmp_ips, ftrace_swap_ips);
4200		4200
4201	start_pg = ftrace_allocate_pages(count);	4201	start_pg = ftrace_allocate_pages(count);
4202	if (!start_pg)	4202	if (!start_pg)
4203	return -ENOMEM;	4203	return -ENOMEM;
4204		4204
4205	mutex_lock(&ftrace_lock);	4205	mutex_lock(&ftrace_lock);
4206		4206
4207	/*	4207	/*
4208	* Core and each module needs their own pages, as	4208	* Core and each module needs their own pages, as
4209	* modules will free them when they are removed.	4209	* modules will free them when they are removed.
4210	* Force a new page to be allocated for modules.	4210	* Force a new page to be allocated for modules.
4211	*/	4211	*/
4212	if (!mod) {	4212	if (!mod) {
4213	WARN_ON(ftrace_pages \|\| ftrace_pages_start);	4213	WARN_ON(ftrace_pages \|\| ftrace_pages_start);
4214	/* First initialization */	4214	/* First initialization */
4215	ftrace_pages = ftrace_pages_start = start_pg;	4215	ftrace_pages = ftrace_pages_start = start_pg;
4216	} else {	4216	} else {
4217	if (!ftrace_pages)	4217	if (!ftrace_pages)
4218	goto out;	4218	goto out;
4219		4219
4220	if (WARN_ON(ftrace_pages->next)) {	4220	if (WARN_ON(ftrace_pages->next)) {
4221	/* Hmm, we have free pages? */	4221	/* Hmm, we have free pages? */
4222	while (ftrace_pages->next)	4222	while (ftrace_pages->next)
4223	ftrace_pages = ftrace_pages->next;	4223	ftrace_pages = ftrace_pages->next;
4224	}	4224	}
4225		4225
4226	ftrace_pages->next = start_pg;	4226	ftrace_pages->next = start_pg;
4227	}	4227	}
4228		4228
4229	p = start;	4229	p = start;
4230	pg = start_pg;	4230	pg = start_pg;
4231	while (p < end) {	4231	while (p < end) {
4232	addr = ftrace_call_adjust(*p++);	4232	addr = ftrace_call_adjust(*p++);
4233	/*	4233	/*
4234	* Some architecture linkers will pad between	4234	* Some architecture linkers will pad between
4235	* the different mcount_loc sections of different	4235	* the different mcount_loc sections of different
4236	* object files to satisfy alignments.	4236	* object files to satisfy alignments.
4237	* Skip any NULL pointers.	4237	* Skip any NULL pointers.
4238	*/	4238	*/
4239	if (!addr)	4239	if (!addr)
4240	continue;	4240	continue;
4241		4241
4242	if (pg->index == pg->size) {	4242	if (pg->index == pg->size) {
4243	/* We should have allocated enough */	4243	/* We should have allocated enough */
4244	if (WARN_ON(!pg->next))	4244	if (WARN_ON(!pg->next))
4245	break;	4245	break;
4246	pg = pg->next;	4246	pg = pg->next;
4247	}	4247	}
4248		4248
4249	rec = &pg->records[pg->index++];	4249	rec = &pg->records[pg->index++];
4250	rec->ip = addr;	4250	rec->ip = addr;
4251	}	4251	}
4252		4252
4253	/* We should have used all pages */	4253	/* We should have used all pages */
4254	WARN_ON(pg->next);	4254	WARN_ON(pg->next);
4255		4255
4256	/* Assign the last page to ftrace_pages */	4256	/* Assign the last page to ftrace_pages */
4257	ftrace_pages = pg;	4257	ftrace_pages = pg;
4258		4258
4259	/*	4259	/*
4260	* We only need to disable interrupts on start up	4260	* We only need to disable interrupts on start up
4261	* because we are modifying code that an interrupt	4261	* because we are modifying code that an interrupt
4262	* may execute, and the modification is not atomic.	4262	* may execute, and the modification is not atomic.
4263	* But for modules, nothing runs the code we modify	4263	* But for modules, nothing runs the code we modify
4264	* until we are finished with it, and there's no	4264	* until we are finished with it, and there's no
4265	* reason to cause large interrupt latencies while we do it.	4265	* reason to cause large interrupt latencies while we do it.
4266	*/	4266	*/
4267	if (!mod)	4267	if (!mod)
4268	local_irq_save(flags);	4268	local_irq_save(flags);
4269	ftrace_update_code(mod, start_pg);	4269	ftrace_update_code(mod, start_pg);
4270	if (!mod)	4270	if (!mod)
4271	local_irq_restore(flags);	4271	local_irq_restore(flags);
4272	ret = 0;	4272	ret = 0;
4273	out:	4273	out:
4274	mutex_unlock(&ftrace_lock);	4274	mutex_unlock(&ftrace_lock);
4275		4275
4276	return ret;	4276	return ret;
4277	}	4277	}
4278		4278
4279	#ifdef CONFIG_MODULES	4279	#ifdef CONFIG_MODULES
4280		4280
4281	#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)	4281	#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
4282		4282
4283	void ftrace_release_mod(struct module *mod)	4283	void ftrace_release_mod(struct module *mod)
4284	{	4284	{
4285	struct dyn_ftrace *rec;	4285	struct dyn_ftrace *rec;
4286	struct ftrace_page **last_pg;	4286	struct ftrace_page **last_pg;
4287	struct ftrace_page *pg;	4287	struct ftrace_page *pg;
4288	int order;	4288	int order;
4289		4289
4290	mutex_lock(&ftrace_lock);	4290	mutex_lock(&ftrace_lock);
4291		4291
4292	if (ftrace_disabled)	4292	if (ftrace_disabled)
4293	goto out_unlock;	4293	goto out_unlock;
4294		4294
4295	/*	4295	/*
4296	* Each module has its own ftrace_pages, remove	4296	* Each module has its own ftrace_pages, remove
4297	* them from the list.	4297	* them from the list.
4298	*/	4298	*/
4299	last_pg = &ftrace_pages_start;	4299	last_pg = &ftrace_pages_start;
4300	for (pg = ftrace_pages_start; pg; pg = *last_pg) {	4300	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
4301	rec = &pg->records[0];	4301	rec = &pg->records[0];
4302	if (within_module_core(rec->ip, mod)) {	4302	if (within_module_core(rec->ip, mod)) {
4303	/*	4303	/*
4304	* As core pages are first, the first	4304	* As core pages are first, the first
4305	* page should never be a module page.	4305	* page should never be a module page.
4306	*/	4306	*/
4307	if (WARN_ON(pg == ftrace_pages_start))	4307	if (WARN_ON(pg == ftrace_pages_start))
4308	goto out_unlock;	4308	goto out_unlock;
4309		4309
4310	/* Check if we are deleting the last page */	4310	/* Check if we are deleting the last page */
4311	if (pg == ftrace_pages)	4311	if (pg == ftrace_pages)
4312	ftrace_pages = next_to_ftrace_page(last_pg);	4312	ftrace_pages = next_to_ftrace_page(last_pg);
4313		4313
4314	*last_pg = pg->next;	4314	*last_pg = pg->next;
4315	order = get_count_order(pg->size / ENTRIES_PER_PAGE);	4315	order = get_count_order(pg->size / ENTRIES_PER_PAGE);
4316	free_pages((unsigned long)pg->records, order);	4316	free_pages((unsigned long)pg->records, order);
4317	kfree(pg);	4317	kfree(pg);
4318	} else	4318	} else
4319	last_pg = &pg->next;	4319	last_pg = &pg->next;
4320	}	4320	}
4321	out_unlock:	4321	out_unlock:
4322	mutex_unlock(&ftrace_lock);	4322	mutex_unlock(&ftrace_lock);
4323	}	4323	}
4324		4324
4325	static void ftrace_init_module(struct module *mod,	4325	static void ftrace_init_module(struct module *mod,
4326	unsigned long start, unsigned long end)	4326	unsigned long start, unsigned long end)
4327	{	4327	{
4328	if (ftrace_disabled \|\| start == end)	4328	if (ftrace_disabled \|\| start == end)
4329	return;	4329	return;
4330	ftrace_process_locs(mod, start, end);	4330	ftrace_process_locs(mod, start, end);
4331	}	4331	}
4332		4332
4333	static int ftrace_module_notify_enter(struct notifier_block *self,	4333	void ftrace_module_init(struct module *mod)
4334	unsigned long val, void *data)
4335	{	4334	{
4336	struct module *mod = data;	4335	ftrace_init_module(mod, mod->ftrace_callsites,
4337		4336	mod->ftrace_callsites +
4338	if (val == MODULE_STATE_COMING)	4337	mod->num_ftrace_callsites);
4339	ftrace_init_module(mod, mod->ftrace_callsites,
4340	mod->ftrace_callsites +
4341	mod->num_ftrace_callsites);
4342	return 0;
4343	}	4338	}
4344		4339
4345	static int ftrace_module_notify_exit(struct notifier_block *self,	4340	static int ftrace_module_notify_exit(struct notifier_block *self,
4346	unsigned long val, void *data)	4341	unsigned long val, void *data)
4347	{	4342	{
4348	struct module *mod = data;	4343	struct module *mod = data;
4349		4344
4350	if (val == MODULE_STATE_GOING)	4345	if (val == MODULE_STATE_GOING)
4351	ftrace_release_mod(mod);	4346	ftrace_release_mod(mod);
4352		4347
4353	return 0;	4348	return 0;
4354	}	4349	}
4355	#else	4350	#else
4356	static int ftrace_module_notify_enter(struct notifier_block *self,
4357	unsigned long val, void *data)
4358	{
4359	return 0;
4360	}
4361	static int ftrace_module_notify_exit(struct notifier_block *self,	4351	static int ftrace_module_notify_exit(struct notifier_block *self,
4362	unsigned long val, void *data)	4352	unsigned long val, void *data)
4363	{	4353	{
4364	return 0;	4354	return 0;
4365	}	4355	}
4366	#endif /* CONFIG_MODULES */	4356	#endif /* CONFIG_MODULES */
4367		4357
4368	struct notifier_block ftrace_module_enter_nb = {
4369	.notifier_call = ftrace_module_notify_enter,
4370	.priority = INT_MAX, /* Run before anything that can use kprobes */
4371	};
4372
4373	struct notifier_block ftrace_module_exit_nb = {	4358	struct notifier_block ftrace_module_exit_nb = {
4374	.notifier_call = ftrace_module_notify_exit,	4359	.notifier_call = ftrace_module_notify_exit,
4375	.priority = INT_MIN, /* Run after anything that can remove kprobes */	4360	.priority = INT_MIN, /* Run after anything that can remove kprobes */
4376	};	4361	};
4377		4362
4378	void __init ftrace_init(void)	4363	void __init ftrace_init(void)
4379	{	4364	{
4380	extern unsigned long __start_mcount_loc[];	4365	extern unsigned long __start_mcount_loc[];
4381	extern unsigned long __stop_mcount_loc[];	4366	extern unsigned long __stop_mcount_loc[];
4382	unsigned long count, flags;	4367	unsigned long count, flags;
4383	int ret;	4368	int ret;
4384		4369
4385	local_irq_save(flags);	4370	local_irq_save(flags);
4386	ret = ftrace_dyn_arch_init();	4371	ret = ftrace_dyn_arch_init();
4387	local_irq_restore(flags);	4372	local_irq_restore(flags);
4388	if (ret)	4373	if (ret)
4389	goto failed;	4374	goto failed;
4390		4375
4391	count = __stop_mcount_loc - __start_mcount_loc;	4376	count = __stop_mcount_loc - __start_mcount_loc;
4392	if (!count) {	4377	if (!count) {
4393	pr_info("ftrace: No functions to be traced?\n");	4378	pr_info("ftrace: No functions to be traced?\n");
4394	goto failed;	4379	goto failed;
4395	}	4380	}
4396		4381
4397	pr_info("ftrace: allocating %ld entries in %ld pages\n",	4382	pr_info("ftrace: allocating %ld entries in %ld pages\n",
4398	count, count / ENTRIES_PER_PAGE + 1);	4383	count, count / ENTRIES_PER_PAGE + 1);
4399		4384
4400	last_ftrace_enabled = ftrace_enabled = 1;	4385	last_ftrace_enabled = ftrace_enabled = 1;
4401		4386
4402	ret = ftrace_process_locs(NULL,	4387	ret = ftrace_process_locs(NULL,
4403	__start_mcount_loc,	4388	__start_mcount_loc,
4404	__stop_mcount_loc);	4389	__stop_mcount_loc);
4405
4406	ret = register_module_notifier(&ftrace_module_enter_nb);
4407	if (ret)
4408	pr_warning("Failed to register trace ftrace module enter notifier\n");
4409		4390
4410	ret = register_module_notifier(&ftrace_module_exit_nb);	4391	ret = register_module_notifier(&ftrace_module_exit_nb);
4411	if (ret)	4392	if (ret)
4412	pr_warning("Failed to register trace ftrace module exit notifier\n");	4393	pr_warning("Failed to register trace ftrace module exit notifier\n");
4413		4394
4414	set_ftrace_early_filters();	4395	set_ftrace_early_filters();
4415		4396
4416	return;	4397	return;
4417	failed:	4398	failed:
4418	ftrace_disabled = 1;	4399	ftrace_disabled = 1;
4419	}	4400	}
4420		4401
4421	#else	4402	#else
4422		4403
4423	static struct ftrace_ops global_ops = {	4404	static struct ftrace_ops global_ops = {
4424	.func = ftrace_stub,	4405	.func = ftrace_stub,
4425	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,	4406	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,
4426	INIT_REGEX_LOCK(global_ops)	4407	INIT_REGEX_LOCK(global_ops)
4427	};	4408	};
4428		4409
4429	static int __init ftrace_nodyn_init(void)	4410	static int __init ftrace_nodyn_init(void)
4430	{	4411	{
4431	ftrace_enabled = 1;	4412	ftrace_enabled = 1;
4432	return 0;	4413	return 0;
4433	}	4414	}
4434	core_initcall(ftrace_nodyn_init);	4415	core_initcall(ftrace_nodyn_init);
4435		4416
4436	static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }	4417	static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
4437	static inline void ftrace_startup_enable(int command) { }	4418	static inline void ftrace_startup_enable(int command) { }
4438	/* Keep as macros so we do not need to define the commands */	4419	/* Keep as macros so we do not need to define the commands */
4439	# define ftrace_startup(ops, command) \	4420	# define ftrace_startup(ops, command) \
4440	({ \	4421	({ \
4441	int ___ret = __register_ftrace_function(ops); \	4422	int ___ret = __register_ftrace_function(ops); \
4442	if (!___ret) \	4423	if (!___ret) \
4443	(ops)->flags \|= FTRACE_OPS_FL_ENABLED; \	4424	(ops)->flags \|= FTRACE_OPS_FL_ENABLED; \
4444	___ret; \	4425	___ret; \
4445	})	4426	})
4446	# define ftrace_shutdown(ops, command) \	4427	# define ftrace_shutdown(ops, command) \
4447	({ \	4428	({ \
4448	int ___ret = __unregister_ftrace_function(ops); \	4429	int ___ret = __unregister_ftrace_function(ops); \
4449	if (!___ret) \	4430	if (!___ret) \
4450	(ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \	4431	(ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \
4451	___ret; \	4432	___ret; \
4452	})	4433	})
4453		4434
4454	# define ftrace_startup_sysctl() do { } while (0)	4435	# define ftrace_startup_sysctl() do { } while (0)
4455	# define ftrace_shutdown_sysctl() do { } while (0)	4436	# define ftrace_shutdown_sysctl() do { } while (0)
4456		4437
4457	static inline int	4438	static inline int
4458	ftrace_ops_test(struct ftrace_ops ops, unsigned long ip, void regs)	4439	ftrace_ops_test(struct ftrace_ops ops, unsigned long ip, void regs)
4459	{	4440	{
4460	return 1;	4441	return 1;
4461	}	4442	}
4462		4443
4463	#endif /* CONFIG_DYNAMIC_FTRACE */	4444	#endif /* CONFIG_DYNAMIC_FTRACE */
4464		4445
4465	static void	4446	static void
4466	ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,	4447	ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
4467	struct ftrace_ops op, struct pt_regs regs)	4448	struct ftrace_ops op, struct pt_regs regs)
4468	{	4449	{
4469	if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))	4450	if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))
4470	return;	4451	return;
4471		4452
4472	/*	4453	/*
4473	* Some of the ops may be dynamically allocated,	4454	* Some of the ops may be dynamically allocated,
4474	* they must be freed after a synchronize_sched().	4455	* they must be freed after a synchronize_sched().
4475	*/	4456	*/
4476	preempt_disable_notrace();	4457	preempt_disable_notrace();
4477	trace_recursion_set(TRACE_CONTROL_BIT);	4458	trace_recursion_set(TRACE_CONTROL_BIT);
4478		4459
4479	/*	4460	/*
4480	* Control funcs (perf) uses RCU. Only trace if	4461	* Control funcs (perf) uses RCU. Only trace if
4481	* RCU is currently active.	4462	* RCU is currently active.
4482	*/	4463	*/
4483	if (!rcu_is_watching())	4464	if (!rcu_is_watching())
4484	goto out;	4465	goto out;
4485		4466
4486	do_for_each_ftrace_op(op, ftrace_control_list) {	4467	do_for_each_ftrace_op(op, ftrace_control_list) {
4487	if (!(op->flags & FTRACE_OPS_FL_STUB) &&	4468	if (!(op->flags & FTRACE_OPS_FL_STUB) &&
4488	!ftrace_function_local_disabled(op) &&	4469	!ftrace_function_local_disabled(op) &&
4489	ftrace_ops_test(op, ip, regs))	4470	ftrace_ops_test(op, ip, regs))
4490	op->func(ip, parent_ip, op, regs);	4471	op->func(ip, parent_ip, op, regs);
4491	} while_for_each_ftrace_op(op);	4472	} while_for_each_ftrace_op(op);
4492	out:	4473	out:
4493	trace_recursion_clear(TRACE_CONTROL_BIT);	4474	trace_recursion_clear(TRACE_CONTROL_BIT);
4494	preempt_enable_notrace();	4475	preempt_enable_notrace();
4495	}	4476	}
4496		4477
4497	static struct ftrace_ops control_ops = {	4478	static struct ftrace_ops control_ops = {
4498	.func = ftrace_ops_control_func,	4479	.func = ftrace_ops_control_func,
4499	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,	4480	.flags = FTRACE_OPS_FL_RECURSION_SAFE \| FTRACE_OPS_FL_INITIALIZED,
4500	INIT_REGEX_LOCK(control_ops)	4481	INIT_REGEX_LOCK(control_ops)
4501	};	4482	};
4502		4483
4503	static inline void	4484	static inline void
4504	__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,	4485	__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4505	struct ftrace_ops ignored, struct pt_regs regs)	4486	struct ftrace_ops ignored, struct pt_regs regs)
4506	{	4487	{
4507	struct ftrace_ops *op;	4488	struct ftrace_ops *op;
4508	int bit;	4489	int bit;
4509		4490
4510	if (function_trace_stop)	4491	if (function_trace_stop)
4511	return;	4492	return;
4512		4493
4513	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);	4494	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
4514	if (bit < 0)	4495	if (bit < 0)
4515	return;	4496	return;
4516		4497
4517	/*	4498	/*
4518	* Some of the ops may be dynamically allocated,	4499	* Some of the ops may be dynamically allocated,
4519	* they must be freed after a synchronize_sched().	4500	* they must be freed after a synchronize_sched().
4520	*/	4501	*/
4521	preempt_disable_notrace();	4502	preempt_disable_notrace();
4522	do_for_each_ftrace_op(op, ftrace_ops_list) {	4503	do_for_each_ftrace_op(op, ftrace_ops_list) {
4523	if (ftrace_ops_test(op, ip, regs))	4504	if (ftrace_ops_test(op, ip, regs))
4524	op->func(ip, parent_ip, op, regs);	4505	op->func(ip, parent_ip, op, regs);
4525	} while_for_each_ftrace_op(op);	4506	} while_for_each_ftrace_op(op);
4526	preempt_enable_notrace();	4507	preempt_enable_notrace();
4527	trace_clear_recursion(bit);	4508	trace_clear_recursion(bit);
4528	}	4509	}
4529		4510
4530	/*	4511	/*
4531	* Some archs only support passing ip and parent_ip. Even though	4512	* Some archs only support passing ip and parent_ip. Even though
4532	* the list function ignores the op parameter, we do not want any	4513	* the list function ignores the op parameter, we do not want any
4533	* C side effects, where a function is called without the caller	4514	* C side effects, where a function is called without the caller
4534	* sending a third parameter.	4515	* sending a third parameter.
4535	* Archs are to support both the regs and ftrace_ops at the same time.	4516	* Archs are to support both the regs and ftrace_ops at the same time.
4536	* If they support ftrace_ops, it is assumed they support regs.	4517	* If they support ftrace_ops, it is assumed they support regs.
4537	* If call backs want to use regs, they must either check for regs	4518	* If call backs want to use regs, they must either check for regs
4538	* being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.	4519	* being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
4539	* Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.	4520	* Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
4540	* An architecture can pass partial regs with ftrace_ops and still	4521	* An architecture can pass partial regs with ftrace_ops and still
4541	* set the ARCH_SUPPORT_FTARCE_OPS.	4522	* set the ARCH_SUPPORT_FTARCE_OPS.
4542	*/	4523	*/
4543	#if ARCH_SUPPORTS_FTRACE_OPS	4524	#if ARCH_SUPPORTS_FTRACE_OPS
4544	static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,	4525	static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4545	struct ftrace_ops op, struct pt_regs regs)	4526	struct ftrace_ops op, struct pt_regs regs)
4546	{	4527	{
4547	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);	4528	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
4548	}	4529	}
4549	#else	4530	#else
4550	static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)	4531	static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
4551	{	4532	{
4552	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);	4533	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
4553	}	4534	}
4554	#endif	4535	#endif
4555		4536
4556	static void clear_ftrace_swapper(void)	4537	static void clear_ftrace_swapper(void)
4557	{	4538	{
4558	struct task_struct *p;	4539	struct task_struct *p;
4559	int cpu;	4540	int cpu;
4560		4541
4561	get_online_cpus();	4542	get_online_cpus();
4562	for_each_online_cpu(cpu) {	4543	for_each_online_cpu(cpu) {
4563	p = idle_task(cpu);	4544	p = idle_task(cpu);
4564	clear_tsk_trace_trace(p);	4545	clear_tsk_trace_trace(p);
4565	}	4546	}
4566	put_online_cpus();	4547	put_online_cpus();
4567	}	4548	}
4568		4549
4569	static void set_ftrace_swapper(void)	4550	static void set_ftrace_swapper(void)
4570	{	4551	{
4571	struct task_struct *p;	4552	struct task_struct *p;
4572	int cpu;	4553	int cpu;
4573		4554
4574	get_online_cpus();	4555	get_online_cpus();
4575	for_each_online_cpu(cpu) {	4556	for_each_online_cpu(cpu) {
4576	p = idle_task(cpu);	4557	p = idle_task(cpu);
4577	set_tsk_trace_trace(p);	4558	set_tsk_trace_trace(p);
4578	}	4559	}
4579	put_online_cpus();	4560	put_online_cpus();
4580	}	4561	}
4581		4562
4582	static void clear_ftrace_pid(struct pid *pid)	4563	static void clear_ftrace_pid(struct pid *pid)
4583	{	4564	{
4584	struct task_struct *p;	4565	struct task_struct *p;
4585		4566
4586	rcu_read_lock();	4567	rcu_read_lock();
4587	do_each_pid_task(pid, PIDTYPE_PID, p) {	4568	do_each_pid_task(pid, PIDTYPE_PID, p) {
4588	clear_tsk_trace_trace(p);	4569	clear_tsk_trace_trace(p);
4589	} while_each_pid_task(pid, PIDTYPE_PID, p);	4570	} while_each_pid_task(pid, PIDTYPE_PID, p);
4590	rcu_read_unlock();	4571	rcu_read_unlock();
4591		4572
4592	put_pid(pid);	4573	put_pid(pid);
4593	}	4574	}
4594		4575
4595	static void set_ftrace_pid(struct pid *pid)	4576	static void set_ftrace_pid(struct pid *pid)
4596	{	4577	{
4597	struct task_struct *p;	4578	struct task_struct *p;
4598		4579
4599	rcu_read_lock();	4580	rcu_read_lock();
4600	do_each_pid_task(pid, PIDTYPE_PID, p) {	4581	do_each_pid_task(pid, PIDTYPE_PID, p) {
4601	set_tsk_trace_trace(p);	4582	set_tsk_trace_trace(p);
4602	} while_each_pid_task(pid, PIDTYPE_PID, p);	4583	} while_each_pid_task(pid, PIDTYPE_PID, p);
4603	rcu_read_unlock();	4584	rcu_read_unlock();
4604	}	4585	}
4605		4586
4606	static void clear_ftrace_pid_task(struct pid *pid)	4587	static void clear_ftrace_pid_task(struct pid *pid)
4607	{	4588	{
4608	if (pid == ftrace_swapper_pid)	4589	if (pid == ftrace_swapper_pid)
4609	clear_ftrace_swapper();	4590	clear_ftrace_swapper();
4610	else	4591	else
4611	clear_ftrace_pid(pid);	4592	clear_ftrace_pid(pid);
4612	}	4593	}
4613		4594
4614	static void set_ftrace_pid_task(struct pid *pid)	4595	static void set_ftrace_pid_task(struct pid *pid)
4615	{	4596	{
4616	if (pid == ftrace_swapper_pid)	4597	if (pid == ftrace_swapper_pid)
4617	set_ftrace_swapper();	4598	set_ftrace_swapper();
4618	else	4599	else
4619	set_ftrace_pid(pid);	4600	set_ftrace_pid(pid);
4620	}	4601	}
4621		4602
4622	static int ftrace_pid_add(int p)	4603	static int ftrace_pid_add(int p)
4623	{	4604	{
4624	struct pid *pid;	4605	struct pid *pid;
4625	struct ftrace_pid *fpid;	4606	struct ftrace_pid *fpid;
4626	int ret = -EINVAL;	4607	int ret = -EINVAL;
4627		4608
4628	mutex_lock(&ftrace_lock);	4609	mutex_lock(&ftrace_lock);
4629		4610
4630	if (!p)	4611	if (!p)
4631	pid = ftrace_swapper_pid;	4612	pid = ftrace_swapper_pid;
4632	else	4613	else
4633	pid = find_get_pid(p);	4614	pid = find_get_pid(p);
4634		4615
4635	if (!pid)	4616	if (!pid)
4636	goto out;	4617	goto out;
4637		4618
4638	ret = 0;	4619	ret = 0;
4639		4620
4640	list_for_each_entry(fpid, &ftrace_pids, list)	4621	list_for_each_entry(fpid, &ftrace_pids, list)
4641	if (fpid->pid == pid)	4622	if (fpid->pid == pid)
4642	goto out_put;	4623	goto out_put;
4643		4624
4644	ret = -ENOMEM;	4625	ret = -ENOMEM;
4645		4626
4646	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);	4627	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
4647	if (!fpid)	4628	if (!fpid)
4648	goto out_put;	4629	goto out_put;
4649		4630
4650	list_add(&fpid->list, &ftrace_pids);	4631	list_add(&fpid->list, &ftrace_pids);
4651	fpid->pid = pid;	4632	fpid->pid = pid;
4652		4633
4653	set_ftrace_pid_task(pid);	4634	set_ftrace_pid_task(pid);
4654		4635
4655	ftrace_update_pid_func();	4636	ftrace_update_pid_func();
4656	ftrace_startup_enable(0);	4637	ftrace_startup_enable(0);
4657		4638
4658	mutex_unlock(&ftrace_lock);	4639	mutex_unlock(&ftrace_lock);
4659	return 0;	4640	return 0;
4660		4641
4661	out_put:	4642	out_put:
4662	if (pid != ftrace_swapper_pid)	4643	if (pid != ftrace_swapper_pid)
4663	put_pid(pid);	4644	put_pid(pid);
4664		4645
4665	out:	4646	out:
4666	mutex_unlock(&ftrace_lock);	4647	mutex_unlock(&ftrace_lock);
4667	return ret;	4648	return ret;
4668	}	4649	}
4669		4650
4670	static void ftrace_pid_reset(void)	4651	static void ftrace_pid_reset(void)
4671	{	4652	{
4672	struct ftrace_pid fpid, safe;	4653	struct ftrace_pid fpid, safe;
4673		4654
4674	mutex_lock(&ftrace_lock);	4655	mutex_lock(&ftrace_lock);
4675	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {	4656	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
4676	struct pid *pid = fpid->pid;	4657	struct pid *pid = fpid->pid;
4677		4658
4678	clear_ftrace_pid_task(pid);	4659	clear_ftrace_pid_task(pid);
4679		4660
4680	list_del(&fpid->list);	4661	list_del(&fpid->list);
4681	kfree(fpid);	4662	kfree(fpid);
4682	}	4663	}
4683		4664
4684	ftrace_update_pid_func();	4665	ftrace_update_pid_func();
4685	ftrace_startup_enable(0);	4666	ftrace_startup_enable(0);
4686		4667
4687	mutex_unlock(&ftrace_lock);	4668	mutex_unlock(&ftrace_lock);
4688	}	4669	}
4689		4670
4690	static void fpid_start(struct seq_file m, loff_t *pos)	4671	static void fpid_start(struct seq_file m, loff_t *pos)
4691	{	4672	{
4692	mutex_lock(&ftrace_lock);	4673	mutex_lock(&ftrace_lock);
4693		4674
4694	if (list_empty(&ftrace_pids) && (!*pos))	4675	if (list_empty(&ftrace_pids) && (!*pos))
4695	return (void *) 1;	4676	return (void *) 1;
4696		4677
4697	return seq_list_start(&ftrace_pids, *pos);	4678	return seq_list_start(&ftrace_pids, *pos);
4698	}	4679	}
4699		4680
4700	static void fpid_next(struct seq_file m, void v, loff_t pos)	4681	static void fpid_next(struct seq_file m, void v, loff_t pos)
4701	{	4682	{
4702	if (v == (void *)1)	4683	if (v == (void *)1)
4703	return NULL;	4684	return NULL;
4704		4685
4705	return seq_list_next(v, &ftrace_pids, pos);	4686	return seq_list_next(v, &ftrace_pids, pos);
4706	}	4687	}
4707		4688
4708	static void fpid_stop(struct seq_file m, void p)	4689	static void fpid_stop(struct seq_file m, void p)
4709	{	4690	{
4710	mutex_unlock(&ftrace_lock);	4691	mutex_unlock(&ftrace_lock);
4711	}	4692	}
4712		4693
4713	static int fpid_show(struct seq_file m, void v)	4694	static int fpid_show(struct seq_file m, void v)
4714	{	4695	{
4715	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);	4696	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
4716		4697
4717	if (v == (void *)1) {	4698	if (v == (void *)1) {
4718	seq_printf(m, "no pid\n");	4699	seq_printf(m, "no pid\n");
4719	return 0;	4700	return 0;
4720	}	4701	}
4721		4702
4722	if (fpid->pid == ftrace_swapper_pid)	4703	if (fpid->pid == ftrace_swapper_pid)
4723	seq_printf(m, "swapper tasks\n");	4704	seq_printf(m, "swapper tasks\n");
4724	else	4705	else
4725	seq_printf(m, "%u\n", pid_vnr(fpid->pid));	4706	seq_printf(m, "%u\n", pid_vnr(fpid->pid));
4726		4707
4727	return 0;	4708	return 0;
4728	}	4709	}
4729		4710
4730	static const struct seq_operations ftrace_pid_sops = {	4711	static const struct seq_operations ftrace_pid_sops = {
4731	.start = fpid_start,	4712	.start = fpid_start,
4732	.next = fpid_next,	4713	.next = fpid_next,
4733	.stop = fpid_stop,	4714	.stop = fpid_stop,
4734	.show = fpid_show,	4715	.show = fpid_show,
4735	};	4716	};
4736		4717
4737	static int	4718	static int
4738	ftrace_pid_open(struct inode inode, struct file file)	4719	ftrace_pid_open(struct inode inode, struct file file)
4739	{	4720	{
4740	int ret = 0;	4721	int ret = 0;
4741		4722
4742	if ((file->f_mode & FMODE_WRITE) &&	4723	if ((file->f_mode & FMODE_WRITE) &&
4743	(file->f_flags & O_TRUNC))	4724	(file->f_flags & O_TRUNC))
4744	ftrace_pid_reset();	4725	ftrace_pid_reset();
4745		4726
4746	if (file->f_mode & FMODE_READ)	4727	if (file->f_mode & FMODE_READ)
4747	ret = seq_open(file, &ftrace_pid_sops);	4728	ret = seq_open(file, &ftrace_pid_sops);
4748		4729
4749	return ret;	4730	return ret;
4750	}	4731	}
4751		4732
4752	static ssize_t	4733	static ssize_t
4753	ftrace_pid_write(struct file filp, const char __user ubuf,	4734	ftrace_pid_write(struct file filp, const char __user ubuf,
4754	size_t cnt, loff_t *ppos)	4735	size_t cnt, loff_t *ppos)
4755	{	4736	{
4756	char buf[64], *tmp;	4737	char buf[64], *tmp;
4757	long val;	4738	long val;
4758	int ret;	4739	int ret;
4759		4740
4760	if (cnt >= sizeof(buf))	4741	if (cnt >= sizeof(buf))
4761	return -EINVAL;	4742	return -EINVAL;
4762		4743
4763	if (copy_from_user(&buf, ubuf, cnt))	4744	if (copy_from_user(&buf, ubuf, cnt))
4764	return -EFAULT;	4745	return -EFAULT;
4765		4746
4766	buf[cnt] = 0;	4747	buf[cnt] = 0;
4767		4748
4768	/*	4749	/*
4769	* Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"	4750	* Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
4770	* to clean the filter quietly.	4751	* to clean the filter quietly.
4771	*/	4752	*/
4772	tmp = strstrip(buf);	4753	tmp = strstrip(buf);
4773	if (strlen(tmp) == 0)	4754	if (strlen(tmp) == 0)
4774	return 1;	4755	return 1;
4775		4756
4776	ret = kstrtol(tmp, 10, &val);	4757	ret = kstrtol(tmp, 10, &val);
4777	if (ret < 0)	4758	if (ret < 0)
4778	return ret;	4759	return ret;
4779		4760
4780	ret = ftrace_pid_add(val);	4761	ret = ftrace_pid_add(val);
4781		4762
4782	return ret ? ret : cnt;	4763	return ret ? ret : cnt;
4783	}	4764	}
4784		4765
4785	static int	4766	static int
4786	ftrace_pid_release(struct inode inode, struct file file)	4767	ftrace_pid_release(struct inode inode, struct file file)
4787	{	4768	{
4788	if (file->f_mode & FMODE_READ)	4769	if (file->f_mode & FMODE_READ)
4789	seq_release(inode, file);	4770	seq_release(inode, file);
4790		4771
4791	return 0;	4772	return 0;
4792	}	4773	}
4793		4774
4794	static const struct file_operations ftrace_pid_fops = {	4775	static const struct file_operations ftrace_pid_fops = {
4795	.open = ftrace_pid_open,	4776	.open = ftrace_pid_open,
4796	.write = ftrace_pid_write,	4777	.write = ftrace_pid_write,
4797	.read = seq_read,	4778	.read = seq_read,
4798	.llseek = tracing_lseek,	4779	.llseek = tracing_lseek,
4799	.release = ftrace_pid_release,	4780	.release = ftrace_pid_release,
4800	};	4781	};
4801		4782
4802	static __init int ftrace_init_debugfs(void)	4783	static __init int ftrace_init_debugfs(void)
4803	{	4784	{
4804	struct dentry *d_tracer;	4785	struct dentry *d_tracer;
4805		4786
4806	d_tracer = tracing_init_dentry();	4787	d_tracer = tracing_init_dentry();
4807	if (!d_tracer)	4788	if (!d_tracer)
4808	return 0;	4789	return 0;
4809		4790
4810	ftrace_init_dyn_debugfs(d_tracer);	4791	ftrace_init_dyn_debugfs(d_tracer);
4811		4792
4812	trace_create_file("set_ftrace_pid", 0644, d_tracer,	4793	trace_create_file("set_ftrace_pid", 0644, d_tracer,
4813	NULL, &ftrace_pid_fops);	4794	NULL, &ftrace_pid_fops);
4814		4795
4815	ftrace_profile_debugfs(d_tracer);	4796	ftrace_profile_debugfs(d_tracer);
4816		4797
4817	return 0;	4798	return 0;
4818	}	4799	}
4819	fs_initcall(ftrace_init_debugfs);	4800	fs_initcall(ftrace_init_debugfs);
4820		4801
4821	/**	4802	/**
4822	* ftrace_kill - kill ftrace	4803	* ftrace_kill - kill ftrace
4823	*	4804	*
4824	* This function should be used by panic code. It stops ftrace	4805	* This function should be used by panic code. It stops ftrace
4825	* but in a not so nice way. If you need to simply kill ftrace	4806	* but in a not so nice way. If you need to simply kill ftrace
4826	* from a non-atomic section, use ftrace_kill.	4807	* from a non-atomic section, use ftrace_kill.
4827	*/	4808	*/
4828	void ftrace_kill(void)	4809	void ftrace_kill(void)
4829	{	4810	{
4830	ftrace_disabled = 1;	4811	ftrace_disabled = 1;
4831	ftrace_enabled = 0;	4812	ftrace_enabled = 0;
4832	clear_ftrace_function();	4813	clear_ftrace_function();
4833	}	4814	}
4834		4815
4835	/**	4816	/**
4836	* Test if ftrace is dead or not.	4817	* Test if ftrace is dead or not.
4837	*/	4818	*/
4838	int ftrace_is_dead(void)	4819	int ftrace_is_dead(void)
4839	{	4820	{
4840	return ftrace_disabled;	4821	return ftrace_disabled;
4841	}	4822	}
4842		4823
4843	/**	4824	/**
4844	* register_ftrace_function - register a function for profiling	4825	* register_ftrace_function - register a function for profiling
4845	* @ops - ops structure that holds the function for profiling.	4826	* @ops - ops structure that holds the function for profiling.
4846	*	4827	*
4847	* Register a function to be called by all functions in the	4828	* Register a function to be called by all functions in the
4848	* kernel.	4829	* kernel.
4849	*	4830	*
4850	* Note: @ops->func and all the functions it calls must be labeled	4831	* Note: @ops->func and all the functions it calls must be labeled
4851	* with "notrace", otherwise it will go into a	4832	* with "notrace", otherwise it will go into a
4852	* recursive loop.	4833	* recursive loop.
4853	*/	4834	*/
4854	int register_ftrace_function(struct ftrace_ops *ops)	4835	int register_ftrace_function(struct ftrace_ops *ops)
4855	{	4836	{
4856	int ret = -1;	4837	int ret = -1;
4857		4838
4858	ftrace_ops_init(ops);	4839	ftrace_ops_init(ops);
4859		4840
4860	mutex_lock(&ftrace_lock);	4841	mutex_lock(&ftrace_lock);
4861		4842
4862	ret = ftrace_startup(ops, 0);	4843	ret = ftrace_startup(ops, 0);
4863		4844
4864	mutex_unlock(&ftrace_lock);	4845	mutex_unlock(&ftrace_lock);
4865		4846
4866	return ret;	4847	return ret;
4867	}	4848	}
4868	EXPORT_SYMBOL_GPL(register_ftrace_function);	4849	EXPORT_SYMBOL_GPL(register_ftrace_function);
4869		4850
4870	/**	4851	/**
4871	* unregister_ftrace_function - unregister a function for profiling.	4852	* unregister_ftrace_function - unregister a function for profiling.
4872	* @ops - ops structure that holds the function to unregister	4853	* @ops - ops structure that holds the function to unregister
4873	*	4854	*
4874	* Unregister a function that was added to be called by ftrace profiling.	4855	* Unregister a function that was added to be called by ftrace profiling.
4875	*/	4856	*/
4876	int unregister_ftrace_function(struct ftrace_ops *ops)	4857	int unregister_ftrace_function(struct ftrace_ops *ops)
4877	{	4858	{
4878	int ret;	4859	int ret;
4879		4860
4880	mutex_lock(&ftrace_lock);	4861	mutex_lock(&ftrace_lock);
4881	ret = ftrace_shutdown(ops, 0);	4862	ret = ftrace_shutdown(ops, 0);
4882	mutex_unlock(&ftrace_lock);	4863	mutex_unlock(&ftrace_lock);
4883		4864
4884	return ret;	4865	return ret;
4885	}	4866	}
4886	EXPORT_SYMBOL_GPL(unregister_ftrace_function);	4867	EXPORT_SYMBOL_GPL(unregister_ftrace_function);
4887		4868
4888	int	4869	int
4889	ftrace_enable_sysctl(struct ctl_table *table, int write,	4870	ftrace_enable_sysctl(struct ctl_table *table, int write,
4890	void __user buffer, size_t lenp,	4871	void __user buffer, size_t lenp,
4891	loff_t *ppos)	4872	loff_t *ppos)
4892	{	4873	{
4893	int ret = -ENODEV;	4874	int ret = -ENODEV;
4894		4875
4895	mutex_lock(&ftrace_lock);	4876	mutex_lock(&ftrace_lock);
4896		4877
4897	if (unlikely(ftrace_disabled))	4878	if (unlikely(ftrace_disabled))
4898	goto out;	4879	goto out;
4899		4880
4900	ret = proc_dointvec(table, write, buffer, lenp, ppos);	4881	ret = proc_dointvec(table, write, buffer, lenp, ppos);
4901		4882
4902	if (ret \|\| !write \|\| (last_ftrace_enabled == !!ftrace_enabled))	4883	if (ret \|\| !write \|\| (last_ftrace_enabled == !!ftrace_enabled))
4903	goto out;	4884	goto out;
4904		4885
4905	last_ftrace_enabled = !!ftrace_enabled;	4886	last_ftrace_enabled = !!ftrace_enabled;
4906		4887
4907	if (ftrace_enabled) {	4888	if (ftrace_enabled) {
4908		4889
4909	ftrace_startup_sysctl();	4890	ftrace_startup_sysctl();
4910		4891
4911	/* we are starting ftrace again */	4892	/* we are starting ftrace again */
4912	if (ftrace_ops_list != &ftrace_list_end)	4893	if (ftrace_ops_list != &ftrace_list_end)
4913	update_ftrace_function();	4894	update_ftrace_function();
4914		4895
4915	} else {	4896	} else {
4916	/* stopping ftrace calls (just send to ftrace_stub) */	4897	/* stopping ftrace calls (just send to ftrace_stub) */
4917	ftrace_trace_function = ftrace_stub;	4898	ftrace_trace_function = ftrace_stub;
4918		4899
4919	ftrace_shutdown_sysctl();	4900	ftrace_shutdown_sysctl();
4920	}	4901	}
4921		4902
4922	out:	4903	out:
4923	mutex_unlock(&ftrace_lock);	4904	mutex_unlock(&ftrace_lock);
4924	return ret;	4905	return ret;
4925	}	4906	}
4926		4907
4927	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	4908	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4928		4909
4929	static int ftrace_graph_active;	4910	static int ftrace_graph_active;
4930	static struct notifier_block ftrace_suspend_notifier;	4911	static struct notifier_block ftrace_suspend_notifier;
4931		4912
4932	int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)	4913	int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
4933	{	4914	{
4934	return 0;	4915	return 0;
4935	}	4916	}
4936		4917
4937	/* The callbacks that hook a function */	4918	/* The callbacks that hook a function */
4938	trace_func_graph_ret_t ftrace_graph_return =	4919	trace_func_graph_ret_t ftrace_graph_return =
4939	(trace_func_graph_ret_t)ftrace_stub;	4920	(trace_func_graph_ret_t)ftrace_stub;
4940	trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;	4921	trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4941	static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;	4922	static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
4942		4923
4943	/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */	4924	/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
4944	static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)	4925	static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
4945	{	4926	{
4946	int i;	4927	int i;
4947	int ret = 0;	4928	int ret = 0;
4948	unsigned long flags;	4929	unsigned long flags;
4949	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;	4930	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
4950	struct task_struct g, t;	4931	struct task_struct g, t;
4951		4932
4952	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {	4933	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
4953	ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH	4934	ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
4954	* sizeof(struct ftrace_ret_stack),	4935	* sizeof(struct ftrace_ret_stack),
4955	GFP_KERNEL);	4936	GFP_KERNEL);
4956	if (!ret_stack_list[i]) {	4937	if (!ret_stack_list[i]) {
4957	start = 0;	4938	start = 0;
4958	end = i;	4939	end = i;
4959	ret = -ENOMEM;	4940	ret = -ENOMEM;
4960	goto free;	4941	goto free;
4961	}	4942	}
4962	}	4943	}
4963		4944
4964	read_lock_irqsave(&tasklist_lock, flags);	4945	read_lock_irqsave(&tasklist_lock, flags);
4965	do_each_thread(g, t) {	4946	do_each_thread(g, t) {
4966	if (start == end) {	4947	if (start == end) {
4967	ret = -EAGAIN;	4948	ret = -EAGAIN;
4968	goto unlock;	4949	goto unlock;
4969	}	4950	}
4970		4951
4971	if (t->ret_stack == NULL) {	4952	if (t->ret_stack == NULL) {
4972	atomic_set(&t->tracing_graph_pause, 0);	4953	atomic_set(&t->tracing_graph_pause, 0);
4973	atomic_set(&t->trace_overrun, 0);	4954	atomic_set(&t->trace_overrun, 0);
4974	t->curr_ret_stack = -1;	4955	t->curr_ret_stack = -1;
4975	/* Make sure the tasks see the -1 first: */	4956	/* Make sure the tasks see the -1 first: */
4976	smp_wmb();	4957	smp_wmb();
4977	t->ret_stack = ret_stack_list[start++];	4958	t->ret_stack = ret_stack_list[start++];
4978	}	4959	}
4979	} while_each_thread(g, t);	4960	} while_each_thread(g, t);
4980		4961
4981	unlock:	4962	unlock:
4982	read_unlock_irqrestore(&tasklist_lock, flags);	4963	read_unlock_irqrestore(&tasklist_lock, flags);
4983	free:	4964	free:
4984	for (i = start; i < end; i++)	4965	for (i = start; i < end; i++)
4985	kfree(ret_stack_list[i]);	4966	kfree(ret_stack_list[i]);
4986	return ret;	4967	return ret;
4987	}	4968	}
4988		4969
4989	static void	4970	static void
4990	ftrace_graph_probe_sched_switch(void *ignore,	4971	ftrace_graph_probe_sched_switch(void *ignore,
4991	struct task_struct prev, struct task_struct next)	4972	struct task_struct prev, struct task_struct next)
4992	{	4973	{
4993	unsigned long long timestamp;	4974	unsigned long long timestamp;
4994	int index;	4975	int index;
4995		4976
4996	/*	4977	/*
4997	* Does the user want to count the time a function was asleep.	4978	* Does the user want to count the time a function was asleep.
4998	* If so, do not update the time stamps.	4979	* If so, do not update the time stamps.
4999	*/	4980	*/
5000	if (trace_flags & TRACE_ITER_SLEEP_TIME)	4981	if (trace_flags & TRACE_ITER_SLEEP_TIME)
5001	return;	4982	return;
5002		4983
5003	timestamp = trace_clock_local();	4984	timestamp = trace_clock_local();
5004		4985
5005	prev->ftrace_timestamp = timestamp;	4986	prev->ftrace_timestamp = timestamp;
5006		4987
5007	/* only process tasks that we timestamped */	4988	/* only process tasks that we timestamped */
5008	if (!next->ftrace_timestamp)	4989	if (!next->ftrace_timestamp)
5009	return;	4990	return;
5010		4991
5011	/*	4992	/*
5012	* Update all the counters in next to make up for the	4993	* Update all the counters in next to make up for the
5013	* time next was sleeping.	4994	* time next was sleeping.
5014	*/	4995	*/
5015	timestamp -= next->ftrace_timestamp;	4996	timestamp -= next->ftrace_timestamp;
5016		4997
5017	for (index = next->curr_ret_stack; index >= 0; index--)	4998	for (index = next->curr_ret_stack; index >= 0; index--)
5018	next->ret_stack[index].calltime += timestamp;	4999	next->ret_stack[index].calltime += timestamp;
5019	}	5000	}
5020		5001
5021	/* Allocate a return stack for each task */	5002	/* Allocate a return stack for each task */
5022	static int start_graph_tracing(void)	5003	static int start_graph_tracing(void)
5023	{	5004	{
5024	struct ftrace_ret_stack **ret_stack_list;	5005	struct ftrace_ret_stack **ret_stack_list;
5025	int ret, cpu;	5006	int ret, cpu;
5026		5007
5027	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *	5008	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
5028	sizeof(struct ftrace_ret_stack *),	5009	sizeof(struct ftrace_ret_stack *),
5029	GFP_KERNEL);	5010	GFP_KERNEL);
5030		5011
5031	if (!ret_stack_list)	5012	if (!ret_stack_list)
5032	return -ENOMEM;	5013	return -ENOMEM;
5033		5014
5034	/* The cpu_boot init_task->ret_stack will never be freed */	5015	/* The cpu_boot init_task->ret_stack will never be freed */
5035	for_each_online_cpu(cpu) {	5016	for_each_online_cpu(cpu) {
5036	if (!idle_task(cpu)->ret_stack)	5017	if (!idle_task(cpu)->ret_stack)
5037	ftrace_graph_init_idle_task(idle_task(cpu), cpu);	5018	ftrace_graph_init_idle_task(idle_task(cpu), cpu);
5038	}	5019	}
5039		5020
5040	do {	5021	do {
5041	ret = alloc_retstack_tasklist(ret_stack_list);	5022	ret = alloc_retstack_tasklist(ret_stack_list);
5042	} while (ret == -EAGAIN);	5023	} while (ret == -EAGAIN);
5043		5024
5044	if (!ret) {	5025	if (!ret) {
5045	ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);	5026	ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5046	if (ret)	5027	if (ret)
5047	pr_info("ftrace_graph: Couldn't activate tracepoint"	5028	pr_info("ftrace_graph: Couldn't activate tracepoint"
5048	" probe to kernel_sched_switch\n");	5029	" probe to kernel_sched_switch\n");
5049	}	5030	}
5050		5031
5051	kfree(ret_stack_list);	5032	kfree(ret_stack_list);
5052	return ret;	5033	return ret;
5053	}	5034	}
5054		5035
5055	/*	5036	/*
5056	* Hibernation protection.	5037	* Hibernation protection.
5057	* The state of the current task is too much unstable during	5038	* The state of the current task is too much unstable during
5058	* suspend/restore to disk. We want to protect against that.	5039	* suspend/restore to disk. We want to protect against that.
5059	*/	5040	*/
5060	static int	5041	static int
5061	ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,	5042	ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
5062	void *unused)	5043	void *unused)
5063	{	5044	{
5064	switch (state) {	5045	switch (state) {
5065	case PM_HIBERNATION_PREPARE:	5046	case PM_HIBERNATION_PREPARE:
5066	pause_graph_tracing();	5047	pause_graph_tracing();
5067	break;	5048	break;
5068		5049
5069	case PM_POST_HIBERNATION:	5050	case PM_POST_HIBERNATION:
5070	unpause_graph_tracing();	5051	unpause_graph_tracing();
5071	break;	5052	break;
5072	}	5053	}
5073	return NOTIFY_DONE;	5054	return NOTIFY_DONE;
5074	}	5055	}
5075		5056
5076	/* Just a place holder for function graph */	5057	/* Just a place holder for function graph */
5077	static struct ftrace_ops fgraph_ops __read_mostly = {	5058	static struct ftrace_ops fgraph_ops __read_mostly = {
5078	.func = ftrace_stub,	5059	.func = ftrace_stub,
5079	.flags = FTRACE_OPS_FL_STUB \| FTRACE_OPS_FL_GLOBAL \|	5060	.flags = FTRACE_OPS_FL_STUB \| FTRACE_OPS_FL_GLOBAL \|
5080	FTRACE_OPS_FL_RECURSION_SAFE,	5061	FTRACE_OPS_FL_RECURSION_SAFE,
5081	};	5062	};
5082		5063
5083	static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)	5064	static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
5084	{	5065	{
5085	if (!ftrace_ops_test(&global_ops, trace->func, NULL))	5066	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
5086	return 0;	5067	return 0;
5087	return __ftrace_graph_entry(trace);	5068	return __ftrace_graph_entry(trace);
5088	}	5069	}
5089		5070
5090	/*	5071	/*
5091	* The function graph tracer should only trace the functions defined	5072	* The function graph tracer should only trace the functions defined
5092	* by set_ftrace_filter and set_ftrace_notrace. If another function	5073	* by set_ftrace_filter and set_ftrace_notrace. If another function
5093	* tracer ops is registered, the graph tracer requires testing the	5074	* tracer ops is registered, the graph tracer requires testing the
5094	* function against the global ops, and not just trace any function	5075	* function against the global ops, and not just trace any function
5095	* that any ftrace_ops registered.	5076	* that any ftrace_ops registered.
5096	*/	5077	*/
5097	static void update_function_graph_func(void)	5078	static void update_function_graph_func(void)
5098	{	5079	{
5099	if (ftrace_ops_list == &ftrace_list_end \|\|	5080	if (ftrace_ops_list == &ftrace_list_end \|\|
5100	(ftrace_ops_list == &global_ops &&	5081	(ftrace_ops_list == &global_ops &&
5101	global_ops.next == &ftrace_list_end))	5082	global_ops.next == &ftrace_list_end))
5102	ftrace_graph_entry = __ftrace_graph_entry;	5083	ftrace_graph_entry = __ftrace_graph_entry;
5103	else	5084	else
5104	ftrace_graph_entry = ftrace_graph_entry_test;	5085	ftrace_graph_entry = ftrace_graph_entry_test;
5105	}	5086	}
5106		5087
5107	int register_ftrace_graph(trace_func_graph_ret_t retfunc,	5088	int register_ftrace_graph(trace_func_graph_ret_t retfunc,
5108	trace_func_graph_ent_t entryfunc)	5089	trace_func_graph_ent_t entryfunc)
5109	{	5090	{
5110	int ret = 0;	5091	int ret = 0;
5111		5092
5112	mutex_lock(&ftrace_lock);	5093	mutex_lock(&ftrace_lock);
5113		5094
5114	/* we currently allow only one tracer registered at a time */	5095	/* we currently allow only one tracer registered at a time */
5115	if (ftrace_graph_active) {	5096	if (ftrace_graph_active) {
5116	ret = -EBUSY;	5097	ret = -EBUSY;
5117	goto out;	5098	goto out;
5118	}	5099	}
5119		5100
5120	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;	5101	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
5121	register_pm_notifier(&ftrace_suspend_notifier);	5102	register_pm_notifier(&ftrace_suspend_notifier);
5122		5103
5123	ftrace_graph_active++;	5104	ftrace_graph_active++;
5124	ret = start_graph_tracing();	5105	ret = start_graph_tracing();
5125	if (ret) {	5106	if (ret) {
5126	ftrace_graph_active--;	5107	ftrace_graph_active--;
5127	goto out;	5108	goto out;
5128	}	5109	}
5129		5110
5130	ftrace_graph_return = retfunc;	5111	ftrace_graph_return = retfunc;
5131		5112
5132	/*	5113	/*
5133	* Update the indirect function to the entryfunc, and the	5114	* Update the indirect function to the entryfunc, and the
5134	* function that gets called to the entry_test first. Then	5115	* function that gets called to the entry_test first. Then
5135	* call the update fgraph entry function to determine if	5116	* call the update fgraph entry function to determine if
5136	* the entryfunc should be called directly or not.	5117	* the entryfunc should be called directly or not.
5137	*/	5118	*/
5138	__ftrace_graph_entry = entryfunc;	5119	__ftrace_graph_entry = entryfunc;
5139	ftrace_graph_entry = ftrace_graph_entry_test;	5120	ftrace_graph_entry = ftrace_graph_entry_test;
5140	update_function_graph_func();	5121	update_function_graph_func();
5141		5122
5142	ret = ftrace_startup(&fgraph_ops, FTRACE_START_FUNC_RET);	5123	ret = ftrace_startup(&fgraph_ops, FTRACE_START_FUNC_RET);
5143		5124
5144	out:	5125	out:
5145	mutex_unlock(&ftrace_lock);	5126	mutex_unlock(&ftrace_lock);
5146	return ret;	5127	return ret;
5147	}	5128	}
5148		5129
5149	void unregister_ftrace_graph(void)	5130	void unregister_ftrace_graph(void)
5150	{	5131	{
5151	mutex_lock(&ftrace_lock);	5132	mutex_lock(&ftrace_lock);
5152		5133
5153	if (unlikely(!ftrace_graph_active))	5134	if (unlikely(!ftrace_graph_active))
5154	goto out;	5135	goto out;
5155		5136
5156	ftrace_graph_active--;	5137	ftrace_graph_active--;
5157	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;	5138	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
5158	ftrace_graph_entry = ftrace_graph_entry_stub;	5139	ftrace_graph_entry = ftrace_graph_entry_stub;
5159	__ftrace_graph_entry = ftrace_graph_entry_stub;	5140	__ftrace_graph_entry = ftrace_graph_entry_stub;
5160	ftrace_shutdown(&fgraph_ops, FTRACE_STOP_FUNC_RET);	5141	ftrace_shutdown(&fgraph_ops, FTRACE_STOP_FUNC_RET);
5161	unregister_pm_notifier(&ftrace_suspend_notifier);	5142	unregister_pm_notifier(&ftrace_suspend_notifier);
5162	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);	5143	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5163		5144
5164	out:	5145	out:
5165	mutex_unlock(&ftrace_lock);	5146	mutex_unlock(&ftrace_lock);
5166	}	5147	}
5167		5148
5168	static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);	5149	static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
5169		5150
5170	static void	5151	static void
5171	graph_init_task(struct task_struct t, struct ftrace_ret_stack ret_stack)	5152	graph_init_task(struct task_struct t, struct ftrace_ret_stack ret_stack)
5172	{	5153	{
5173	atomic_set(&t->tracing_graph_pause, 0);	5154	atomic_set(&t->tracing_graph_pause, 0);
5174	atomic_set(&t->trace_overrun, 0);	5155	atomic_set(&t->trace_overrun, 0);
5175	t->ftrace_timestamp = 0;	5156	t->ftrace_timestamp = 0;
5176	/* make curr_ret_stack visible before we add the ret_stack */	5157	/* make curr_ret_stack visible before we add the ret_stack */
5177	smp_wmb();	5158	smp_wmb();
5178	t->ret_stack = ret_stack;	5159	t->ret_stack = ret_stack;
5179	}	5160	}
5180		5161
5181	/*	5162	/*
5182	* Allocate a return stack for the idle task. May be the first	5163	* Allocate a return stack for the idle task. May be the first
5183	* time through, or it may be done by CPU hotplug online.	5164	* time through, or it may be done by CPU hotplug online.
5184	*/	5165	*/
5185	void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)	5166	void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
5186	{	5167	{
5187	t->curr_ret_stack = -1;	5168	t->curr_ret_stack = -1;
5188	/*	5169	/*
5189	* The idle task has no parent, it either has its own	5170	* The idle task has no parent, it either has its own
5190	* stack or no stack at all.	5171	* stack or no stack at all.
5191	*/	5172	*/
5192	if (t->ret_stack)	5173	if (t->ret_stack)
5193	WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));	5174	WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
5194		5175
5195	if (ftrace_graph_active) {	5176	if (ftrace_graph_active) {
5196	struct ftrace_ret_stack *ret_stack;	5177	struct ftrace_ret_stack *ret_stack;
5197		5178
5198	ret_stack = per_cpu(idle_ret_stack, cpu);	5179	ret_stack = per_cpu(idle_ret_stack, cpu);
5199	if (!ret_stack) {	5180	if (!ret_stack) {
5200	ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH	5181	ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
5201	* sizeof(struct ftrace_ret_stack),	5182	* sizeof(struct ftrace_ret_stack),
5202	GFP_KERNEL);	5183	GFP_KERNEL);
5203	if (!ret_stack)	5184	if (!ret_stack)
5204	return;	5185	return;
5205	per_cpu(idle_ret_stack, cpu) = ret_stack;	5186	per_cpu(idle_ret_stack, cpu) = ret_stack;
5206	}	5187	}
5207	graph_init_task(t, ret_stack);	5188	graph_init_task(t, ret_stack);
5208	}	5189	}
5209	}	5190	}
5210		5191
5211	/* Allocate a return stack for newly created task */	5192	/* Allocate a return stack for newly created task */
5212	void ftrace_graph_init_task(struct task_struct *t)	5193	void ftrace_graph_init_task(struct task_struct *t)
5213	{	5194	{
5214	/* Make sure we do not use the parent ret_stack */	5195	/* Make sure we do not use the parent ret_stack */
5215	t->ret_stack = NULL;	5196	t->ret_stack = NULL;
5216	t->curr_ret_stack = -1;	5197	t->curr_ret_stack = -1;
5217		5198
5218	if (ftrace_graph_active) {	5199	if (ftrace_graph_active) {
5219	struct ftrace_ret_stack *ret_stack;	5200	struct ftrace_ret_stack *ret_stack;
5220		5201
5221	ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH	5202	ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
5222	* sizeof(struct ftrace_ret_stack),	5203	* sizeof(struct ftrace_ret_stack),
5223	GFP_KERNEL);	5204	GFP_KERNEL);
5224	if (!ret_stack)	5205	if (!ret_stack)
5225	return;	5206	return;
5226	graph_init_task(t, ret_stack);	5207	graph_init_task(t, ret_stack);
5227	}	5208	}
5228	}	5209	}
5229		5210
5230	void ftrace_graph_exit_task(struct task_struct *t)	5211	void ftrace_graph_exit_task(struct task_struct *t)
5231	{	5212	{
5232	struct ftrace_ret_stack *ret_stack = t->ret_stack;	5213	struct ftrace_ret_stack *ret_stack = t->ret_stack;
5233		5214
5234	t->ret_stack = NULL;	5215	t->ret_stack = NULL;
5235	/* NULL must become visible to IRQs before we free it: */	5216	/* NULL must become visible to IRQs before we free it: */
5236	barrier();	5217	barrier();
5237		5218
5238	kfree(ret_stack);	5219	kfree(ret_stack);
5239	}	5220	}
5240		5221
5241	void ftrace_graph_stop(void)	5222	void ftrace_graph_stop(void)
5242	{	5223	{
5243	ftrace_stop();	5224	ftrace_stop();
5244	}	5225	}
5245	#endif	5226	#endif
5246		5227

Link here
Eric Lee 2015-10-21 04:46:20 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2015-10-21 04:46:47 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-01-07 07:05:19 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-01-18 12:28:20 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-01-20 17:31:46 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-01-20 18:04:44 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-01-21 10:41:01 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-02-04 11:24:29 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-08-17 08:28:29 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2016-08-24 02:31:58 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2017-03-31 15:45:25 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2018-09-03 10:53:35 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel
Link here
Eric Lee 2018-09-03 11:06:10 UTC

mentioned in commit 21f585

_mentioned in commit 21f585_

Choose File ... File name...
Cancel