Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 751e9983ee276cb150e8812b1d995f6035a63878

Authored by Li Zefan
Committed by Steven Rostedt
1 parent b82a4045f7
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

ftrace: Fix MATCH_END_ONLY function filter 

For '*foo' pattern, we should allow any string ending with
'foo', but ftrace filter incorrectly disallows strings
like bar_foo_foo:

  # echo '*io' > set_ftrace_filter
  # cat set_ftrace_filter | grep 'req_bio_endio'
  # cat available_filter_functions | grep 'req_bio_endio'
  req_bio_endio

Signed-off-by: Li Zefan <lizf@cn.fujitsu.com>
LKML-Reference: <4B4E870E.6060607@cn.fujitsu.com>
Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

Showing 1 changed file with 3 additions and 3 deletions Inline Diff

kernel/trace/ftrace.c
Diff comments   View file @ 751e998
1 /* 1 /*
2 * Infrastructure for profiling code inserted by 'gcc -pg'. 2 * Infrastructure for profiling code inserted by 'gcc -pg'.
3 * 3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6 * 6 *
7 * Originally ported from the -rt patch by: 7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9 * 9 *
10 * Based on code in the latency_tracer, that is: 10 * Based on code in the latency_tracer, that is:
11 * 11 *
12 * Copyright (C) 2004-2006 Ingo Molnar 12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III 13 * Copyright (C) 2004 William Lee Irwin III
14 */ 14 */
15 15
16 #include <linux/stop_machine.h> 16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h> 17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h> 18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h> 19 #include <linux/seq_file.h>
20 #include <linux/suspend.h> 20 #include <linux/suspend.h>
21 #include <linux/debugfs.h> 21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h> 22 #include <linux/hardirq.h>
23 #include <linux/kthread.h> 23 #include <linux/kthread.h>
24 #include <linux/uaccess.h> 24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h> 25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h> 26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h> 27 #include <linux/sysctl.h>
28 #include <linux/ctype.h> 28 #include <linux/ctype.h>
29 #include <linux/list.h> 29 #include <linux/list.h>
30 #include <linux/hash.h> 30 #include <linux/hash.h>
31 31
32 #include <trace/events/sched.h> 32 #include <trace/events/sched.h>
33 33
34 #include <asm/ftrace.h> 34 #include <asm/ftrace.h>
35 #include <asm/setup.h> 35 #include <asm/setup.h>
36 36
37 #include "trace_output.h" 37 #include "trace_output.h"
38 #include "trace_stat.h" 38 #include "trace_stat.h"
39 39
40 #define FTRACE_WARN_ON(cond) \ 40 #define FTRACE_WARN_ON(cond) \
41 do { \ 41 do { \
42 if (WARN_ON(cond)) \ 42 if (WARN_ON(cond)) \
43 ftrace_kill(); \ 43 ftrace_kill(); \
44 } while (0) 44 } while (0)
45 45
46 #define FTRACE_WARN_ON_ONCE(cond) \ 46 #define FTRACE_WARN_ON_ONCE(cond) \
47 do { \ 47 do { \
48 if (WARN_ON_ONCE(cond)) \ 48 if (WARN_ON_ONCE(cond)) \
49 ftrace_kill(); \ 49 ftrace_kill(); \
50 } while (0) 50 } while (0)
51 51
52 /* hash bits for specific function selection */ 52 /* hash bits for specific function selection */
53 #define FTRACE_HASH_BITS 7 53 #define FTRACE_HASH_BITS 7
54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) 54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
55 55
56 /* ftrace_enabled is a method to turn ftrace on or off */ 56 /* ftrace_enabled is a method to turn ftrace on or off */
57 int ftrace_enabled __read_mostly; 57 int ftrace_enabled __read_mostly;
58 static int last_ftrace_enabled; 58 static int last_ftrace_enabled;
59 59
60 /* Quick disabling of function tracer. */ 60 /* Quick disabling of function tracer. */
61 int function_trace_stop; 61 int function_trace_stop;
62 62
63 /* List for set_ftrace_pid's pids. */ 63 /* List for set_ftrace_pid's pids. */
64 LIST_HEAD(ftrace_pids); 64 LIST_HEAD(ftrace_pids);
65 struct ftrace_pid { 65 struct ftrace_pid {
66 struct list_head list; 66 struct list_head list;
67 struct pid *pid; 67 struct pid *pid;
68 }; 68 };
69 69
70 /* 70 /*
71 * ftrace_disabled is set when an anomaly is discovered. 71 * ftrace_disabled is set when an anomaly is discovered.
72 * ftrace_disabled is much stronger than ftrace_enabled. 72 * ftrace_disabled is much stronger than ftrace_enabled.
73 */ 73 */
74 static int ftrace_disabled __read_mostly; 74 static int ftrace_disabled __read_mostly;
75 75
76 static DEFINE_MUTEX(ftrace_lock); 76 static DEFINE_MUTEX(ftrace_lock);
77 77
78 static struct ftrace_ops ftrace_list_end __read_mostly = 78 static struct ftrace_ops ftrace_list_end __read_mostly =
79 { 79 {
80 .func = ftrace_stub, 80 .func = ftrace_stub,
81 }; 81 };
82 82
83 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; 83 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
84 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 84 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
85 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; 85 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
86 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; 86 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
87 87
88 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 88 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
89 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); 89 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
90 #endif 90 #endif
91 91
92 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) 92 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
93 { 93 {
94 struct ftrace_ops *op = ftrace_list; 94 struct ftrace_ops *op = ftrace_list;
95 95
96 /* in case someone actually ports this to alpha! */ 96 /* in case someone actually ports this to alpha! */
97 read_barrier_depends(); 97 read_barrier_depends();
98 98
99 while (op != &ftrace_list_end) { 99 while (op != &ftrace_list_end) {
100 /* silly alpha */ 100 /* silly alpha */
101 read_barrier_depends(); 101 read_barrier_depends();
102 op->func(ip, parent_ip); 102 op->func(ip, parent_ip);
103 op = op->next; 103 op = op->next;
104 }; 104 };
105 } 105 }
106 106
107 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip) 107 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
108 { 108 {
109 if (!test_tsk_trace_trace(current)) 109 if (!test_tsk_trace_trace(current))
110 return; 110 return;
111 111
112 ftrace_pid_function(ip, parent_ip); 112 ftrace_pid_function(ip, parent_ip);
113 } 113 }
114 114
115 static void set_ftrace_pid_function(ftrace_func_t func) 115 static void set_ftrace_pid_function(ftrace_func_t func)
116 { 116 {
117 /* do not set ftrace_pid_function to itself! */ 117 /* do not set ftrace_pid_function to itself! */
118 if (func != ftrace_pid_func) 118 if (func != ftrace_pid_func)
119 ftrace_pid_function = func; 119 ftrace_pid_function = func;
120 } 120 }
121 121
122 /** 122 /**
123 * clear_ftrace_function - reset the ftrace function 123 * clear_ftrace_function - reset the ftrace function
124 * 124 *
125 * This NULLs the ftrace function and in essence stops 125 * This NULLs the ftrace function and in essence stops
126 * tracing. There may be lag 126 * tracing. There may be lag
127 */ 127 */
128 void clear_ftrace_function(void) 128 void clear_ftrace_function(void)
129 { 129 {
130 ftrace_trace_function = ftrace_stub; 130 ftrace_trace_function = ftrace_stub;
131 __ftrace_trace_function = ftrace_stub; 131 __ftrace_trace_function = ftrace_stub;
132 ftrace_pid_function = ftrace_stub; 132 ftrace_pid_function = ftrace_stub;
133 } 133 }
134 134
135 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 135 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
136 /* 136 /*
137 * For those archs that do not test ftrace_trace_stop in their 137 * For those archs that do not test ftrace_trace_stop in their
138 * mcount call site, we need to do it from C. 138 * mcount call site, we need to do it from C.
139 */ 139 */
140 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip) 140 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
141 { 141 {
142 if (function_trace_stop) 142 if (function_trace_stop)
143 return; 143 return;
144 144
145 __ftrace_trace_function(ip, parent_ip); 145 __ftrace_trace_function(ip, parent_ip);
146 } 146 }
147 #endif 147 #endif
148 148
149 static int __register_ftrace_function(struct ftrace_ops *ops) 149 static int __register_ftrace_function(struct ftrace_ops *ops)
150 { 150 {
151 ops->next = ftrace_list; 151 ops->next = ftrace_list;
152 /* 152 /*
153 * We are entering ops into the ftrace_list but another 153 * We are entering ops into the ftrace_list but another
154 * CPU might be walking that list. We need to make sure 154 * CPU might be walking that list. We need to make sure
155 * the ops->next pointer is valid before another CPU sees 155 * the ops->next pointer is valid before another CPU sees
156 * the ops pointer included into the ftrace_list. 156 * the ops pointer included into the ftrace_list.
157 */ 157 */
158 smp_wmb(); 158 smp_wmb();
159 ftrace_list = ops; 159 ftrace_list = ops;
160 160
161 if (ftrace_enabled) { 161 if (ftrace_enabled) {
162 ftrace_func_t func; 162 ftrace_func_t func;
163 163
164 if (ops->next == &ftrace_list_end) 164 if (ops->next == &ftrace_list_end)
165 func = ops->func; 165 func = ops->func;
166 else 166 else
167 func = ftrace_list_func; 167 func = ftrace_list_func;
168 168
169 if (!list_empty(&ftrace_pids)) { 169 if (!list_empty(&ftrace_pids)) {
170 set_ftrace_pid_function(func); 170 set_ftrace_pid_function(func);
171 func = ftrace_pid_func; 171 func = ftrace_pid_func;
172 } 172 }
173 173
174 /* 174 /*
175 * For one func, simply call it directly. 175 * For one func, simply call it directly.
176 * For more than one func, call the chain. 176 * For more than one func, call the chain.
177 */ 177 */
178 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 178 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
179 ftrace_trace_function = func; 179 ftrace_trace_function = func;
180 #else 180 #else
181 __ftrace_trace_function = func; 181 __ftrace_trace_function = func;
182 ftrace_trace_function = ftrace_test_stop_func; 182 ftrace_trace_function = ftrace_test_stop_func;
183 #endif 183 #endif
184 } 184 }
185 185
186 return 0; 186 return 0;
187 } 187 }
188 188
189 static int __unregister_ftrace_function(struct ftrace_ops *ops) 189 static int __unregister_ftrace_function(struct ftrace_ops *ops)
190 { 190 {
191 struct ftrace_ops **p; 191 struct ftrace_ops **p;
192 192
193 /* 193 /*
194 * If we are removing the last function, then simply point 194 * If we are removing the last function, then simply point
195 * to the ftrace_stub. 195 * to the ftrace_stub.
196 */ 196 */
197 if (ftrace_list == ops && ops->next == &ftrace_list_end) { 197 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
198 ftrace_trace_function = ftrace_stub; 198 ftrace_trace_function = ftrace_stub;
199 ftrace_list = &ftrace_list_end; 199 ftrace_list = &ftrace_list_end;
200 return 0; 200 return 0;
201 } 201 }
202 202
203 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) 203 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
204 if (*p == ops) 204 if (*p == ops)
205 break; 205 break;
206 206
207 if (*p != ops) 207 if (*p != ops)
208 return -1; 208 return -1;
209 209
210 *p = (*p)->next; 210 *p = (*p)->next;
211 211
212 if (ftrace_enabled) { 212 if (ftrace_enabled) {
213 /* If we only have one func left, then call that directly */ 213 /* If we only have one func left, then call that directly */
214 if (ftrace_list->next == &ftrace_list_end) { 214 if (ftrace_list->next == &ftrace_list_end) {
215 ftrace_func_t func = ftrace_list->func; 215 ftrace_func_t func = ftrace_list->func;
216 216
217 if (!list_empty(&ftrace_pids)) { 217 if (!list_empty(&ftrace_pids)) {
218 set_ftrace_pid_function(func); 218 set_ftrace_pid_function(func);
219 func = ftrace_pid_func; 219 func = ftrace_pid_func;
220 } 220 }
221 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 221 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
222 ftrace_trace_function = func; 222 ftrace_trace_function = func;
223 #else 223 #else
224 __ftrace_trace_function = func; 224 __ftrace_trace_function = func;
225 #endif 225 #endif
226 } 226 }
227 } 227 }
228 228
229 return 0; 229 return 0;
230 } 230 }
231 231
232 static void ftrace_update_pid_func(void) 232 static void ftrace_update_pid_func(void)
233 { 233 {
234 ftrace_func_t func; 234 ftrace_func_t func;
235 235
236 if (ftrace_trace_function == ftrace_stub) 236 if (ftrace_trace_function == ftrace_stub)
237 return; 237 return;
238 238
239 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 239 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
240 func = ftrace_trace_function; 240 func = ftrace_trace_function;
241 #else 241 #else
242 func = __ftrace_trace_function; 242 func = __ftrace_trace_function;
243 #endif 243 #endif
244 244
245 if (!list_empty(&ftrace_pids)) { 245 if (!list_empty(&ftrace_pids)) {
246 set_ftrace_pid_function(func); 246 set_ftrace_pid_function(func);
247 func = ftrace_pid_func; 247 func = ftrace_pid_func;
248 } else { 248 } else {
249 if (func == ftrace_pid_func) 249 if (func == ftrace_pid_func)
250 func = ftrace_pid_function; 250 func = ftrace_pid_function;
251 } 251 }
252 252
253 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 253 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
254 ftrace_trace_function = func; 254 ftrace_trace_function = func;
255 #else 255 #else
256 __ftrace_trace_function = func; 256 __ftrace_trace_function = func;
257 #endif 257 #endif
258 } 258 }
259 259
260 #ifdef CONFIG_FUNCTION_PROFILER 260 #ifdef CONFIG_FUNCTION_PROFILER
261 struct ftrace_profile { 261 struct ftrace_profile {
262 struct hlist_node node; 262 struct hlist_node node;
263 unsigned long ip; 263 unsigned long ip;
264 unsigned long counter; 264 unsigned long counter;
265 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 265 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
266 unsigned long long time; 266 unsigned long long time;
267 #endif 267 #endif
268 }; 268 };
269 269
270 struct ftrace_profile_page { 270 struct ftrace_profile_page {
271 struct ftrace_profile_page *next; 271 struct ftrace_profile_page *next;
272 unsigned long index; 272 unsigned long index;
273 struct ftrace_profile records[]; 273 struct ftrace_profile records[];
274 }; 274 };
275 275
276 struct ftrace_profile_stat { 276 struct ftrace_profile_stat {
277 atomic_t disabled; 277 atomic_t disabled;
278 struct hlist_head *hash; 278 struct hlist_head *hash;
279 struct ftrace_profile_page *pages; 279 struct ftrace_profile_page *pages;
280 struct ftrace_profile_page *start; 280 struct ftrace_profile_page *start;
281 struct tracer_stat stat; 281 struct tracer_stat stat;
282 }; 282 };
283 283
284 #define PROFILE_RECORDS_SIZE \ 284 #define PROFILE_RECORDS_SIZE \
285 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 285 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
286 286
287 #define PROFILES_PER_PAGE \ 287 #define PROFILES_PER_PAGE \
288 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 288 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
289 289
290 static int ftrace_profile_bits __read_mostly; 290 static int ftrace_profile_bits __read_mostly;
291 static int ftrace_profile_enabled __read_mostly; 291 static int ftrace_profile_enabled __read_mostly;
292 292
293 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 293 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
294 static DEFINE_MUTEX(ftrace_profile_lock); 294 static DEFINE_MUTEX(ftrace_profile_lock);
295 295
296 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 296 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
297 297
298 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */ 298 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
299 299
300 static void * 300 static void *
301 function_stat_next(void *v, int idx) 301 function_stat_next(void *v, int idx)
302 { 302 {
303 struct ftrace_profile *rec = v; 303 struct ftrace_profile *rec = v;
304 struct ftrace_profile_page *pg; 304 struct ftrace_profile_page *pg;
305 305
306 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 306 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
307 307
308 again: 308 again:
309 if (idx != 0) 309 if (idx != 0)
310 rec++; 310 rec++;
311 311
312 if ((void *)rec >= (void *)&pg->records[pg->index]) { 312 if ((void *)rec >= (void *)&pg->records[pg->index]) {
313 pg = pg->next; 313 pg = pg->next;
314 if (!pg) 314 if (!pg)
315 return NULL; 315 return NULL;
316 rec = &pg->records[0]; 316 rec = &pg->records[0];
317 if (!rec->counter) 317 if (!rec->counter)
318 goto again; 318 goto again;
319 } 319 }
320 320
321 return rec; 321 return rec;
322 } 322 }
323 323
324 static void *function_stat_start(struct tracer_stat *trace) 324 static void *function_stat_start(struct tracer_stat *trace)
325 { 325 {
326 struct ftrace_profile_stat *stat = 326 struct ftrace_profile_stat *stat =
327 container_of(trace, struct ftrace_profile_stat, stat); 327 container_of(trace, struct ftrace_profile_stat, stat);
328 328
329 if (!stat || !stat->start) 329 if (!stat || !stat->start)
330 return NULL; 330 return NULL;
331 331
332 return function_stat_next(&stat->start->records[0], 0); 332 return function_stat_next(&stat->start->records[0], 0);
333 } 333 }
334 334
335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
336 /* function graph compares on total time */ 336 /* function graph compares on total time */
337 static int function_stat_cmp(void *p1, void *p2) 337 static int function_stat_cmp(void *p1, void *p2)
338 { 338 {
339 struct ftrace_profile *a = p1; 339 struct ftrace_profile *a = p1;
340 struct ftrace_profile *b = p2; 340 struct ftrace_profile *b = p2;
341 341
342 if (a->time < b->time) 342 if (a->time < b->time)
343 return -1; 343 return -1;
344 if (a->time > b->time) 344 if (a->time > b->time)
345 return 1; 345 return 1;
346 else 346 else
347 return 0; 347 return 0;
348 } 348 }
349 #else 349 #else
350 /* not function graph compares against hits */ 350 /* not function graph compares against hits */
351 static int function_stat_cmp(void *p1, void *p2) 351 static int function_stat_cmp(void *p1, void *p2)
352 { 352 {
353 struct ftrace_profile *a = p1; 353 struct ftrace_profile *a = p1;
354 struct ftrace_profile *b = p2; 354 struct ftrace_profile *b = p2;
355 355
356 if (a->counter < b->counter) 356 if (a->counter < b->counter)
357 return -1; 357 return -1;
358 if (a->counter > b->counter) 358 if (a->counter > b->counter)
359 return 1; 359 return 1;
360 else 360 else
361 return 0; 361 return 0;
362 } 362 }
363 #endif 363 #endif
364 364
365 static int function_stat_headers(struct seq_file *m) 365 static int function_stat_headers(struct seq_file *m)
366 { 366 {
367 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 367 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
368 seq_printf(m, " Function " 368 seq_printf(m, " Function "
369 "Hit Time Avg\n" 369 "Hit Time Avg\n"
370 " -------- " 370 " -------- "
371 "--- ---- ---\n"); 371 "--- ---- ---\n");
372 #else 372 #else
373 seq_printf(m, " Function Hit\n" 373 seq_printf(m, " Function Hit\n"
374 " -------- ---\n"); 374 " -------- ---\n");
375 #endif 375 #endif
376 return 0; 376 return 0;
377 } 377 }
378 378
379 static int function_stat_show(struct seq_file *m, void *v) 379 static int function_stat_show(struct seq_file *m, void *v)
380 { 380 {
381 struct ftrace_profile *rec = v; 381 struct ftrace_profile *rec = v;
382 char str[KSYM_SYMBOL_LEN]; 382 char str[KSYM_SYMBOL_LEN];
383 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 383 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
384 static DEFINE_MUTEX(mutex); 384 static DEFINE_MUTEX(mutex);
385 static struct trace_seq s; 385 static struct trace_seq s;
386 unsigned long long avg; 386 unsigned long long avg;
387 #endif 387 #endif
388 388
389 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 389 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
390 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 390 seq_printf(m, " %-30.30s %10lu", str, rec->counter);
391 391
392 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 392 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
393 seq_printf(m, " "); 393 seq_printf(m, " ");
394 avg = rec->time; 394 avg = rec->time;
395 do_div(avg, rec->counter); 395 do_div(avg, rec->counter);
396 396
397 mutex_lock(&mutex); 397 mutex_lock(&mutex);
398 trace_seq_init(&s); 398 trace_seq_init(&s);
399 trace_print_graph_duration(rec->time, &s); 399 trace_print_graph_duration(rec->time, &s);
400 trace_seq_puts(&s, " "); 400 trace_seq_puts(&s, " ");
401 trace_print_graph_duration(avg, &s); 401 trace_print_graph_duration(avg, &s);
402 trace_print_seq(m, &s); 402 trace_print_seq(m, &s);
403 mutex_unlock(&mutex); 403 mutex_unlock(&mutex);
404 #endif 404 #endif
405 seq_putc(m, '\n'); 405 seq_putc(m, '\n');
406 406
407 return 0; 407 return 0;
408 } 408 }
409 409
410 static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 410 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
411 { 411 {
412 struct ftrace_profile_page *pg; 412 struct ftrace_profile_page *pg;
413 413
414 pg = stat->pages = stat->start; 414 pg = stat->pages = stat->start;
415 415
416 while (pg) { 416 while (pg) {
417 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 417 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
418 pg->index = 0; 418 pg->index = 0;
419 pg = pg->next; 419 pg = pg->next;
420 } 420 }
421 421
422 memset(stat->hash, 0, 422 memset(stat->hash, 0,
423 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 423 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
424 } 424 }
425 425
426 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 426 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
427 { 427 {
428 struct ftrace_profile_page *pg; 428 struct ftrace_profile_page *pg;
429 int functions; 429 int functions;
430 int pages; 430 int pages;
431 int i; 431 int i;
432 432
433 /* If we already allocated, do nothing */ 433 /* If we already allocated, do nothing */
434 if (stat->pages) 434 if (stat->pages)
435 return 0; 435 return 0;
436 436
437 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 437 stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
438 if (!stat->pages) 438 if (!stat->pages)
439 return -ENOMEM; 439 return -ENOMEM;
440 440
441 #ifdef CONFIG_DYNAMIC_FTRACE 441 #ifdef CONFIG_DYNAMIC_FTRACE
442 functions = ftrace_update_tot_cnt; 442 functions = ftrace_update_tot_cnt;
443 #else 443 #else
444 /* 444 /*
445 * We do not know the number of functions that exist because 445 * We do not know the number of functions that exist because
446 * dynamic tracing is what counts them. With past experience 446 * dynamic tracing is what counts them. With past experience
447 * we have around 20K functions. That should be more than enough. 447 * we have around 20K functions. That should be more than enough.
448 * It is highly unlikely we will execute every function in 448 * It is highly unlikely we will execute every function in
449 * the kernel. 449 * the kernel.
450 */ 450 */
451 functions = 20000; 451 functions = 20000;
452 #endif 452 #endif
453 453
454 pg = stat->start = stat->pages; 454 pg = stat->start = stat->pages;
455 455
456 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 456 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
457 457
458 for (i = 0; i < pages; i++) { 458 for (i = 0; i < pages; i++) {
459 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 459 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
460 if (!pg->next) 460 if (!pg->next)
461 goto out_free; 461 goto out_free;
462 pg = pg->next; 462 pg = pg->next;
463 } 463 }
464 464
465 return 0; 465 return 0;
466 466
467 out_free: 467 out_free:
468 pg = stat->start; 468 pg = stat->start;
469 while (pg) { 469 while (pg) {
470 unsigned long tmp = (unsigned long)pg; 470 unsigned long tmp = (unsigned long)pg;
471 471
472 pg = pg->next; 472 pg = pg->next;
473 free_page(tmp); 473 free_page(tmp);
474 } 474 }
475 475
476 free_page((unsigned long)stat->pages); 476 free_page((unsigned long)stat->pages);
477 stat->pages = NULL; 477 stat->pages = NULL;
478 stat->start = NULL; 478 stat->start = NULL;
479 479
480 return -ENOMEM; 480 return -ENOMEM;
481 } 481 }
482 482
483 static int ftrace_profile_init_cpu(int cpu) 483 static int ftrace_profile_init_cpu(int cpu)
484 { 484 {
485 struct ftrace_profile_stat *stat; 485 struct ftrace_profile_stat *stat;
486 int size; 486 int size;
487 487
488 stat = &per_cpu(ftrace_profile_stats, cpu); 488 stat = &per_cpu(ftrace_profile_stats, cpu);
489 489
490 if (stat->hash) { 490 if (stat->hash) {
491 /* If the profile is already created, simply reset it */ 491 /* If the profile is already created, simply reset it */
492 ftrace_profile_reset(stat); 492 ftrace_profile_reset(stat);
493 return 0; 493 return 0;
494 } 494 }
495 495
496 /* 496 /*
497 * We are profiling all functions, but usually only a few thousand 497 * We are profiling all functions, but usually only a few thousand
498 * functions are hit. We'll make a hash of 1024 items. 498 * functions are hit. We'll make a hash of 1024 items.
499 */ 499 */
500 size = FTRACE_PROFILE_HASH_SIZE; 500 size = FTRACE_PROFILE_HASH_SIZE;
501 501
502 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL); 502 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
503 503
504 if (!stat->hash) 504 if (!stat->hash)
505 return -ENOMEM; 505 return -ENOMEM;
506 506
507 if (!ftrace_profile_bits) { 507 if (!ftrace_profile_bits) {
508 size--; 508 size--;
509 509
510 for (; size; size >>= 1) 510 for (; size; size >>= 1)
511 ftrace_profile_bits++; 511 ftrace_profile_bits++;
512 } 512 }
513 513
514 /* Preallocate the function profiling pages */ 514 /* Preallocate the function profiling pages */
515 if (ftrace_profile_pages_init(stat) < 0) { 515 if (ftrace_profile_pages_init(stat) < 0) {
516 kfree(stat->hash); 516 kfree(stat->hash);
517 stat->hash = NULL; 517 stat->hash = NULL;
518 return -ENOMEM; 518 return -ENOMEM;
519 } 519 }
520 520
521 return 0; 521 return 0;
522 } 522 }
523 523
524 static int ftrace_profile_init(void) 524 static int ftrace_profile_init(void)
525 { 525 {
526 int cpu; 526 int cpu;
527 int ret = 0; 527 int ret = 0;
528 528
529 for_each_online_cpu(cpu) { 529 for_each_online_cpu(cpu) {
530 ret = ftrace_profile_init_cpu(cpu); 530 ret = ftrace_profile_init_cpu(cpu);
531 if (ret) 531 if (ret)
532 break; 532 break;
533 } 533 }
534 534
535 return ret; 535 return ret;
536 } 536 }
537 537
538 /* interrupts must be disabled */ 538 /* interrupts must be disabled */
539 static struct ftrace_profile * 539 static struct ftrace_profile *
540 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 540 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
541 { 541 {
542 struct ftrace_profile *rec; 542 struct ftrace_profile *rec;
543 struct hlist_head *hhd; 543 struct hlist_head *hhd;
544 struct hlist_node *n; 544 struct hlist_node *n;
545 unsigned long key; 545 unsigned long key;
546 546
547 key = hash_long(ip, ftrace_profile_bits); 547 key = hash_long(ip, ftrace_profile_bits);
548 hhd = &stat->hash[key]; 548 hhd = &stat->hash[key];
549 549
550 if (hlist_empty(hhd)) 550 if (hlist_empty(hhd))
551 return NULL; 551 return NULL;
552 552
553 hlist_for_each_entry_rcu(rec, n, hhd, node) { 553 hlist_for_each_entry_rcu(rec, n, hhd, node) {
554 if (rec->ip == ip) 554 if (rec->ip == ip)
555 return rec; 555 return rec;
556 } 556 }
557 557
558 return NULL; 558 return NULL;
559 } 559 }
560 560
561 static void ftrace_add_profile(struct ftrace_profile_stat *stat, 561 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
562 struct ftrace_profile *rec) 562 struct ftrace_profile *rec)
563 { 563 {
564 unsigned long key; 564 unsigned long key;
565 565
566 key = hash_long(rec->ip, ftrace_profile_bits); 566 key = hash_long(rec->ip, ftrace_profile_bits);
567 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 567 hlist_add_head_rcu(&rec->node, &stat->hash[key]);
568 } 568 }
569 569
570 /* 570 /*
571 * The memory is already allocated, this simply finds a new record to use. 571 * The memory is already allocated, this simply finds a new record to use.
572 */ 572 */
573 static struct ftrace_profile * 573 static struct ftrace_profile *
574 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 574 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
575 { 575 {
576 struct ftrace_profile *rec = NULL; 576 struct ftrace_profile *rec = NULL;
577 577
578 /* prevent recursion (from NMIs) */ 578 /* prevent recursion (from NMIs) */
579 if (atomic_inc_return(&stat->disabled) != 1) 579 if (atomic_inc_return(&stat->disabled) != 1)
580 goto out; 580 goto out;
581 581
582 /* 582 /*
583 * Try to find the function again since an NMI 583 * Try to find the function again since an NMI
584 * could have added it 584 * could have added it
585 */ 585 */
586 rec = ftrace_find_profiled_func(stat, ip); 586 rec = ftrace_find_profiled_func(stat, ip);
587 if (rec) 587 if (rec)
588 goto out; 588 goto out;
589 589
590 if (stat->pages->index == PROFILES_PER_PAGE) { 590 if (stat->pages->index == PROFILES_PER_PAGE) {
591 if (!stat->pages->next) 591 if (!stat->pages->next)
592 goto out; 592 goto out;
593 stat->pages = stat->pages->next; 593 stat->pages = stat->pages->next;
594 } 594 }
595 595
596 rec = &stat->pages->records[stat->pages->index++]; 596 rec = &stat->pages->records[stat->pages->index++];
597 rec->ip = ip; 597 rec->ip = ip;
598 ftrace_add_profile(stat, rec); 598 ftrace_add_profile(stat, rec);
599 599
600 out: 600 out:
601 atomic_dec(&stat->disabled); 601 atomic_dec(&stat->disabled);
602 602
603 return rec; 603 return rec;
604 } 604 }
605 605
606 static void 606 static void
607 function_profile_call(unsigned long ip, unsigned long parent_ip) 607 function_profile_call(unsigned long ip, unsigned long parent_ip)
608 { 608 {
609 struct ftrace_profile_stat *stat; 609 struct ftrace_profile_stat *stat;
610 struct ftrace_profile *rec; 610 struct ftrace_profile *rec;
611 unsigned long flags; 611 unsigned long flags;
612 612
613 if (!ftrace_profile_enabled) 613 if (!ftrace_profile_enabled)
614 return; 614 return;
615 615
616 local_irq_save(flags); 616 local_irq_save(flags);
617 617
618 stat = &__get_cpu_var(ftrace_profile_stats); 618 stat = &__get_cpu_var(ftrace_profile_stats);
619 if (!stat->hash || !ftrace_profile_enabled) 619 if (!stat->hash || !ftrace_profile_enabled)
620 goto out; 620 goto out;
621 621
622 rec = ftrace_find_profiled_func(stat, ip); 622 rec = ftrace_find_profiled_func(stat, ip);
623 if (!rec) { 623 if (!rec) {
624 rec = ftrace_profile_alloc(stat, ip); 624 rec = ftrace_profile_alloc(stat, ip);
625 if (!rec) 625 if (!rec)
626 goto out; 626 goto out;
627 } 627 }
628 628
629 rec->counter++; 629 rec->counter++;
630 out: 630 out:
631 local_irq_restore(flags); 631 local_irq_restore(flags);
632 } 632 }
633 633
634 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 634 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
635 static int profile_graph_entry(struct ftrace_graph_ent *trace) 635 static int profile_graph_entry(struct ftrace_graph_ent *trace)
636 { 636 {
637 function_profile_call(trace->func, 0); 637 function_profile_call(trace->func, 0);
638 return 1; 638 return 1;
639 } 639 }
640 640
641 static void profile_graph_return(struct ftrace_graph_ret *trace) 641 static void profile_graph_return(struct ftrace_graph_ret *trace)
642 { 642 {
643 struct ftrace_profile_stat *stat; 643 struct ftrace_profile_stat *stat;
644 unsigned long long calltime; 644 unsigned long long calltime;
645 struct ftrace_profile *rec; 645 struct ftrace_profile *rec;
646 unsigned long flags; 646 unsigned long flags;
647 647
648 local_irq_save(flags); 648 local_irq_save(flags);
649 stat = &__get_cpu_var(ftrace_profile_stats); 649 stat = &__get_cpu_var(ftrace_profile_stats);
650 if (!stat->hash || !ftrace_profile_enabled) 650 if (!stat->hash || !ftrace_profile_enabled)
651 goto out; 651 goto out;
652 652
653 calltime = trace->rettime - trace->calltime; 653 calltime = trace->rettime - trace->calltime;
654 654
655 if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) { 655 if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
656 int index; 656 int index;
657 657
658 index = trace->depth; 658 index = trace->depth;
659 659
660 /* Append this call time to the parent time to subtract */ 660 /* Append this call time to the parent time to subtract */
661 if (index) 661 if (index)
662 current->ret_stack[index - 1].subtime += calltime; 662 current->ret_stack[index - 1].subtime += calltime;
663 663
664 if (current->ret_stack[index].subtime < calltime) 664 if (current->ret_stack[index].subtime < calltime)
665 calltime -= current->ret_stack[index].subtime; 665 calltime -= current->ret_stack[index].subtime;
666 else 666 else
667 calltime = 0; 667 calltime = 0;
668 } 668 }
669 669
670 rec = ftrace_find_profiled_func(stat, trace->func); 670 rec = ftrace_find_profiled_func(stat, trace->func);
671 if (rec) 671 if (rec)
672 rec->time += calltime; 672 rec->time += calltime;
673 673
674 out: 674 out:
675 local_irq_restore(flags); 675 local_irq_restore(flags);
676 } 676 }
677 677
678 static int register_ftrace_profiler(void) 678 static int register_ftrace_profiler(void)
679 { 679 {
680 return register_ftrace_graph(&profile_graph_return, 680 return register_ftrace_graph(&profile_graph_return,
681 &profile_graph_entry); 681 &profile_graph_entry);
682 } 682 }
683 683
684 static void unregister_ftrace_profiler(void) 684 static void unregister_ftrace_profiler(void)
685 { 685 {
686 unregister_ftrace_graph(); 686 unregister_ftrace_graph();
687 } 687 }
688 #else 688 #else
689 static struct ftrace_ops ftrace_profile_ops __read_mostly = 689 static struct ftrace_ops ftrace_profile_ops __read_mostly =
690 { 690 {
691 .func = function_profile_call, 691 .func = function_profile_call,
692 }; 692 };
693 693
694 static int register_ftrace_profiler(void) 694 static int register_ftrace_profiler(void)
695 { 695 {
696 return register_ftrace_function(&ftrace_profile_ops); 696 return register_ftrace_function(&ftrace_profile_ops);
697 } 697 }
698 698
699 static void unregister_ftrace_profiler(void) 699 static void unregister_ftrace_profiler(void)
700 { 700 {
701 unregister_ftrace_function(&ftrace_profile_ops); 701 unregister_ftrace_function(&ftrace_profile_ops);
702 } 702 }
703 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 703 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
704 704
705 static ssize_t 705 static ssize_t
706 ftrace_profile_write(struct file *filp, const char __user *ubuf, 706 ftrace_profile_write(struct file *filp, const char __user *ubuf,
707 size_t cnt, loff_t *ppos) 707 size_t cnt, loff_t *ppos)
708 { 708 {
709 unsigned long val; 709 unsigned long val;
710 char buf[64]; /* big enough to hold a number */ 710 char buf[64]; /* big enough to hold a number */
711 int ret; 711 int ret;
712 712
713 if (cnt >= sizeof(buf)) 713 if (cnt >= sizeof(buf))
714 return -EINVAL; 714 return -EINVAL;
715 715
716 if (copy_from_user(&buf, ubuf, cnt)) 716 if (copy_from_user(&buf, ubuf, cnt))
717 return -EFAULT; 717 return -EFAULT;
718 718
719 buf[cnt] = 0; 719 buf[cnt] = 0;
720 720
721 ret = strict_strtoul(buf, 10, &val); 721 ret = strict_strtoul(buf, 10, &val);
722 if (ret < 0) 722 if (ret < 0)
723 return ret; 723 return ret;
724 724
725 val = !!val; 725 val = !!val;
726 726
727 mutex_lock(&ftrace_profile_lock); 727 mutex_lock(&ftrace_profile_lock);
728 if (ftrace_profile_enabled ^ val) { 728 if (ftrace_profile_enabled ^ val) {
729 if (val) { 729 if (val) {
730 ret = ftrace_profile_init(); 730 ret = ftrace_profile_init();
731 if (ret < 0) { 731 if (ret < 0) {
732 cnt = ret; 732 cnt = ret;
733 goto out; 733 goto out;
734 } 734 }
735 735
736 ret = register_ftrace_profiler(); 736 ret = register_ftrace_profiler();
737 if (ret < 0) { 737 if (ret < 0) {
738 cnt = ret; 738 cnt = ret;
739 goto out; 739 goto out;
740 } 740 }
741 ftrace_profile_enabled = 1; 741 ftrace_profile_enabled = 1;
742 } else { 742 } else {
743 ftrace_profile_enabled = 0; 743 ftrace_profile_enabled = 0;
744 /* 744 /*
745 * unregister_ftrace_profiler calls stop_machine 745 * unregister_ftrace_profiler calls stop_machine
746 * so this acts like an synchronize_sched. 746 * so this acts like an synchronize_sched.
747 */ 747 */
748 unregister_ftrace_profiler(); 748 unregister_ftrace_profiler();
749 } 749 }
750 } 750 }
751 out: 751 out:
752 mutex_unlock(&ftrace_profile_lock); 752 mutex_unlock(&ftrace_profile_lock);
753 753
754 *ppos += cnt; 754 *ppos += cnt;
755 755
756 return cnt; 756 return cnt;
757 } 757 }
758 758
759 static ssize_t 759 static ssize_t
760 ftrace_profile_read(struct file *filp, char __user *ubuf, 760 ftrace_profile_read(struct file *filp, char __user *ubuf,
761 size_t cnt, loff_t *ppos) 761 size_t cnt, loff_t *ppos)
762 { 762 {
763 char buf[64]; /* big enough to hold a number */ 763 char buf[64]; /* big enough to hold a number */
764 int r; 764 int r;
765 765
766 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 766 r = sprintf(buf, "%u\n", ftrace_profile_enabled);
767 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 767 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
768 } 768 }
769 769
770 static const struct file_operations ftrace_profile_fops = { 770 static const struct file_operations ftrace_profile_fops = {
771 .open = tracing_open_generic, 771 .open = tracing_open_generic,
772 .read = ftrace_profile_read, 772 .read = ftrace_profile_read,
773 .write = ftrace_profile_write, 773 .write = ftrace_profile_write,
774 }; 774 };
775 775
776 /* used to initialize the real stat files */ 776 /* used to initialize the real stat files */
777 static struct tracer_stat function_stats __initdata = { 777 static struct tracer_stat function_stats __initdata = {
778 .name = "functions", 778 .name = "functions",
779 .stat_start = function_stat_start, 779 .stat_start = function_stat_start,
780 .stat_next = function_stat_next, 780 .stat_next = function_stat_next,
781 .stat_cmp = function_stat_cmp, 781 .stat_cmp = function_stat_cmp,
782 .stat_headers = function_stat_headers, 782 .stat_headers = function_stat_headers,
783 .stat_show = function_stat_show 783 .stat_show = function_stat_show
784 }; 784 };
785 785
786 static __init void ftrace_profile_debugfs(struct dentry *d_tracer) 786 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
787 { 787 {
788 struct ftrace_profile_stat *stat; 788 struct ftrace_profile_stat *stat;
789 struct dentry *entry; 789 struct dentry *entry;
790 char *name; 790 char *name;
791 int ret; 791 int ret;
792 int cpu; 792 int cpu;
793 793
794 for_each_possible_cpu(cpu) { 794 for_each_possible_cpu(cpu) {
795 stat = &per_cpu(ftrace_profile_stats, cpu); 795 stat = &per_cpu(ftrace_profile_stats, cpu);
796 796
797 /* allocate enough for function name + cpu number */ 797 /* allocate enough for function name + cpu number */
798 name = kmalloc(32, GFP_KERNEL); 798 name = kmalloc(32, GFP_KERNEL);
799 if (!name) { 799 if (!name) {
800 /* 800 /*
801 * The files created are permanent, if something happens 801 * The files created are permanent, if something happens
802 * we still do not free memory. 802 * we still do not free memory.
803 */ 803 */
804 WARN(1, 804 WARN(1,
805 "Could not allocate stat file for cpu %d\n", 805 "Could not allocate stat file for cpu %d\n",
806 cpu); 806 cpu);
807 return; 807 return;
808 } 808 }
809 stat->stat = function_stats; 809 stat->stat = function_stats;
810 snprintf(name, 32, "function%d", cpu); 810 snprintf(name, 32, "function%d", cpu);
811 stat->stat.name = name; 811 stat->stat.name = name;
812 ret = register_stat_tracer(&stat->stat); 812 ret = register_stat_tracer(&stat->stat);
813 if (ret) { 813 if (ret) {
814 WARN(1, 814 WARN(1,
815 "Could not register function stat for cpu %d\n", 815 "Could not register function stat for cpu %d\n",
816 cpu); 816 cpu);
817 kfree(name); 817 kfree(name);
818 return; 818 return;
819 } 819 }
820 } 820 }
821 821
822 entry = debugfs_create_file("function_profile_enabled", 0644, 822 entry = debugfs_create_file("function_profile_enabled", 0644,
823 d_tracer, NULL, &ftrace_profile_fops); 823 d_tracer, NULL, &ftrace_profile_fops);
824 if (!entry) 824 if (!entry)
825 pr_warning("Could not create debugfs " 825 pr_warning("Could not create debugfs "
826 "'function_profile_enabled' entry\n"); 826 "'function_profile_enabled' entry\n");
827 } 827 }
828 828
829 #else /* CONFIG_FUNCTION_PROFILER */ 829 #else /* CONFIG_FUNCTION_PROFILER */
830 static __init void ftrace_profile_debugfs(struct dentry *d_tracer) 830 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
831 { 831 {
832 } 832 }
833 #endif /* CONFIG_FUNCTION_PROFILER */ 833 #endif /* CONFIG_FUNCTION_PROFILER */
834 834
835 static struct pid * const ftrace_swapper_pid = &init_struct_pid; 835 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
836 836
837 #ifdef CONFIG_DYNAMIC_FTRACE 837 #ifdef CONFIG_DYNAMIC_FTRACE
838 838
839 #ifndef CONFIG_FTRACE_MCOUNT_RECORD 839 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
840 # error Dynamic ftrace depends on MCOUNT_RECORD 840 # error Dynamic ftrace depends on MCOUNT_RECORD
841 #endif 841 #endif
842 842
843 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly; 843 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
844 844
845 struct ftrace_func_probe { 845 struct ftrace_func_probe {
846 struct hlist_node node; 846 struct hlist_node node;
847 struct ftrace_probe_ops *ops; 847 struct ftrace_probe_ops *ops;
848 unsigned long flags; 848 unsigned long flags;
849 unsigned long ip; 849 unsigned long ip;
850 void *data; 850 void *data;
851 struct rcu_head rcu; 851 struct rcu_head rcu;
852 }; 852 };
853 853
854 enum { 854 enum {
855 FTRACE_ENABLE_CALLS = (1 << 0), 855 FTRACE_ENABLE_CALLS = (1 << 0),
856 FTRACE_DISABLE_CALLS = (1 << 1), 856 FTRACE_DISABLE_CALLS = (1 << 1),
857 FTRACE_UPDATE_TRACE_FUNC = (1 << 2), 857 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
858 FTRACE_ENABLE_MCOUNT = (1 << 3), 858 FTRACE_ENABLE_MCOUNT = (1 << 3),
859 FTRACE_DISABLE_MCOUNT = (1 << 4), 859 FTRACE_DISABLE_MCOUNT = (1 << 4),
860 FTRACE_START_FUNC_RET = (1 << 5), 860 FTRACE_START_FUNC_RET = (1 << 5),
861 FTRACE_STOP_FUNC_RET = (1 << 6), 861 FTRACE_STOP_FUNC_RET = (1 << 6),
862 }; 862 };
863 863
864 static int ftrace_filtered; 864 static int ftrace_filtered;
865 865
866 static struct dyn_ftrace *ftrace_new_addrs; 866 static struct dyn_ftrace *ftrace_new_addrs;
867 867
868 static DEFINE_MUTEX(ftrace_regex_lock); 868 static DEFINE_MUTEX(ftrace_regex_lock);
869 869
870 struct ftrace_page { 870 struct ftrace_page {
871 struct ftrace_page *next; 871 struct ftrace_page *next;
872 int index; 872 int index;
873 struct dyn_ftrace records[]; 873 struct dyn_ftrace records[];
874 }; 874 };
875 875
876 #define ENTRIES_PER_PAGE \ 876 #define ENTRIES_PER_PAGE \
877 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace)) 877 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
878 878
879 /* estimate from running different kernels */ 879 /* estimate from running different kernels */
880 #define NR_TO_INIT 10000 880 #define NR_TO_INIT 10000
881 881
882 static struct ftrace_page *ftrace_pages_start; 882 static struct ftrace_page *ftrace_pages_start;
883 static struct ftrace_page *ftrace_pages; 883 static struct ftrace_page *ftrace_pages;
884 884
885 static struct dyn_ftrace *ftrace_free_records; 885 static struct dyn_ftrace *ftrace_free_records;
886 886
887 /* 887 /*
888 * This is a double for. Do not use 'break' to break out of the loop, 888 * This is a double for. Do not use 'break' to break out of the loop,
889 * you must use a goto. 889 * you must use a goto.
890 */ 890 */
891 #define do_for_each_ftrace_rec(pg, rec) \ 891 #define do_for_each_ftrace_rec(pg, rec) \
892 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 892 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
893 int _____i; \ 893 int _____i; \
894 for (_____i = 0; _____i < pg->index; _____i++) { \ 894 for (_____i = 0; _____i < pg->index; _____i++) { \
895 rec = &pg->records[_____i]; 895 rec = &pg->records[_____i];
896 896
897 #define while_for_each_ftrace_rec() \ 897 #define while_for_each_ftrace_rec() \
898 } \ 898 } \
899 } 899 }
900 900
901 #ifdef CONFIG_KPROBES 901 #ifdef CONFIG_KPROBES
902 902
903 static int frozen_record_count; 903 static int frozen_record_count;
904 904
905 static inline void freeze_record(struct dyn_ftrace *rec) 905 static inline void freeze_record(struct dyn_ftrace *rec)
906 { 906 {
907 if (!(rec->flags & FTRACE_FL_FROZEN)) { 907 if (!(rec->flags & FTRACE_FL_FROZEN)) {
908 rec->flags |= FTRACE_FL_FROZEN; 908 rec->flags |= FTRACE_FL_FROZEN;
909 frozen_record_count++; 909 frozen_record_count++;
910 } 910 }
911 } 911 }
912 912
913 static inline void unfreeze_record(struct dyn_ftrace *rec) 913 static inline void unfreeze_record(struct dyn_ftrace *rec)
914 { 914 {
915 if (rec->flags & FTRACE_FL_FROZEN) { 915 if (rec->flags & FTRACE_FL_FROZEN) {
916 rec->flags &= ~FTRACE_FL_FROZEN; 916 rec->flags &= ~FTRACE_FL_FROZEN;
917 frozen_record_count--; 917 frozen_record_count--;
918 } 918 }
919 } 919 }
920 920
921 static inline int record_frozen(struct dyn_ftrace *rec) 921 static inline int record_frozen(struct dyn_ftrace *rec)
922 { 922 {
923 return rec->flags & FTRACE_FL_FROZEN; 923 return rec->flags & FTRACE_FL_FROZEN;
924 } 924 }
925 #else 925 #else
926 # define freeze_record(rec) ({ 0; }) 926 # define freeze_record(rec) ({ 0; })
927 # define unfreeze_record(rec) ({ 0; }) 927 # define unfreeze_record(rec) ({ 0; })
928 # define record_frozen(rec) ({ 0; }) 928 # define record_frozen(rec) ({ 0; })
929 #endif /* CONFIG_KPROBES */ 929 #endif /* CONFIG_KPROBES */
930 930
931 static void ftrace_free_rec(struct dyn_ftrace *rec) 931 static void ftrace_free_rec(struct dyn_ftrace *rec)
932 { 932 {
933 rec->freelist = ftrace_free_records; 933 rec->freelist = ftrace_free_records;
934 ftrace_free_records = rec; 934 ftrace_free_records = rec;
935 rec->flags |= FTRACE_FL_FREE; 935 rec->flags |= FTRACE_FL_FREE;
936 } 936 }
937 937
938 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) 938 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
939 { 939 {
940 struct dyn_ftrace *rec; 940 struct dyn_ftrace *rec;
941 941
942 /* First check for freed records */ 942 /* First check for freed records */
943 if (ftrace_free_records) { 943 if (ftrace_free_records) {
944 rec = ftrace_free_records; 944 rec = ftrace_free_records;
945 945
946 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) { 946 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
947 FTRACE_WARN_ON_ONCE(1); 947 FTRACE_WARN_ON_ONCE(1);
948 ftrace_free_records = NULL; 948 ftrace_free_records = NULL;
949 return NULL; 949 return NULL;
950 } 950 }
951 951
952 ftrace_free_records = rec->freelist; 952 ftrace_free_records = rec->freelist;
953 memset(rec, 0, sizeof(*rec)); 953 memset(rec, 0, sizeof(*rec));
954 return rec; 954 return rec;
955 } 955 }
956 956
957 if (ftrace_pages->index == ENTRIES_PER_PAGE) { 957 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
958 if (!ftrace_pages->next) { 958 if (!ftrace_pages->next) {
959 /* allocate another page */ 959 /* allocate another page */
960 ftrace_pages->next = 960 ftrace_pages->next =
961 (void *)get_zeroed_page(GFP_KERNEL); 961 (void *)get_zeroed_page(GFP_KERNEL);
962 if (!ftrace_pages->next) 962 if (!ftrace_pages->next)
963 return NULL; 963 return NULL;
964 } 964 }
965 ftrace_pages = ftrace_pages->next; 965 ftrace_pages = ftrace_pages->next;
966 } 966 }
967 967
968 return &ftrace_pages->records[ftrace_pages->index++]; 968 return &ftrace_pages->records[ftrace_pages->index++];
969 } 969 }
970 970
971 static struct dyn_ftrace * 971 static struct dyn_ftrace *
972 ftrace_record_ip(unsigned long ip) 972 ftrace_record_ip(unsigned long ip)
973 { 973 {
974 struct dyn_ftrace *rec; 974 struct dyn_ftrace *rec;
975 975
976 if (ftrace_disabled) 976 if (ftrace_disabled)
977 return NULL; 977 return NULL;
978 978
979 rec = ftrace_alloc_dyn_node(ip); 979 rec = ftrace_alloc_dyn_node(ip);
980 if (!rec) 980 if (!rec)
981 return NULL; 981 return NULL;
982 982
983 rec->ip = ip; 983 rec->ip = ip;
984 rec->newlist = ftrace_new_addrs; 984 rec->newlist = ftrace_new_addrs;
985 ftrace_new_addrs = rec; 985 ftrace_new_addrs = rec;
986 986
987 return rec; 987 return rec;
988 } 988 }
989 989
990 static void print_ip_ins(const char *fmt, unsigned char *p) 990 static void print_ip_ins(const char *fmt, unsigned char *p)
991 { 991 {
992 int i; 992 int i;
993 993
994 printk(KERN_CONT "%s", fmt); 994 printk(KERN_CONT "%s", fmt);
995 995
996 for (i = 0; i < MCOUNT_INSN_SIZE; i++) 996 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
997 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); 997 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
998 } 998 }
999 999
1000 static void ftrace_bug(int failed, unsigned long ip) 1000 static void ftrace_bug(int failed, unsigned long ip)
1001 { 1001 {
1002 switch (failed) { 1002 switch (failed) {
1003 case -EFAULT: 1003 case -EFAULT:
1004 FTRACE_WARN_ON_ONCE(1); 1004 FTRACE_WARN_ON_ONCE(1);
1005 pr_info("ftrace faulted on modifying "); 1005 pr_info("ftrace faulted on modifying ");
1006 print_ip_sym(ip); 1006 print_ip_sym(ip);
1007 break; 1007 break;
1008 case -EINVAL: 1008 case -EINVAL:
1009 FTRACE_WARN_ON_ONCE(1); 1009 FTRACE_WARN_ON_ONCE(1);
1010 pr_info("ftrace failed to modify "); 1010 pr_info("ftrace failed to modify ");
1011 print_ip_sym(ip); 1011 print_ip_sym(ip);
1012 print_ip_ins(" actual: ", (unsigned char *)ip); 1012 print_ip_ins(" actual: ", (unsigned char *)ip);
1013 printk(KERN_CONT "\n"); 1013 printk(KERN_CONT "\n");
1014 break; 1014 break;
1015 case -EPERM: 1015 case -EPERM:
1016 FTRACE_WARN_ON_ONCE(1); 1016 FTRACE_WARN_ON_ONCE(1);
1017 pr_info("ftrace faulted on writing "); 1017 pr_info("ftrace faulted on writing ");
1018 print_ip_sym(ip); 1018 print_ip_sym(ip);
1019 break; 1019 break;
1020 default: 1020 default:
1021 FTRACE_WARN_ON_ONCE(1); 1021 FTRACE_WARN_ON_ONCE(1);
1022 pr_info("ftrace faulted on unknown error "); 1022 pr_info("ftrace faulted on unknown error ");
1023 print_ip_sym(ip); 1023 print_ip_sym(ip);
1024 } 1024 }
1025 } 1025 }
1026 1026
1027 1027
1028 static int 1028 static int
1029 __ftrace_replace_code(struct dyn_ftrace *rec, int enable) 1029 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1030 { 1030 {
1031 unsigned long ftrace_addr; 1031 unsigned long ftrace_addr;
1032 unsigned long flag = 0UL; 1032 unsigned long flag = 0UL;
1033 1033
1034 ftrace_addr = (unsigned long)FTRACE_ADDR; 1034 ftrace_addr = (unsigned long)FTRACE_ADDR;
1035 1035
1036 /* 1036 /*
1037 * If this record is not to be traced or we want to disable it, 1037 * If this record is not to be traced or we want to disable it,
1038 * then disable it. 1038 * then disable it.
1039 * 1039 *
1040 * If we want to enable it and filtering is off, then enable it. 1040 * If we want to enable it and filtering is off, then enable it.
1041 * 1041 *
1042 * If we want to enable it and filtering is on, enable it only if 1042 * If we want to enable it and filtering is on, enable it only if
1043 * it's filtered 1043 * it's filtered
1044 */ 1044 */
1045 if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) { 1045 if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
1046 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER)) 1046 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1047 flag = FTRACE_FL_ENABLED; 1047 flag = FTRACE_FL_ENABLED;
1048 } 1048 }
1049 1049
1050 /* If the state of this record hasn't changed, then do nothing */ 1050 /* If the state of this record hasn't changed, then do nothing */
1051 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 1051 if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1052 return 0; 1052 return 0;
1053 1053
1054 if (flag) { 1054 if (flag) {
1055 rec->flags |= FTRACE_FL_ENABLED; 1055 rec->flags |= FTRACE_FL_ENABLED;
1056 return ftrace_make_call(rec, ftrace_addr); 1056 return ftrace_make_call(rec, ftrace_addr);
1057 } 1057 }
1058 1058
1059 rec->flags &= ~FTRACE_FL_ENABLED; 1059 rec->flags &= ~FTRACE_FL_ENABLED;
1060 return ftrace_make_nop(NULL, rec, ftrace_addr); 1060 return ftrace_make_nop(NULL, rec, ftrace_addr);
1061 } 1061 }
1062 1062
1063 static void ftrace_replace_code(int enable) 1063 static void ftrace_replace_code(int enable)
1064 { 1064 {
1065 struct dyn_ftrace *rec; 1065 struct dyn_ftrace *rec;
1066 struct ftrace_page *pg; 1066 struct ftrace_page *pg;
1067 int failed; 1067 int failed;
1068 1068
1069 do_for_each_ftrace_rec(pg, rec) { 1069 do_for_each_ftrace_rec(pg, rec) {
1070 /* 1070 /*
1071 * Skip over free records, records that have 1071 * Skip over free records, records that have
1072 * failed and not converted. 1072 * failed and not converted.
1073 */ 1073 */
1074 if (rec->flags & FTRACE_FL_FREE || 1074 if (rec->flags & FTRACE_FL_FREE ||
1075 rec->flags & FTRACE_FL_FAILED || 1075 rec->flags & FTRACE_FL_FAILED ||
1076 !(rec->flags & FTRACE_FL_CONVERTED)) 1076 !(rec->flags & FTRACE_FL_CONVERTED))
1077 continue; 1077 continue;
1078 1078
1079 /* ignore updates to this record's mcount site */ 1079 /* ignore updates to this record's mcount site */
1080 if (get_kprobe((void *)rec->ip)) { 1080 if (get_kprobe((void *)rec->ip)) {
1081 freeze_record(rec); 1081 freeze_record(rec);
1082 continue; 1082 continue;
1083 } else { 1083 } else {
1084 unfreeze_record(rec); 1084 unfreeze_record(rec);
1085 } 1085 }
1086 1086
1087 failed = __ftrace_replace_code(rec, enable); 1087 failed = __ftrace_replace_code(rec, enable);
1088 if (failed) { 1088 if (failed) {
1089 rec->flags |= FTRACE_FL_FAILED; 1089 rec->flags |= FTRACE_FL_FAILED;
1090 ftrace_bug(failed, rec->ip); 1090 ftrace_bug(failed, rec->ip);
1091 /* Stop processing */ 1091 /* Stop processing */
1092 return; 1092 return;
1093 } 1093 }
1094 } while_for_each_ftrace_rec(); 1094 } while_for_each_ftrace_rec();
1095 } 1095 }
1096 1096
1097 static int 1097 static int
1098 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) 1098 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1099 { 1099 {
1100 unsigned long ip; 1100 unsigned long ip;
1101 int ret; 1101 int ret;
1102 1102
1103 ip = rec->ip; 1103 ip = rec->ip;
1104 1104
1105 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); 1105 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1106 if (ret) { 1106 if (ret) {
1107 ftrace_bug(ret, ip); 1107 ftrace_bug(ret, ip);
1108 rec->flags |= FTRACE_FL_FAILED; 1108 rec->flags |= FTRACE_FL_FAILED;
1109 return 0; 1109 return 0;
1110 } 1110 }
1111 return 1; 1111 return 1;
1112 } 1112 }
1113 1113
1114 /* 1114 /*
1115 * archs can override this function if they must do something 1115 * archs can override this function if they must do something
1116 * before the modifying code is performed. 1116 * before the modifying code is performed.
1117 */ 1117 */
1118 int __weak ftrace_arch_code_modify_prepare(void) 1118 int __weak ftrace_arch_code_modify_prepare(void)
1119 { 1119 {
1120 return 0; 1120 return 0;
1121 } 1121 }
1122 1122
1123 /* 1123 /*
1124 * archs can override this function if they must do something 1124 * archs can override this function if they must do something
1125 * after the modifying code is performed. 1125 * after the modifying code is performed.
1126 */ 1126 */
1127 int __weak ftrace_arch_code_modify_post_process(void) 1127 int __weak ftrace_arch_code_modify_post_process(void)
1128 { 1128 {
1129 return 0; 1129 return 0;
1130 } 1130 }
1131 1131
1132 static int __ftrace_modify_code(void *data) 1132 static int __ftrace_modify_code(void *data)
1133 { 1133 {
1134 int *command = data; 1134 int *command = data;
1135 1135
1136 if (*command & FTRACE_ENABLE_CALLS) 1136 if (*command & FTRACE_ENABLE_CALLS)
1137 ftrace_replace_code(1); 1137 ftrace_replace_code(1);
1138 else if (*command & FTRACE_DISABLE_CALLS) 1138 else if (*command & FTRACE_DISABLE_CALLS)
1139 ftrace_replace_code(0); 1139 ftrace_replace_code(0);
1140 1140
1141 if (*command & FTRACE_UPDATE_TRACE_FUNC) 1141 if (*command & FTRACE_UPDATE_TRACE_FUNC)
1142 ftrace_update_ftrace_func(ftrace_trace_function); 1142 ftrace_update_ftrace_func(ftrace_trace_function);
1143 1143
1144 if (*command & FTRACE_START_FUNC_RET) 1144 if (*command & FTRACE_START_FUNC_RET)
1145 ftrace_enable_ftrace_graph_caller(); 1145 ftrace_enable_ftrace_graph_caller();
1146 else if (*command & FTRACE_STOP_FUNC_RET) 1146 else if (*command & FTRACE_STOP_FUNC_RET)
1147 ftrace_disable_ftrace_graph_caller(); 1147 ftrace_disable_ftrace_graph_caller();
1148 1148
1149 return 0; 1149 return 0;
1150 } 1150 }
1151 1151
1152 static void ftrace_run_update_code(int command) 1152 static void ftrace_run_update_code(int command)
1153 { 1153 {
1154 int ret; 1154 int ret;
1155 1155
1156 ret = ftrace_arch_code_modify_prepare(); 1156 ret = ftrace_arch_code_modify_prepare();
1157 FTRACE_WARN_ON(ret); 1157 FTRACE_WARN_ON(ret);
1158 if (ret) 1158 if (ret)
1159 return; 1159 return;
1160 1160
1161 stop_machine(__ftrace_modify_code, &command, NULL); 1161 stop_machine(__ftrace_modify_code, &command, NULL);
1162 1162
1163 ret = ftrace_arch_code_modify_post_process(); 1163 ret = ftrace_arch_code_modify_post_process();
1164 FTRACE_WARN_ON(ret); 1164 FTRACE_WARN_ON(ret);
1165 } 1165 }
1166 1166
1167 static ftrace_func_t saved_ftrace_func; 1167 static ftrace_func_t saved_ftrace_func;
1168 static int ftrace_start_up; 1168 static int ftrace_start_up;
1169 1169
1170 static void ftrace_startup_enable(int command) 1170 static void ftrace_startup_enable(int command)
1171 { 1171 {
1172 if (saved_ftrace_func != ftrace_trace_function) { 1172 if (saved_ftrace_func != ftrace_trace_function) {
1173 saved_ftrace_func = ftrace_trace_function; 1173 saved_ftrace_func = ftrace_trace_function;
1174 command |= FTRACE_UPDATE_TRACE_FUNC; 1174 command |= FTRACE_UPDATE_TRACE_FUNC;
1175 } 1175 }
1176 1176
1177 if (!command || !ftrace_enabled) 1177 if (!command || !ftrace_enabled)
1178 return; 1178 return;
1179 1179
1180 ftrace_run_update_code(command); 1180 ftrace_run_update_code(command);
1181 } 1181 }
1182 1182
1183 static void ftrace_startup(int command) 1183 static void ftrace_startup(int command)
1184 { 1184 {
1185 if (unlikely(ftrace_disabled)) 1185 if (unlikely(ftrace_disabled))
1186 return; 1186 return;
1187 1187
1188 ftrace_start_up++; 1188 ftrace_start_up++;
1189 command |= FTRACE_ENABLE_CALLS; 1189 command |= FTRACE_ENABLE_CALLS;
1190 1190
1191 ftrace_startup_enable(command); 1191 ftrace_startup_enable(command);
1192 } 1192 }
1193 1193
1194 static void ftrace_shutdown(int command) 1194 static void ftrace_shutdown(int command)
1195 { 1195 {
1196 if (unlikely(ftrace_disabled)) 1196 if (unlikely(ftrace_disabled))
1197 return; 1197 return;
1198 1198
1199 ftrace_start_up--; 1199 ftrace_start_up--;
1200 /* 1200 /*
1201 * Just warn in case of unbalance, no need to kill ftrace, it's not 1201 * Just warn in case of unbalance, no need to kill ftrace, it's not
1202 * critical but the ftrace_call callers may be never nopped again after 1202 * critical but the ftrace_call callers may be never nopped again after
1203 * further ftrace uses. 1203 * further ftrace uses.
1204 */ 1204 */
1205 WARN_ON_ONCE(ftrace_start_up < 0); 1205 WARN_ON_ONCE(ftrace_start_up < 0);
1206 1206
1207 if (!ftrace_start_up) 1207 if (!ftrace_start_up)
1208 command |= FTRACE_DISABLE_CALLS; 1208 command |= FTRACE_DISABLE_CALLS;
1209 1209
1210 if (saved_ftrace_func != ftrace_trace_function) { 1210 if (saved_ftrace_func != ftrace_trace_function) {
1211 saved_ftrace_func = ftrace_trace_function; 1211 saved_ftrace_func = ftrace_trace_function;
1212 command |= FTRACE_UPDATE_TRACE_FUNC; 1212 command |= FTRACE_UPDATE_TRACE_FUNC;
1213 } 1213 }
1214 1214
1215 if (!command || !ftrace_enabled) 1215 if (!command || !ftrace_enabled)
1216 return; 1216 return;
1217 1217
1218 ftrace_run_update_code(command); 1218 ftrace_run_update_code(command);
1219 } 1219 }
1220 1220
1221 static void ftrace_startup_sysctl(void) 1221 static void ftrace_startup_sysctl(void)
1222 { 1222 {
1223 int command = FTRACE_ENABLE_MCOUNT; 1223 int command = FTRACE_ENABLE_MCOUNT;
1224 1224
1225 if (unlikely(ftrace_disabled)) 1225 if (unlikely(ftrace_disabled))
1226 return; 1226 return;
1227 1227
1228 /* Force update next time */ 1228 /* Force update next time */
1229 saved_ftrace_func = NULL; 1229 saved_ftrace_func = NULL;
1230 /* ftrace_start_up is true if we want ftrace running */ 1230 /* ftrace_start_up is true if we want ftrace running */
1231 if (ftrace_start_up) 1231 if (ftrace_start_up)
1232 command |= FTRACE_ENABLE_CALLS; 1232 command |= FTRACE_ENABLE_CALLS;
1233 1233
1234 ftrace_run_update_code(command); 1234 ftrace_run_update_code(command);
1235 } 1235 }
1236 1236
1237 static void ftrace_shutdown_sysctl(void) 1237 static void ftrace_shutdown_sysctl(void)
1238 { 1238 {
1239 int command = FTRACE_DISABLE_MCOUNT; 1239 int command = FTRACE_DISABLE_MCOUNT;
1240 1240
1241 if (unlikely(ftrace_disabled)) 1241 if (unlikely(ftrace_disabled))
1242 return; 1242 return;
1243 1243
1244 /* ftrace_start_up is true if ftrace is running */ 1244 /* ftrace_start_up is true if ftrace is running */
1245 if (ftrace_start_up) 1245 if (ftrace_start_up)
1246 command |= FTRACE_DISABLE_CALLS; 1246 command |= FTRACE_DISABLE_CALLS;
1247 1247
1248 ftrace_run_update_code(command); 1248 ftrace_run_update_code(command);
1249 } 1249 }
1250 1250
1251 static cycle_t ftrace_update_time; 1251 static cycle_t ftrace_update_time;
1252 static unsigned long ftrace_update_cnt; 1252 static unsigned long ftrace_update_cnt;
1253 unsigned long ftrace_update_tot_cnt; 1253 unsigned long ftrace_update_tot_cnt;
1254 1254
1255 static int ftrace_update_code(struct module *mod) 1255 static int ftrace_update_code(struct module *mod)
1256 { 1256 {
1257 struct dyn_ftrace *p; 1257 struct dyn_ftrace *p;
1258 cycle_t start, stop; 1258 cycle_t start, stop;
1259 1259
1260 start = ftrace_now(raw_smp_processor_id()); 1260 start = ftrace_now(raw_smp_processor_id());
1261 ftrace_update_cnt = 0; 1261 ftrace_update_cnt = 0;
1262 1262
1263 while (ftrace_new_addrs) { 1263 while (ftrace_new_addrs) {
1264 1264
1265 /* If something went wrong, bail without enabling anything */ 1265 /* If something went wrong, bail without enabling anything */
1266 if (unlikely(ftrace_disabled)) 1266 if (unlikely(ftrace_disabled))
1267 return -1; 1267 return -1;
1268 1268
1269 p = ftrace_new_addrs; 1269 p = ftrace_new_addrs;
1270 ftrace_new_addrs = p->newlist; 1270 ftrace_new_addrs = p->newlist;
1271 p->flags = 0L; 1271 p->flags = 0L;
1272 1272
1273 /* 1273 /*
1274 * Do the initial record convertion from mcount jump 1274 * Do the initial record convertion from mcount jump
1275 * to the NOP instructions. 1275 * to the NOP instructions.
1276 */ 1276 */
1277 if (!ftrace_code_disable(mod, p)) { 1277 if (!ftrace_code_disable(mod, p)) {
1278 ftrace_free_rec(p); 1278 ftrace_free_rec(p);
1279 continue; 1279 continue;
1280 } 1280 }
1281 1281
1282 p->flags |= FTRACE_FL_CONVERTED; 1282 p->flags |= FTRACE_FL_CONVERTED;
1283 ftrace_update_cnt++; 1283 ftrace_update_cnt++;
1284 1284
1285 /* 1285 /*
1286 * If the tracing is enabled, go ahead and enable the record. 1286 * If the tracing is enabled, go ahead and enable the record.
1287 * 1287 *
1288 * The reason not to enable the record immediatelly is the 1288 * The reason not to enable the record immediatelly is the
1289 * inherent check of ftrace_make_nop/ftrace_make_call for 1289 * inherent check of ftrace_make_nop/ftrace_make_call for
1290 * correct previous instructions. Making first the NOP 1290 * correct previous instructions. Making first the NOP
1291 * conversion puts the module to the correct state, thus 1291 * conversion puts the module to the correct state, thus
1292 * passing the ftrace_make_call check. 1292 * passing the ftrace_make_call check.
1293 */ 1293 */
1294 if (ftrace_start_up) { 1294 if (ftrace_start_up) {
1295 int failed = __ftrace_replace_code(p, 1); 1295 int failed = __ftrace_replace_code(p, 1);
1296 if (failed) { 1296 if (failed) {
1297 ftrace_bug(failed, p->ip); 1297 ftrace_bug(failed, p->ip);
1298 ftrace_free_rec(p); 1298 ftrace_free_rec(p);
1299 } 1299 }
1300 } 1300 }
1301 } 1301 }
1302 1302
1303 stop = ftrace_now(raw_smp_processor_id()); 1303 stop = ftrace_now(raw_smp_processor_id());
1304 ftrace_update_time = stop - start; 1304 ftrace_update_time = stop - start;
1305 ftrace_update_tot_cnt += ftrace_update_cnt; 1305 ftrace_update_tot_cnt += ftrace_update_cnt;
1306 1306
1307 return 0; 1307 return 0;
1308 } 1308 }
1309 1309
1310 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) 1310 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1311 { 1311 {
1312 struct ftrace_page *pg; 1312 struct ftrace_page *pg;
1313 int cnt; 1313 int cnt;
1314 int i; 1314 int i;
1315 1315
1316 /* allocate a few pages */ 1316 /* allocate a few pages */
1317 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL); 1317 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1318 if (!ftrace_pages_start) 1318 if (!ftrace_pages_start)
1319 return -1; 1319 return -1;
1320 1320
1321 /* 1321 /*
1322 * Allocate a few more pages. 1322 * Allocate a few more pages.
1323 * 1323 *
1324 * TODO: have some parser search vmlinux before 1324 * TODO: have some parser search vmlinux before
1325 * final linking to find all calls to ftrace. 1325 * final linking to find all calls to ftrace.
1326 * Then we can: 1326 * Then we can:
1327 * a) know how many pages to allocate. 1327 * a) know how many pages to allocate.
1328 * and/or 1328 * and/or
1329 * b) set up the table then. 1329 * b) set up the table then.
1330 * 1330 *
1331 * The dynamic code is still necessary for 1331 * The dynamic code is still necessary for
1332 * modules. 1332 * modules.
1333 */ 1333 */
1334 1334
1335 pg = ftrace_pages = ftrace_pages_start; 1335 pg = ftrace_pages = ftrace_pages_start;
1336 1336
1337 cnt = num_to_init / ENTRIES_PER_PAGE; 1337 cnt = num_to_init / ENTRIES_PER_PAGE;
1338 pr_info("ftrace: allocating %ld entries in %d pages\n", 1338 pr_info("ftrace: allocating %ld entries in %d pages\n",
1339 num_to_init, cnt + 1); 1339 num_to_init, cnt + 1);
1340 1340
1341 for (i = 0; i < cnt; i++) { 1341 for (i = 0; i < cnt; i++) {
1342 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 1342 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1343 1343
1344 /* If we fail, we'll try later anyway */ 1344 /* If we fail, we'll try later anyway */
1345 if (!pg->next) 1345 if (!pg->next)
1346 break; 1346 break;
1347 1347
1348 pg = pg->next; 1348 pg = pg->next;
1349 } 1349 }
1350 1350
1351 return 0; 1351 return 0;
1352 } 1352 }
1353 1353
1354 enum { 1354 enum {
1355 FTRACE_ITER_FILTER = (1 << 0), 1355 FTRACE_ITER_FILTER = (1 << 0),
1356 FTRACE_ITER_NOTRACE = (1 << 1), 1356 FTRACE_ITER_NOTRACE = (1 << 1),
1357 FTRACE_ITER_FAILURES = (1 << 2), 1357 FTRACE_ITER_FAILURES = (1 << 2),
1358 FTRACE_ITER_PRINTALL = (1 << 3), 1358 FTRACE_ITER_PRINTALL = (1 << 3),
1359 FTRACE_ITER_HASH = (1 << 4), 1359 FTRACE_ITER_HASH = (1 << 4),
1360 }; 1360 };
1361 1361
1362 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 1362 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1363 1363
1364 struct ftrace_iterator { 1364 struct ftrace_iterator {
1365 struct ftrace_page *pg; 1365 struct ftrace_page *pg;
1366 int hidx; 1366 int hidx;
1367 int idx; 1367 int idx;
1368 unsigned flags; 1368 unsigned flags;
1369 struct trace_parser parser; 1369 struct trace_parser parser;
1370 }; 1370 };
1371 1371
1372 static void * 1372 static void *
1373 t_hash_next(struct seq_file *m, void *v, loff_t *pos) 1373 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1374 { 1374 {
1375 struct ftrace_iterator *iter = m->private; 1375 struct ftrace_iterator *iter = m->private;
1376 struct hlist_node *hnd = v; 1376 struct hlist_node *hnd = v;
1377 struct hlist_head *hhd; 1377 struct hlist_head *hhd;
1378 1378
1379 WARN_ON(!(iter->flags & FTRACE_ITER_HASH)); 1379 WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1380 1380
1381 (*pos)++; 1381 (*pos)++;
1382 1382
1383 retry: 1383 retry:
1384 if (iter->hidx >= FTRACE_FUNC_HASHSIZE) 1384 if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1385 return NULL; 1385 return NULL;
1386 1386
1387 hhd = &ftrace_func_hash[iter->hidx]; 1387 hhd = &ftrace_func_hash[iter->hidx];
1388 1388
1389 if (hlist_empty(hhd)) { 1389 if (hlist_empty(hhd)) {
1390 iter->hidx++; 1390 iter->hidx++;
1391 hnd = NULL; 1391 hnd = NULL;
1392 goto retry; 1392 goto retry;
1393 } 1393 }
1394 1394
1395 if (!hnd) 1395 if (!hnd)
1396 hnd = hhd->first; 1396 hnd = hhd->first;
1397 else { 1397 else {
1398 hnd = hnd->next; 1398 hnd = hnd->next;
1399 if (!hnd) { 1399 if (!hnd) {
1400 iter->hidx++; 1400 iter->hidx++;
1401 goto retry; 1401 goto retry;
1402 } 1402 }
1403 } 1403 }
1404 1404
1405 return hnd; 1405 return hnd;
1406 } 1406 }
1407 1407
1408 static void *t_hash_start(struct seq_file *m, loff_t *pos) 1408 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1409 { 1409 {
1410 struct ftrace_iterator *iter = m->private; 1410 struct ftrace_iterator *iter = m->private;
1411 void *p = NULL; 1411 void *p = NULL;
1412 loff_t l; 1412 loff_t l;
1413 1413
1414 if (!(iter->flags & FTRACE_ITER_HASH)) 1414 if (!(iter->flags & FTRACE_ITER_HASH))
1415 *pos = 0; 1415 *pos = 0;
1416 1416
1417 iter->flags |= FTRACE_ITER_HASH; 1417 iter->flags |= FTRACE_ITER_HASH;
1418 1418
1419 iter->hidx = 0; 1419 iter->hidx = 0;
1420 for (l = 0; l <= *pos; ) { 1420 for (l = 0; l <= *pos; ) {
1421 p = t_hash_next(m, p, &l); 1421 p = t_hash_next(m, p, &l);
1422 if (!p) 1422 if (!p)
1423 break; 1423 break;
1424 } 1424 }
1425 return p; 1425 return p;
1426 } 1426 }
1427 1427
1428 static int t_hash_show(struct seq_file *m, void *v) 1428 static int t_hash_show(struct seq_file *m, void *v)
1429 { 1429 {
1430 struct ftrace_func_probe *rec; 1430 struct ftrace_func_probe *rec;
1431 struct hlist_node *hnd = v; 1431 struct hlist_node *hnd = v;
1432 1432
1433 rec = hlist_entry(hnd, struct ftrace_func_probe, node); 1433 rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1434 1434
1435 if (rec->ops->print) 1435 if (rec->ops->print)
1436 return rec->ops->print(m, rec->ip, rec->ops, rec->data); 1436 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1437 1437
1438 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); 1438 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1439 1439
1440 if (rec->data) 1440 if (rec->data)
1441 seq_printf(m, ":%p", rec->data); 1441 seq_printf(m, ":%p", rec->data);
1442 seq_putc(m, '\n'); 1442 seq_putc(m, '\n');
1443 1443
1444 return 0; 1444 return 0;
1445 } 1445 }
1446 1446
1447 static void * 1447 static void *
1448 t_next(struct seq_file *m, void *v, loff_t *pos) 1448 t_next(struct seq_file *m, void *v, loff_t *pos)
1449 { 1449 {
1450 struct ftrace_iterator *iter = m->private; 1450 struct ftrace_iterator *iter = m->private;
1451 struct dyn_ftrace *rec = NULL; 1451 struct dyn_ftrace *rec = NULL;
1452 1452
1453 if (iter->flags & FTRACE_ITER_HASH) 1453 if (iter->flags & FTRACE_ITER_HASH)
1454 return t_hash_next(m, v, pos); 1454 return t_hash_next(m, v, pos);
1455 1455
1456 (*pos)++; 1456 (*pos)++;
1457 1457
1458 if (iter->flags & FTRACE_ITER_PRINTALL) 1458 if (iter->flags & FTRACE_ITER_PRINTALL)
1459 return NULL; 1459 return NULL;
1460 1460
1461 retry: 1461 retry:
1462 if (iter->idx >= iter->pg->index) { 1462 if (iter->idx >= iter->pg->index) {
1463 if (iter->pg->next) { 1463 if (iter->pg->next) {
1464 iter->pg = iter->pg->next; 1464 iter->pg = iter->pg->next;
1465 iter->idx = 0; 1465 iter->idx = 0;
1466 goto retry; 1466 goto retry;
1467 } 1467 }
1468 } else { 1468 } else {
1469 rec = &iter->pg->records[iter->idx++]; 1469 rec = &iter->pg->records[iter->idx++];
1470 if ((rec->flags & FTRACE_FL_FREE) || 1470 if ((rec->flags & FTRACE_FL_FREE) ||
1471 1471
1472 (!(iter->flags & FTRACE_ITER_FAILURES) && 1472 (!(iter->flags & FTRACE_ITER_FAILURES) &&
1473 (rec->flags & FTRACE_FL_FAILED)) || 1473 (rec->flags & FTRACE_FL_FAILED)) ||
1474 1474
1475 ((iter->flags & FTRACE_ITER_FAILURES) && 1475 ((iter->flags & FTRACE_ITER_FAILURES) &&
1476 !(rec->flags & FTRACE_FL_FAILED)) || 1476 !(rec->flags & FTRACE_FL_FAILED)) ||
1477 1477
1478 ((iter->flags & FTRACE_ITER_FILTER) && 1478 ((iter->flags & FTRACE_ITER_FILTER) &&
1479 !(rec->flags & FTRACE_FL_FILTER)) || 1479 !(rec->flags & FTRACE_FL_FILTER)) ||
1480 1480
1481 ((iter->flags & FTRACE_ITER_NOTRACE) && 1481 ((iter->flags & FTRACE_ITER_NOTRACE) &&
1482 !(rec->flags & FTRACE_FL_NOTRACE))) { 1482 !(rec->flags & FTRACE_FL_NOTRACE))) {
1483 rec = NULL; 1483 rec = NULL;
1484 goto retry; 1484 goto retry;
1485 } 1485 }
1486 } 1486 }
1487 1487
1488 return rec; 1488 return rec;
1489 } 1489 }
1490 1490
1491 static void *t_start(struct seq_file *m, loff_t *pos) 1491 static void *t_start(struct seq_file *m, loff_t *pos)
1492 { 1492 {
1493 struct ftrace_iterator *iter = m->private; 1493 struct ftrace_iterator *iter = m->private;
1494 void *p = NULL; 1494 void *p = NULL;
1495 loff_t l; 1495 loff_t l;
1496 1496
1497 mutex_lock(&ftrace_lock); 1497 mutex_lock(&ftrace_lock);
1498 /* 1498 /*
1499 * For set_ftrace_filter reading, if we have the filter 1499 * For set_ftrace_filter reading, if we have the filter
1500 * off, we can short cut and just print out that all 1500 * off, we can short cut and just print out that all
1501 * functions are enabled. 1501 * functions are enabled.
1502 */ 1502 */
1503 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) { 1503 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1504 if (*pos > 0) 1504 if (*pos > 0)
1505 return t_hash_start(m, pos); 1505 return t_hash_start(m, pos);
1506 iter->flags |= FTRACE_ITER_PRINTALL; 1506 iter->flags |= FTRACE_ITER_PRINTALL;
1507 return iter; 1507 return iter;
1508 } 1508 }
1509 1509
1510 if (iter->flags & FTRACE_ITER_HASH) 1510 if (iter->flags & FTRACE_ITER_HASH)
1511 return t_hash_start(m, pos); 1511 return t_hash_start(m, pos);
1512 1512
1513 iter->pg = ftrace_pages_start; 1513 iter->pg = ftrace_pages_start;
1514 iter->idx = 0; 1514 iter->idx = 0;
1515 for (l = 0; l <= *pos; ) { 1515 for (l = 0; l <= *pos; ) {
1516 p = t_next(m, p, &l); 1516 p = t_next(m, p, &l);
1517 if (!p) 1517 if (!p)
1518 break; 1518 break;
1519 } 1519 }
1520 1520
1521 if (!p && iter->flags & FTRACE_ITER_FILTER) 1521 if (!p && iter->flags & FTRACE_ITER_FILTER)
1522 return t_hash_start(m, pos); 1522 return t_hash_start(m, pos);
1523 1523
1524 return p; 1524 return p;
1525 } 1525 }
1526 1526
1527 static void t_stop(struct seq_file *m, void *p) 1527 static void t_stop(struct seq_file *m, void *p)
1528 { 1528 {
1529 mutex_unlock(&ftrace_lock); 1529 mutex_unlock(&ftrace_lock);
1530 } 1530 }
1531 1531
1532 static int t_show(struct seq_file *m, void *v) 1532 static int t_show(struct seq_file *m, void *v)
1533 { 1533 {
1534 struct ftrace_iterator *iter = m->private; 1534 struct ftrace_iterator *iter = m->private;
1535 struct dyn_ftrace *rec = v; 1535 struct dyn_ftrace *rec = v;
1536 1536
1537 if (iter->flags & FTRACE_ITER_HASH) 1537 if (iter->flags & FTRACE_ITER_HASH)
1538 return t_hash_show(m, v); 1538 return t_hash_show(m, v);
1539 1539
1540 if (iter->flags & FTRACE_ITER_PRINTALL) { 1540 if (iter->flags & FTRACE_ITER_PRINTALL) {
1541 seq_printf(m, "#### all functions enabled ####\n"); 1541 seq_printf(m, "#### all functions enabled ####\n");
1542 return 0; 1542 return 0;
1543 } 1543 }
1544 1544
1545 if (!rec) 1545 if (!rec)
1546 return 0; 1546 return 0;
1547 1547
1548 seq_printf(m, "%ps\n", (void *)rec->ip); 1548 seq_printf(m, "%ps\n", (void *)rec->ip);
1549 1549
1550 return 0; 1550 return 0;
1551 } 1551 }
1552 1552
1553 static const struct seq_operations show_ftrace_seq_ops = { 1553 static const struct seq_operations show_ftrace_seq_ops = {
1554 .start = t_start, 1554 .start = t_start,
1555 .next = t_next, 1555 .next = t_next,
1556 .stop = t_stop, 1556 .stop = t_stop,
1557 .show = t_show, 1557 .show = t_show,
1558 }; 1558 };
1559 1559
1560 static int 1560 static int
1561 ftrace_avail_open(struct inode *inode, struct file *file) 1561 ftrace_avail_open(struct inode *inode, struct file *file)
1562 { 1562 {
1563 struct ftrace_iterator *iter; 1563 struct ftrace_iterator *iter;
1564 int ret; 1564 int ret;
1565 1565
1566 if (unlikely(ftrace_disabled)) 1566 if (unlikely(ftrace_disabled))
1567 return -ENODEV; 1567 return -ENODEV;
1568 1568
1569 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1569 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1570 if (!iter) 1570 if (!iter)
1571 return -ENOMEM; 1571 return -ENOMEM;
1572 1572
1573 iter->pg = ftrace_pages_start; 1573 iter->pg = ftrace_pages_start;
1574 1574
1575 ret = seq_open(file, &show_ftrace_seq_ops); 1575 ret = seq_open(file, &show_ftrace_seq_ops);
1576 if (!ret) { 1576 if (!ret) {
1577 struct seq_file *m = file->private_data; 1577 struct seq_file *m = file->private_data;
1578 1578
1579 m->private = iter; 1579 m->private = iter;
1580 } else { 1580 } else {
1581 kfree(iter); 1581 kfree(iter);
1582 } 1582 }
1583 1583
1584 return ret; 1584 return ret;
1585 } 1585 }
1586 1586
1587 static int 1587 static int
1588 ftrace_failures_open(struct inode *inode, struct file *file) 1588 ftrace_failures_open(struct inode *inode, struct file *file)
1589 { 1589 {
1590 int ret; 1590 int ret;
1591 struct seq_file *m; 1591 struct seq_file *m;
1592 struct ftrace_iterator *iter; 1592 struct ftrace_iterator *iter;
1593 1593
1594 ret = ftrace_avail_open(inode, file); 1594 ret = ftrace_avail_open(inode, file);
1595 if (!ret) { 1595 if (!ret) {
1596 m = (struct seq_file *)file->private_data; 1596 m = (struct seq_file *)file->private_data;
1597 iter = (struct ftrace_iterator *)m->private; 1597 iter = (struct ftrace_iterator *)m->private;
1598 iter->flags = FTRACE_ITER_FAILURES; 1598 iter->flags = FTRACE_ITER_FAILURES;
1599 } 1599 }
1600 1600
1601 return ret; 1601 return ret;
1602 } 1602 }
1603 1603
1604 1604
1605 static void ftrace_filter_reset(int enable) 1605 static void ftrace_filter_reset(int enable)
1606 { 1606 {
1607 struct ftrace_page *pg; 1607 struct ftrace_page *pg;
1608 struct dyn_ftrace *rec; 1608 struct dyn_ftrace *rec;
1609 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1609 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1610 1610
1611 mutex_lock(&ftrace_lock); 1611 mutex_lock(&ftrace_lock);
1612 if (enable) 1612 if (enable)
1613 ftrace_filtered = 0; 1613 ftrace_filtered = 0;
1614 do_for_each_ftrace_rec(pg, rec) { 1614 do_for_each_ftrace_rec(pg, rec) {
1615 if (rec->flags & FTRACE_FL_FAILED) 1615 if (rec->flags & FTRACE_FL_FAILED)
1616 continue; 1616 continue;
1617 rec->flags &= ~type; 1617 rec->flags &= ~type;
1618 } while_for_each_ftrace_rec(); 1618 } while_for_each_ftrace_rec();
1619 mutex_unlock(&ftrace_lock); 1619 mutex_unlock(&ftrace_lock);
1620 } 1620 }
1621 1621
1622 static int 1622 static int
1623 ftrace_regex_open(struct inode *inode, struct file *file, int enable) 1623 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1624 { 1624 {
1625 struct ftrace_iterator *iter; 1625 struct ftrace_iterator *iter;
1626 int ret = 0; 1626 int ret = 0;
1627 1627
1628 if (unlikely(ftrace_disabled)) 1628 if (unlikely(ftrace_disabled))
1629 return -ENODEV; 1629 return -ENODEV;
1630 1630
1631 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1631 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1632 if (!iter) 1632 if (!iter)
1633 return -ENOMEM; 1633 return -ENOMEM;
1634 1634
1635 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) { 1635 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1636 kfree(iter); 1636 kfree(iter);
1637 return -ENOMEM; 1637 return -ENOMEM;
1638 } 1638 }
1639 1639
1640 mutex_lock(&ftrace_regex_lock); 1640 mutex_lock(&ftrace_regex_lock);
1641 if ((file->f_mode & FMODE_WRITE) && 1641 if ((file->f_mode & FMODE_WRITE) &&
1642 (file->f_flags & O_TRUNC)) 1642 (file->f_flags & O_TRUNC))
1643 ftrace_filter_reset(enable); 1643 ftrace_filter_reset(enable);
1644 1644
1645 if (file->f_mode & FMODE_READ) { 1645 if (file->f_mode & FMODE_READ) {
1646 iter->pg = ftrace_pages_start; 1646 iter->pg = ftrace_pages_start;
1647 iter->flags = enable ? FTRACE_ITER_FILTER : 1647 iter->flags = enable ? FTRACE_ITER_FILTER :
1648 FTRACE_ITER_NOTRACE; 1648 FTRACE_ITER_NOTRACE;
1649 1649
1650 ret = seq_open(file, &show_ftrace_seq_ops); 1650 ret = seq_open(file, &show_ftrace_seq_ops);
1651 if (!ret) { 1651 if (!ret) {
1652 struct seq_file *m = file->private_data; 1652 struct seq_file *m = file->private_data;
1653 m->private = iter; 1653 m->private = iter;
1654 } else { 1654 } else {
1655 trace_parser_put(&iter->parser); 1655 trace_parser_put(&iter->parser);
1656 kfree(iter); 1656 kfree(iter);
1657 } 1657 }
1658 } else 1658 } else
1659 file->private_data = iter; 1659 file->private_data = iter;
1660 mutex_unlock(&ftrace_regex_lock); 1660 mutex_unlock(&ftrace_regex_lock);
1661 1661
1662 return ret; 1662 return ret;
1663 } 1663 }
1664 1664
1665 static int 1665 static int
1666 ftrace_filter_open(struct inode *inode, struct file *file) 1666 ftrace_filter_open(struct inode *inode, struct file *file)
1667 { 1667 {
1668 return ftrace_regex_open(inode, file, 1); 1668 return ftrace_regex_open(inode, file, 1);
1669 } 1669 }
1670 1670
1671 static int 1671 static int
1672 ftrace_notrace_open(struct inode *inode, struct file *file) 1672 ftrace_notrace_open(struct inode *inode, struct file *file)
1673 { 1673 {
1674 return ftrace_regex_open(inode, file, 0); 1674 return ftrace_regex_open(inode, file, 0);
1675 } 1675 }
1676 1676
1677 static loff_t 1677 static loff_t
1678 ftrace_regex_lseek(struct file *file, loff_t offset, int origin) 1678 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1679 { 1679 {
1680 loff_t ret; 1680 loff_t ret;
1681 1681
1682 if (file->f_mode & FMODE_READ) 1682 if (file->f_mode & FMODE_READ)
1683 ret = seq_lseek(file, offset, origin); 1683 ret = seq_lseek(file, offset, origin);
1684 else 1684 else
1685 file->f_pos = ret = 1; 1685 file->f_pos = ret = 1;
1686 1686
1687 return ret; 1687 return ret;
1688 } 1688 }
1689 1689
1690 static int ftrace_match(char *str, char *regex, int len, int type) 1690 static int ftrace_match(char *str, char *regex, int len, int type)
1691 { 1691 {
1692 int matched = 0; 1692 int matched = 0;
1693 char *ptr; 1693 int slen;
1694 1694
1695 switch (type) { 1695 switch (type) {
1696 case MATCH_FULL: 1696 case MATCH_FULL:
1697 if (strcmp(str, regex) == 0) 1697 if (strcmp(str, regex) == 0)
1698 matched = 1; 1698 matched = 1;
1699 break; 1699 break;
1700 case MATCH_FRONT_ONLY: 1700 case MATCH_FRONT_ONLY:
1701 if (strncmp(str, regex, len) == 0) 1701 if (strncmp(str, regex, len) == 0)
1702 matched = 1; 1702 matched = 1;
1703 break; 1703 break;
1704 case MATCH_MIDDLE_ONLY: 1704 case MATCH_MIDDLE_ONLY:
1705 if (strstr(str, regex)) 1705 if (strstr(str, regex))
1706 matched = 1; 1706 matched = 1;
1707 break; 1707 break;
1708 case MATCH_END_ONLY: 1708 case MATCH_END_ONLY:
1709 ptr = strstr(str, regex); 1709 slen = strlen(str);
1710 if (ptr && (ptr[len] == 0)) 1710 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
1711 matched = 1; 1711 matched = 1;
1712 break; 1712 break;
1713 } 1713 }
1714 1714
1715 return matched; 1715 return matched;
1716 } 1716 }
1717 1717
1718 static int 1718 static int
1719 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type) 1719 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1720 { 1720 {
1721 char str[KSYM_SYMBOL_LEN]; 1721 char str[KSYM_SYMBOL_LEN];
1722 1722
1723 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 1723 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1724 return ftrace_match(str, regex, len, type); 1724 return ftrace_match(str, regex, len, type);
1725 } 1725 }
1726 1726
1727 static int ftrace_match_records(char *buff, int len, int enable) 1727 static int ftrace_match_records(char *buff, int len, int enable)
1728 { 1728 {
1729 unsigned int search_len; 1729 unsigned int search_len;
1730 struct ftrace_page *pg; 1730 struct ftrace_page *pg;
1731 struct dyn_ftrace *rec; 1731 struct dyn_ftrace *rec;
1732 unsigned long flag; 1732 unsigned long flag;
1733 char *search; 1733 char *search;
1734 int type; 1734 int type;
1735 int not; 1735 int not;
1736 int found = 0; 1736 int found = 0;
1737 1737
1738 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1738 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1739 type = filter_parse_regex(buff, len, &search, &not); 1739 type = filter_parse_regex(buff, len, &search, &not);
1740 1740
1741 search_len = strlen(search); 1741 search_len = strlen(search);
1742 1742
1743 mutex_lock(&ftrace_lock); 1743 mutex_lock(&ftrace_lock);
1744 do_for_each_ftrace_rec(pg, rec) { 1744 do_for_each_ftrace_rec(pg, rec) {
1745 1745
1746 if (rec->flags & FTRACE_FL_FAILED) 1746 if (rec->flags & FTRACE_FL_FAILED)
1747 continue; 1747 continue;
1748 1748
1749 if (ftrace_match_record(rec, search, search_len, type)) { 1749 if (ftrace_match_record(rec, search, search_len, type)) {
1750 if (not) 1750 if (not)
1751 rec->flags &= ~flag; 1751 rec->flags &= ~flag;
1752 else 1752 else
1753 rec->flags |= flag; 1753 rec->flags |= flag;
1754 found = 1; 1754 found = 1;
1755 } 1755 }
1756 /* 1756 /*
1757 * Only enable filtering if we have a function that 1757 * Only enable filtering if we have a function that
1758 * is filtered on. 1758 * is filtered on.
1759 */ 1759 */
1760 if (enable && (rec->flags & FTRACE_FL_FILTER)) 1760 if (enable && (rec->flags & FTRACE_FL_FILTER))
1761 ftrace_filtered = 1; 1761 ftrace_filtered = 1;
1762 } while_for_each_ftrace_rec(); 1762 } while_for_each_ftrace_rec();
1763 mutex_unlock(&ftrace_lock); 1763 mutex_unlock(&ftrace_lock);
1764 1764
1765 return found; 1765 return found;
1766 } 1766 }
1767 1767
1768 static int 1768 static int
1769 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod, 1769 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1770 char *regex, int len, int type) 1770 char *regex, int len, int type)
1771 { 1771 {
1772 char str[KSYM_SYMBOL_LEN]; 1772 char str[KSYM_SYMBOL_LEN];
1773 char *modname; 1773 char *modname;
1774 1774
1775 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); 1775 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1776 1776
1777 if (!modname || strcmp(modname, mod)) 1777 if (!modname || strcmp(modname, mod))
1778 return 0; 1778 return 0;
1779 1779
1780 /* blank search means to match all funcs in the mod */ 1780 /* blank search means to match all funcs in the mod */
1781 if (len) 1781 if (len)
1782 return ftrace_match(str, regex, len, type); 1782 return ftrace_match(str, regex, len, type);
1783 else 1783 else
1784 return 1; 1784 return 1;
1785 } 1785 }
1786 1786
1787 static int ftrace_match_module_records(char *buff, char *mod, int enable) 1787 static int ftrace_match_module_records(char *buff, char *mod, int enable)
1788 { 1788 {
1789 unsigned search_len = 0; 1789 unsigned search_len = 0;
1790 struct ftrace_page *pg; 1790 struct ftrace_page *pg;
1791 struct dyn_ftrace *rec; 1791 struct dyn_ftrace *rec;
1792 int type = MATCH_FULL; 1792 int type = MATCH_FULL;
1793 char *search = buff; 1793 char *search = buff;
1794 unsigned long flag; 1794 unsigned long flag;
1795 int not = 0; 1795 int not = 0;
1796 int found = 0; 1796 int found = 0;
1797 1797
1798 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1798 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1799 1799
1800 /* blank or '*' mean the same */ 1800 /* blank or '*' mean the same */
1801 if (strcmp(buff, "*") == 0) 1801 if (strcmp(buff, "*") == 0)
1802 buff[0] = 0; 1802 buff[0] = 0;
1803 1803
1804 /* handle the case of 'dont filter this module' */ 1804 /* handle the case of 'dont filter this module' */
1805 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) { 1805 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1806 buff[0] = 0; 1806 buff[0] = 0;
1807 not = 1; 1807 not = 1;
1808 } 1808 }
1809 1809
1810 if (strlen(buff)) { 1810 if (strlen(buff)) {
1811 type = filter_parse_regex(buff, strlen(buff), &search, &not); 1811 type = filter_parse_regex(buff, strlen(buff), &search, &not);
1812 search_len = strlen(search); 1812 search_len = strlen(search);
1813 } 1813 }
1814 1814
1815 mutex_lock(&ftrace_lock); 1815 mutex_lock(&ftrace_lock);
1816 do_for_each_ftrace_rec(pg, rec) { 1816 do_for_each_ftrace_rec(pg, rec) {
1817 1817
1818 if (rec->flags & FTRACE_FL_FAILED) 1818 if (rec->flags & FTRACE_FL_FAILED)
1819 continue; 1819 continue;
1820 1820
1821 if (ftrace_match_module_record(rec, mod, 1821 if (ftrace_match_module_record(rec, mod,
1822 search, search_len, type)) { 1822 search, search_len, type)) {
1823 if (not) 1823 if (not)
1824 rec->flags &= ~flag; 1824 rec->flags &= ~flag;
1825 else 1825 else
1826 rec->flags |= flag; 1826 rec->flags |= flag;
1827 found = 1; 1827 found = 1;
1828 } 1828 }
1829 if (enable && (rec->flags & FTRACE_FL_FILTER)) 1829 if (enable && (rec->flags & FTRACE_FL_FILTER))
1830 ftrace_filtered = 1; 1830 ftrace_filtered = 1;
1831 1831
1832 } while_for_each_ftrace_rec(); 1832 } while_for_each_ftrace_rec();
1833 mutex_unlock(&ftrace_lock); 1833 mutex_unlock(&ftrace_lock);
1834 1834
1835 return found; 1835 return found;
1836 } 1836 }
1837 1837
1838 /* 1838 /*
1839 * We register the module command as a template to show others how 1839 * We register the module command as a template to show others how
1840 * to register the a command as well. 1840 * to register the a command as well.
1841 */ 1841 */
1842 1842
1843 static int 1843 static int
1844 ftrace_mod_callback(char *func, char *cmd, char *param, int enable) 1844 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1845 { 1845 {
1846 char *mod; 1846 char *mod;
1847 1847
1848 /* 1848 /*
1849 * cmd == 'mod' because we only registered this func 1849 * cmd == 'mod' because we only registered this func
1850 * for the 'mod' ftrace_func_command. 1850 * for the 'mod' ftrace_func_command.
1851 * But if you register one func with multiple commands, 1851 * But if you register one func with multiple commands,
1852 * you can tell which command was used by the cmd 1852 * you can tell which command was used by the cmd
1853 * parameter. 1853 * parameter.
1854 */ 1854 */
1855 1855
1856 /* we must have a module name */ 1856 /* we must have a module name */
1857 if (!param) 1857 if (!param)
1858 return -EINVAL; 1858 return -EINVAL;
1859 1859
1860 mod = strsep(&param, ":"); 1860 mod = strsep(&param, ":");
1861 if (!strlen(mod)) 1861 if (!strlen(mod))
1862 return -EINVAL; 1862 return -EINVAL;
1863 1863
1864 if (ftrace_match_module_records(func, mod, enable)) 1864 if (ftrace_match_module_records(func, mod, enable))
1865 return 0; 1865 return 0;
1866 return -EINVAL; 1866 return -EINVAL;
1867 } 1867 }
1868 1868
1869 static struct ftrace_func_command ftrace_mod_cmd = { 1869 static struct ftrace_func_command ftrace_mod_cmd = {
1870 .name = "mod", 1870 .name = "mod",
1871 .func = ftrace_mod_callback, 1871 .func = ftrace_mod_callback,
1872 }; 1872 };
1873 1873
1874 static int __init ftrace_mod_cmd_init(void) 1874 static int __init ftrace_mod_cmd_init(void)
1875 { 1875 {
1876 return register_ftrace_command(&ftrace_mod_cmd); 1876 return register_ftrace_command(&ftrace_mod_cmd);
1877 } 1877 }
1878 device_initcall(ftrace_mod_cmd_init); 1878 device_initcall(ftrace_mod_cmd_init);
1879 1879
1880 static void 1880 static void
1881 function_trace_probe_call(unsigned long ip, unsigned long parent_ip) 1881 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1882 { 1882 {
1883 struct ftrace_func_probe *entry; 1883 struct ftrace_func_probe *entry;
1884 struct hlist_head *hhd; 1884 struct hlist_head *hhd;
1885 struct hlist_node *n; 1885 struct hlist_node *n;
1886 unsigned long key; 1886 unsigned long key;
1887 int resched; 1887 int resched;
1888 1888
1889 key = hash_long(ip, FTRACE_HASH_BITS); 1889 key = hash_long(ip, FTRACE_HASH_BITS);
1890 1890
1891 hhd = &ftrace_func_hash[key]; 1891 hhd = &ftrace_func_hash[key];
1892 1892
1893 if (hlist_empty(hhd)) 1893 if (hlist_empty(hhd))
1894 return; 1894 return;
1895 1895
1896 /* 1896 /*
1897 * Disable preemption for these calls to prevent a RCU grace 1897 * Disable preemption for these calls to prevent a RCU grace
1898 * period. This syncs the hash iteration and freeing of items 1898 * period. This syncs the hash iteration and freeing of items
1899 * on the hash. rcu_read_lock is too dangerous here. 1899 * on the hash. rcu_read_lock is too dangerous here.
1900 */ 1900 */
1901 resched = ftrace_preempt_disable(); 1901 resched = ftrace_preempt_disable();
1902 hlist_for_each_entry_rcu(entry, n, hhd, node) { 1902 hlist_for_each_entry_rcu(entry, n, hhd, node) {
1903 if (entry->ip == ip) 1903 if (entry->ip == ip)
1904 entry->ops->func(ip, parent_ip, &entry->data); 1904 entry->ops->func(ip, parent_ip, &entry->data);
1905 } 1905 }
1906 ftrace_preempt_enable(resched); 1906 ftrace_preempt_enable(resched);
1907 } 1907 }
1908 1908
1909 static struct ftrace_ops trace_probe_ops __read_mostly = 1909 static struct ftrace_ops trace_probe_ops __read_mostly =
1910 { 1910 {
1911 .func = function_trace_probe_call, 1911 .func = function_trace_probe_call,
1912 }; 1912 };
1913 1913
1914 static int ftrace_probe_registered; 1914 static int ftrace_probe_registered;
1915 1915
1916 static void __enable_ftrace_function_probe(void) 1916 static void __enable_ftrace_function_probe(void)
1917 { 1917 {
1918 int i; 1918 int i;
1919 1919
1920 if (ftrace_probe_registered) 1920 if (ftrace_probe_registered)
1921 return; 1921 return;
1922 1922
1923 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 1923 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1924 struct hlist_head *hhd = &ftrace_func_hash[i]; 1924 struct hlist_head *hhd = &ftrace_func_hash[i];
1925 if (hhd->first) 1925 if (hhd->first)
1926 break; 1926 break;
1927 } 1927 }
1928 /* Nothing registered? */ 1928 /* Nothing registered? */
1929 if (i == FTRACE_FUNC_HASHSIZE) 1929 if (i == FTRACE_FUNC_HASHSIZE)
1930 return; 1930 return;
1931 1931
1932 __register_ftrace_function(&trace_probe_ops); 1932 __register_ftrace_function(&trace_probe_ops);
1933 ftrace_startup(0); 1933 ftrace_startup(0);
1934 ftrace_probe_registered = 1; 1934 ftrace_probe_registered = 1;
1935 } 1935 }
1936 1936
1937 static void __disable_ftrace_function_probe(void) 1937 static void __disable_ftrace_function_probe(void)
1938 { 1938 {
1939 int i; 1939 int i;
1940 1940
1941 if (!ftrace_probe_registered) 1941 if (!ftrace_probe_registered)
1942 return; 1942 return;
1943 1943
1944 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 1944 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1945 struct hlist_head *hhd = &ftrace_func_hash[i]; 1945 struct hlist_head *hhd = &ftrace_func_hash[i];
1946 if (hhd->first) 1946 if (hhd->first)
1947 return; 1947 return;
1948 } 1948 }
1949 1949
1950 /* no more funcs left */ 1950 /* no more funcs left */
1951 __unregister_ftrace_function(&trace_probe_ops); 1951 __unregister_ftrace_function(&trace_probe_ops);
1952 ftrace_shutdown(0); 1952 ftrace_shutdown(0);
1953 ftrace_probe_registered = 0; 1953 ftrace_probe_registered = 0;
1954 } 1954 }
1955 1955
1956 1956
1957 static void ftrace_free_entry_rcu(struct rcu_head *rhp) 1957 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1958 { 1958 {
1959 struct ftrace_func_probe *entry = 1959 struct ftrace_func_probe *entry =
1960 container_of(rhp, struct ftrace_func_probe, rcu); 1960 container_of(rhp, struct ftrace_func_probe, rcu);
1961 1961
1962 if (entry->ops->free) 1962 if (entry->ops->free)
1963 entry->ops->free(&entry->data); 1963 entry->ops->free(&entry->data);
1964 kfree(entry); 1964 kfree(entry);
1965 } 1965 }
1966 1966
1967 1967
1968 int 1968 int
1969 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 1969 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1970 void *data) 1970 void *data)
1971 { 1971 {
1972 struct ftrace_func_probe *entry; 1972 struct ftrace_func_probe *entry;
1973 struct ftrace_page *pg; 1973 struct ftrace_page *pg;
1974 struct dyn_ftrace *rec; 1974 struct dyn_ftrace *rec;
1975 int type, len, not; 1975 int type, len, not;
1976 unsigned long key; 1976 unsigned long key;
1977 int count = 0; 1977 int count = 0;
1978 char *search; 1978 char *search;
1979 1979
1980 type = filter_parse_regex(glob, strlen(glob), &search, &not); 1980 type = filter_parse_regex(glob, strlen(glob), &search, &not);
1981 len = strlen(search); 1981 len = strlen(search);
1982 1982
1983 /* we do not support '!' for function probes */ 1983 /* we do not support '!' for function probes */
1984 if (WARN_ON(not)) 1984 if (WARN_ON(not))
1985 return -EINVAL; 1985 return -EINVAL;
1986 1986
1987 mutex_lock(&ftrace_lock); 1987 mutex_lock(&ftrace_lock);
1988 do_for_each_ftrace_rec(pg, rec) { 1988 do_for_each_ftrace_rec(pg, rec) {
1989 1989
1990 if (rec->flags & FTRACE_FL_FAILED) 1990 if (rec->flags & FTRACE_FL_FAILED)
1991 continue; 1991 continue;
1992 1992
1993 if (!ftrace_match_record(rec, search, len, type)) 1993 if (!ftrace_match_record(rec, search, len, type))
1994 continue; 1994 continue;
1995 1995
1996 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1996 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1997 if (!entry) { 1997 if (!entry) {
1998 /* If we did not process any, then return error */ 1998 /* If we did not process any, then return error */
1999 if (!count) 1999 if (!count)
2000 count = -ENOMEM; 2000 count = -ENOMEM;
2001 goto out_unlock; 2001 goto out_unlock;
2002 } 2002 }
2003 2003
2004 count++; 2004 count++;
2005 2005
2006 entry->data = data; 2006 entry->data = data;
2007 2007
2008 /* 2008 /*
2009 * The caller might want to do something special 2009 * The caller might want to do something special
2010 * for each function we find. We call the callback 2010 * for each function we find. We call the callback
2011 * to give the caller an opportunity to do so. 2011 * to give the caller an opportunity to do so.
2012 */ 2012 */
2013 if (ops->callback) { 2013 if (ops->callback) {
2014 if (ops->callback(rec->ip, &entry->data) < 0) { 2014 if (ops->callback(rec->ip, &entry->data) < 0) {
2015 /* caller does not like this func */ 2015 /* caller does not like this func */
2016 kfree(entry); 2016 kfree(entry);
2017 continue; 2017 continue;
2018 } 2018 }
2019 } 2019 }
2020 2020
2021 entry->ops = ops; 2021 entry->ops = ops;
2022 entry->ip = rec->ip; 2022 entry->ip = rec->ip;
2023 2023
2024 key = hash_long(entry->ip, FTRACE_HASH_BITS); 2024 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2025 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]); 2025 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2026 2026
2027 } while_for_each_ftrace_rec(); 2027 } while_for_each_ftrace_rec();
2028 __enable_ftrace_function_probe(); 2028 __enable_ftrace_function_probe();
2029 2029
2030 out_unlock: 2030 out_unlock:
2031 mutex_unlock(&ftrace_lock); 2031 mutex_unlock(&ftrace_lock);
2032 2032
2033 return count; 2033 return count;
2034 } 2034 }
2035 2035
2036 enum { 2036 enum {
2037 PROBE_TEST_FUNC = 1, 2037 PROBE_TEST_FUNC = 1,
2038 PROBE_TEST_DATA = 2 2038 PROBE_TEST_DATA = 2
2039 }; 2039 };
2040 2040
2041 static void 2041 static void
2042 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 2042 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2043 void *data, int flags) 2043 void *data, int flags)
2044 { 2044 {
2045 struct ftrace_func_probe *entry; 2045 struct ftrace_func_probe *entry;
2046 struct hlist_node *n, *tmp; 2046 struct hlist_node *n, *tmp;
2047 char str[KSYM_SYMBOL_LEN]; 2047 char str[KSYM_SYMBOL_LEN];
2048 int type = MATCH_FULL; 2048 int type = MATCH_FULL;
2049 int i, len = 0; 2049 int i, len = 0;
2050 char *search; 2050 char *search;
2051 2051
2052 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) 2052 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2053 glob = NULL; 2053 glob = NULL;
2054 else if (glob) { 2054 else if (glob) {
2055 int not; 2055 int not;
2056 2056
2057 type = filter_parse_regex(glob, strlen(glob), &search, &not); 2057 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2058 len = strlen(search); 2058 len = strlen(search);
2059 2059
2060 /* we do not support '!' for function probes */ 2060 /* we do not support '!' for function probes */
2061 if (WARN_ON(not)) 2061 if (WARN_ON(not))
2062 return; 2062 return;
2063 } 2063 }
2064 2064
2065 mutex_lock(&ftrace_lock); 2065 mutex_lock(&ftrace_lock);
2066 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 2066 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2067 struct hlist_head *hhd = &ftrace_func_hash[i]; 2067 struct hlist_head *hhd = &ftrace_func_hash[i];
2068 2068
2069 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) { 2069 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2070 2070
2071 /* break up if statements for readability */ 2071 /* break up if statements for readability */
2072 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops) 2072 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2073 continue; 2073 continue;
2074 2074
2075 if ((flags & PROBE_TEST_DATA) && entry->data != data) 2075 if ((flags & PROBE_TEST_DATA) && entry->data != data)
2076 continue; 2076 continue;
2077 2077
2078 /* do this last, since it is the most expensive */ 2078 /* do this last, since it is the most expensive */
2079 if (glob) { 2079 if (glob) {
2080 kallsyms_lookup(entry->ip, NULL, NULL, 2080 kallsyms_lookup(entry->ip, NULL, NULL,
2081 NULL, str); 2081 NULL, str);
2082 if (!ftrace_match(str, glob, len, type)) 2082 if (!ftrace_match(str, glob, len, type))
2083 continue; 2083 continue;
2084 } 2084 }
2085 2085
2086 hlist_del(&entry->node); 2086 hlist_del(&entry->node);
2087 call_rcu(&entry->rcu, ftrace_free_entry_rcu); 2087 call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2088 } 2088 }
2089 } 2089 }
2090 __disable_ftrace_function_probe(); 2090 __disable_ftrace_function_probe();
2091 mutex_unlock(&ftrace_lock); 2091 mutex_unlock(&ftrace_lock);
2092 } 2092 }
2093 2093
2094 void 2094 void
2095 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 2095 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2096 void *data) 2096 void *data)
2097 { 2097 {
2098 __unregister_ftrace_function_probe(glob, ops, data, 2098 __unregister_ftrace_function_probe(glob, ops, data,
2099 PROBE_TEST_FUNC | PROBE_TEST_DATA); 2099 PROBE_TEST_FUNC | PROBE_TEST_DATA);
2100 } 2100 }
2101 2101
2102 void 2102 void
2103 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) 2103 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2104 { 2104 {
2105 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC); 2105 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2106 } 2106 }
2107 2107
2108 void unregister_ftrace_function_probe_all(char *glob) 2108 void unregister_ftrace_function_probe_all(char *glob)
2109 { 2109 {
2110 __unregister_ftrace_function_probe(glob, NULL, NULL, 0); 2110 __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2111 } 2111 }
2112 2112
2113 static LIST_HEAD(ftrace_commands); 2113 static LIST_HEAD(ftrace_commands);
2114 static DEFINE_MUTEX(ftrace_cmd_mutex); 2114 static DEFINE_MUTEX(ftrace_cmd_mutex);
2115 2115
2116 int register_ftrace_command(struct ftrace_func_command *cmd) 2116 int register_ftrace_command(struct ftrace_func_command *cmd)
2117 { 2117 {
2118 struct ftrace_func_command *p; 2118 struct ftrace_func_command *p;
2119 int ret = 0; 2119 int ret = 0;
2120 2120
2121 mutex_lock(&ftrace_cmd_mutex); 2121 mutex_lock(&ftrace_cmd_mutex);
2122 list_for_each_entry(p, &ftrace_commands, list) { 2122 list_for_each_entry(p, &ftrace_commands, list) {
2123 if (strcmp(cmd->name, p->name) == 0) { 2123 if (strcmp(cmd->name, p->name) == 0) {
2124 ret = -EBUSY; 2124 ret = -EBUSY;
2125 goto out_unlock; 2125 goto out_unlock;
2126 } 2126 }
2127 } 2127 }
2128 list_add(&cmd->list, &ftrace_commands); 2128 list_add(&cmd->list, &ftrace_commands);
2129 out_unlock: 2129 out_unlock:
2130 mutex_unlock(&ftrace_cmd_mutex); 2130 mutex_unlock(&ftrace_cmd_mutex);
2131 2131
2132 return ret; 2132 return ret;
2133 } 2133 }
2134 2134
2135 int unregister_ftrace_command(struct ftrace_func_command *cmd) 2135 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2136 { 2136 {
2137 struct ftrace_func_command *p, *n; 2137 struct ftrace_func_command *p, *n;
2138 int ret = -ENODEV; 2138 int ret = -ENODEV;
2139 2139
2140 mutex_lock(&ftrace_cmd_mutex); 2140 mutex_lock(&ftrace_cmd_mutex);
2141 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 2141 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2142 if (strcmp(cmd->name, p->name) == 0) { 2142 if (strcmp(cmd->name, p->name) == 0) {
2143 ret = 0; 2143 ret = 0;
2144 list_del_init(&p->list); 2144 list_del_init(&p->list);
2145 goto out_unlock; 2145 goto out_unlock;
2146 } 2146 }
2147 } 2147 }
2148 out_unlock: 2148 out_unlock:
2149 mutex_unlock(&ftrace_cmd_mutex); 2149 mutex_unlock(&ftrace_cmd_mutex);
2150 2150
2151 return ret; 2151 return ret;
2152 } 2152 }
2153 2153
2154 static int ftrace_process_regex(char *buff, int len, int enable) 2154 static int ftrace_process_regex(char *buff, int len, int enable)
2155 { 2155 {
2156 char *func, *command, *next = buff; 2156 char *func, *command, *next = buff;
2157 struct ftrace_func_command *p; 2157 struct ftrace_func_command *p;
2158 int ret = -EINVAL; 2158 int ret = -EINVAL;
2159 2159
2160 func = strsep(&next, ":"); 2160 func = strsep(&next, ":");
2161 2161
2162 if (!next) { 2162 if (!next) {
2163 if (ftrace_match_records(func, len, enable)) 2163 if (ftrace_match_records(func, len, enable))
2164 return 0; 2164 return 0;
2165 return ret; 2165 return ret;
2166 } 2166 }
2167 2167
2168 /* command found */ 2168 /* command found */
2169 2169
2170 command = strsep(&next, ":"); 2170 command = strsep(&next, ":");
2171 2171
2172 mutex_lock(&ftrace_cmd_mutex); 2172 mutex_lock(&ftrace_cmd_mutex);
2173 list_for_each_entry(p, &ftrace_commands, list) { 2173 list_for_each_entry(p, &ftrace_commands, list) {
2174 if (strcmp(p->name, command) == 0) { 2174 if (strcmp(p->name, command) == 0) {
2175 ret = p->func(func, command, next, enable); 2175 ret = p->func(func, command, next, enable);
2176 goto out_unlock; 2176 goto out_unlock;
2177 } 2177 }
2178 } 2178 }
2179 out_unlock: 2179 out_unlock:
2180 mutex_unlock(&ftrace_cmd_mutex); 2180 mutex_unlock(&ftrace_cmd_mutex);
2181 2181
2182 return ret; 2182 return ret;
2183 } 2183 }
2184 2184
2185 static ssize_t 2185 static ssize_t
2186 ftrace_regex_write(struct file *file, const char __user *ubuf, 2186 ftrace_regex_write(struct file *file, const char __user *ubuf,
2187 size_t cnt, loff_t *ppos, int enable) 2187 size_t cnt, loff_t *ppos, int enable)
2188 { 2188 {
2189 struct ftrace_iterator *iter; 2189 struct ftrace_iterator *iter;
2190 struct trace_parser *parser; 2190 struct trace_parser *parser;
2191 ssize_t ret, read; 2191 ssize_t ret, read;
2192 2192
2193 if (!cnt) 2193 if (!cnt)
2194 return 0; 2194 return 0;
2195 2195
2196 mutex_lock(&ftrace_regex_lock); 2196 mutex_lock(&ftrace_regex_lock);
2197 2197
2198 if (file->f_mode & FMODE_READ) { 2198 if (file->f_mode & FMODE_READ) {
2199 struct seq_file *m = file->private_data; 2199 struct seq_file *m = file->private_data;
2200 iter = m->private; 2200 iter = m->private;
2201 } else 2201 } else
2202 iter = file->private_data; 2202 iter = file->private_data;
2203 2203
2204 parser = &iter->parser; 2204 parser = &iter->parser;
2205 read = trace_get_user(parser, ubuf, cnt, ppos); 2205 read = trace_get_user(parser, ubuf, cnt, ppos);
2206 2206
2207 if (read >= 0 && trace_parser_loaded(parser) && 2207 if (read >= 0 && trace_parser_loaded(parser) &&
2208 !trace_parser_cont(parser)) { 2208 !trace_parser_cont(parser)) {
2209 ret = ftrace_process_regex(parser->buffer, 2209 ret = ftrace_process_regex(parser->buffer,
2210 parser->idx, enable); 2210 parser->idx, enable);
2211 trace_parser_clear(parser); 2211 trace_parser_clear(parser);
2212 if (ret) 2212 if (ret)
2213 goto out_unlock; 2213 goto out_unlock;
2214 } 2214 }
2215 2215
2216 ret = read; 2216 ret = read;
2217 out_unlock: 2217 out_unlock:
2218 mutex_unlock(&ftrace_regex_lock); 2218 mutex_unlock(&ftrace_regex_lock);
2219 2219
2220 return ret; 2220 return ret;
2221 } 2221 }
2222 2222
2223 static ssize_t 2223 static ssize_t
2224 ftrace_filter_write(struct file *file, const char __user *ubuf, 2224 ftrace_filter_write(struct file *file, const char __user *ubuf,
2225 size_t cnt, loff_t *ppos) 2225 size_t cnt, loff_t *ppos)
2226 { 2226 {
2227 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 2227 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2228 } 2228 }
2229 2229
2230 static ssize_t 2230 static ssize_t
2231 ftrace_notrace_write(struct file *file, const char __user *ubuf, 2231 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2232 size_t cnt, loff_t *ppos) 2232 size_t cnt, loff_t *ppos)
2233 { 2233 {
2234 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 2234 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2235 } 2235 }
2236 2236
2237 static void 2237 static void
2238 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable) 2238 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2239 { 2239 {
2240 if (unlikely(ftrace_disabled)) 2240 if (unlikely(ftrace_disabled))
2241 return; 2241 return;
2242 2242
2243 mutex_lock(&ftrace_regex_lock); 2243 mutex_lock(&ftrace_regex_lock);
2244 if (reset) 2244 if (reset)
2245 ftrace_filter_reset(enable); 2245 ftrace_filter_reset(enable);
2246 if (buf) 2246 if (buf)
2247 ftrace_match_records(buf, len, enable); 2247 ftrace_match_records(buf, len, enable);
2248 mutex_unlock(&ftrace_regex_lock); 2248 mutex_unlock(&ftrace_regex_lock);
2249 } 2249 }
2250 2250
2251 /** 2251 /**
2252 * ftrace_set_filter - set a function to filter on in ftrace 2252 * ftrace_set_filter - set a function to filter on in ftrace
2253 * @buf - the string that holds the function filter text. 2253 * @buf - the string that holds the function filter text.
2254 * @len - the length of the string. 2254 * @len - the length of the string.
2255 * @reset - non zero to reset all filters before applying this filter. 2255 * @reset - non zero to reset all filters before applying this filter.
2256 * 2256 *
2257 * Filters denote which functions should be enabled when tracing is enabled. 2257 * Filters denote which functions should be enabled when tracing is enabled.
2258 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 2258 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2259 */ 2259 */
2260 void ftrace_set_filter(unsigned char *buf, int len, int reset) 2260 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2261 { 2261 {
2262 ftrace_set_regex(buf, len, reset, 1); 2262 ftrace_set_regex(buf, len, reset, 1);
2263 } 2263 }
2264 2264
2265 /** 2265 /**
2266 * ftrace_set_notrace - set a function to not trace in ftrace 2266 * ftrace_set_notrace - set a function to not trace in ftrace
2267 * @buf - the string that holds the function notrace text. 2267 * @buf - the string that holds the function notrace text.
2268 * @len - the length of the string. 2268 * @len - the length of the string.
2269 * @reset - non zero to reset all filters before applying this filter. 2269 * @reset - non zero to reset all filters before applying this filter.
2270 * 2270 *
2271 * Notrace Filters denote which functions should not be enabled when tracing 2271 * Notrace Filters denote which functions should not be enabled when tracing
2272 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 2272 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2273 * for tracing. 2273 * for tracing.
2274 */ 2274 */
2275 void ftrace_set_notrace(unsigned char *buf, int len, int reset) 2275 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2276 { 2276 {
2277 ftrace_set_regex(buf, len, reset, 0); 2277 ftrace_set_regex(buf, len, reset, 0);
2278 } 2278 }
2279 2279
2280 /* 2280 /*
2281 * command line interface to allow users to set filters on boot up. 2281 * command line interface to allow users to set filters on boot up.
2282 */ 2282 */
2283 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 2283 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
2284 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 2284 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2285 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 2285 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2286 2286
2287 static int __init set_ftrace_notrace(char *str) 2287 static int __init set_ftrace_notrace(char *str)
2288 { 2288 {
2289 strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 2289 strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2290 return 1; 2290 return 1;
2291 } 2291 }
2292 __setup("ftrace_notrace=", set_ftrace_notrace); 2292 __setup("ftrace_notrace=", set_ftrace_notrace);
2293 2293
2294 static int __init set_ftrace_filter(char *str) 2294 static int __init set_ftrace_filter(char *str)
2295 { 2295 {
2296 strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 2296 strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2297 return 1; 2297 return 1;
2298 } 2298 }
2299 __setup("ftrace_filter=", set_ftrace_filter); 2299 __setup("ftrace_filter=", set_ftrace_filter);
2300 2300
2301 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2301 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2302 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 2302 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2303 static int __init set_graph_function(char *str) 2303 static int __init set_graph_function(char *str)
2304 { 2304 {
2305 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 2305 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2306 return 1; 2306 return 1;
2307 } 2307 }
2308 __setup("ftrace_graph_filter=", set_graph_function); 2308 __setup("ftrace_graph_filter=", set_graph_function);
2309 2309
2310 static void __init set_ftrace_early_graph(char *buf) 2310 static void __init set_ftrace_early_graph(char *buf)
2311 { 2311 {
2312 int ret; 2312 int ret;
2313 char *func; 2313 char *func;
2314 2314
2315 while (buf) { 2315 while (buf) {
2316 func = strsep(&buf, ","); 2316 func = strsep(&buf, ",");
2317 /* we allow only one expression at a time */ 2317 /* we allow only one expression at a time */
2318 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, 2318 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2319 func); 2319 func);
2320 if (ret) 2320 if (ret)
2321 printk(KERN_DEBUG "ftrace: function %s not " 2321 printk(KERN_DEBUG "ftrace: function %s not "
2322 "traceable\n", func); 2322 "traceable\n", func);
2323 } 2323 }
2324 } 2324 }
2325 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2325 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2326 2326
2327 static void __init set_ftrace_early_filter(char *buf, int enable) 2327 static void __init set_ftrace_early_filter(char *buf, int enable)
2328 { 2328 {
2329 char *func; 2329 char *func;
2330 2330
2331 while (buf) { 2331 while (buf) {
2332 func = strsep(&buf, ","); 2332 func = strsep(&buf, ",");
2333 ftrace_set_regex(func, strlen(func), 0, enable); 2333 ftrace_set_regex(func, strlen(func), 0, enable);
2334 } 2334 }
2335 } 2335 }
2336 2336
2337 static void __init set_ftrace_early_filters(void) 2337 static void __init set_ftrace_early_filters(void)
2338 { 2338 {
2339 if (ftrace_filter_buf[0]) 2339 if (ftrace_filter_buf[0])
2340 set_ftrace_early_filter(ftrace_filter_buf, 1); 2340 set_ftrace_early_filter(ftrace_filter_buf, 1);
2341 if (ftrace_notrace_buf[0]) 2341 if (ftrace_notrace_buf[0])
2342 set_ftrace_early_filter(ftrace_notrace_buf, 0); 2342 set_ftrace_early_filter(ftrace_notrace_buf, 0);
2343 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2343 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2344 if (ftrace_graph_buf[0]) 2344 if (ftrace_graph_buf[0])
2345 set_ftrace_early_graph(ftrace_graph_buf); 2345 set_ftrace_early_graph(ftrace_graph_buf);
2346 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2346 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2347 } 2347 }
2348 2348
2349 static int 2349 static int
2350 ftrace_regex_release(struct inode *inode, struct file *file, int enable) 2350 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2351 { 2351 {
2352 struct seq_file *m = (struct seq_file *)file->private_data; 2352 struct seq_file *m = (struct seq_file *)file->private_data;
2353 struct ftrace_iterator *iter; 2353 struct ftrace_iterator *iter;
2354 struct trace_parser *parser; 2354 struct trace_parser *parser;
2355 2355
2356 mutex_lock(&ftrace_regex_lock); 2356 mutex_lock(&ftrace_regex_lock);
2357 if (file->f_mode & FMODE_READ) { 2357 if (file->f_mode & FMODE_READ) {
2358 iter = m->private; 2358 iter = m->private;
2359 2359
2360 seq_release(inode, file); 2360 seq_release(inode, file);
2361 } else 2361 } else
2362 iter = file->private_data; 2362 iter = file->private_data;
2363 2363
2364 parser = &iter->parser; 2364 parser = &iter->parser;
2365 if (trace_parser_loaded(parser)) { 2365 if (trace_parser_loaded(parser)) {
2366 parser->buffer[parser->idx] = 0; 2366 parser->buffer[parser->idx] = 0;
2367 ftrace_match_records(parser->buffer, parser->idx, enable); 2367 ftrace_match_records(parser->buffer, parser->idx, enable);
2368 } 2368 }
2369 2369
2370 mutex_lock(&ftrace_lock); 2370 mutex_lock(&ftrace_lock);
2371 if (ftrace_start_up && ftrace_enabled) 2371 if (ftrace_start_up && ftrace_enabled)
2372 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 2372 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2373 mutex_unlock(&ftrace_lock); 2373 mutex_unlock(&ftrace_lock);
2374 2374
2375 trace_parser_put(parser); 2375 trace_parser_put(parser);
2376 kfree(iter); 2376 kfree(iter);
2377 2377
2378 mutex_unlock(&ftrace_regex_lock); 2378 mutex_unlock(&ftrace_regex_lock);
2379 return 0; 2379 return 0;
2380 } 2380 }
2381 2381
2382 static int 2382 static int
2383 ftrace_filter_release(struct inode *inode, struct file *file) 2383 ftrace_filter_release(struct inode *inode, struct file *file)
2384 { 2384 {
2385 return ftrace_regex_release(inode, file, 1); 2385 return ftrace_regex_release(inode, file, 1);
2386 } 2386 }
2387 2387
2388 static int 2388 static int
2389 ftrace_notrace_release(struct inode *inode, struct file *file) 2389 ftrace_notrace_release(struct inode *inode, struct file *file)
2390 { 2390 {
2391 return ftrace_regex_release(inode, file, 0); 2391 return ftrace_regex_release(inode, file, 0);
2392 } 2392 }
2393 2393
2394 static const struct file_operations ftrace_avail_fops = { 2394 static const struct file_operations ftrace_avail_fops = {
2395 .open = ftrace_avail_open, 2395 .open = ftrace_avail_open,
2396 .read = seq_read, 2396 .read = seq_read,
2397 .llseek = seq_lseek, 2397 .llseek = seq_lseek,
2398 .release = seq_release_private, 2398 .release = seq_release_private,
2399 }; 2399 };
2400 2400
2401 static const struct file_operations ftrace_failures_fops = { 2401 static const struct file_operations ftrace_failures_fops = {
2402 .open = ftrace_failures_open, 2402 .open = ftrace_failures_open,
2403 .read = seq_read, 2403 .read = seq_read,
2404 .llseek = seq_lseek, 2404 .llseek = seq_lseek,
2405 .release = seq_release_private, 2405 .release = seq_release_private,
2406 }; 2406 };
2407 2407
2408 static const struct file_operations ftrace_filter_fops = { 2408 static const struct file_operations ftrace_filter_fops = {
2409 .open = ftrace_filter_open, 2409 .open = ftrace_filter_open,
2410 .read = seq_read, 2410 .read = seq_read,
2411 .write = ftrace_filter_write, 2411 .write = ftrace_filter_write,
2412 .llseek = ftrace_regex_lseek, 2412 .llseek = ftrace_regex_lseek,
2413 .release = ftrace_filter_release, 2413 .release = ftrace_filter_release,
2414 }; 2414 };
2415 2415
2416 static const struct file_operations ftrace_notrace_fops = { 2416 static const struct file_operations ftrace_notrace_fops = {
2417 .open = ftrace_notrace_open, 2417 .open = ftrace_notrace_open,
2418 .read = seq_read, 2418 .read = seq_read,
2419 .write = ftrace_notrace_write, 2419 .write = ftrace_notrace_write,
2420 .llseek = ftrace_regex_lseek, 2420 .llseek = ftrace_regex_lseek,
2421 .release = ftrace_notrace_release, 2421 .release = ftrace_notrace_release,
2422 }; 2422 };
2423 2423
2424 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2424 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2425 2425
2426 static DEFINE_MUTEX(graph_lock); 2426 static DEFINE_MUTEX(graph_lock);
2427 2427
2428 int ftrace_graph_count; 2428 int ftrace_graph_count;
2429 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; 2429 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2430 2430
2431 static void * 2431 static void *
2432 __g_next(struct seq_file *m, loff_t *pos) 2432 __g_next(struct seq_file *m, loff_t *pos)
2433 { 2433 {
2434 if (*pos >= ftrace_graph_count) 2434 if (*pos >= ftrace_graph_count)
2435 return NULL; 2435 return NULL;
2436 return &ftrace_graph_funcs[*pos]; 2436 return &ftrace_graph_funcs[*pos];
2437 } 2437 }
2438 2438
2439 static void * 2439 static void *
2440 g_next(struct seq_file *m, void *v, loff_t *pos) 2440 g_next(struct seq_file *m, void *v, loff_t *pos)
2441 { 2441 {
2442 (*pos)++; 2442 (*pos)++;
2443 return __g_next(m, pos); 2443 return __g_next(m, pos);
2444 } 2444 }
2445 2445
2446 static void *g_start(struct seq_file *m, loff_t *pos) 2446 static void *g_start(struct seq_file *m, loff_t *pos)
2447 { 2447 {
2448 mutex_lock(&graph_lock); 2448 mutex_lock(&graph_lock);
2449 2449
2450 /* Nothing, tell g_show to print all functions are enabled */ 2450 /* Nothing, tell g_show to print all functions are enabled */
2451 if (!ftrace_graph_count && !*pos) 2451 if (!ftrace_graph_count && !*pos)
2452 return (void *)1; 2452 return (void *)1;
2453 2453
2454 return __g_next(m, pos); 2454 return __g_next(m, pos);
2455 } 2455 }
2456 2456
2457 static void g_stop(struct seq_file *m, void *p) 2457 static void g_stop(struct seq_file *m, void *p)
2458 { 2458 {
2459 mutex_unlock(&graph_lock); 2459 mutex_unlock(&graph_lock);
2460 } 2460 }
2461 2461
2462 static int g_show(struct seq_file *m, void *v) 2462 static int g_show(struct seq_file *m, void *v)
2463 { 2463 {
2464 unsigned long *ptr = v; 2464 unsigned long *ptr = v;
2465 2465
2466 if (!ptr) 2466 if (!ptr)
2467 return 0; 2467 return 0;
2468 2468
2469 if (ptr == (unsigned long *)1) { 2469 if (ptr == (unsigned long *)1) {
2470 seq_printf(m, "#### all functions enabled ####\n"); 2470 seq_printf(m, "#### all functions enabled ####\n");
2471 return 0; 2471 return 0;
2472 } 2472 }
2473 2473
2474 seq_printf(m, "%ps\n", (void *)*ptr); 2474 seq_printf(m, "%ps\n", (void *)*ptr);
2475 2475
2476 return 0; 2476 return 0;
2477 } 2477 }
2478 2478
2479 static const struct seq_operations ftrace_graph_seq_ops = { 2479 static const struct seq_operations ftrace_graph_seq_ops = {
2480 .start = g_start, 2480 .start = g_start,
2481 .next = g_next, 2481 .next = g_next,
2482 .stop = g_stop, 2482 .stop = g_stop,
2483 .show = g_show, 2483 .show = g_show,
2484 }; 2484 };
2485 2485
2486 static int 2486 static int
2487 ftrace_graph_open(struct inode *inode, struct file *file) 2487 ftrace_graph_open(struct inode *inode, struct file *file)
2488 { 2488 {
2489 int ret = 0; 2489 int ret = 0;
2490 2490
2491 if (unlikely(ftrace_disabled)) 2491 if (unlikely(ftrace_disabled))
2492 return -ENODEV; 2492 return -ENODEV;
2493 2493
2494 mutex_lock(&graph_lock); 2494 mutex_lock(&graph_lock);
2495 if ((file->f_mode & FMODE_WRITE) && 2495 if ((file->f_mode & FMODE_WRITE) &&
2496 (file->f_flags & O_TRUNC)) { 2496 (file->f_flags & O_TRUNC)) {
2497 ftrace_graph_count = 0; 2497 ftrace_graph_count = 0;
2498 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs)); 2498 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2499 } 2499 }
2500 mutex_unlock(&graph_lock); 2500 mutex_unlock(&graph_lock);
2501 2501
2502 if (file->f_mode & FMODE_READ) 2502 if (file->f_mode & FMODE_READ)
2503 ret = seq_open(file, &ftrace_graph_seq_ops); 2503 ret = seq_open(file, &ftrace_graph_seq_ops);
2504 2504
2505 return ret; 2505 return ret;
2506 } 2506 }
2507 2507
2508 static int 2508 static int
2509 ftrace_graph_release(struct inode *inode, struct file *file) 2509 ftrace_graph_release(struct inode *inode, struct file *file)
2510 { 2510 {
2511 if (file->f_mode & FMODE_READ) 2511 if (file->f_mode & FMODE_READ)
2512 seq_release(inode, file); 2512 seq_release(inode, file);
2513 return 0; 2513 return 0;
2514 } 2514 }
2515 2515
2516 static int 2516 static int
2517 ftrace_set_func(unsigned long *array, int *idx, char *buffer) 2517 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2518 { 2518 {
2519 struct dyn_ftrace *rec; 2519 struct dyn_ftrace *rec;
2520 struct ftrace_page *pg; 2520 struct ftrace_page *pg;
2521 int search_len; 2521 int search_len;
2522 int found = 0; 2522 int found = 0;
2523 int type, not; 2523 int type, not;
2524 char *search; 2524 char *search;
2525 bool exists; 2525 bool exists;
2526 int i; 2526 int i;
2527 2527
2528 if (ftrace_disabled) 2528 if (ftrace_disabled)
2529 return -ENODEV; 2529 return -ENODEV;
2530 2530
2531 /* decode regex */ 2531 /* decode regex */
2532 type = filter_parse_regex(buffer, strlen(buffer), &search, &not); 2532 type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2533 if (not) 2533 if (not)
2534 return -EINVAL; 2534 return -EINVAL;
2535 2535
2536 search_len = strlen(search); 2536 search_len = strlen(search);
2537 2537
2538 mutex_lock(&ftrace_lock); 2538 mutex_lock(&ftrace_lock);
2539 do_for_each_ftrace_rec(pg, rec) { 2539 do_for_each_ftrace_rec(pg, rec) {
2540 2540
2541 if (*idx >= FTRACE_GRAPH_MAX_FUNCS) 2541 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2542 break; 2542 break;
2543 2543
2544 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) 2544 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2545 continue; 2545 continue;
2546 2546
2547 if (ftrace_match_record(rec, search, search_len, type)) { 2547 if (ftrace_match_record(rec, search, search_len, type)) {
2548 /* ensure it is not already in the array */ 2548 /* ensure it is not already in the array */
2549 exists = false; 2549 exists = false;
2550 for (i = 0; i < *idx; i++) 2550 for (i = 0; i < *idx; i++)
2551 if (array[i] == rec->ip) { 2551 if (array[i] == rec->ip) {
2552 exists = true; 2552 exists = true;
2553 break; 2553 break;
2554 } 2554 }
2555 if (!exists) 2555 if (!exists)
2556 array[(*idx)++] = rec->ip; 2556 array[(*idx)++] = rec->ip;
2557 found = 1; 2557 found = 1;
2558 } 2558 }
2559 } while_for_each_ftrace_rec(); 2559 } while_for_each_ftrace_rec();
2560 2560
2561 mutex_unlock(&ftrace_lock); 2561 mutex_unlock(&ftrace_lock);
2562 2562
2563 return found ? 0 : -EINVAL; 2563 return found ? 0 : -EINVAL;
2564 } 2564 }
2565 2565
2566 static ssize_t 2566 static ssize_t
2567 ftrace_graph_write(struct file *file, const char __user *ubuf, 2567 ftrace_graph_write(struct file *file, const char __user *ubuf,
2568 size_t cnt, loff_t *ppos) 2568 size_t cnt, loff_t *ppos)
2569 { 2569 {
2570 struct trace_parser parser; 2570 struct trace_parser parser;
2571 ssize_t read, ret; 2571 ssize_t read, ret;
2572 2572
2573 if (!cnt || cnt < 0) 2573 if (!cnt || cnt < 0)
2574 return 0; 2574 return 0;
2575 2575
2576 mutex_lock(&graph_lock); 2576 mutex_lock(&graph_lock);
2577 2577
2578 if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) { 2578 if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2579 ret = -EBUSY; 2579 ret = -EBUSY;
2580 goto out_unlock; 2580 goto out_unlock;
2581 } 2581 }
2582 2582
2583 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) { 2583 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2584 ret = -ENOMEM; 2584 ret = -ENOMEM;
2585 goto out_unlock; 2585 goto out_unlock;
2586 } 2586 }
2587 2587
2588 read = trace_get_user(&parser, ubuf, cnt, ppos); 2588 read = trace_get_user(&parser, ubuf, cnt, ppos);
2589 2589
2590 if (read >= 0 && trace_parser_loaded((&parser))) { 2590 if (read >= 0 && trace_parser_loaded((&parser))) {
2591 parser.buffer[parser.idx] = 0; 2591 parser.buffer[parser.idx] = 0;
2592 2592
2593 /* we allow only one expression at a time */ 2593 /* we allow only one expression at a time */
2594 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, 2594 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2595 parser.buffer); 2595 parser.buffer);
2596 if (ret) 2596 if (ret)
2597 goto out_free; 2597 goto out_free;
2598 } 2598 }
2599 2599
2600 ret = read; 2600 ret = read;
2601 2601
2602 out_free: 2602 out_free:
2603 trace_parser_put(&parser); 2603 trace_parser_put(&parser);
2604 out_unlock: 2604 out_unlock:
2605 mutex_unlock(&graph_lock); 2605 mutex_unlock(&graph_lock);
2606 2606
2607 return ret; 2607 return ret;
2608 } 2608 }
2609 2609
2610 static const struct file_operations ftrace_graph_fops = { 2610 static const struct file_operations ftrace_graph_fops = {
2611 .open = ftrace_graph_open, 2611 .open = ftrace_graph_open,
2612 .read = seq_read, 2612 .read = seq_read,
2613 .write = ftrace_graph_write, 2613 .write = ftrace_graph_write,
2614 .release = ftrace_graph_release, 2614 .release = ftrace_graph_release,
2615 }; 2615 };
2616 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2616 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2617 2617
2618 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) 2618 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2619 { 2619 {
2620 2620
2621 trace_create_file("available_filter_functions", 0444, 2621 trace_create_file("available_filter_functions", 0444,
2622 d_tracer, NULL, &ftrace_avail_fops); 2622 d_tracer, NULL, &ftrace_avail_fops);
2623 2623
2624 trace_create_file("failures", 0444, 2624 trace_create_file("failures", 0444,
2625 d_tracer, NULL, &ftrace_failures_fops); 2625 d_tracer, NULL, &ftrace_failures_fops);
2626 2626
2627 trace_create_file("set_ftrace_filter", 0644, d_tracer, 2627 trace_create_file("set_ftrace_filter", 0644, d_tracer,
2628 NULL, &ftrace_filter_fops); 2628 NULL, &ftrace_filter_fops);
2629 2629
2630 trace_create_file("set_ftrace_notrace", 0644, d_tracer, 2630 trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2631 NULL, &ftrace_notrace_fops); 2631 NULL, &ftrace_notrace_fops);
2632 2632
2633 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 2633 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2634 trace_create_file("set_graph_function", 0444, d_tracer, 2634 trace_create_file("set_graph_function", 0444, d_tracer,
2635 NULL, 2635 NULL,
2636 &ftrace_graph_fops); 2636 &ftrace_graph_fops);
2637 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2637 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2638 2638
2639 return 0; 2639 return 0;
2640 } 2640 }
2641 2641
2642 static int ftrace_process_locs(struct module *mod, 2642 static int ftrace_process_locs(struct module *mod,
2643 unsigned long *start, 2643 unsigned long *start,
2644 unsigned long *end) 2644 unsigned long *end)
2645 { 2645 {
2646 unsigned long *p; 2646 unsigned long *p;
2647 unsigned long addr; 2647 unsigned long addr;
2648 unsigned long flags; 2648 unsigned long flags;
2649 2649
2650 mutex_lock(&ftrace_lock); 2650 mutex_lock(&ftrace_lock);
2651 p = start; 2651 p = start;
2652 while (p < end) { 2652 while (p < end) {
2653 addr = ftrace_call_adjust(*p++); 2653 addr = ftrace_call_adjust(*p++);
2654 /* 2654 /*
2655 * Some architecture linkers will pad between 2655 * Some architecture linkers will pad between
2656 * the different mcount_loc sections of different 2656 * the different mcount_loc sections of different
2657 * object files to satisfy alignments. 2657 * object files to satisfy alignments.
2658 * Skip any NULL pointers. 2658 * Skip any NULL pointers.
2659 */ 2659 */
2660 if (!addr) 2660 if (!addr)
2661 continue; 2661 continue;
2662 ftrace_record_ip(addr); 2662 ftrace_record_ip(addr);
2663 } 2663 }
2664 2664
2665 /* disable interrupts to prevent kstop machine */ 2665 /* disable interrupts to prevent kstop machine */
2666 local_irq_save(flags); 2666 local_irq_save(flags);
2667 ftrace_update_code(mod); 2667 ftrace_update_code(mod);
2668 local_irq_restore(flags); 2668 local_irq_restore(flags);
2669 mutex_unlock(&ftrace_lock); 2669 mutex_unlock(&ftrace_lock);
2670 2670
2671 return 0; 2671 return 0;
2672 } 2672 }
2673 2673
2674 #ifdef CONFIG_MODULES 2674 #ifdef CONFIG_MODULES
2675 void ftrace_release_mod(struct module *mod) 2675 void ftrace_release_mod(struct module *mod)
2676 { 2676 {
2677 struct dyn_ftrace *rec; 2677 struct dyn_ftrace *rec;
2678 struct ftrace_page *pg; 2678 struct ftrace_page *pg;
2679 2679
2680 if (ftrace_disabled) 2680 if (ftrace_disabled)
2681 return; 2681 return;
2682 2682
2683 mutex_lock(&ftrace_lock); 2683 mutex_lock(&ftrace_lock);
2684 do_for_each_ftrace_rec(pg, rec) { 2684 do_for_each_ftrace_rec(pg, rec) {
2685 if (within_module_core(rec->ip, mod)) { 2685 if (within_module_core(rec->ip, mod)) {
2686 /* 2686 /*
2687 * rec->ip is changed in ftrace_free_rec() 2687 * rec->ip is changed in ftrace_free_rec()
2688 * It should not between s and e if record was freed. 2688 * It should not between s and e if record was freed.
2689 */ 2689 */
2690 FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE); 2690 FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2691 ftrace_free_rec(rec); 2691 ftrace_free_rec(rec);
2692 } 2692 }
2693 } while_for_each_ftrace_rec(); 2693 } while_for_each_ftrace_rec();
2694 mutex_unlock(&ftrace_lock); 2694 mutex_unlock(&ftrace_lock);
2695 } 2695 }
2696 2696
2697 static void ftrace_init_module(struct module *mod, 2697 static void ftrace_init_module(struct module *mod,
2698 unsigned long *start, unsigned long *end) 2698 unsigned long *start, unsigned long *end)
2699 { 2699 {
2700 if (ftrace_disabled || start == end) 2700 if (ftrace_disabled || start == end)
2701 return; 2701 return;
2702 ftrace_process_locs(mod, start, end); 2702 ftrace_process_locs(mod, start, end);
2703 } 2703 }
2704 2704
2705 static int ftrace_module_notify(struct notifier_block *self, 2705 static int ftrace_module_notify(struct notifier_block *self,
2706 unsigned long val, void *data) 2706 unsigned long val, void *data)
2707 { 2707 {
2708 struct module *mod = data; 2708 struct module *mod = data;
2709 2709
2710 switch (val) { 2710 switch (val) {
2711 case MODULE_STATE_COMING: 2711 case MODULE_STATE_COMING:
2712 ftrace_init_module(mod, mod->ftrace_callsites, 2712 ftrace_init_module(mod, mod->ftrace_callsites,
2713 mod->ftrace_callsites + 2713 mod->ftrace_callsites +
2714 mod->num_ftrace_callsites); 2714 mod->num_ftrace_callsites);
2715 break; 2715 break;
2716 case MODULE_STATE_GOING: 2716 case MODULE_STATE_GOING:
2717 ftrace_release_mod(mod); 2717 ftrace_release_mod(mod);
2718 break; 2718 break;
2719 } 2719 }
2720 2720
2721 return 0; 2721 return 0;
2722 } 2722 }
2723 #else 2723 #else
2724 static int ftrace_module_notify(struct notifier_block *self, 2724 static int ftrace_module_notify(struct notifier_block *self,
2725 unsigned long val, void *data) 2725 unsigned long val, void *data)
2726 { 2726 {
2727 return 0; 2727 return 0;
2728 } 2728 }
2729 #endif /* CONFIG_MODULES */ 2729 #endif /* CONFIG_MODULES */
2730 2730
2731 struct notifier_block ftrace_module_nb = { 2731 struct notifier_block ftrace_module_nb = {
2732 .notifier_call = ftrace_module_notify, 2732 .notifier_call = ftrace_module_notify,
2733 .priority = 0, 2733 .priority = 0,
2734 }; 2734 };
2735 2735
2736 extern unsigned long __start_mcount_loc[]; 2736 extern unsigned long __start_mcount_loc[];
2737 extern unsigned long __stop_mcount_loc[]; 2737 extern unsigned long __stop_mcount_loc[];
2738 2738
2739 void __init ftrace_init(void) 2739 void __init ftrace_init(void)
2740 { 2740 {
2741 unsigned long count, addr, flags; 2741 unsigned long count, addr, flags;
2742 int ret; 2742 int ret;
2743 2743
2744 /* Keep the ftrace pointer to the stub */ 2744 /* Keep the ftrace pointer to the stub */
2745 addr = (unsigned long)ftrace_stub; 2745 addr = (unsigned long)ftrace_stub;
2746 2746
2747 local_irq_save(flags); 2747 local_irq_save(flags);
2748 ftrace_dyn_arch_init(&addr); 2748 ftrace_dyn_arch_init(&addr);
2749 local_irq_restore(flags); 2749 local_irq_restore(flags);
2750 2750
2751 /* ftrace_dyn_arch_init places the return code in addr */ 2751 /* ftrace_dyn_arch_init places the return code in addr */
2752 if (addr) 2752 if (addr)
2753 goto failed; 2753 goto failed;
2754 2754
2755 count = __stop_mcount_loc - __start_mcount_loc; 2755 count = __stop_mcount_loc - __start_mcount_loc;
2756 2756
2757 ret = ftrace_dyn_table_alloc(count); 2757 ret = ftrace_dyn_table_alloc(count);
2758 if (ret) 2758 if (ret)
2759 goto failed; 2759 goto failed;
2760 2760
2761 last_ftrace_enabled = ftrace_enabled = 1; 2761 last_ftrace_enabled = ftrace_enabled = 1;
2762 2762
2763 ret = ftrace_process_locs(NULL, 2763 ret = ftrace_process_locs(NULL,
2764 __start_mcount_loc, 2764 __start_mcount_loc,
2765 __stop_mcount_loc); 2765 __stop_mcount_loc);
2766 2766
2767 ret = register_module_notifier(&ftrace_module_nb); 2767 ret = register_module_notifier(&ftrace_module_nb);
2768 if (ret) 2768 if (ret)
2769 pr_warning("Failed to register trace ftrace module notifier\n"); 2769 pr_warning("Failed to register trace ftrace module notifier\n");
2770 2770
2771 set_ftrace_early_filters(); 2771 set_ftrace_early_filters();
2772 2772
2773 return; 2773 return;
2774 failed: 2774 failed:
2775 ftrace_disabled = 1; 2775 ftrace_disabled = 1;
2776 } 2776 }
2777 2777
2778 #else 2778 #else
2779 2779
2780 static int __init ftrace_nodyn_init(void) 2780 static int __init ftrace_nodyn_init(void)
2781 { 2781 {
2782 ftrace_enabled = 1; 2782 ftrace_enabled = 1;
2783 return 0; 2783 return 0;
2784 } 2784 }
2785 device_initcall(ftrace_nodyn_init); 2785 device_initcall(ftrace_nodyn_init);
2786 2786
2787 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; } 2787 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2788 static inline void ftrace_startup_enable(int command) { } 2788 static inline void ftrace_startup_enable(int command) { }
2789 /* Keep as macros so we do not need to define the commands */ 2789 /* Keep as macros so we do not need to define the commands */
2790 # define ftrace_startup(command) do { } while (0) 2790 # define ftrace_startup(command) do { } while (0)
2791 # define ftrace_shutdown(command) do { } while (0) 2791 # define ftrace_shutdown(command) do { } while (0)
2792 # define ftrace_startup_sysctl() do { } while (0) 2792 # define ftrace_startup_sysctl() do { } while (0)
2793 # define ftrace_shutdown_sysctl() do { } while (0) 2793 # define ftrace_shutdown_sysctl() do { } while (0)
2794 #endif /* CONFIG_DYNAMIC_FTRACE */ 2794 #endif /* CONFIG_DYNAMIC_FTRACE */
2795 2795
2796 static void clear_ftrace_swapper(void) 2796 static void clear_ftrace_swapper(void)
2797 { 2797 {
2798 struct task_struct *p; 2798 struct task_struct *p;
2799 int cpu; 2799 int cpu;
2800 2800
2801 get_online_cpus(); 2801 get_online_cpus();
2802 for_each_online_cpu(cpu) { 2802 for_each_online_cpu(cpu) {
2803 p = idle_task(cpu); 2803 p = idle_task(cpu);
2804 clear_tsk_trace_trace(p); 2804 clear_tsk_trace_trace(p);
2805 } 2805 }
2806 put_online_cpus(); 2806 put_online_cpus();
2807 } 2807 }
2808 2808
2809 static void set_ftrace_swapper(void) 2809 static void set_ftrace_swapper(void)
2810 { 2810 {
2811 struct task_struct *p; 2811 struct task_struct *p;
2812 int cpu; 2812 int cpu;
2813 2813
2814 get_online_cpus(); 2814 get_online_cpus();
2815 for_each_online_cpu(cpu) { 2815 for_each_online_cpu(cpu) {
2816 p = idle_task(cpu); 2816 p = idle_task(cpu);
2817 set_tsk_trace_trace(p); 2817 set_tsk_trace_trace(p);
2818 } 2818 }
2819 put_online_cpus(); 2819 put_online_cpus();
2820 } 2820 }
2821 2821
2822 static void clear_ftrace_pid(struct pid *pid) 2822 static void clear_ftrace_pid(struct pid *pid)
2823 { 2823 {
2824 struct task_struct *p; 2824 struct task_struct *p;
2825 2825
2826 rcu_read_lock(); 2826 rcu_read_lock();
2827 do_each_pid_task(pid, PIDTYPE_PID, p) { 2827 do_each_pid_task(pid, PIDTYPE_PID, p) {
2828 clear_tsk_trace_trace(p); 2828 clear_tsk_trace_trace(p);
2829 } while_each_pid_task(pid, PIDTYPE_PID, p); 2829 } while_each_pid_task(pid, PIDTYPE_PID, p);
2830 rcu_read_unlock(); 2830 rcu_read_unlock();
2831 2831
2832 put_pid(pid); 2832 put_pid(pid);
2833 } 2833 }
2834 2834
2835 static void set_ftrace_pid(struct pid *pid) 2835 static void set_ftrace_pid(struct pid *pid)
2836 { 2836 {
2837 struct task_struct *p; 2837 struct task_struct *p;
2838 2838
2839 rcu_read_lock(); 2839 rcu_read_lock();
2840 do_each_pid_task(pid, PIDTYPE_PID, p) { 2840 do_each_pid_task(pid, PIDTYPE_PID, p) {
2841 set_tsk_trace_trace(p); 2841 set_tsk_trace_trace(p);
2842 } while_each_pid_task(pid, PIDTYPE_PID, p); 2842 } while_each_pid_task(pid, PIDTYPE_PID, p);
2843 rcu_read_unlock(); 2843 rcu_read_unlock();
2844 } 2844 }
2845 2845
2846 static void clear_ftrace_pid_task(struct pid *pid) 2846 static void clear_ftrace_pid_task(struct pid *pid)
2847 { 2847 {
2848 if (pid == ftrace_swapper_pid) 2848 if (pid == ftrace_swapper_pid)
2849 clear_ftrace_swapper(); 2849 clear_ftrace_swapper();
2850 else 2850 else
2851 clear_ftrace_pid(pid); 2851 clear_ftrace_pid(pid);
2852 } 2852 }
2853 2853
2854 static void set_ftrace_pid_task(struct pid *pid) 2854 static void set_ftrace_pid_task(struct pid *pid)
2855 { 2855 {
2856 if (pid == ftrace_swapper_pid) 2856 if (pid == ftrace_swapper_pid)
2857 set_ftrace_swapper(); 2857 set_ftrace_swapper();
2858 else 2858 else
2859 set_ftrace_pid(pid); 2859 set_ftrace_pid(pid);
2860 } 2860 }
2861 2861
2862 static int ftrace_pid_add(int p) 2862 static int ftrace_pid_add(int p)
2863 { 2863 {
2864 struct pid *pid; 2864 struct pid *pid;
2865 struct ftrace_pid *fpid; 2865 struct ftrace_pid *fpid;
2866 int ret = -EINVAL; 2866 int ret = -EINVAL;
2867 2867
2868 mutex_lock(&ftrace_lock); 2868 mutex_lock(&ftrace_lock);
2869 2869
2870 if (!p) 2870 if (!p)
2871 pid = ftrace_swapper_pid; 2871 pid = ftrace_swapper_pid;
2872 else 2872 else
2873 pid = find_get_pid(p); 2873 pid = find_get_pid(p);
2874 2874
2875 if (!pid) 2875 if (!pid)
2876 goto out; 2876 goto out;
2877 2877
2878 ret = 0; 2878 ret = 0;
2879 2879
2880 list_for_each_entry(fpid, &ftrace_pids, list) 2880 list_for_each_entry(fpid, &ftrace_pids, list)
2881 if (fpid->pid == pid) 2881 if (fpid->pid == pid)
2882 goto out_put; 2882 goto out_put;
2883 2883
2884 ret = -ENOMEM; 2884 ret = -ENOMEM;
2885 2885
2886 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL); 2886 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
2887 if (!fpid) 2887 if (!fpid)
2888 goto out_put; 2888 goto out_put;
2889 2889
2890 list_add(&fpid->list, &ftrace_pids); 2890 list_add(&fpid->list, &ftrace_pids);
2891 fpid->pid = pid; 2891 fpid->pid = pid;
2892 2892
2893 set_ftrace_pid_task(pid); 2893 set_ftrace_pid_task(pid);
2894 2894
2895 ftrace_update_pid_func(); 2895 ftrace_update_pid_func();
2896 ftrace_startup_enable(0); 2896 ftrace_startup_enable(0);
2897 2897
2898 mutex_unlock(&ftrace_lock); 2898 mutex_unlock(&ftrace_lock);
2899 return 0; 2899 return 0;
2900 2900
2901 out_put: 2901 out_put:
2902 if (pid != ftrace_swapper_pid) 2902 if (pid != ftrace_swapper_pid)
2903 put_pid(pid); 2903 put_pid(pid);
2904 2904
2905 out: 2905 out:
2906 mutex_unlock(&ftrace_lock); 2906 mutex_unlock(&ftrace_lock);
2907 return ret; 2907 return ret;
2908 } 2908 }
2909 2909
2910 static void ftrace_pid_reset(void) 2910 static void ftrace_pid_reset(void)
2911 { 2911 {
2912 struct ftrace_pid *fpid, *safe; 2912 struct ftrace_pid *fpid, *safe;
2913 2913
2914 mutex_lock(&ftrace_lock); 2914 mutex_lock(&ftrace_lock);
2915 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) { 2915 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
2916 struct pid *pid = fpid->pid; 2916 struct pid *pid = fpid->pid;
2917 2917
2918 clear_ftrace_pid_task(pid); 2918 clear_ftrace_pid_task(pid);
2919 2919
2920 list_del(&fpid->list); 2920 list_del(&fpid->list);
2921 kfree(fpid); 2921 kfree(fpid);
2922 } 2922 }
2923 2923
2924 ftrace_update_pid_func(); 2924 ftrace_update_pid_func();
2925 ftrace_startup_enable(0); 2925 ftrace_startup_enable(0);
2926 2926
2927 mutex_unlock(&ftrace_lock); 2927 mutex_unlock(&ftrace_lock);
2928 } 2928 }
2929 2929
2930 static void *fpid_start(struct seq_file *m, loff_t *pos) 2930 static void *fpid_start(struct seq_file *m, loff_t *pos)
2931 { 2931 {
2932 mutex_lock(&ftrace_lock); 2932 mutex_lock(&ftrace_lock);
2933 2933
2934 if (list_empty(&ftrace_pids) && (!*pos)) 2934 if (list_empty(&ftrace_pids) && (!*pos))
2935 return (void *) 1; 2935 return (void *) 1;
2936 2936
2937 return seq_list_start(&ftrace_pids, *pos); 2937 return seq_list_start(&ftrace_pids, *pos);
2938 } 2938 }
2939 2939
2940 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 2940 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
2941 { 2941 {
2942 if (v == (void *)1) 2942 if (v == (void *)1)
2943 return NULL; 2943 return NULL;
2944 2944
2945 return seq_list_next(v, &ftrace_pids, pos); 2945 return seq_list_next(v, &ftrace_pids, pos);
2946 } 2946 }
2947 2947
2948 static void fpid_stop(struct seq_file *m, void *p) 2948 static void fpid_stop(struct seq_file *m, void *p)
2949 { 2949 {
2950 mutex_unlock(&ftrace_lock); 2950 mutex_unlock(&ftrace_lock);
2951 } 2951 }
2952 2952
2953 static int fpid_show(struct seq_file *m, void *v) 2953 static int fpid_show(struct seq_file *m, void *v)
2954 { 2954 {
2955 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list); 2955 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
2956 2956
2957 if (v == (void *)1) { 2957 if (v == (void *)1) {
2958 seq_printf(m, "no pid\n"); 2958 seq_printf(m, "no pid\n");
2959 return 0; 2959 return 0;
2960 } 2960 }
2961 2961
2962 if (fpid->pid == ftrace_swapper_pid) 2962 if (fpid->pid == ftrace_swapper_pid)
2963 seq_printf(m, "swapper tasks\n"); 2963 seq_printf(m, "swapper tasks\n");
2964 else 2964 else
2965 seq_printf(m, "%u\n", pid_vnr(fpid->pid)); 2965 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
2966 2966
2967 return 0; 2967 return 0;
2968 } 2968 }
2969 2969
2970 static const struct seq_operations ftrace_pid_sops = { 2970 static const struct seq_operations ftrace_pid_sops = {
2971 .start = fpid_start, 2971 .start = fpid_start,
2972 .next = fpid_next, 2972 .next = fpid_next,
2973 .stop = fpid_stop, 2973 .stop = fpid_stop,
2974 .show = fpid_show, 2974 .show = fpid_show,
2975 }; 2975 };
2976 2976
2977 static int 2977 static int
2978 ftrace_pid_open(struct inode *inode, struct file *file) 2978 ftrace_pid_open(struct inode *inode, struct file *file)
2979 { 2979 {
2980 int ret = 0; 2980 int ret = 0;
2981 2981
2982 if ((file->f_mode & FMODE_WRITE) && 2982 if ((file->f_mode & FMODE_WRITE) &&
2983 (file->f_flags & O_TRUNC)) 2983 (file->f_flags & O_TRUNC))
2984 ftrace_pid_reset(); 2984 ftrace_pid_reset();
2985 2985
2986 if (file->f_mode & FMODE_READ) 2986 if (file->f_mode & FMODE_READ)
2987 ret = seq_open(file, &ftrace_pid_sops); 2987 ret = seq_open(file, &ftrace_pid_sops);
2988 2988
2989 return ret; 2989 return ret;
2990 } 2990 }
2991 2991
2992 static ssize_t 2992 static ssize_t
2993 ftrace_pid_write(struct file *filp, const char __user *ubuf, 2993 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2994 size_t cnt, loff_t *ppos) 2994 size_t cnt, loff_t *ppos)
2995 { 2995 {
2996 char buf[64], *tmp; 2996 char buf[64], *tmp;
2997 long val; 2997 long val;
2998 int ret; 2998 int ret;
2999 2999
3000 if (cnt >= sizeof(buf)) 3000 if (cnt >= sizeof(buf))
3001 return -EINVAL; 3001 return -EINVAL;
3002 3002
3003 if (copy_from_user(&buf, ubuf, cnt)) 3003 if (copy_from_user(&buf, ubuf, cnt))
3004 return -EFAULT; 3004 return -EFAULT;
3005 3005
3006 buf[cnt] = 0; 3006 buf[cnt] = 0;
3007 3007
3008 /* 3008 /*
3009 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid" 3009 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3010 * to clean the filter quietly. 3010 * to clean the filter quietly.
3011 */ 3011 */
3012 tmp = strstrip(buf); 3012 tmp = strstrip(buf);
3013 if (strlen(tmp) == 0) 3013 if (strlen(tmp) == 0)
3014 return 1; 3014 return 1;
3015 3015
3016 ret = strict_strtol(tmp, 10, &val); 3016 ret = strict_strtol(tmp, 10, &val);
3017 if (ret < 0) 3017 if (ret < 0)
3018 return ret; 3018 return ret;
3019 3019
3020 ret = ftrace_pid_add(val); 3020 ret = ftrace_pid_add(val);
3021 3021
3022 return ret ? ret : cnt; 3022 return ret ? ret : cnt;
3023 } 3023 }
3024 3024
3025 static int 3025 static int
3026 ftrace_pid_release(struct inode *inode, struct file *file) 3026 ftrace_pid_release(struct inode *inode, struct file *file)
3027 { 3027 {
3028 if (file->f_mode & FMODE_READ) 3028 if (file->f_mode & FMODE_READ)
3029 seq_release(inode, file); 3029 seq_release(inode, file);
3030 3030
3031 return 0; 3031 return 0;
3032 } 3032 }
3033 3033
3034 static const struct file_operations ftrace_pid_fops = { 3034 static const struct file_operations ftrace_pid_fops = {
3035 .open = ftrace_pid_open, 3035 .open = ftrace_pid_open,
3036 .write = ftrace_pid_write, 3036 .write = ftrace_pid_write,
3037 .read = seq_read, 3037 .read = seq_read,
3038 .llseek = seq_lseek, 3038 .llseek = seq_lseek,
3039 .release = ftrace_pid_release, 3039 .release = ftrace_pid_release,
3040 }; 3040 };
3041 3041
3042 static __init int ftrace_init_debugfs(void) 3042 static __init int ftrace_init_debugfs(void)
3043 { 3043 {
3044 struct dentry *d_tracer; 3044 struct dentry *d_tracer;
3045 3045
3046 d_tracer = tracing_init_dentry(); 3046 d_tracer = tracing_init_dentry();
3047 if (!d_tracer) 3047 if (!d_tracer)
3048 return 0; 3048 return 0;
3049 3049
3050 ftrace_init_dyn_debugfs(d_tracer); 3050 ftrace_init_dyn_debugfs(d_tracer);
3051 3051
3052 trace_create_file("set_ftrace_pid", 0644, d_tracer, 3052 trace_create_file("set_ftrace_pid", 0644, d_tracer,
3053 NULL, &ftrace_pid_fops); 3053 NULL, &ftrace_pid_fops);
3054 3054
3055 ftrace_profile_debugfs(d_tracer); 3055 ftrace_profile_debugfs(d_tracer);
3056 3056
3057 return 0; 3057 return 0;
3058 } 3058 }
3059 fs_initcall(ftrace_init_debugfs); 3059 fs_initcall(ftrace_init_debugfs);
3060 3060
3061 /** 3061 /**
3062 * ftrace_kill - kill ftrace 3062 * ftrace_kill - kill ftrace
3063 * 3063 *
3064 * This function should be used by panic code. It stops ftrace 3064 * This function should be used by panic code. It stops ftrace
3065 * but in a not so nice way. If you need to simply kill ftrace 3065 * but in a not so nice way. If you need to simply kill ftrace
3066 * from a non-atomic section, use ftrace_kill. 3066 * from a non-atomic section, use ftrace_kill.
3067 */ 3067 */
3068 void ftrace_kill(void) 3068 void ftrace_kill(void)
3069 { 3069 {
3070 ftrace_disabled = 1; 3070 ftrace_disabled = 1;
3071 ftrace_enabled = 0; 3071 ftrace_enabled = 0;
3072 clear_ftrace_function(); 3072 clear_ftrace_function();
3073 } 3073 }
3074 3074
3075 /** 3075 /**
3076 * register_ftrace_function - register a function for profiling 3076 * register_ftrace_function - register a function for profiling
3077 * @ops - ops structure that holds the function for profiling. 3077 * @ops - ops structure that holds the function for profiling.
3078 * 3078 *
3079 * Register a function to be called by all functions in the 3079 * Register a function to be called by all functions in the
3080 * kernel. 3080 * kernel.
3081 * 3081 *
3082 * Note: @ops->func and all the functions it calls must be labeled 3082 * Note: @ops->func and all the functions it calls must be labeled
3083 * with "notrace", otherwise it will go into a 3083 * with "notrace", otherwise it will go into a
3084 * recursive loop. 3084 * recursive loop.
3085 */ 3085 */
3086 int register_ftrace_function(struct ftrace_ops *ops) 3086 int register_ftrace_function(struct ftrace_ops *ops)
3087 { 3087 {
3088 int ret; 3088 int ret;
3089 3089
3090 if (unlikely(ftrace_disabled)) 3090 if (unlikely(ftrace_disabled))
3091 return -1; 3091 return -1;
3092 3092
3093 mutex_lock(&ftrace_lock); 3093 mutex_lock(&ftrace_lock);
3094 3094
3095 ret = __register_ftrace_function(ops); 3095 ret = __register_ftrace_function(ops);
3096 ftrace_startup(0); 3096 ftrace_startup(0);
3097 3097
3098 mutex_unlock(&ftrace_lock); 3098 mutex_unlock(&ftrace_lock);
3099 return ret; 3099 return ret;
3100 } 3100 }
3101 3101
3102 /** 3102 /**
3103 * unregister_ftrace_function - unregister a function for profiling. 3103 * unregister_ftrace_function - unregister a function for profiling.
3104 * @ops - ops structure that holds the function to unregister 3104 * @ops - ops structure that holds the function to unregister
3105 * 3105 *
3106 * Unregister a function that was added to be called by ftrace profiling. 3106 * Unregister a function that was added to be called by ftrace profiling.
3107 */ 3107 */
3108 int unregister_ftrace_function(struct ftrace_ops *ops) 3108 int unregister_ftrace_function(struct ftrace_ops *ops)
3109 { 3109 {
3110 int ret; 3110 int ret;
3111 3111
3112 mutex_lock(&ftrace_lock); 3112 mutex_lock(&ftrace_lock);
3113 ret = __unregister_ftrace_function(ops); 3113 ret = __unregister_ftrace_function(ops);
3114 ftrace_shutdown(0); 3114 ftrace_shutdown(0);
3115 mutex_unlock(&ftrace_lock); 3115 mutex_unlock(&ftrace_lock);
3116 3116
3117 return ret; 3117 return ret;
3118 } 3118 }
3119 3119
3120 int 3120 int
3121 ftrace_enable_sysctl(struct ctl_table *table, int write, 3121 ftrace_enable_sysctl(struct ctl_table *table, int write,
3122 void __user *buffer, size_t *lenp, 3122 void __user *buffer, size_t *lenp,
3123 loff_t *ppos) 3123 loff_t *ppos)
3124 { 3124 {
3125 int ret; 3125 int ret;
3126 3126
3127 if (unlikely(ftrace_disabled)) 3127 if (unlikely(ftrace_disabled))
3128 return -ENODEV; 3128 return -ENODEV;
3129 3129
3130 mutex_lock(&ftrace_lock); 3130 mutex_lock(&ftrace_lock);
3131 3131
3132 ret = proc_dointvec(table, write, buffer, lenp, ppos); 3132 ret = proc_dointvec(table, write, buffer, lenp, ppos);
3133 3133
3134 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 3134 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3135 goto out; 3135 goto out;
3136 3136
3137 last_ftrace_enabled = !!ftrace_enabled; 3137 last_ftrace_enabled = !!ftrace_enabled;
3138 3138
3139 if (ftrace_enabled) { 3139 if (ftrace_enabled) {
3140 3140
3141 ftrace_startup_sysctl(); 3141 ftrace_startup_sysctl();
3142 3142
3143 /* we are starting ftrace again */ 3143 /* we are starting ftrace again */
3144 if (ftrace_list != &ftrace_list_end) { 3144 if (ftrace_list != &ftrace_list_end) {
3145 if (ftrace_list->next == &ftrace_list_end) 3145 if (ftrace_list->next == &ftrace_list_end)
3146 ftrace_trace_function = ftrace_list->func; 3146 ftrace_trace_function = ftrace_list->func;
3147 else 3147 else
3148 ftrace_trace_function = ftrace_list_func; 3148 ftrace_trace_function = ftrace_list_func;
3149 } 3149 }
3150 3150
3151 } else { 3151 } else {
3152 /* stopping ftrace calls (just send to ftrace_stub) */ 3152 /* stopping ftrace calls (just send to ftrace_stub) */
3153 ftrace_trace_function = ftrace_stub; 3153 ftrace_trace_function = ftrace_stub;
3154 3154
3155 ftrace_shutdown_sysctl(); 3155 ftrace_shutdown_sysctl();
3156 } 3156 }
3157 3157
3158 out: 3158 out:
3159 mutex_unlock(&ftrace_lock); 3159 mutex_unlock(&ftrace_lock);
3160 return ret; 3160 return ret;
3161 } 3161 }
3162 3162
3163 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 3163 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3164 3164
3165 static int ftrace_graph_active; 3165 static int ftrace_graph_active;
3166 static struct notifier_block ftrace_suspend_notifier; 3166 static struct notifier_block ftrace_suspend_notifier;
3167 3167
3168 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) 3168 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3169 { 3169 {
3170 return 0; 3170 return 0;
3171 } 3171 }
3172 3172
3173 /* The callbacks that hook a function */ 3173 /* The callbacks that hook a function */
3174 trace_func_graph_ret_t ftrace_graph_return = 3174 trace_func_graph_ret_t ftrace_graph_return =
3175 (trace_func_graph_ret_t)ftrace_stub; 3175 (trace_func_graph_ret_t)ftrace_stub;
3176 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; 3176 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3177 3177
3178 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ 3178 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3179 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) 3179 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3180 { 3180 {
3181 int i; 3181 int i;
3182 int ret = 0; 3182 int ret = 0;
3183 unsigned long flags; 3183 unsigned long flags;
3184 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; 3184 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3185 struct task_struct *g, *t; 3185 struct task_struct *g, *t;
3186 3186
3187 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { 3187 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3188 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH 3188 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3189 * sizeof(struct ftrace_ret_stack), 3189 * sizeof(struct ftrace_ret_stack),
3190 GFP_KERNEL); 3190 GFP_KERNEL);
3191 if (!ret_stack_list[i]) { 3191 if (!ret_stack_list[i]) {
3192 start = 0; 3192 start = 0;
3193 end = i; 3193 end = i;
3194 ret = -ENOMEM; 3194 ret = -ENOMEM;
3195 goto free; 3195 goto free;
3196 } 3196 }
3197 } 3197 }
3198 3198
3199 read_lock_irqsave(&tasklist_lock, flags); 3199 read_lock_irqsave(&tasklist_lock, flags);
3200 do_each_thread(g, t) { 3200 do_each_thread(g, t) {
3201 if (start == end) { 3201 if (start == end) {
3202 ret = -EAGAIN; 3202 ret = -EAGAIN;
3203 goto unlock; 3203 goto unlock;
3204 } 3204 }
3205 3205
3206 if (t->ret_stack == NULL) { 3206 if (t->ret_stack == NULL) {
3207 atomic_set(&t->tracing_graph_pause, 0); 3207 atomic_set(&t->tracing_graph_pause, 0);
3208 atomic_set(&t->trace_overrun, 0); 3208 atomic_set(&t->trace_overrun, 0);
3209 t->curr_ret_stack = -1; 3209 t->curr_ret_stack = -1;
3210 /* Make sure the tasks see the -1 first: */ 3210 /* Make sure the tasks see the -1 first: */
3211 smp_wmb(); 3211 smp_wmb();
3212 t->ret_stack = ret_stack_list[start++]; 3212 t->ret_stack = ret_stack_list[start++];
3213 } 3213 }
3214 } while_each_thread(g, t); 3214 } while_each_thread(g, t);
3215 3215
3216 unlock: 3216 unlock:
3217 read_unlock_irqrestore(&tasklist_lock, flags); 3217 read_unlock_irqrestore(&tasklist_lock, flags);
3218 free: 3218 free:
3219 for (i = start; i < end; i++) 3219 for (i = start; i < end; i++)
3220 kfree(ret_stack_list[i]); 3220 kfree(ret_stack_list[i]);
3221 return ret; 3221 return ret;
3222 } 3222 }
3223 3223
3224 static void 3224 static void
3225 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev, 3225 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3226 struct task_struct *next) 3226 struct task_struct *next)
3227 { 3227 {
3228 unsigned long long timestamp; 3228 unsigned long long timestamp;
3229 int index; 3229 int index;
3230 3230
3231 /* 3231 /*
3232 * Does the user want to count the time a function was asleep. 3232 * Does the user want to count the time a function was asleep.
3233 * If so, do not update the time stamps. 3233 * If so, do not update the time stamps.
3234 */ 3234 */
3235 if (trace_flags & TRACE_ITER_SLEEP_TIME) 3235 if (trace_flags & TRACE_ITER_SLEEP_TIME)
3236 return; 3236 return;
3237 3237
3238 timestamp = trace_clock_local(); 3238 timestamp = trace_clock_local();
3239 3239
3240 prev->ftrace_timestamp = timestamp; 3240 prev->ftrace_timestamp = timestamp;
3241 3241
3242 /* only process tasks that we timestamped */ 3242 /* only process tasks that we timestamped */
3243 if (!next->ftrace_timestamp) 3243 if (!next->ftrace_timestamp)
3244 return; 3244 return;
3245 3245
3246 /* 3246 /*
3247 * Update all the counters in next to make up for the 3247 * Update all the counters in next to make up for the
3248 * time next was sleeping. 3248 * time next was sleeping.
3249 */ 3249 */
3250 timestamp -= next->ftrace_timestamp; 3250 timestamp -= next->ftrace_timestamp;
3251 3251
3252 for (index = next->curr_ret_stack; index >= 0; index--) 3252 for (index = next->curr_ret_stack; index >= 0; index--)
3253 next->ret_stack[index].calltime += timestamp; 3253 next->ret_stack[index].calltime += timestamp;
3254 } 3254 }
3255 3255
3256 /* Allocate a return stack for each task */ 3256 /* Allocate a return stack for each task */
3257 static int start_graph_tracing(void) 3257 static int start_graph_tracing(void)
3258 { 3258 {
3259 struct ftrace_ret_stack **ret_stack_list; 3259 struct ftrace_ret_stack **ret_stack_list;
3260 int ret, cpu; 3260 int ret, cpu;
3261 3261
3262 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE * 3262 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3263 sizeof(struct ftrace_ret_stack *), 3263 sizeof(struct ftrace_ret_stack *),
3264 GFP_KERNEL); 3264 GFP_KERNEL);
3265 3265
3266 if (!ret_stack_list) 3266 if (!ret_stack_list)
3267 return -ENOMEM; 3267 return -ENOMEM;
3268 3268
3269 /* The cpu_boot init_task->ret_stack will never be freed */ 3269 /* The cpu_boot init_task->ret_stack will never be freed */
3270 for_each_online_cpu(cpu) { 3270 for_each_online_cpu(cpu) {
3271 if (!idle_task(cpu)->ret_stack) 3271 if (!idle_task(cpu)->ret_stack)
3272 ftrace_graph_init_task(idle_task(cpu)); 3272 ftrace_graph_init_task(idle_task(cpu));
3273 } 3273 }
3274 3274
3275 do { 3275 do {
3276 ret = alloc_retstack_tasklist(ret_stack_list); 3276 ret = alloc_retstack_tasklist(ret_stack_list);
3277 } while (ret == -EAGAIN); 3277 } while (ret == -EAGAIN);
3278 3278
3279 if (!ret) { 3279 if (!ret) {
3280 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch); 3280 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3281 if (ret) 3281 if (ret)
3282 pr_info("ftrace_graph: Couldn't activate tracepoint" 3282 pr_info("ftrace_graph: Couldn't activate tracepoint"
3283 " probe to kernel_sched_switch\n"); 3283 " probe to kernel_sched_switch\n");
3284 } 3284 }
3285 3285
3286 kfree(ret_stack_list); 3286 kfree(ret_stack_list);
3287 return ret; 3287 return ret;
3288 } 3288 }
3289 3289
3290 /* 3290 /*
3291 * Hibernation protection. 3291 * Hibernation protection.
3292 * The state of the current task is too much unstable during 3292 * The state of the current task is too much unstable during
3293 * suspend/restore to disk. We want to protect against that. 3293 * suspend/restore to disk. We want to protect against that.
3294 */ 3294 */
3295 static int 3295 static int
3296 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, 3296 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3297 void *unused) 3297 void *unused)
3298 { 3298 {
3299 switch (state) { 3299 switch (state) {
3300 case PM_HIBERNATION_PREPARE: 3300 case PM_HIBERNATION_PREPARE:
3301 pause_graph_tracing(); 3301 pause_graph_tracing();
3302 break; 3302 break;
3303 3303
3304 case PM_POST_HIBERNATION: 3304 case PM_POST_HIBERNATION:
3305 unpause_graph_tracing(); 3305 unpause_graph_tracing();
3306 break; 3306 break;
3307 } 3307 }
3308 return NOTIFY_DONE; 3308 return NOTIFY_DONE;
3309 } 3309 }
3310 3310
3311 int register_ftrace_graph(trace_func_graph_ret_t retfunc, 3311 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3312 trace_func_graph_ent_t entryfunc) 3312 trace_func_graph_ent_t entryfunc)
3313 { 3313 {
3314 int ret = 0; 3314 int ret = 0;
3315 3315
3316 mutex_lock(&ftrace_lock); 3316 mutex_lock(&ftrace_lock);
3317 3317
3318 /* we currently allow only one tracer registered at a time */ 3318 /* we currently allow only one tracer registered at a time */
3319 if (ftrace_graph_active) { 3319 if (ftrace_graph_active) {
3320 ret = -EBUSY; 3320 ret = -EBUSY;
3321 goto out; 3321 goto out;
3322 } 3322 }
3323 3323
3324 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call; 3324 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3325 register_pm_notifier(&ftrace_suspend_notifier); 3325 register_pm_notifier(&ftrace_suspend_notifier);
3326 3326
3327 ftrace_graph_active++; 3327 ftrace_graph_active++;
3328 ret = start_graph_tracing(); 3328 ret = start_graph_tracing();
3329 if (ret) { 3329 if (ret) {
3330 ftrace_graph_active--; 3330 ftrace_graph_active--;
3331 goto out; 3331 goto out;
3332 } 3332 }
3333 3333
3334 ftrace_graph_return = retfunc; 3334 ftrace_graph_return = retfunc;
3335 ftrace_graph_entry = entryfunc; 3335 ftrace_graph_entry = entryfunc;
3336 3336
3337 ftrace_startup(FTRACE_START_FUNC_RET); 3337 ftrace_startup(FTRACE_START_FUNC_RET);
3338 3338
3339 out: 3339 out:
3340 mutex_unlock(&ftrace_lock); 3340 mutex_unlock(&ftrace_lock);
3341 return ret; 3341 return ret;
3342 } 3342 }
3343 3343
3344 void unregister_ftrace_graph(void) 3344 void unregister_ftrace_graph(void)
3345 { 3345 {
3346 mutex_lock(&ftrace_lock); 3346 mutex_lock(&ftrace_lock);
3347 3347
3348 if (unlikely(!ftrace_graph_active)) 3348 if (unlikely(!ftrace_graph_active))
3349 goto out; 3349 goto out;
3350 3350
3351 ftrace_graph_active--; 3351 ftrace_graph_active--;
3352 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch); 3352 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3353 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 3353 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3354 ftrace_graph_entry = ftrace_graph_entry_stub; 3354 ftrace_graph_entry = ftrace_graph_entry_stub;
3355 ftrace_shutdown(FTRACE_STOP_FUNC_RET); 3355 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3356 unregister_pm_notifier(&ftrace_suspend_notifier); 3356 unregister_pm_notifier(&ftrace_suspend_notifier);
3357 3357
3358 out: 3358 out:
3359 mutex_unlock(&ftrace_lock); 3359 mutex_unlock(&ftrace_lock);
3360 } 3360 }
3361 3361
3362 /* Allocate a return stack for newly created task */ 3362 /* Allocate a return stack for newly created task */
3363 void ftrace_graph_init_task(struct task_struct *t) 3363 void ftrace_graph_init_task(struct task_struct *t)
3364 { 3364 {
3365 /* Make sure we do not use the parent ret_stack */ 3365 /* Make sure we do not use the parent ret_stack */
3366 t->ret_stack = NULL; 3366 t->ret_stack = NULL;
3367 3367
3368 if (ftrace_graph_active) { 3368 if (ftrace_graph_active) {
3369 struct ftrace_ret_stack *ret_stack; 3369 struct ftrace_ret_stack *ret_stack;
3370 3370
3371 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 3371 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3372 * sizeof(struct ftrace_ret_stack), 3372 * sizeof(struct ftrace_ret_stack),
3373 GFP_KERNEL); 3373 GFP_KERNEL);
3374 if (!ret_stack) 3374 if (!ret_stack)
3375 return; 3375 return;
3376 t->curr_ret_stack = -1; 3376 t->curr_ret_stack = -1;
3377 atomic_set(&t->tracing_graph_pause, 0); 3377 atomic_set(&t->tracing_graph_pause, 0);
3378 atomic_set(&t->trace_overrun, 0); 3378 atomic_set(&t->trace_overrun, 0);
3379 t->ftrace_timestamp = 0; 3379 t->ftrace_timestamp = 0;
3380 /* make curr_ret_stack visable before we add the ret_stack */ 3380 /* make curr_ret_stack visable before we add the ret_stack */
3381 smp_wmb(); 3381 smp_wmb();
3382 t->ret_stack = ret_stack; 3382 t->ret_stack = ret_stack;
3383 } 3383 }
3384 } 3384 }
3385 3385
3386 void ftrace_graph_exit_task(struct task_struct *t) 3386 void ftrace_graph_exit_task(struct task_struct *t)
3387 { 3387 {
3388 struct ftrace_ret_stack *ret_stack = t->ret_stack; 3388 struct ftrace_ret_stack *ret_stack = t->ret_stack;
3389 3389
3390 t->ret_stack = NULL; 3390 t->ret_stack = NULL;
3391 /* NULL must become visible to IRQs before we free it: */ 3391 /* NULL must become visible to IRQs before we free it: */
3392 barrier(); 3392 barrier();
3393 3393
3394 kfree(ret_stack); 3394 kfree(ret_stack);
3395 } 3395 }
3396 3396
3397 void ftrace_graph_stop(void) 3397 void ftrace_graph_stop(void)
3398 { 3398 {
3399 ftrace_stop(); 3399 ftrace_stop();
3400 } 3400 }
3401 #endif 3401 #endif
3402 3402