Blame view
kernel/trace/ftrace.c
97.3 KB
16444a8a4 ftrace: add basic... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * * Copyright (C) 2004-2006 Ingo Molnar * Copyright (C) 2004 William Lee Irwin III */ |
3d0833953 ftrace: dynamic e... |
15 16 17 |
#include <linux/stop_machine.h> #include <linux/clocksource.h> #include <linux/kallsyms.h> |
5072c59fd ftrace: add filte... |
18 |
#include <linux/seq_file.h> |
4a2b8dda3 tracing/function-... |
19 |
#include <linux/suspend.h> |
5072c59fd ftrace: add filte... |
20 |
#include <linux/debugfs.h> |
3d0833953 ftrace: dynamic e... |
21 |
#include <linux/hardirq.h> |
2d8b820b2 ftrace: cleanups |
22 |
#include <linux/kthread.h> |
5072c59fd ftrace: add filte... |
23 |
#include <linux/uaccess.h> |
5855fead9 ftrace: Use bsear... |
24 |
#include <linux/bsearch.h> |
56d82e000 kernel: Add <linu... |
25 |
#include <linux/module.h> |
2d8b820b2 ftrace: cleanups |
26 |
#include <linux/ftrace.h> |
b0fc494fa ftrace: add ftrac... |
27 |
#include <linux/sysctl.h> |
5a0e3ad6a include cleanup: ... |
28 |
#include <linux/slab.h> |
5072c59fd ftrace: add filte... |
29 |
#include <linux/ctype.h> |
68950619f ftrace: Sort the ... |
30 |
#include <linux/sort.h> |
3d0833953 ftrace: dynamic e... |
31 |
#include <linux/list.h> |
59df055f1 ftrace: trace dif... |
32 |
#include <linux/hash.h> |
3f379b03f ftrace: Replace r... |
33 |
#include <linux/rcupdate.h> |
3d0833953 ftrace: dynamic e... |
34 |
|
ad8d75fff tracing/events: m... |
35 |
#include <trace/events/sched.h> |
8aef2d285 function-graph: i... |
36 |
|
2af15d6a4 ftrace: add kerne... |
37 |
#include <asm/setup.h> |
395a59d0f ftrace: store mco... |
38 |
|
0706f1c48 tracing: adding f... |
39 |
#include "trace_output.h" |
bac429f03 tracing: add func... |
40 |
#include "trace_stat.h" |
16444a8a4 ftrace: add basic... |
41 |
|
6912896e9 ftrace: add ftrac... |
42 |
#define FTRACE_WARN_ON(cond) \ |
0778d9ad3 ftrace: Make FTRA... |
43 44 45 |
({ \ int ___r = cond; \ if (WARN_ON(___r)) \ |
6912896e9 ftrace: add ftrac... |
46 |
ftrace_kill(); \ |
0778d9ad3 ftrace: Make FTRA... |
47 48 |
___r; \ }) |
6912896e9 ftrace: add ftrac... |
49 50 |
#define FTRACE_WARN_ON_ONCE(cond) \ |
0778d9ad3 ftrace: Make FTRA... |
51 52 53 |
({ \ int ___r = cond; \ if (WARN_ON_ONCE(___r)) \ |
6912896e9 ftrace: add ftrac... |
54 |
ftrace_kill(); \ |
0778d9ad3 ftrace: Make FTRA... |
55 56 |
___r; \ }) |
6912896e9 ftrace: add ftrac... |
57 |
|
8fc0c701c ftrace: show sele... |
58 59 60 |
/* hash bits for specific function selection */ #define FTRACE_HASH_BITS 7 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) |
33dc9b126 ftrace: Separate ... |
61 62 |
#define FTRACE_HASH_DEFAULT_BITS 10 #define FTRACE_HASH_MAX_BITS 12 |
8fc0c701c ftrace: show sele... |
63 |
|
4eebcc81a ftrace: disable t... |
64 65 |
/* ftrace_enabled is a method to turn ftrace on or off */ int ftrace_enabled __read_mostly; |
d61f82d06 ftrace: use dynam... |
66 |
static int last_ftrace_enabled; |
b0fc494fa ftrace: add ftrac... |
67 |
|
60a7ecf42 ftrace: add quick... |
68 69 |
/* Quick disabling of function tracer. */ int function_trace_stop; |
756d17ee7 tracing: Support ... |
70 71 72 73 74 75 |
/* List for set_ftrace_pid's pids. */ LIST_HEAD(ftrace_pids); struct ftrace_pid { struct list_head list; struct pid *pid; }; |
4eebcc81a ftrace: disable t... |
76 77 78 79 80 |
/* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */ static int ftrace_disabled __read_mostly; |
52baf1192 ftrace: convert f... |
81 |
static DEFINE_MUTEX(ftrace_lock); |
b0fc494fa ftrace: add ftrac... |
82 |
|
bd38c0e6f ftrace: Fixed an ... |
83 |
static struct ftrace_ops ftrace_list_end __read_mostly = { |
fb9fb015e tracing: clean up... |
84 |
.func = ftrace_stub, |
16444a8a4 ftrace: add basic... |
85 |
}; |
b848914ce ftrace: Implement... |
86 87 |
static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end; static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end; |
16444a8a4 ftrace: add basic... |
88 |
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; |
6331c28c9 ftrace: Fix dynam... |
89 |
static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub; |
60a7ecf42 ftrace: add quick... |
90 |
ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; |
df4fc3155 ftrace: add funct... |
91 |
ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; |
2b499381b ftrace: Have glob... |
92 |
static struct ftrace_ops global_ops; |
16444a8a4 ftrace: add basic... |
93 |
|
b848914ce ftrace: Implement... |
94 95 |
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip); |
3f379b03f ftrace: Replace r... |
96 |
/* |
b848914ce ftrace: Implement... |
97 |
* Traverse the ftrace_global_list, invoking all entries. The reason that we |
3f379b03f ftrace: Replace r... |
98 99 100 |
* can use rcu_dereference_raw() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period * mechanism. The rcu_dereference_raw() calls are needed to handle |
b848914ce ftrace: Implement... |
101 |
* concurrent insertions into the ftrace_global_list. |
3f379b03f ftrace: Replace r... |
102 103 104 |
* * Silly Alpha and silly pointer-speculation compiler optimizations! */ |
b848914ce ftrace: Implement... |
105 106 |
static void ftrace_global_list_func(unsigned long ip, unsigned long parent_ip) |
16444a8a4 ftrace: add basic... |
107 |
{ |
b1cff0ad1 ftrace: Add inter... |
108 109 110 111 |
struct ftrace_ops *op; if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT))) return; |
16444a8a4 ftrace: add basic... |
112 |
|
b1cff0ad1 ftrace: Add inter... |
113 114 |
trace_recursion_set(TRACE_GLOBAL_BIT); op = rcu_dereference_raw(ftrace_global_list); /*see above*/ |
16444a8a4 ftrace: add basic... |
115 |
while (op != &ftrace_list_end) { |
16444a8a4 ftrace: add basic... |
116 |
op->func(ip, parent_ip); |
3f379b03f ftrace: Replace r... |
117 |
op = rcu_dereference_raw(op->next); /*see above*/ |
16444a8a4 ftrace: add basic... |
118 |
}; |
b1cff0ad1 ftrace: Add inter... |
119 |
trace_recursion_clear(TRACE_GLOBAL_BIT); |
16444a8a4 ftrace: add basic... |
120 |
} |
df4fc3155 ftrace: add funct... |
121 122 |
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip) { |
0ef8cde56 ftrace: use task ... |
123 |
if (!test_tsk_trace_trace(current)) |
df4fc3155 ftrace: add funct... |
124 125 126 127 128 129 130 131 132 133 134 |
return; ftrace_pid_function(ip, parent_ip); } static void set_ftrace_pid_function(ftrace_func_t func) { /* do not set ftrace_pid_function to itself! */ if (func != ftrace_pid_func) ftrace_pid_function = func; } |
16444a8a4 ftrace: add basic... |
135 |
/** |
3d0833953 ftrace: dynamic e... |
136 |
* clear_ftrace_function - reset the ftrace function |
16444a8a4 ftrace: add basic... |
137 |
* |
3d0833953 ftrace: dynamic e... |
138 139 |
* This NULLs the ftrace function and in essence stops * tracing. There may be lag |
16444a8a4 ftrace: add basic... |
140 |
*/ |
3d0833953 ftrace: dynamic e... |
141 |
void clear_ftrace_function(void) |
16444a8a4 ftrace: add basic... |
142 |
{ |
3d0833953 ftrace: dynamic e... |
143 |
ftrace_trace_function = ftrace_stub; |
60a7ecf42 ftrace: add quick... |
144 |
__ftrace_trace_function = ftrace_stub; |
6331c28c9 ftrace: Fix dynam... |
145 |
__ftrace_trace_function_delay = ftrace_stub; |
df4fc3155 ftrace: add funct... |
146 |
ftrace_pid_function = ftrace_stub; |
3d0833953 ftrace: dynamic e... |
147 |
} |
60a7ecf42 ftrace: add quick... |
148 149 150 151 152 153 154 155 156 157 158 159 160 |
#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST /* * For those archs that do not test ftrace_trace_stop in their * mcount call site, we need to do it from C. */ static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip) { if (function_trace_stop) return; __ftrace_trace_function(ip, parent_ip); } #endif |
2b499381b ftrace: Have glob... |
161 |
static void update_global_ops(void) |
491d0dcfb ftrace: Consolida... |
162 163 164 165 166 167 168 169 |
{ ftrace_func_t func; /* * If there's only one function registered, then call that * function directly. Otherwise, we need to iterate over the * registered callers. */ |
b848914ce ftrace: Implement... |
170 171 172 |
if (ftrace_global_list == &ftrace_list_end || ftrace_global_list->next == &ftrace_list_end) func = ftrace_global_list->func; |
491d0dcfb ftrace: Consolida... |
173 |
else |
b848914ce ftrace: Implement... |
174 |
func = ftrace_global_list_func; |
491d0dcfb ftrace: Consolida... |
175 176 177 178 179 180 |
/* If we filter on pids, update to use the pid function */ if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } |
2b499381b ftrace: Have glob... |
181 182 183 184 185 186 187 188 189 |
global_ops.func = func; } static void update_ftrace_function(void) { ftrace_func_t func; update_global_ops(); |
cdbe61bfe ftrace: Allow dyn... |
190 191 192 193 194 |
/* * If we are at the end of the list and this ops is * not dynamic, then have the mcount trampoline call * the function directly */ |
b848914ce ftrace: Implement... |
195 |
if (ftrace_ops_list == &ftrace_list_end || |
cdbe61bfe ftrace: Allow dyn... |
196 197 |
(ftrace_ops_list->next == &ftrace_list_end && !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC))) |
b848914ce ftrace: Implement... |
198 199 200 |
func = ftrace_ops_list->func; else func = ftrace_ops_list_func; |
2b499381b ftrace: Have glob... |
201 |
|
491d0dcfb ftrace: Consolida... |
202 203 204 |
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST ftrace_trace_function = func; #else |
6331c28c9 ftrace: Fix dynam... |
205 206 207 208 |
#ifdef CONFIG_DYNAMIC_FTRACE /* do not update till all functions have been modified */ __ftrace_trace_function_delay = func; #else |
491d0dcfb ftrace: Consolida... |
209 |
__ftrace_trace_function = func; |
6331c28c9 ftrace: Fix dynam... |
210 |
#endif |
491d0dcfb ftrace: Consolida... |
211 212 213 |
ftrace_trace_function = ftrace_test_stop_func; #endif } |
2b499381b ftrace: Have glob... |
214 |
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) |
3d0833953 ftrace: dynamic e... |
215 |
{ |
2b499381b ftrace: Have glob... |
216 |
ops->next = *list; |
16444a8a4 ftrace: add basic... |
217 |
/* |
b848914ce ftrace: Implement... |
218 |
* We are entering ops into the list but another |
16444a8a4 ftrace: add basic... |
219 220 |
* CPU might be walking that list. We need to make sure * the ops->next pointer is valid before another CPU sees |
b848914ce ftrace: Implement... |
221 |
* the ops pointer included into the list. |
16444a8a4 ftrace: add basic... |
222 |
*/ |
2b499381b ftrace: Have glob... |
223 |
rcu_assign_pointer(*list, ops); |
16444a8a4 ftrace: add basic... |
224 |
} |
2b499381b ftrace: Have glob... |
225 |
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) |
16444a8a4 ftrace: add basic... |
226 |
{ |
16444a8a4 ftrace: add basic... |
227 |
struct ftrace_ops **p; |
16444a8a4 ftrace: add basic... |
228 229 |
/* |
3d0833953 ftrace: dynamic e... |
230 231 |
* If we are removing the last function, then simply point * to the ftrace_stub. |
16444a8a4 ftrace: add basic... |
232 |
*/ |
2b499381b ftrace: Have glob... |
233 234 |
if (*list == ops && ops->next == &ftrace_list_end) { *list = &ftrace_list_end; |
e6ea44e9b ftrace: consolida... |
235 |
return 0; |
16444a8a4 ftrace: add basic... |
236 |
} |
2b499381b ftrace: Have glob... |
237 |
for (p = list; *p != &ftrace_list_end; p = &(*p)->next) |
16444a8a4 ftrace: add basic... |
238 239 |
if (*p == ops) break; |
e6ea44e9b ftrace: consolida... |
240 241 |
if (*p != ops) return -1; |
16444a8a4 ftrace: add basic... |
242 243 |
*p = (*p)->next; |
2b499381b ftrace: Have glob... |
244 245 |
return 0; } |
16444a8a4 ftrace: add basic... |
246 |
|
2b499381b ftrace: Have glob... |
247 248 249 250 251 252 253 |
static int __register_ftrace_function(struct ftrace_ops *ops) { if (ftrace_disabled) return -ENODEV; if (FTRACE_WARN_ON(ops == &global_ops)) return -EINVAL; |
b848914ce ftrace: Implement... |
254 255 |
if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) return -EBUSY; |
cdbe61bfe ftrace: Allow dyn... |
256 257 |
if (!core_kernel_data((unsigned long)ops)) ops->flags |= FTRACE_OPS_FL_DYNAMIC; |
b848914ce ftrace: Implement... |
258 259 260 261 262 263 264 265 |
if (ops->flags & FTRACE_OPS_FL_GLOBAL) { int first = ftrace_global_list == &ftrace_list_end; add_ftrace_ops(&ftrace_global_list, ops); ops->flags |= FTRACE_OPS_FL_ENABLED; if (first) add_ftrace_ops(&ftrace_ops_list, &global_ops); } else add_ftrace_ops(&ftrace_ops_list, ops); |
2b499381b ftrace: Have glob... |
266 267 268 269 270 271 272 273 274 275 276 277 |
if (ftrace_enabled) update_ftrace_function(); return 0; } static int __unregister_ftrace_function(struct ftrace_ops *ops) { int ret; if (ftrace_disabled) return -ENODEV; |
b848914ce ftrace: Implement... |
278 279 |
if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) return -EBUSY; |
2b499381b ftrace: Have glob... |
280 281 |
if (FTRACE_WARN_ON(ops == &global_ops)) return -EINVAL; |
b848914ce ftrace: Implement... |
282 283 284 285 286 287 288 289 |
if (ops->flags & FTRACE_OPS_FL_GLOBAL) { ret = remove_ftrace_ops(&ftrace_global_list, ops); if (!ret && ftrace_global_list == &ftrace_list_end) ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops); if (!ret) ops->flags &= ~FTRACE_OPS_FL_ENABLED; } else ret = remove_ftrace_ops(&ftrace_ops_list, ops); |
2b499381b ftrace: Have glob... |
290 291 |
if (ret < 0) return ret; |
b848914ce ftrace: Implement... |
292 |
|
491d0dcfb ftrace: Consolida... |
293 294 |
if (ftrace_enabled) update_ftrace_function(); |
16444a8a4 ftrace: add basic... |
295 |
|
cdbe61bfe ftrace: Allow dyn... |
296 297 298 299 300 301 |
/* * Dynamic ops may be freed, we must make sure that all * callers are done before leaving this function. */ if (ops->flags & FTRACE_OPS_FL_DYNAMIC) synchronize_sched(); |
e6ea44e9b ftrace: consolida... |
302 |
return 0; |
3d0833953 ftrace: dynamic e... |
303 |
} |
df4fc3155 ftrace: add funct... |
304 305 |
static void ftrace_update_pid_func(void) { |
491d0dcfb ftrace: Consolida... |
306 |
/* Only do something if we are tracing something */ |
df4fc3155 ftrace: add funct... |
307 |
if (ftrace_trace_function == ftrace_stub) |
10dd3ebe2 tracing: fix dead... |
308 |
return; |
df4fc3155 ftrace: add funct... |
309 |
|
491d0dcfb ftrace: Consolida... |
310 |
update_ftrace_function(); |
df4fc3155 ftrace: add funct... |
311 |
} |
493762fc5 tracing: move fun... |
312 313 314 315 316 |
#ifdef CONFIG_FUNCTION_PROFILER struct ftrace_profile { struct hlist_node node; unsigned long ip; unsigned long counter; |
0706f1c48 tracing: adding f... |
317 318 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER unsigned long long time; |
e330b3bcd tracing: Show sam... |
319 |
unsigned long long time_squared; |
0706f1c48 tracing: adding f... |
320 |
#endif |
8fc0c701c ftrace: show sele... |
321 |
}; |
493762fc5 tracing: move fun... |
322 323 324 325 |
struct ftrace_profile_page { struct ftrace_profile_page *next; unsigned long index; struct ftrace_profile records[]; |
d61f82d06 ftrace: use dynam... |
326 |
}; |
cafb168a1 tracing: make the... |
327 328 329 330 331 332 333 |
struct ftrace_profile_stat { atomic_t disabled; struct hlist_head *hash; struct ftrace_profile_page *pages; struct ftrace_profile_page *start; struct tracer_stat stat; }; |
493762fc5 tracing: move fun... |
334 335 |
#define PROFILE_RECORDS_SIZE \ (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) |
5072c59fd ftrace: add filte... |
336 |
|
493762fc5 tracing: move fun... |
337 338 |
#define PROFILES_PER_PAGE \ (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) |
3d0833953 ftrace: dynamic e... |
339 |
|
fb9fb015e tracing: clean up... |
340 341 342 343 |
static int ftrace_profile_bits __read_mostly; static int ftrace_profile_enabled __read_mostly; /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ |
bac429f03 tracing: add func... |
344 |
static DEFINE_MUTEX(ftrace_profile_lock); |
cafb168a1 tracing: make the... |
345 |
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); |
493762fc5 tracing: move fun... |
346 347 |
#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */ |
bac429f03 tracing: add func... |
348 349 350 |
static void * function_stat_next(void *v, int idx) { |
493762fc5 tracing: move fun... |
351 352 |
struct ftrace_profile *rec = v; struct ftrace_profile_page *pg; |
bac429f03 tracing: add func... |
353 |
|
493762fc5 tracing: move fun... |
354 |
pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); |
bac429f03 tracing: add func... |
355 356 |
again: |
0296e4254 ftrace: Fix the o... |
357 358 |
if (idx != 0) rec++; |
bac429f03 tracing: add func... |
359 360 361 362 363 |
if ((void *)rec >= (void *)&pg->records[pg->index]) { pg = pg->next; if (!pg) return NULL; rec = &pg->records[0]; |
493762fc5 tracing: move fun... |
364 365 |
if (!rec->counter) goto again; |
bac429f03 tracing: add func... |
366 |
} |
bac429f03 tracing: add func... |
367 368 369 370 371 |
return rec; } static void *function_stat_start(struct tracer_stat *trace) { |
cafb168a1 tracing: make the... |
372 373 374 375 376 377 378 |
struct ftrace_profile_stat *stat = container_of(trace, struct ftrace_profile_stat, stat); if (!stat || !stat->start) return NULL; return function_stat_next(&stat->start->records[0], 0); |
bac429f03 tracing: add func... |
379 |
} |
0706f1c48 tracing: adding f... |
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER /* function graph compares on total time */ static int function_stat_cmp(void *p1, void *p2) { struct ftrace_profile *a = p1; struct ftrace_profile *b = p2; if (a->time < b->time) return -1; if (a->time > b->time) return 1; else return 0; } #else /* not function graph compares against hits */ |
bac429f03 tracing: add func... |
396 397 |
static int function_stat_cmp(void *p1, void *p2) { |
493762fc5 tracing: move fun... |
398 399 |
struct ftrace_profile *a = p1; struct ftrace_profile *b = p2; |
bac429f03 tracing: add func... |
400 401 402 403 404 405 406 407 |
if (a->counter < b->counter) return -1; if (a->counter > b->counter) return 1; else return 0; } |
0706f1c48 tracing: adding f... |
408 |
#endif |
bac429f03 tracing: add func... |
409 410 411 |
static int function_stat_headers(struct seq_file *m) { |
0706f1c48 tracing: adding f... |
412 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER |
34886c8bc tracing: add aver... |
413 |
seq_printf(m, " Function " |
e330b3bcd tracing: Show sam... |
414 415 |
"Hit Time Avg s^2 " |
34886c8bc tracing: add aver... |
416 |
" -------- " |
e330b3bcd tracing: Show sam... |
417 418 |
"--- ---- --- --- "); |
0706f1c48 tracing: adding f... |
419 |
#else |
bac429f03 tracing: add func... |
420 421 422 423 |
seq_printf(m, " Function Hit " " -------- --- "); |
0706f1c48 tracing: adding f... |
424 |
#endif |
bac429f03 tracing: add func... |
425 426 427 428 429 |
return 0; } static int function_stat_show(struct seq_file *m, void *v) { |
493762fc5 tracing: move fun... |
430 |
struct ftrace_profile *rec = v; |
bac429f03 tracing: add func... |
431 |
char str[KSYM_SYMBOL_LEN]; |
3aaba20f2 tracing: Fix a ra... |
432 |
int ret = 0; |
0706f1c48 tracing: adding f... |
433 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER |
34886c8bc tracing: add aver... |
434 435 |
static struct trace_seq s; unsigned long long avg; |
e330b3bcd tracing: Show sam... |
436 |
unsigned long long stddev; |
0706f1c48 tracing: adding f... |
437 |
#endif |
3aaba20f2 tracing: Fix a ra... |
438 439 440 441 442 443 444 |
mutex_lock(&ftrace_profile_lock); /* we raced with function_profile_reset() */ if (unlikely(rec->counter == 0)) { ret = -EBUSY; goto out; } |
bac429f03 tracing: add func... |
445 446 |
kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); |
0706f1c48 tracing: adding f... |
447 448 449 450 |
seq_printf(m, " %-30.30s %10lu", str, rec->counter); #ifdef CONFIG_FUNCTION_GRAPH_TRACER seq_printf(m, " "); |
34886c8bc tracing: add aver... |
451 452 |
avg = rec->time; do_div(avg, rec->counter); |
e330b3bcd tracing: Show sam... |
453 454 455 456 457 458 459 460 461 462 463 |
/* Sample standard deviation (s^2) */ if (rec->counter <= 1) stddev = 0; else { stddev = rec->time_squared - rec->counter * avg * avg; /* * Divide only 1000 for ns^2 -> us^2 conversion. * trace_print_graph_duration will divide 1000 again. */ do_div(stddev, (rec->counter - 1) * 1000); } |
34886c8bc tracing: add aver... |
464 465 466 467 |
trace_seq_init(&s); trace_print_graph_duration(rec->time, &s); trace_seq_puts(&s, " "); trace_print_graph_duration(avg, &s); |
e330b3bcd tracing: Show sam... |
468 469 |
trace_seq_puts(&s, " "); trace_print_graph_duration(stddev, &s); |
0706f1c48 tracing: adding f... |
470 |
trace_print_seq(m, &s); |
0706f1c48 tracing: adding f... |
471 472 473 |
#endif seq_putc(m, ' '); |
3aaba20f2 tracing: Fix a ra... |
474 475 |
out: mutex_unlock(&ftrace_profile_lock); |
bac429f03 tracing: add func... |
476 |
|
3aaba20f2 tracing: Fix a ra... |
477 |
return ret; |
bac429f03 tracing: add func... |
478 |
} |
cafb168a1 tracing: make the... |
479 |
static void ftrace_profile_reset(struct ftrace_profile_stat *stat) |
bac429f03 tracing: add func... |
480 |
{ |
493762fc5 tracing: move fun... |
481 |
struct ftrace_profile_page *pg; |
bac429f03 tracing: add func... |
482 |
|
cafb168a1 tracing: make the... |
483 |
pg = stat->pages = stat->start; |
bac429f03 tracing: add func... |
484 |
|
493762fc5 tracing: move fun... |
485 486 487 488 |
while (pg) { memset(pg->records, 0, PROFILE_RECORDS_SIZE); pg->index = 0; pg = pg->next; |
bac429f03 tracing: add func... |
489 |
} |
cafb168a1 tracing: make the... |
490 |
memset(stat->hash, 0, |
493762fc5 tracing: move fun... |
491 492 |
FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); } |
bac429f03 tracing: add func... |
493 |
|
cafb168a1 tracing: make the... |
494 |
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) |
493762fc5 tracing: move fun... |
495 496 |
{ struct ftrace_profile_page *pg; |
318e0a73c tracing: remove o... |
497 498 |
int functions; int pages; |
493762fc5 tracing: move fun... |
499 |
int i; |
bac429f03 tracing: add func... |
500 |
|
493762fc5 tracing: move fun... |
501 |
/* If we already allocated, do nothing */ |
cafb168a1 tracing: make the... |
502 |
if (stat->pages) |
493762fc5 tracing: move fun... |
503 |
return 0; |
bac429f03 tracing: add func... |
504 |
|
cafb168a1 tracing: make the... |
505 506 |
stat->pages = (void *)get_zeroed_page(GFP_KERNEL); if (!stat->pages) |
493762fc5 tracing: move fun... |
507 |
return -ENOMEM; |
bac429f03 tracing: add func... |
508 |
|
318e0a73c tracing: remove o... |
509 510 511 512 513 514 515 516 517 518 519 520 |
#ifdef CONFIG_DYNAMIC_FTRACE functions = ftrace_update_tot_cnt; #else /* * We do not know the number of functions that exist because * dynamic tracing is what counts them. With past experience * we have around 20K functions. That should be more than enough. * It is highly unlikely we will execute every function in * the kernel. */ functions = 20000; #endif |
cafb168a1 tracing: make the... |
521 |
pg = stat->start = stat->pages; |
bac429f03 tracing: add func... |
522 |
|
318e0a73c tracing: remove o... |
523 524 525 |
pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); for (i = 0; i < pages; i++) { |
493762fc5 tracing: move fun... |
526 |
pg->next = (void *)get_zeroed_page(GFP_KERNEL); |
493762fc5 tracing: move fun... |
527 |
if (!pg->next) |
318e0a73c tracing: remove o... |
528 |
goto out_free; |
493762fc5 tracing: move fun... |
529 530 531 532 |
pg = pg->next; } return 0; |
318e0a73c tracing: remove o... |
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 |
out_free: pg = stat->start; while (pg) { unsigned long tmp = (unsigned long)pg; pg = pg->next; free_page(tmp); } free_page((unsigned long)stat->pages); stat->pages = NULL; stat->start = NULL; return -ENOMEM; |
bac429f03 tracing: add func... |
548 |
} |
cafb168a1 tracing: make the... |
549 |
static int ftrace_profile_init_cpu(int cpu) |
bac429f03 tracing: add func... |
550 |
{ |
cafb168a1 tracing: make the... |
551 |
struct ftrace_profile_stat *stat; |
493762fc5 tracing: move fun... |
552 |
int size; |
bac429f03 tracing: add func... |
553 |
|
cafb168a1 tracing: make the... |
554 555 556 |
stat = &per_cpu(ftrace_profile_stats, cpu); if (stat->hash) { |
493762fc5 tracing: move fun... |
557 |
/* If the profile is already created, simply reset it */ |
cafb168a1 tracing: make the... |
558 |
ftrace_profile_reset(stat); |
493762fc5 tracing: move fun... |
559 560 |
return 0; } |
bac429f03 tracing: add func... |
561 |
|
493762fc5 tracing: move fun... |
562 563 564 565 566 |
/* * We are profiling all functions, but usually only a few thousand * functions are hit. We'll make a hash of 1024 items. */ size = FTRACE_PROFILE_HASH_SIZE; |
bac429f03 tracing: add func... |
567 |
|
cafb168a1 tracing: make the... |
568 |
stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL); |
493762fc5 tracing: move fun... |
569 |
|
cafb168a1 tracing: make the... |
570 |
if (!stat->hash) |
493762fc5 tracing: move fun... |
571 |
return -ENOMEM; |
cafb168a1 tracing: make the... |
572 573 |
if (!ftrace_profile_bits) { size--; |
493762fc5 tracing: move fun... |
574 |
|
cafb168a1 tracing: make the... |
575 576 577 |
for (; size; size >>= 1) ftrace_profile_bits++; } |
493762fc5 tracing: move fun... |
578 |
|
318e0a73c tracing: remove o... |
579 |
/* Preallocate the function profiling pages */ |
cafb168a1 tracing: make the... |
580 581 582 |
if (ftrace_profile_pages_init(stat) < 0) { kfree(stat->hash); stat->hash = NULL; |
493762fc5 tracing: move fun... |
583 584 585 586 |
return -ENOMEM; } return 0; |
bac429f03 tracing: add func... |
587 |
} |
cafb168a1 tracing: make the... |
588 589 590 591 592 593 594 595 596 597 598 599 600 |
static int ftrace_profile_init(void) { int cpu; int ret = 0; for_each_online_cpu(cpu) { ret = ftrace_profile_init_cpu(cpu); if (ret) break; } return ret; } |
493762fc5 tracing: move fun... |
601 |
/* interrupts must be disabled */ |
cafb168a1 tracing: make the... |
602 603 |
static struct ftrace_profile * ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) |
bac429f03 tracing: add func... |
604 |
{ |
493762fc5 tracing: move fun... |
605 |
struct ftrace_profile *rec; |
bac429f03 tracing: add func... |
606 607 |
struct hlist_head *hhd; struct hlist_node *n; |
bac429f03 tracing: add func... |
608 |
unsigned long key; |
bac429f03 tracing: add func... |
609 |
key = hash_long(ip, ftrace_profile_bits); |
cafb168a1 tracing: make the... |
610 |
hhd = &stat->hash[key]; |
bac429f03 tracing: add func... |
611 612 613 |
if (hlist_empty(hhd)) return NULL; |
bac429f03 tracing: add func... |
614 615 |
hlist_for_each_entry_rcu(rec, n, hhd, node) { if (rec->ip == ip) |
493762fc5 tracing: move fun... |
616 617 618 619 620 |
return rec; } return NULL; } |
cafb168a1 tracing: make the... |
621 622 |
static void ftrace_add_profile(struct ftrace_profile_stat *stat, struct ftrace_profile *rec) |
493762fc5 tracing: move fun... |
623 624 625 626 |
{ unsigned long key; key = hash_long(rec->ip, ftrace_profile_bits); |
cafb168a1 tracing: make the... |
627 |
hlist_add_head_rcu(&rec->node, &stat->hash[key]); |
493762fc5 tracing: move fun... |
628 |
} |
318e0a73c tracing: remove o... |
629 630 631 |
/* * The memory is already allocated, this simply finds a new record to use. */ |
493762fc5 tracing: move fun... |
632 |
static struct ftrace_profile * |
318e0a73c tracing: remove o... |
633 |
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) |
493762fc5 tracing: move fun... |
634 635 |
{ struct ftrace_profile *rec = NULL; |
318e0a73c tracing: remove o... |
636 |
/* prevent recursion (from NMIs) */ |
cafb168a1 tracing: make the... |
637 |
if (atomic_inc_return(&stat->disabled) != 1) |
493762fc5 tracing: move fun... |
638 |
goto out; |
493762fc5 tracing: move fun... |
639 |
/* |
318e0a73c tracing: remove o... |
640 641 |
* Try to find the function again since an NMI * could have added it |
493762fc5 tracing: move fun... |
642 |
*/ |
cafb168a1 tracing: make the... |
643 |
rec = ftrace_find_profiled_func(stat, ip); |
493762fc5 tracing: move fun... |
644 |
if (rec) |
cafb168a1 tracing: make the... |
645 |
goto out; |
493762fc5 tracing: move fun... |
646 |
|
cafb168a1 tracing: make the... |
647 648 649 650 |
if (stat->pages->index == PROFILES_PER_PAGE) { if (!stat->pages->next) goto out; stat->pages = stat->pages->next; |
bac429f03 tracing: add func... |
651 |
} |
493762fc5 tracing: move fun... |
652 |
|
cafb168a1 tracing: make the... |
653 |
rec = &stat->pages->records[stat->pages->index++]; |
493762fc5 tracing: move fun... |
654 |
rec->ip = ip; |
cafb168a1 tracing: make the... |
655 |
ftrace_add_profile(stat, rec); |
493762fc5 tracing: move fun... |
656 |
|
bac429f03 tracing: add func... |
657 |
out: |
cafb168a1 tracing: make the... |
658 |
atomic_dec(&stat->disabled); |
bac429f03 tracing: add func... |
659 660 661 662 663 664 665 |
return rec; } static void function_profile_call(unsigned long ip, unsigned long parent_ip) { |
cafb168a1 tracing: make the... |
666 |
struct ftrace_profile_stat *stat; |
493762fc5 tracing: move fun... |
667 |
struct ftrace_profile *rec; |
bac429f03 tracing: add func... |
668 669 670 671 672 673 |
unsigned long flags; if (!ftrace_profile_enabled) return; local_irq_save(flags); |
cafb168a1 tracing: make the... |
674 675 |
stat = &__get_cpu_var(ftrace_profile_stats); |
0f6ce3de4 ftrace: do not pr... |
676 |
if (!stat->hash || !ftrace_profile_enabled) |
cafb168a1 tracing: make the... |
677 678 679 |
goto out; rec = ftrace_find_profiled_func(stat, ip); |
493762fc5 tracing: move fun... |
680 |
if (!rec) { |
318e0a73c tracing: remove o... |
681 |
rec = ftrace_profile_alloc(stat, ip); |
493762fc5 tracing: move fun... |
682 683 684 |
if (!rec) goto out; } |
bac429f03 tracing: add func... |
685 686 687 688 689 |
rec->counter++; out: local_irq_restore(flags); } |
0706f1c48 tracing: adding f... |
690 691 692 693 694 695 696 697 698 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER static int profile_graph_entry(struct ftrace_graph_ent *trace) { function_profile_call(trace->func, 0); return 1; } static void profile_graph_return(struct ftrace_graph_ret *trace) { |
cafb168a1 tracing: make the... |
699 |
struct ftrace_profile_stat *stat; |
a2a16d6a3 function-graph: a... |
700 |
unsigned long long calltime; |
0706f1c48 tracing: adding f... |
701 |
struct ftrace_profile *rec; |
cafb168a1 tracing: make the... |
702 |
unsigned long flags; |
0706f1c48 tracing: adding f... |
703 704 |
local_irq_save(flags); |
cafb168a1 tracing: make the... |
705 |
stat = &__get_cpu_var(ftrace_profile_stats); |
0f6ce3de4 ftrace: do not pr... |
706 |
if (!stat->hash || !ftrace_profile_enabled) |
cafb168a1 tracing: make the... |
707 |
goto out; |
37e44bc50 tracing: Fix slee... |
708 709 710 |
/* If the calltime was zero'd ignore it */ if (!trace->calltime) goto out; |
a2a16d6a3 function-graph: a... |
711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 |
calltime = trace->rettime - trace->calltime; if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) { int index; index = trace->depth; /* Append this call time to the parent time to subtract */ if (index) current->ret_stack[index - 1].subtime += calltime; if (current->ret_stack[index].subtime < calltime) calltime -= current->ret_stack[index].subtime; else calltime = 0; } |
cafb168a1 tracing: make the... |
727 |
rec = ftrace_find_profiled_func(stat, trace->func); |
e330b3bcd tracing: Show sam... |
728 |
if (rec) { |
a2a16d6a3 function-graph: a... |
729 |
rec->time += calltime; |
e330b3bcd tracing: Show sam... |
730 731 |
rec->time_squared += calltime * calltime; } |
a2a16d6a3 function-graph: a... |
732 |
|
cafb168a1 tracing: make the... |
733 |
out: |
0706f1c48 tracing: adding f... |
734 735 736 737 738 739 740 741 742 743 744 745 746 747 |
local_irq_restore(flags); } static int register_ftrace_profiler(void) { return register_ftrace_graph(&profile_graph_return, &profile_graph_entry); } static void unregister_ftrace_profiler(void) { unregister_ftrace_graph(); } #else |
bd38c0e6f ftrace: Fixed an ... |
748 |
static struct ftrace_ops ftrace_profile_ops __read_mostly = { |
fb9fb015e tracing: clean up... |
749 |
.func = function_profile_call, |
bac429f03 tracing: add func... |
750 |
}; |
0706f1c48 tracing: adding f... |
751 752 753 754 755 756 757 758 759 760 |
static int register_ftrace_profiler(void) { return register_ftrace_function(&ftrace_profile_ops); } static void unregister_ftrace_profiler(void) { unregister_ftrace_function(&ftrace_profile_ops); } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
bac429f03 tracing: add func... |
761 762 763 764 765 |
static ssize_t ftrace_profile_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { unsigned long val; |
bac429f03 tracing: add func... |
766 |
int ret; |
22fe9b54d tracing: Convert ... |
767 768 |
ret = kstrtoul_from_user(ubuf, cnt, 10, &val); if (ret) |
bac429f03 tracing: add func... |
769 770 771 772 773 774 775 |
return ret; val = !!val; mutex_lock(&ftrace_profile_lock); if (ftrace_profile_enabled ^ val) { if (val) { |
493762fc5 tracing: move fun... |
776 777 778 779 780 |
ret = ftrace_profile_init(); if (ret < 0) { cnt = ret; goto out; } |
0706f1c48 tracing: adding f... |
781 782 783 784 785 |
ret = register_ftrace_profiler(); if (ret < 0) { cnt = ret; goto out; } |
bac429f03 tracing: add func... |
786 787 788 |
ftrace_profile_enabled = 1; } else { ftrace_profile_enabled = 0; |
0f6ce3de4 ftrace: do not pr... |
789 790 791 792 |
/* * unregister_ftrace_profiler calls stop_machine * so this acts like an synchronize_sched. */ |
0706f1c48 tracing: adding f... |
793 |
unregister_ftrace_profiler(); |
bac429f03 tracing: add func... |
794 795 |
} } |
493762fc5 tracing: move fun... |
796 |
out: |
bac429f03 tracing: add func... |
797 |
mutex_unlock(&ftrace_profile_lock); |
cf8517cf9 tracing: Update *... |
798 |
*ppos += cnt; |
bac429f03 tracing: add func... |
799 800 801 |
return cnt; } |
493762fc5 tracing: move fun... |
802 803 804 805 |
static ssize_t ftrace_profile_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { |
fb9fb015e tracing: clean up... |
806 |
char buf[64]; /* big enough to hold a number */ |
493762fc5 tracing: move fun... |
807 808 809 810 811 812 |
int r; r = sprintf(buf, "%u ", ftrace_profile_enabled); return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); } |
bac429f03 tracing: add func... |
813 814 815 816 |
static const struct file_operations ftrace_profile_fops = { .open = tracing_open_generic, .read = ftrace_profile_read, .write = ftrace_profile_write, |
6038f373a llseek: automatic... |
817 |
.llseek = default_llseek, |
bac429f03 tracing: add func... |
818 |
}; |
cafb168a1 tracing: make the... |
819 820 |
/* used to initialize the real stat files */ static struct tracer_stat function_stats __initdata = { |
fb9fb015e tracing: clean up... |
821 822 823 824 825 826 |
.name = "functions", .stat_start = function_stat_start, .stat_next = function_stat_next, .stat_cmp = function_stat_cmp, .stat_headers = function_stat_headers, .stat_show = function_stat_show |
cafb168a1 tracing: make the... |
827 |
}; |
6ab5d668b tracing/function-... |
828 |
static __init void ftrace_profile_debugfs(struct dentry *d_tracer) |
bac429f03 tracing: add func... |
829 |
{ |
cafb168a1 tracing: make the... |
830 |
struct ftrace_profile_stat *stat; |
bac429f03 tracing: add func... |
831 |
struct dentry *entry; |
cafb168a1 tracing: make the... |
832 |
char *name; |
bac429f03 tracing: add func... |
833 |
int ret; |
cafb168a1 tracing: make the... |
834 835 836 837 838 839 840 841 842 843 844 845 |
int cpu; for_each_possible_cpu(cpu) { stat = &per_cpu(ftrace_profile_stats, cpu); /* allocate enough for function name + cpu number */ name = kmalloc(32, GFP_KERNEL); if (!name) { /* * The files created are permanent, if something happens * we still do not free memory. */ |
cafb168a1 tracing: make the... |
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 |
WARN(1, "Could not allocate stat file for cpu %d ", cpu); return; } stat->stat = function_stats; snprintf(name, 32, "function%d", cpu); stat->stat.name = name; ret = register_stat_tracer(&stat->stat); if (ret) { WARN(1, "Could not register function stat for cpu %d ", cpu); kfree(name); return; } |
bac429f03 tracing: add func... |
864 865 866 867 868 869 870 871 872 |
} entry = debugfs_create_file("function_profile_enabled", 0644, d_tracer, NULL, &ftrace_profile_fops); if (!entry) pr_warning("Could not create debugfs " "'function_profile_enabled' entry "); } |
bac429f03 tracing: add func... |
873 |
#else /* CONFIG_FUNCTION_PROFILER */ |
6ab5d668b tracing/function-... |
874 |
static __init void ftrace_profile_debugfs(struct dentry *d_tracer) |
bac429f03 tracing: add func... |
875 876 |
{ } |
bac429f03 tracing: add func... |
877 |
#endif /* CONFIG_FUNCTION_PROFILER */ |
493762fc5 tracing: move fun... |
878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 |
static struct pid * const ftrace_swapper_pid = &init_struct_pid; #ifdef CONFIG_DYNAMIC_FTRACE #ifndef CONFIG_FTRACE_MCOUNT_RECORD # error Dynamic ftrace depends on MCOUNT_RECORD #endif static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly; struct ftrace_func_probe { struct hlist_node node; struct ftrace_probe_ops *ops; unsigned long flags; unsigned long ip; void *data; struct rcu_head rcu; }; |
b448c4e3a ftrace: Replace F... |
896 897 898 899 900 901 902 903 904 |
struct ftrace_func_entry { struct hlist_node hlist; unsigned long ip; }; struct ftrace_hash { unsigned long size_bits; struct hlist_head *buckets; unsigned long count; |
07fd5515f ftrace: Free hash... |
905 |
struct rcu_head rcu; |
b448c4e3a ftrace: Replace F... |
906 |
}; |
33dc9b126 ftrace: Separate ... |
907 908 909 910 911 912 913 914 915 |
/* * We make these constant because no one should touch them, * but they are used as the default "empty hash", to avoid allocating * it all the time. These are in a read only section such that if * anyone does try to modify it, it will cause an exception. */ static const struct hlist_head empty_buckets[1]; static const struct ftrace_hash empty_hash = { .buckets = (struct hlist_head *)empty_buckets, |
1cf41dd79 ftrace: Use hash ... |
916 |
}; |
33dc9b126 ftrace: Separate ... |
917 |
#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) |
493762fc5 tracing: move fun... |
918 |
|
2b499381b ftrace: Have glob... |
919 |
static struct ftrace_ops global_ops = { |
f45948e89 ftrace: Create a ... |
920 |
.func = ftrace_stub, |
33dc9b126 ftrace: Separate ... |
921 922 |
.notrace_hash = EMPTY_HASH, .filter_hash = EMPTY_HASH, |
f45948e89 ftrace: Create a ... |
923 |
}; |
493762fc5 tracing: move fun... |
924 925 926 927 |
static DEFINE_MUTEX(ftrace_regex_lock); struct ftrace_page { struct ftrace_page *next; |
a79008755 ftrace: Allocate ... |
928 |
struct dyn_ftrace *records; |
493762fc5 tracing: move fun... |
929 |
int index; |
a79008755 ftrace: Allocate ... |
930 |
int size; |
493762fc5 tracing: move fun... |
931 |
}; |
85ae32ae0 ftrace: Replace r... |
932 |
static struct ftrace_page *ftrace_new_pgs; |
a79008755 ftrace: Allocate ... |
933 934 |
#define ENTRY_SIZE sizeof(struct dyn_ftrace) #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE) |
493762fc5 tracing: move fun... |
935 936 937 938 939 940 |
/* estimate from running different kernels */ #define NR_TO_INIT 10000 static struct ftrace_page *ftrace_pages_start; static struct ftrace_page *ftrace_pages; |
06a51d930 ftrace: Create ft... |
941 942 943 944 |
static bool ftrace_hash_empty(struct ftrace_hash *hash) { return !hash || !hash->count; } |
b448c4e3a ftrace: Replace F... |
945 946 947 948 949 950 951 |
static struct ftrace_func_entry * ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) { unsigned long key; struct ftrace_func_entry *entry; struct hlist_head *hhd; struct hlist_node *n; |
06a51d930 ftrace: Create ft... |
952 |
if (ftrace_hash_empty(hash)) |
b448c4e3a ftrace: Replace F... |
953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 |
return NULL; if (hash->size_bits > 0) key = hash_long(ip, hash->size_bits); else key = 0; hhd = &hash->buckets[key]; hlist_for_each_entry_rcu(entry, n, hhd, hlist) { if (entry->ip == ip) return entry; } return NULL; } |
33dc9b126 ftrace: Separate ... |
968 969 |
static void __add_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) |
b448c4e3a ftrace: Replace F... |
970 |
{ |
b448c4e3a ftrace: Replace F... |
971 972 |
struct hlist_head *hhd; unsigned long key; |
b448c4e3a ftrace: Replace F... |
973 |
if (hash->size_bits) |
33dc9b126 ftrace: Separate ... |
974 |
key = hash_long(entry->ip, hash->size_bits); |
b448c4e3a ftrace: Replace F... |
975 976 |
else key = 0; |
b448c4e3a ftrace: Replace F... |
977 978 979 |
hhd = &hash->buckets[key]; hlist_add_head(&entry->hlist, hhd); hash->count++; |
33dc9b126 ftrace: Separate ... |
980 981 982 983 984 985 986 987 988 989 990 991 |
} static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip) { struct ftrace_func_entry *entry; entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return -ENOMEM; entry->ip = ip; __add_hash_entry(hash, entry); |
b448c4e3a ftrace: Replace F... |
992 993 994 995 996 |
return 0; } static void |
33dc9b126 ftrace: Separate ... |
997 |
free_hash_entry(struct ftrace_hash *hash, |
b448c4e3a ftrace: Replace F... |
998 999 1000 1001 1002 1003 |
struct ftrace_func_entry *entry) { hlist_del(&entry->hlist); kfree(entry); hash->count--; } |
33dc9b126 ftrace: Separate ... |
1004 1005 1006 1007 1008 1009 1010 |
static void remove_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) { hlist_del(&entry->hlist); hash->count--; } |
b448c4e3a ftrace: Replace F... |
1011 1012 1013 1014 1015 1016 1017 |
static void ftrace_hash_clear(struct ftrace_hash *hash) { struct hlist_head *hhd; struct hlist_node *tp, *tn; struct ftrace_func_entry *entry; int size = 1 << hash->size_bits; int i; |
33dc9b126 ftrace: Separate ... |
1018 1019 |
if (!hash->count) return; |
b448c4e3a ftrace: Replace F... |
1020 1021 1022 |
for (i = 0; i < size; i++) { hhd = &hash->buckets[i]; hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) |
33dc9b126 ftrace: Separate ... |
1023 |
free_hash_entry(hash, entry); |
b448c4e3a ftrace: Replace F... |
1024 1025 1026 |
} FTRACE_WARN_ON(hash->count); } |
33dc9b126 ftrace: Separate ... |
1027 1028 1029 1030 1031 1032 1033 1034 |
static void free_ftrace_hash(struct ftrace_hash *hash) { if (!hash || hash == EMPTY_HASH) return; ftrace_hash_clear(hash); kfree(hash->buckets); kfree(hash); } |
07fd5515f ftrace: Free hash... |
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 |
static void __free_ftrace_hash_rcu(struct rcu_head *rcu) { struct ftrace_hash *hash; hash = container_of(rcu, struct ftrace_hash, rcu); free_ftrace_hash(hash); } static void free_ftrace_hash_rcu(struct ftrace_hash *hash) { if (!hash || hash == EMPTY_HASH) return; call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu); } |
33dc9b126 ftrace: Separate ... |
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 |
static struct ftrace_hash *alloc_ftrace_hash(int size_bits) { struct ftrace_hash *hash; int size; hash = kzalloc(sizeof(*hash), GFP_KERNEL); if (!hash) return NULL; size = 1 << size_bits; hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL); if (!hash->buckets) { kfree(hash); return NULL; } hash->size_bits = size_bits; return hash; } static struct ftrace_hash * alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) { struct ftrace_func_entry *entry; struct ftrace_hash *new_hash; struct hlist_node *tp; int size; int ret; int i; new_hash = alloc_ftrace_hash(size_bits); if (!new_hash) return NULL; /* Empty hash? */ |
06a51d930 ftrace: Create ft... |
1086 |
if (ftrace_hash_empty(hash)) |
33dc9b126 ftrace: Separate ... |
1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 |
return new_hash; size = 1 << hash->size_bits; for (i = 0; i < size; i++) { hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) { ret = add_hash_entry(new_hash, entry->ip); if (ret < 0) goto free_hash; } } FTRACE_WARN_ON(new_hash->count != hash->count); return new_hash; free_hash: free_ftrace_hash(new_hash); return NULL; } |
41fb61c2d ftrace: Balance r... |
1106 1107 1108 1109 |
static void ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash); static void ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash); |
33dc9b126 ftrace: Separate ... |
1110 |
static int |
41fb61c2d ftrace: Balance r... |
1111 1112 |
ftrace_hash_move(struct ftrace_ops *ops, int enable, struct ftrace_hash **dst, struct ftrace_hash *src) |
33dc9b126 ftrace: Separate ... |
1113 1114 1115 1116 |
{ struct ftrace_func_entry *entry; struct hlist_node *tp, *tn; struct hlist_head *hhd; |
07fd5515f ftrace: Free hash... |
1117 1118 |
struct ftrace_hash *old_hash; struct ftrace_hash *new_hash; |
33dc9b126 ftrace: Separate ... |
1119 1120 1121 |
unsigned long key; int size = src->count; int bits = 0; |
41fb61c2d ftrace: Balance r... |
1122 |
int ret; |
33dc9b126 ftrace: Separate ... |
1123 1124 1125 |
int i; /* |
41fb61c2d ftrace: Balance r... |
1126 1127 1128 1129 1130 1131 |
* Remove the current set, update the hash and add * them back. */ ftrace_hash_rec_disable(ops, enable); /* |
33dc9b126 ftrace: Separate ... |
1132 1133 1134 1135 |
* If the new source is empty, just free dst and assign it * the empty_hash. */ if (!src->count) { |
07fd5515f ftrace: Free hash... |
1136 1137 |
free_ftrace_hash_rcu(*dst); rcu_assign_pointer(*dst, EMPTY_HASH); |
d4d34b981 ftrace: Fix hash ... |
1138 1139 1140 |
/* still need to update the function records */ ret = 0; goto out; |
33dc9b126 ftrace: Separate ... |
1141 |
} |
33dc9b126 ftrace: Separate ... |
1142 1143 1144 1145 1146 1147 1148 1149 1150 |
/* * Make the hash size about 1/2 the # found */ for (size /= 2; size; size >>= 1) bits++; /* Don't allocate too much */ if (bits > FTRACE_HASH_MAX_BITS) bits = FTRACE_HASH_MAX_BITS; |
41fb61c2d ftrace: Balance r... |
1151 |
ret = -ENOMEM; |
07fd5515f ftrace: Free hash... |
1152 1153 |
new_hash = alloc_ftrace_hash(bits); if (!new_hash) |
41fb61c2d ftrace: Balance r... |
1154 |
goto out; |
33dc9b126 ftrace: Separate ... |
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 |
size = 1 << src->size_bits; for (i = 0; i < size; i++) { hhd = &src->buckets[i]; hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) { if (bits > 0) key = hash_long(entry->ip, bits); else key = 0; remove_hash_entry(src, entry); |
07fd5515f ftrace: Free hash... |
1165 |
__add_hash_entry(new_hash, entry); |
33dc9b126 ftrace: Separate ... |
1166 1167 |
} } |
07fd5515f ftrace: Free hash... |
1168 1169 1170 |
old_hash = *dst; rcu_assign_pointer(*dst, new_hash); free_ftrace_hash_rcu(old_hash); |
41fb61c2d ftrace: Balance r... |
1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 |
ret = 0; out: /* * Enable regardless of ret: * On success, we enable the new hash. * On failure, we re-enable the original hash. */ ftrace_hash_rec_enable(ops, enable); return ret; |
33dc9b126 ftrace: Separate ... |
1181 |
} |
493762fc5 tracing: move fun... |
1182 |
/* |
b848914ce ftrace: Implement... |
1183 1184 1185 1186 1187 1188 1189 |
* Test the hashes for this ops to see if we want to call * the ops->func or not. * * It's a match if the ip is in the ops->filter_hash or * the filter_hash does not exist or is empty, * AND * the ip is not in the ops->notrace_hash. |
cdbe61bfe ftrace: Allow dyn... |
1190 1191 1192 |
* * This needs to be called with preemption disabled as * the hashes are freed with call_rcu_sched(). |
b848914ce ftrace: Implement... |
1193 1194 1195 1196 1197 1198 1199 |
*/ static int ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip) { struct ftrace_hash *filter_hash; struct ftrace_hash *notrace_hash; int ret; |
b848914ce ftrace: Implement... |
1200 1201 |
filter_hash = rcu_dereference_raw(ops->filter_hash); notrace_hash = rcu_dereference_raw(ops->notrace_hash); |
06a51d930 ftrace: Create ft... |
1202 |
if ((ftrace_hash_empty(filter_hash) || |
b848914ce ftrace: Implement... |
1203 |
ftrace_lookup_ip(filter_hash, ip)) && |
06a51d930 ftrace: Create ft... |
1204 |
(ftrace_hash_empty(notrace_hash) || |
b848914ce ftrace: Implement... |
1205 1206 1207 1208 |
!ftrace_lookup_ip(notrace_hash, ip))) ret = 1; else ret = 0; |
b848914ce ftrace: Implement... |
1209 1210 1211 1212 1213 |
return ret; } /* |
493762fc5 tracing: move fun... |
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 |
* This is a double for. Do not use 'break' to break out of the loop, * you must use a goto. */ #define do_for_each_ftrace_rec(pg, rec) \ for (pg = ftrace_pages_start; pg; pg = pg->next) { \ int _____i; \ for (_____i = 0; _____i < pg->index; _____i++) { \ rec = &pg->records[_____i]; #define while_for_each_ftrace_rec() \ } \ } |
5855fead9 ftrace: Use bsear... |
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 |
static int ftrace_cmp_recs(const void *a, const void *b) { const struct dyn_ftrace *reca = a; const struct dyn_ftrace *recb = b; if (reca->ip > recb->ip) return 1; if (reca->ip < recb->ip) return -1; return 0; } |
c88fd8634 ftrace: Allow arc... |
1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 |
/** * ftrace_location - return true if the ip giving is a traced location * @ip: the instruction pointer to check * * Returns 1 if @ip given is a pointer to a ftrace location. * That is, the instruction that is either a NOP or call to * the function tracer. It checks the ftrace internal tables to * determine if the address belongs or not. */ int ftrace_location(unsigned long ip) { struct ftrace_page *pg; struct dyn_ftrace *rec; |
5855fead9 ftrace: Use bsear... |
1251 |
struct dyn_ftrace key; |
c88fd8634 ftrace: Allow arc... |
1252 |
|
5855fead9 ftrace: Use bsear... |
1253 1254 1255 1256 1257 1258 1259 |
key.ip = ip; for (pg = ftrace_pages_start; pg; pg = pg->next) { rec = bsearch(&key, pg->records, pg->index, sizeof(struct dyn_ftrace), ftrace_cmp_recs); if (rec) |
c88fd8634 ftrace: Allow arc... |
1260 |
return 1; |
5855fead9 ftrace: Use bsear... |
1261 |
} |
c88fd8634 ftrace: Allow arc... |
1262 1263 1264 |
return 0; } |
ed926f9b3 ftrace: Use count... |
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 |
static void __ftrace_hash_rec_update(struct ftrace_ops *ops, int filter_hash, bool inc) { struct ftrace_hash *hash; struct ftrace_hash *other_hash; struct ftrace_page *pg; struct dyn_ftrace *rec; int count = 0; int all = 0; /* Only update if the ops has been registered */ if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) return; /* * In the filter_hash case: * If the count is zero, we update all records. * Otherwise we just update the items in the hash. * * In the notrace_hash case: * We enable the update in the hash. * As disabling notrace means enabling the tracing, * and enabling notrace means disabling, the inc variable * gets inversed. */ if (filter_hash) { hash = ops->filter_hash; other_hash = ops->notrace_hash; |
06a51d930 ftrace: Create ft... |
1294 |
if (ftrace_hash_empty(hash)) |
ed926f9b3 ftrace: Use count... |
1295 1296 1297 1298 1299 1300 1301 1302 1303 |
all = 1; } else { inc = !inc; hash = ops->notrace_hash; other_hash = ops->filter_hash; /* * If the notrace hash has no items, * then there's nothing to do. */ |
06a51d930 ftrace: Create ft... |
1304 |
if (ftrace_hash_empty(hash)) |
ed926f9b3 ftrace: Use count... |
1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 |
return; } do_for_each_ftrace_rec(pg, rec) { int in_other_hash = 0; int in_hash = 0; int match = 0; if (all) { /* * Only the filter_hash affects all records. * Update if the record is not in the notrace hash. */ |
b848914ce ftrace: Implement... |
1318 |
if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) |
ed926f9b3 ftrace: Use count... |
1319 1320 |
match = 1; } else { |
06a51d930 ftrace: Create ft... |
1321 1322 |
in_hash = !!ftrace_lookup_ip(hash, rec->ip); in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); |
ed926f9b3 ftrace: Use count... |
1323 1324 1325 1326 1327 1328 1329 |
/* * */ if (filter_hash && in_hash && !in_other_hash) match = 1; else if (!filter_hash && in_hash && |
06a51d930 ftrace: Create ft... |
1330 |
(in_other_hash || ftrace_hash_empty(other_hash))) |
ed926f9b3 ftrace: Use count... |
1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 |
match = 1; } if (!match) continue; if (inc) { rec->flags++; if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX)) return; } else { if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0)) return; rec->flags--; } count++; /* Shortcut, if we handled all records, we are done. */ if (!all && count == hash->count) return; } while_for_each_ftrace_rec(); } static void ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash) { __ftrace_hash_rec_update(ops, filter_hash, 0); } static void ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash) { __ftrace_hash_rec_update(ops, filter_hash, 1); } |
e309b41dd ftrace: remove no... |
1363 |
static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) |
3c1720f00 ftrace: move memo... |
1364 |
{ |
a79008755 ftrace: Allocate ... |
1365 1366 1367 1368 |
if (ftrace_pages->index == ftrace_pages->size) { /* We should have allocated enough */ if (WARN_ON(!ftrace_pages->next)) return NULL; |
3c1720f00 ftrace: move memo... |
1369 1370 1371 1372 1373 |
ftrace_pages = ftrace_pages->next; } return &ftrace_pages->records[ftrace_pages->index++]; } |
08f5ac906 ftrace: remove ft... |
1374 |
static struct dyn_ftrace * |
d61f82d06 ftrace: use dynam... |
1375 |
ftrace_record_ip(unsigned long ip) |
3d0833953 ftrace: dynamic e... |
1376 |
{ |
08f5ac906 ftrace: remove ft... |
1377 |
struct dyn_ftrace *rec; |
3d0833953 ftrace: dynamic e... |
1378 |
|
f3c7ac40a ftrace: remove co... |
1379 |
if (ftrace_disabled) |
08f5ac906 ftrace: remove ft... |
1380 |
return NULL; |
3d0833953 ftrace: dynamic e... |
1381 |
|
08f5ac906 ftrace: remove ft... |
1382 1383 1384 |
rec = ftrace_alloc_dyn_node(ip); if (!rec) return NULL; |
3d0833953 ftrace: dynamic e... |
1385 |
|
08f5ac906 ftrace: remove ft... |
1386 |
rec->ip = ip; |
3d0833953 ftrace: dynamic e... |
1387 |
|
08f5ac906 ftrace: remove ft... |
1388 |
return rec; |
3d0833953 ftrace: dynamic e... |
1389 |
} |
b17e8a37a ftrace: disable f... |
1390 1391 1392 1393 1394 1395 1396 1397 1398 |
static void print_ip_ins(const char *fmt, unsigned char *p) { int i; printk(KERN_CONT "%s", fmt); for (i = 0; i < MCOUNT_INSN_SIZE; i++) printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); } |
c88fd8634 ftrace: Allow arc... |
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 |
/** * ftrace_bug - report and shutdown function tracer * @failed: The failed type (EFAULT, EINVAL, EPERM) * @ip: The address that failed * * The arch code that enables or disables the function tracing * can call ftrace_bug() when it has detected a problem in * modifying the code. @failed should be one of either: * EFAULT - if the problem happens on reading the @ip address * EINVAL - if what is read at @ip is not what was expected * EPERM - if the problem happens on writting to the @ip address */ void ftrace_bug(int failed, unsigned long ip) |
b17e8a37a ftrace: disable f... |
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 |
{ switch (failed) { case -EFAULT: FTRACE_WARN_ON_ONCE(1); pr_info("ftrace faulted on modifying "); print_ip_sym(ip); break; case -EINVAL: FTRACE_WARN_ON_ONCE(1); pr_info("ftrace failed to modify "); print_ip_sym(ip); |
b17e8a37a ftrace: disable f... |
1423 |
print_ip_ins(" actual: ", (unsigned char *)ip); |
b17e8a37a ftrace: disable f... |
1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 |
printk(KERN_CONT " "); break; case -EPERM: FTRACE_WARN_ON_ONCE(1); pr_info("ftrace faulted on writing "); print_ip_sym(ip); break; default: FTRACE_WARN_ON_ONCE(1); pr_info("ftrace faulted on unknown error "); print_ip_sym(ip); } } |
3c1720f00 ftrace: move memo... |
1438 |
|
2cfa19780 ftrace/alternativ... |
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 |
/* Return 1 if the address range is reserved for ftrace */ int ftrace_text_reserved(void *start, void *end) { struct dyn_ftrace *rec; struct ftrace_page *pg; do_for_each_ftrace_rec(pg, rec) { if (rec->ip <= (unsigned long)end && rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start) return 1; } while_for_each_ftrace_rec(); return 0; } |
c88fd8634 ftrace: Allow arc... |
1452 |
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) |
5072c59fd ftrace: add filte... |
1453 |
{ |
64fbcd162 tracing/function:... |
1454 |
unsigned long flag = 0UL; |
e7d3737ea tracing/function-... |
1455 |
|
982c350b9 ftrace: fix dyn f... |
1456 |
/* |
30fb6aa74 ftrace: Fix unreg... |
1457 |
* If we are updating calls: |
982c350b9 ftrace: fix dyn f... |
1458 |
* |
ed926f9b3 ftrace: Use count... |
1459 1460 |
* If the record has a ref count, then we need to enable it * because someone is using it. |
982c350b9 ftrace: fix dyn f... |
1461 |
* |
ed926f9b3 ftrace: Use count... |
1462 1463 |
* Otherwise we make sure its disabled. * |
30fb6aa74 ftrace: Fix unreg... |
1464 |
* If we are disabling calls, then disable all records that |
ed926f9b3 ftrace: Use count... |
1465 |
* are enabled. |
982c350b9 ftrace: fix dyn f... |
1466 |
*/ |
c88fd8634 ftrace: Allow arc... |
1467 |
if (enable && (rec->flags & ~FTRACE_FL_MASK)) |
ed926f9b3 ftrace: Use count... |
1468 |
flag = FTRACE_FL_ENABLED; |
982c350b9 ftrace: fix dyn f... |
1469 |
|
64fbcd162 tracing/function:... |
1470 1471 |
/* If the state of this record hasn't changed, then do nothing */ if ((rec->flags & FTRACE_FL_ENABLED) == flag) |
c88fd8634 ftrace: Allow arc... |
1472 |
return FTRACE_UPDATE_IGNORE; |
982c350b9 ftrace: fix dyn f... |
1473 |
|
64fbcd162 tracing/function:... |
1474 |
if (flag) { |
c88fd8634 ftrace: Allow arc... |
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 |
if (update) rec->flags |= FTRACE_FL_ENABLED; return FTRACE_UPDATE_MAKE_CALL; } if (update) rec->flags &= ~FTRACE_FL_ENABLED; return FTRACE_UPDATE_MAKE_NOP; } /** * ftrace_update_record, set a record that now is tracing or not * @rec: the record to update * @enable: set to 1 if the record is tracing, zero to force disable * * The records that represent all functions that can be traced need * to be updated when tracing has been enabled. */ int ftrace_update_record(struct dyn_ftrace *rec, int enable) { return ftrace_check_record(rec, enable, 1); } /** * ftrace_test_record, check if the record has been enabled or not * @rec: the record to test * @enable: set to 1 to check if enabled, 0 if it is disabled * * The arch code may need to test if a record is already set to * tracing to determine how to modify the function code that it * represents. */ int ftrace_test_record(struct dyn_ftrace *rec, int enable) { return ftrace_check_record(rec, enable, 0); } static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable) { unsigned long ftrace_addr; int ret; ftrace_addr = (unsigned long)FTRACE_ADDR; ret = ftrace_update_record(rec, enable); switch (ret) { case FTRACE_UPDATE_IGNORE: return 0; case FTRACE_UPDATE_MAKE_CALL: |
64fbcd162 tracing/function:... |
1528 |
return ftrace_make_call(rec, ftrace_addr); |
c88fd8634 ftrace: Allow arc... |
1529 1530 1531 |
case FTRACE_UPDATE_MAKE_NOP: return ftrace_make_nop(NULL, rec, ftrace_addr); |
5072c59fd ftrace: add filte... |
1532 |
} |
c88fd8634 ftrace: Allow arc... |
1533 |
return -1; /* unknow ftrace bug */ |
5072c59fd ftrace: add filte... |
1534 |
} |
30fb6aa74 ftrace: Fix unreg... |
1535 |
static void ftrace_replace_code(int update) |
3c1720f00 ftrace: move memo... |
1536 |
{ |
3c1720f00 ftrace: move memo... |
1537 1538 |
struct dyn_ftrace *rec; struct ftrace_page *pg; |
6a24a244c ftrace: clean up ... |
1539 |
int failed; |
3c1720f00 ftrace: move memo... |
1540 |
|
45a4a2372 ftrace: Remove FT... |
1541 1542 |
if (unlikely(ftrace_disabled)) return; |
265c831cb ftrace: add do_fo... |
1543 |
do_for_each_ftrace_rec(pg, rec) { |
30fb6aa74 ftrace: Fix unreg... |
1544 |
failed = __ftrace_replace_code(rec, update); |
fa9d13cf1 ftrace: don't try... |
1545 |
if (failed) { |
3279ba37d ftrace: check for... |
1546 1547 1548 |
ftrace_bug(failed, rec->ip); /* Stop processing */ return; |
3c1720f00 ftrace: move memo... |
1549 |
} |
265c831cb ftrace: add do_fo... |
1550 |
} while_for_each_ftrace_rec(); |
3c1720f00 ftrace: move memo... |
1551 |
} |
c88fd8634 ftrace: Allow arc... |
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 |
struct ftrace_rec_iter { struct ftrace_page *pg; int index; }; /** * ftrace_rec_iter_start, start up iterating over traced functions * * Returns an iterator handle that is used to iterate over all * the records that represent address locations where functions * are traced. * * May return NULL if no records are available. */ struct ftrace_rec_iter *ftrace_rec_iter_start(void) { /* * We only use a single iterator. * Protected by the ftrace_lock mutex. */ static struct ftrace_rec_iter ftrace_rec_iter; struct ftrace_rec_iter *iter = &ftrace_rec_iter; iter->pg = ftrace_pages_start; iter->index = 0; /* Could have empty pages */ while (iter->pg && !iter->pg->index) iter->pg = iter->pg->next; if (!iter->pg) return NULL; return iter; } /** * ftrace_rec_iter_next, get the next record to process. * @iter: The handle to the iterator. * * Returns the next iterator after the given iterator @iter. */ struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter) { iter->index++; if (iter->index >= iter->pg->index) { iter->pg = iter->pg->next; iter->index = 0; /* Could have empty pages */ while (iter->pg && !iter->pg->index) iter->pg = iter->pg->next; } if (!iter->pg) return NULL; return iter; } /** * ftrace_rec_iter_record, get the record at the iterator location * @iter: The current iterator location * * Returns the record that the current @iter is at. */ struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter) { return &iter->pg->records[iter->index]; } |
492a7ea5b ftrace: fix updat... |
1623 |
static int |
31e889098 ftrace: pass modu... |
1624 |
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) |
3c1720f00 ftrace: move memo... |
1625 1626 |
{ unsigned long ip; |
593eb8a2d ftrace: return er... |
1627 |
int ret; |
3c1720f00 ftrace: move memo... |
1628 1629 |
ip = rec->ip; |
45a4a2372 ftrace: Remove FT... |
1630 1631 |
if (unlikely(ftrace_disabled)) return 0; |
25aac9dc7 ftrace, ia64: exp... |
1632 |
ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); |
593eb8a2d ftrace: return er... |
1633 |
if (ret) { |
31e889098 ftrace: pass modu... |
1634 |
ftrace_bug(ret, ip); |
492a7ea5b ftrace: fix updat... |
1635 |
return 0; |
37ad50841 ftrace - fix dyna... |
1636 |
} |
492a7ea5b ftrace: fix updat... |
1637 |
return 1; |
3c1720f00 ftrace: move memo... |
1638 |
} |
000ab6911 ftrace: allow arc... |
1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 |
/* * archs can override this function if they must do something * before the modifying code is performed. */ int __weak ftrace_arch_code_modify_prepare(void) { return 0; } /* * archs can override this function if they must do something * after the modifying code is performed. */ int __weak ftrace_arch_code_modify_post_process(void) { return 0; } |
e309b41dd ftrace: remove no... |
1656 |
static int __ftrace_modify_code(void *data) |
3d0833953 ftrace: dynamic e... |
1657 |
{ |
d61f82d06 ftrace: use dynam... |
1658 |
int *command = data; |
30fb6aa74 ftrace: Fix unreg... |
1659 |
if (*command & FTRACE_UPDATE_CALLS) |
d61f82d06 ftrace: use dynam... |
1660 |
ftrace_replace_code(1); |
a35832446 ring-buffer: buff... |
1661 |
else if (*command & FTRACE_DISABLE_CALLS) |
d61f82d06 ftrace: use dynam... |
1662 1663 1664 1665 |
ftrace_replace_code(0); if (*command & FTRACE_UPDATE_TRACE_FUNC) ftrace_update_ftrace_func(ftrace_trace_function); |
5a45cfe1c ftrace: use code ... |
1666 1667 1668 1669 |
if (*command & FTRACE_START_FUNC_RET) ftrace_enable_ftrace_graph_caller(); else if (*command & FTRACE_STOP_FUNC_RET) ftrace_disable_ftrace_graph_caller(); |
d61f82d06 ftrace: use dynam... |
1670 |
return 0; |
3d0833953 ftrace: dynamic e... |
1671 |
} |
c88fd8634 ftrace: Allow arc... |
1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 |
/** * ftrace_run_stop_machine, go back to the stop machine method * @command: The command to tell ftrace what to do * * If an arch needs to fall back to the stop machine method, the * it can call this function. */ void ftrace_run_stop_machine(int command) { stop_machine(__ftrace_modify_code, &command, NULL); } /** * arch_ftrace_update_code, modify the code to trace or not trace * @command: The command that needs to be done * * Archs can override this function if it does not need to * run stop_machine() to modify code. */ void __weak arch_ftrace_update_code(int command) { ftrace_run_stop_machine(command); } |
e309b41dd ftrace: remove no... |
1695 |
static void ftrace_run_update_code(int command) |
3d0833953 ftrace: dynamic e... |
1696 |
{ |
000ab6911 ftrace: allow arc... |
1697 1698 1699 1700 1701 1702 |
int ret; ret = ftrace_arch_code_modify_prepare(); FTRACE_WARN_ON(ret); if (ret) return; |
c88fd8634 ftrace: Allow arc... |
1703 1704 1705 1706 1707 |
/* * Do not call function tracer while we update the code. * We are in stop machine. */ function_trace_stop++; |
000ab6911 ftrace: allow arc... |
1708 |
|
c88fd8634 ftrace: Allow arc... |
1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 |
/* * By default we use stop_machine() to modify the code. * But archs can do what ever they want as long as it * is safe. The stop_machine() is the safest, but also * produces the most overhead. */ arch_ftrace_update_code(command); #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST /* * For archs that call ftrace_test_stop_func(), we must * wait till after we update all the function callers * before we update the callback. This keeps different * ops that record different functions from corrupting * each other. */ __ftrace_trace_function = __ftrace_trace_function_delay; #endif function_trace_stop--; |
000ab6911 ftrace: allow arc... |
1728 1729 1730 |
ret = ftrace_arch_code_modify_post_process(); FTRACE_WARN_ON(ret); |
3d0833953 ftrace: dynamic e... |
1731 |
} |
d61f82d06 ftrace: use dynam... |
1732 |
static ftrace_func_t saved_ftrace_func; |
60a7ecf42 ftrace: add quick... |
1733 |
static int ftrace_start_up; |
b848914ce ftrace: Implement... |
1734 |
static int global_start_up; |
df4fc3155 ftrace: add funct... |
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 |
static void ftrace_startup_enable(int command) { if (saved_ftrace_func != ftrace_trace_function) { saved_ftrace_func = ftrace_trace_function; command |= FTRACE_UPDATE_TRACE_FUNC; } if (!command || !ftrace_enabled) return; ftrace_run_update_code(command); } |
d61f82d06 ftrace: use dynam... |
1748 |
|
a1cd61735 ftrace: Have ftra... |
1749 |
static int ftrace_startup(struct ftrace_ops *ops, int command) |
3d0833953 ftrace: dynamic e... |
1750 |
{ |
b848914ce ftrace: Implement... |
1751 |
bool hash_enable = true; |
4eebcc81a ftrace: disable t... |
1752 |
if (unlikely(ftrace_disabled)) |
a1cd61735 ftrace: Have ftra... |
1753 |
return -ENODEV; |
4eebcc81a ftrace: disable t... |
1754 |
|
60a7ecf42 ftrace: add quick... |
1755 |
ftrace_start_up++; |
30fb6aa74 ftrace: Fix unreg... |
1756 |
command |= FTRACE_UPDATE_CALLS; |
d61f82d06 ftrace: use dynam... |
1757 |
|
b848914ce ftrace: Implement... |
1758 1759 1760 1761 1762 1763 1764 1765 |
/* ops marked global share the filter hashes */ if (ops->flags & FTRACE_OPS_FL_GLOBAL) { ops = &global_ops; /* Don't update hash if global is already set */ if (global_start_up) hash_enable = false; global_start_up++; } |
ed926f9b3 ftrace: Use count... |
1766 |
ops->flags |= FTRACE_OPS_FL_ENABLED; |
b848914ce ftrace: Implement... |
1767 |
if (hash_enable) |
ed926f9b3 ftrace: Use count... |
1768 |
ftrace_hash_rec_enable(ops, 1); |
df4fc3155 ftrace: add funct... |
1769 |
ftrace_startup_enable(command); |
a1cd61735 ftrace: Have ftra... |
1770 1771 |
return 0; |
3d0833953 ftrace: dynamic e... |
1772 |
} |
bd69c30b1 ftrace: Add ops p... |
1773 |
static void ftrace_shutdown(struct ftrace_ops *ops, int command) |
3d0833953 ftrace: dynamic e... |
1774 |
{ |
b848914ce ftrace: Implement... |
1775 |
bool hash_disable = true; |
4eebcc81a ftrace: disable t... |
1776 1777 |
if (unlikely(ftrace_disabled)) return; |
60a7ecf42 ftrace: add quick... |
1778 |
ftrace_start_up--; |
9ea1a153a tracing/urgent: w... |
1779 1780 1781 1782 1783 1784 |
/* * Just warn in case of unbalance, no need to kill ftrace, it's not * critical but the ftrace_call callers may be never nopped again after * further ftrace uses. */ WARN_ON_ONCE(ftrace_start_up < 0); |
b848914ce ftrace: Implement... |
1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 |
if (ops->flags & FTRACE_OPS_FL_GLOBAL) { ops = &global_ops; global_start_up--; WARN_ON_ONCE(global_start_up < 0); /* Don't update hash if global still has users */ if (global_start_up) { WARN_ON_ONCE(!ftrace_start_up); hash_disable = false; } } if (hash_disable) |
ed926f9b3 ftrace: Use count... |
1797 |
ftrace_hash_rec_disable(ops, 1); |
b848914ce ftrace: Implement... |
1798 |
if (ops != &global_ops || !global_start_up) |
ed926f9b3 ftrace: Use count... |
1799 |
ops->flags &= ~FTRACE_OPS_FL_ENABLED; |
b848914ce ftrace: Implement... |
1800 |
|
30fb6aa74 ftrace: Fix unreg... |
1801 |
command |= FTRACE_UPDATE_CALLS; |
3d0833953 ftrace: dynamic e... |
1802 |
|
d61f82d06 ftrace: use dynam... |
1803 1804 1805 1806 |
if (saved_ftrace_func != ftrace_trace_function) { saved_ftrace_func = ftrace_trace_function; command |= FTRACE_UPDATE_TRACE_FUNC; } |
3d0833953 ftrace: dynamic e... |
1807 |
|
d61f82d06 ftrace: use dynam... |
1808 |
if (!command || !ftrace_enabled) |
e6ea44e9b ftrace: consolida... |
1809 |
return; |
d61f82d06 ftrace: use dynam... |
1810 1811 |
ftrace_run_update_code(command); |
3d0833953 ftrace: dynamic e... |
1812 |
} |
e309b41dd ftrace: remove no... |
1813 |
static void ftrace_startup_sysctl(void) |
b0fc494fa ftrace: add ftrac... |
1814 |
{ |
4eebcc81a ftrace: disable t... |
1815 1816 |
if (unlikely(ftrace_disabled)) return; |
d61f82d06 ftrace: use dynam... |
1817 1818 |
/* Force update next time */ saved_ftrace_func = NULL; |
60a7ecf42 ftrace: add quick... |
1819 1820 |
/* ftrace_start_up is true if we want ftrace running */ if (ftrace_start_up) |
30fb6aa74 ftrace: Fix unreg... |
1821 |
ftrace_run_update_code(FTRACE_UPDATE_CALLS); |
b0fc494fa ftrace: add ftrac... |
1822 |
} |
e309b41dd ftrace: remove no... |
1823 |
static void ftrace_shutdown_sysctl(void) |
b0fc494fa ftrace: add ftrac... |
1824 |
{ |
4eebcc81a ftrace: disable t... |
1825 1826 |
if (unlikely(ftrace_disabled)) return; |
60a7ecf42 ftrace: add quick... |
1827 1828 |
/* ftrace_start_up is true if ftrace is running */ if (ftrace_start_up) |
79e406d7b tracing: Remove l... |
1829 |
ftrace_run_update_code(FTRACE_DISABLE_CALLS); |
b0fc494fa ftrace: add ftrac... |
1830 |
} |
3d0833953 ftrace: dynamic e... |
1831 1832 1833 |
static cycle_t ftrace_update_time; static unsigned long ftrace_update_cnt; unsigned long ftrace_update_tot_cnt; |
f7bc8b61f ftrace: Fix regre... |
1834 1835 1836 1837 1838 |
static int ops_traces_mod(struct ftrace_ops *ops) { struct ftrace_hash *hash; hash = ops->filter_hash; |
06a51d930 ftrace: Create ft... |
1839 |
return ftrace_hash_empty(hash); |
f7bc8b61f ftrace: Fix regre... |
1840 |
} |
31e889098 ftrace: pass modu... |
1841 |
static int ftrace_update_code(struct module *mod) |
3d0833953 ftrace: dynamic e... |
1842 |
{ |
85ae32ae0 ftrace: Replace r... |
1843 |
struct ftrace_page *pg; |
e94142a67 ftrace: remove st... |
1844 |
struct dyn_ftrace *p; |
f22f9a89c ftrace: avoid mod... |
1845 |
cycle_t start, stop; |
f7bc8b61f ftrace: Fix regre... |
1846 |
unsigned long ref = 0; |
85ae32ae0 ftrace: Replace r... |
1847 |
int i; |
f7bc8b61f ftrace: Fix regre... |
1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 |
/* * When adding a module, we need to check if tracers are * currently enabled and if they are set to trace all functions. * If they are, we need to enable the module functions as well * as update the reference counts for those function records. */ if (mod) { struct ftrace_ops *ops; for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { if (ops->flags & FTRACE_OPS_FL_ENABLED && ops_traces_mod(ops)) ref++; } } |
3d0833953 ftrace: dynamic e... |
1865 |
|
750ed1a40 ftrace: timestamp... |
1866 |
start = ftrace_now(raw_smp_processor_id()); |
3d0833953 ftrace: dynamic e... |
1867 |
ftrace_update_cnt = 0; |
85ae32ae0 ftrace: Replace r... |
1868 |
for (pg = ftrace_new_pgs; pg; pg = pg->next) { |
3d0833953 ftrace: dynamic e... |
1869 |
|
85ae32ae0 ftrace: Replace r... |
1870 1871 1872 1873 |
for (i = 0; i < pg->index; i++) { /* If something went wrong, bail without enabling anything */ if (unlikely(ftrace_disabled)) return -1; |
f22f9a89c ftrace: avoid mod... |
1874 |
|
85ae32ae0 ftrace: Replace r... |
1875 1876 |
p = &pg->records[i]; p->flags = ref; |
f22f9a89c ftrace: avoid mod... |
1877 |
|
85ae32ae0 ftrace: Replace r... |
1878 1879 1880 1881 1882 1883 |
/* * Do the initial record conversion from mcount jump * to the NOP instructions. */ if (!ftrace_code_disable(mod, p)) break; |
5cb084bb1 tracing: Enable r... |
1884 |
|
85ae32ae0 ftrace: Replace r... |
1885 |
ftrace_update_cnt++; |
5cb084bb1 tracing: Enable r... |
1886 |
|
85ae32ae0 ftrace: Replace r... |
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 |
/* * If the tracing is enabled, go ahead and enable the record. * * The reason not to enable the record immediatelly is the * inherent check of ftrace_make_nop/ftrace_make_call for * correct previous instructions. Making first the NOP * conversion puts the module to the correct state, thus * passing the ftrace_make_call check. */ if (ftrace_start_up && ref) { int failed = __ftrace_replace_code(p, 1); if (failed) ftrace_bug(failed, p->ip); } |
5cb084bb1 tracing: Enable r... |
1901 |
} |
3d0833953 ftrace: dynamic e... |
1902 |
} |
85ae32ae0 ftrace: Replace r... |
1903 |
ftrace_new_pgs = NULL; |
750ed1a40 ftrace: timestamp... |
1904 |
stop = ftrace_now(raw_smp_processor_id()); |
3d0833953 ftrace: dynamic e... |
1905 1906 |
ftrace_update_time = stop - start; ftrace_update_tot_cnt += ftrace_update_cnt; |
16444a8a4 ftrace: add basic... |
1907 1908 |
return 0; } |
a79008755 ftrace: Allocate ... |
1909 |
static int ftrace_allocate_records(struct ftrace_page *pg, int count) |
3c1720f00 ftrace: move memo... |
1910 |
{ |
a79008755 ftrace: Allocate ... |
1911 |
int order; |
3c1720f00 ftrace: move memo... |
1912 |
int cnt; |
3c1720f00 ftrace: move memo... |
1913 |
|
a79008755 ftrace: Allocate ... |
1914 1915 1916 1917 |
if (WARN_ON(!count)) return -EINVAL; order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE)); |
3c1720f00 ftrace: move memo... |
1918 1919 |
/* |
a79008755 ftrace: Allocate ... |
1920 1921 |
* We want to fill as much as possible. No more than a page * may be empty. |
3c1720f00 ftrace: move memo... |
1922 |
*/ |
a79008755 ftrace: Allocate ... |
1923 1924 |
while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE) order--; |
3c1720f00 ftrace: move memo... |
1925 |
|
a79008755 ftrace: Allocate ... |
1926 1927 |
again: pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); |
3c1720f00 ftrace: move memo... |
1928 |
|
a79008755 ftrace: Allocate ... |
1929 1930 1931 1932 1933 1934 1935 |
if (!pg->records) { /* if we can't allocate this size, try something smaller */ if (!order) return -ENOMEM; order >>= 1; goto again; } |
3c1720f00 ftrace: move memo... |
1936 |
|
a79008755 ftrace: Allocate ... |
1937 1938 |
cnt = (PAGE_SIZE << order) / ENTRY_SIZE; pg->size = cnt; |
3c1720f00 ftrace: move memo... |
1939 |
|
a79008755 ftrace: Allocate ... |
1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 |
if (cnt > count) cnt = count; return cnt; } static struct ftrace_page * ftrace_allocate_pages(unsigned long num_to_init) { struct ftrace_page *start_pg; struct ftrace_page *pg; int order; int cnt; if (!num_to_init) return 0; start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); if (!pg) return NULL; /* * Try to allocate as much as possible in one continues * location that fills in all of the space. We want to * waste as little space as possible. */ for (;;) { cnt = ftrace_allocate_records(pg, num_to_init); if (cnt < 0) goto free_pages; num_to_init -= cnt; if (!num_to_init) |
3c1720f00 ftrace: move memo... |
1973 |
break; |
a79008755 ftrace: Allocate ... |
1974 1975 1976 |
pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); if (!pg->next) goto free_pages; |
3c1720f00 ftrace: move memo... |
1977 1978 |
pg = pg->next; } |
a79008755 ftrace: Allocate ... |
1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 |
return start_pg; free_pages: while (start_pg) { order = get_count_order(pg->size / ENTRIES_PER_PAGE); free_pages((unsigned long)pg->records, order); start_pg = pg->next; kfree(pg); pg = start_pg; } pr_info("ftrace: FAILED to allocate memory for functions "); return NULL; } static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) { int cnt; if (!num_to_init) { pr_info("ftrace: No functions to be traced? "); return -1; } cnt = num_to_init / ENTRIES_PER_PAGE; pr_info("ftrace: allocating %ld entries in %d pages ", num_to_init, cnt + 1); |
3c1720f00 ftrace: move memo... |
2008 2009 |
return 0; } |
5072c59fd ftrace: add filte... |
2010 2011 2012 |
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ struct ftrace_iterator { |
98c4fd046 tracing: Keep tra... |
2013 |
loff_t pos; |
4aeb69672 tracing: Replace ... |
2014 2015 2016 2017 2018 |
loff_t func_pos; struct ftrace_page *pg; struct dyn_ftrace *func; struct ftrace_func_probe *probe; struct trace_parser parser; |
1cf41dd79 ftrace: Use hash ... |
2019 |
struct ftrace_hash *hash; |
33dc9b126 ftrace: Separate ... |
2020 |
struct ftrace_ops *ops; |
4aeb69672 tracing: Replace ... |
2021 2022 2023 |
int hidx; int idx; unsigned flags; |
5072c59fd ftrace: add filte... |
2024 |
}; |
e309b41dd ftrace: remove no... |
2025 |
static void * |
4aeb69672 tracing: Replace ... |
2026 |
t_hash_next(struct seq_file *m, loff_t *pos) |
8fc0c701c ftrace: show sele... |
2027 2028 |
{ struct ftrace_iterator *iter = m->private; |
4aeb69672 tracing: Replace ... |
2029 |
struct hlist_node *hnd = NULL; |
8fc0c701c ftrace: show sele... |
2030 |
struct hlist_head *hhd; |
8fc0c701c ftrace: show sele... |
2031 |
(*pos)++; |
98c4fd046 tracing: Keep tra... |
2032 |
iter->pos = *pos; |
8fc0c701c ftrace: show sele... |
2033 |
|
4aeb69672 tracing: Replace ... |
2034 2035 |
if (iter->probe) hnd = &iter->probe->node; |
8fc0c701c ftrace: show sele... |
2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 |
retry: if (iter->hidx >= FTRACE_FUNC_HASHSIZE) return NULL; hhd = &ftrace_func_hash[iter->hidx]; if (hlist_empty(hhd)) { iter->hidx++; hnd = NULL; goto retry; } if (!hnd) hnd = hhd->first; else { hnd = hnd->next; if (!hnd) { iter->hidx++; goto retry; } } |
4aeb69672 tracing: Replace ... |
2057 2058 2059 2060 2061 2062 |
if (WARN_ON_ONCE(!hnd)) return NULL; iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node); return iter; |
8fc0c701c ftrace: show sele... |
2063 2064 2065 2066 2067 2068 |
} static void *t_hash_start(struct seq_file *m, loff_t *pos) { struct ftrace_iterator *iter = m->private; void *p = NULL; |
d82d62444 ftrace: Fix t_has... |
2069 |
loff_t l; |
69a3083c4 ftrace: Decouple ... |
2070 2071 |
if (!(iter->flags & FTRACE_ITER_DO_HASH)) return NULL; |
2bccfffd1 tracing: Do not r... |
2072 2073 |
if (iter->func_pos > *pos) return NULL; |
8fc0c701c ftrace: show sele... |
2074 |
|
d82d62444 ftrace: Fix t_has... |
2075 |
iter->hidx = 0; |
2bccfffd1 tracing: Do not r... |
2076 |
for (l = 0; l <= (*pos - iter->func_pos); ) { |
4aeb69672 tracing: Replace ... |
2077 |
p = t_hash_next(m, &l); |
d82d62444 ftrace: Fix t_has... |
2078 2079 2080 |
if (!p) break; } |
4aeb69672 tracing: Replace ... |
2081 2082 |
if (!p) return NULL; |
98c4fd046 tracing: Keep tra... |
2083 2084 |
/* Only set this if we have an item */ iter->flags |= FTRACE_ITER_HASH; |
4aeb69672 tracing: Replace ... |
2085 |
return iter; |
8fc0c701c ftrace: show sele... |
2086 |
} |
4aeb69672 tracing: Replace ... |
2087 2088 |
static int t_hash_show(struct seq_file *m, struct ftrace_iterator *iter) |
8fc0c701c ftrace: show sele... |
2089 |
{ |
b6887d791 ftrace: rename _h... |
2090 |
struct ftrace_func_probe *rec; |
8fc0c701c ftrace: show sele... |
2091 |
|
4aeb69672 tracing: Replace ... |
2092 2093 2094 |
rec = iter->probe; if (WARN_ON_ONCE(!rec)) return -EIO; |
8fc0c701c ftrace: show sele... |
2095 |
|
809dcf29c ftrace: add prett... |
2096 2097 |
if (rec->ops->print) return rec->ops->print(m, rec->ip, rec->ops, rec->data); |
b375a11a2 tracing: switch f... |
2098 |
seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); |
8fc0c701c ftrace: show sele... |
2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 |
if (rec->data) seq_printf(m, ":%p", rec->data); seq_putc(m, ' '); return 0; } static void * |
5072c59fd ftrace: add filte... |
2109 2110 2111 |
t_next(struct seq_file *m, void *v, loff_t *pos) { struct ftrace_iterator *iter = m->private; |
fc13cb0ce ftrace: Allow oth... |
2112 |
struct ftrace_ops *ops = iter->ops; |
5072c59fd ftrace: add filte... |
2113 |
struct dyn_ftrace *rec = NULL; |
45a4a2372 ftrace: Remove FT... |
2114 2115 |
if (unlikely(ftrace_disabled)) return NULL; |
8fc0c701c ftrace: show sele... |
2116 |
if (iter->flags & FTRACE_ITER_HASH) |
4aeb69672 tracing: Replace ... |
2117 |
return t_hash_next(m, pos); |
8fc0c701c ftrace: show sele... |
2118 |
|
5072c59fd ftrace: add filte... |
2119 |
(*pos)++; |
1106b6997 tracing: Fix set_... |
2120 |
iter->pos = iter->func_pos = *pos; |
5072c59fd ftrace: add filte... |
2121 |
|
0c75a3ed6 ftrace: state tha... |
2122 |
if (iter->flags & FTRACE_ITER_PRINTALL) |
57c072c71 tracing: Fix read... |
2123 |
return t_hash_start(m, pos); |
0c75a3ed6 ftrace: state tha... |
2124 |
|
5072c59fd ftrace: add filte... |
2125 2126 2127 2128 2129 2130 2131 2132 2133 |
retry: if (iter->idx >= iter->pg->index) { if (iter->pg->next) { iter->pg = iter->pg->next; iter->idx = 0; goto retry; } } else { rec = &iter->pg->records[iter->idx++]; |
320823098 ftrace: Remove us... |
2134 |
if (((iter->flags & FTRACE_ITER_FILTER) && |
f45948e89 ftrace: Create a ... |
2135 |
!(ftrace_lookup_ip(ops->filter_hash, rec->ip))) || |
0183fb1c9 ftrace: fix set_f... |
2136 |
|
41c52c0db ftrace: set_ftrac... |
2137 |
((iter->flags & FTRACE_ITER_NOTRACE) && |
647bcd03d ftrace: Add enabl... |
2138 2139 2140 2141 |
!ftrace_lookup_ip(ops->notrace_hash, rec->ip)) || ((iter->flags & FTRACE_ITER_ENABLED) && !(rec->flags & ~FTRACE_FL_MASK))) { |
5072c59fd ftrace: add filte... |
2142 2143 2144 2145 |
rec = NULL; goto retry; } } |
4aeb69672 tracing: Replace ... |
2146 |
if (!rec) |
57c072c71 tracing: Fix read... |
2147 |
return t_hash_start(m, pos); |
4aeb69672 tracing: Replace ... |
2148 2149 2150 2151 |
iter->func = rec; return iter; |
5072c59fd ftrace: add filte... |
2152 |
} |
98c4fd046 tracing: Keep tra... |
2153 2154 2155 2156 2157 |
static void reset_iter_read(struct ftrace_iterator *iter) { iter->pos = 0; iter->func_pos = 0; iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH); |
5072c59fd ftrace: add filte... |
2158 2159 2160 2161 2162 |
} static void *t_start(struct seq_file *m, loff_t *pos) { struct ftrace_iterator *iter = m->private; |
fc13cb0ce ftrace: Allow oth... |
2163 |
struct ftrace_ops *ops = iter->ops; |
5072c59fd ftrace: add filte... |
2164 |
void *p = NULL; |
694ce0a54 ftrace: Don't man... |
2165 |
loff_t l; |
5072c59fd ftrace: add filte... |
2166 |
|
8fc0c701c ftrace: show sele... |
2167 |
mutex_lock(&ftrace_lock); |
45a4a2372 ftrace: Remove FT... |
2168 2169 2170 |
if (unlikely(ftrace_disabled)) return NULL; |
0c75a3ed6 ftrace: state tha... |
2171 |
/* |
98c4fd046 tracing: Keep tra... |
2172 2173 2174 2175 2176 2177 |
* If an lseek was done, then reset and start from beginning. */ if (*pos < iter->pos) reset_iter_read(iter); /* |
0c75a3ed6 ftrace: state tha... |
2178 2179 2180 2181 |
* For set_ftrace_filter reading, if we have the filter * off, we can short cut and just print out that all * functions are enabled. */ |
06a51d930 ftrace: Create ft... |
2182 2183 |
if (iter->flags & FTRACE_ITER_FILTER && ftrace_hash_empty(ops->filter_hash)) { |
0c75a3ed6 ftrace: state tha... |
2184 |
if (*pos > 0) |
8fc0c701c ftrace: show sele... |
2185 |
return t_hash_start(m, pos); |
0c75a3ed6 ftrace: state tha... |
2186 |
iter->flags |= FTRACE_ITER_PRINTALL; |
df0916255 tracing: t_start:... |
2187 2188 |
/* reset in case of seek/pread */ iter->flags &= ~FTRACE_ITER_HASH; |
0c75a3ed6 ftrace: state tha... |
2189 2190 |
return iter; } |
8fc0c701c ftrace: show sele... |
2191 2192 |
if (iter->flags & FTRACE_ITER_HASH) return t_hash_start(m, pos); |
98c4fd046 tracing: Keep tra... |
2193 2194 2195 2196 2197 |
/* * Unfortunately, we need to restart at ftrace_pages_start * every time we let go of the ftrace_mutex. This is because * those pointers can change without the lock. */ |
694ce0a54 ftrace: Don't man... |
2198 2199 2200 2201 2202 2203 |
iter->pg = ftrace_pages_start; iter->idx = 0; for (l = 0; l <= *pos; ) { p = t_next(m, p, &l); if (!p) break; |
50cdaf08a ftrace: improve s... |
2204 |
} |
5821e1b74 function tracing:... |
2205 |
|
69a3083c4 ftrace: Decouple ... |
2206 2207 |
if (!p) return t_hash_start(m, pos); |
4aeb69672 tracing: Replace ... |
2208 2209 |
return iter; |
5072c59fd ftrace: add filte... |
2210 2211 2212 2213 |
} static void t_stop(struct seq_file *m, void *p) { |
8fc0c701c ftrace: show sele... |
2214 |
mutex_unlock(&ftrace_lock); |
5072c59fd ftrace: add filte... |
2215 2216 2217 2218 |
} static int t_show(struct seq_file *m, void *v) { |
0c75a3ed6 ftrace: state tha... |
2219 |
struct ftrace_iterator *iter = m->private; |
4aeb69672 tracing: Replace ... |
2220 |
struct dyn_ftrace *rec; |
5072c59fd ftrace: add filte... |
2221 |
|
8fc0c701c ftrace: show sele... |
2222 |
if (iter->flags & FTRACE_ITER_HASH) |
4aeb69672 tracing: Replace ... |
2223 |
return t_hash_show(m, iter); |
8fc0c701c ftrace: show sele... |
2224 |
|
0c75a3ed6 ftrace: state tha... |
2225 2226 2227 2228 2229 |
if (iter->flags & FTRACE_ITER_PRINTALL) { seq_printf(m, "#### all functions enabled #### "); return 0; } |
4aeb69672 tracing: Replace ... |
2230 |
rec = iter->func; |
5072c59fd ftrace: add filte... |
2231 2232 |
if (!rec) return 0; |
647bcd03d ftrace: Add enabl... |
2233 2234 2235 2236 2237 2238 |
seq_printf(m, "%ps", (void *)rec->ip); if (iter->flags & FTRACE_ITER_ENABLED) seq_printf(m, " (%ld)", rec->flags & ~FTRACE_FL_MASK); seq_printf(m, " "); |
5072c59fd ftrace: add filte... |
2239 2240 2241 |
return 0; } |
88e9d34c7 seq_file: constif... |
2242 |
static const struct seq_operations show_ftrace_seq_ops = { |
5072c59fd ftrace: add filte... |
2243 2244 2245 2246 2247 |
.start = t_start, .next = t_next, .stop = t_stop, .show = t_show, }; |
e309b41dd ftrace: remove no... |
2248 |
static int |
5072c59fd ftrace: add filte... |
2249 2250 2251 2252 |
ftrace_avail_open(struct inode *inode, struct file *file) { struct ftrace_iterator *iter; int ret; |
4eebcc81a ftrace: disable t... |
2253 2254 |
if (unlikely(ftrace_disabled)) return -ENODEV; |
5072c59fd ftrace: add filte... |
2255 2256 2257 2258 2259 |
iter = kzalloc(sizeof(*iter), GFP_KERNEL); if (!iter) return -ENOMEM; iter->pg = ftrace_pages_start; |
fc13cb0ce ftrace: Allow oth... |
2260 |
iter->ops = &global_ops; |
5072c59fd ftrace: add filte... |
2261 2262 2263 2264 |
ret = seq_open(file, &show_ftrace_seq_ops); if (!ret) { struct seq_file *m = file->private_data; |
4bf39a941 ftrace: cleanups |
2265 |
|
5072c59fd ftrace: add filte... |
2266 |
m->private = iter; |
4bf39a941 ftrace: cleanups |
2267 |
} else { |
5072c59fd ftrace: add filte... |
2268 |
kfree(iter); |
4bf39a941 ftrace: cleanups |
2269 |
} |
5072c59fd ftrace: add filte... |
2270 2271 2272 |
return ret; } |
647bcd03d ftrace: Add enabl... |
2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 |
static int ftrace_enabled_open(struct inode *inode, struct file *file) { struct ftrace_iterator *iter; int ret; if (unlikely(ftrace_disabled)) return -ENODEV; iter = kzalloc(sizeof(*iter), GFP_KERNEL); if (!iter) return -ENOMEM; iter->pg = ftrace_pages_start; iter->flags = FTRACE_ITER_ENABLED; |
fc13cb0ce ftrace: Allow oth... |
2288 |
iter->ops = &global_ops; |
647bcd03d ftrace: Add enabl... |
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 |
ret = seq_open(file, &show_ftrace_seq_ops); if (!ret) { struct seq_file *m = file->private_data; m->private = iter; } else { kfree(iter); } return ret; } |
1cf41dd79 ftrace: Use hash ... |
2301 |
static void ftrace_filter_reset(struct ftrace_hash *hash) |
5072c59fd ftrace: add filte... |
2302 |
{ |
52baf1192 ftrace: convert f... |
2303 |
mutex_lock(&ftrace_lock); |
1cf41dd79 ftrace: Use hash ... |
2304 |
ftrace_hash_clear(hash); |
52baf1192 ftrace: convert f... |
2305 |
mutex_unlock(&ftrace_lock); |
5072c59fd ftrace: add filte... |
2306 |
} |
fc13cb0ce ftrace: Allow oth... |
2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 |
/** * ftrace_regex_open - initialize function tracer filter files * @ops: The ftrace_ops that hold the hash filters * @flag: The type of filter to process * @inode: The inode, usually passed in to your open routine * @file: The file, usually passed in to your open routine * * ftrace_regex_open() initializes the filter files for the * @ops. Depending on @flag it may process the filter hash or * the notrace hash of @ops. With this called from the open * routine, you can use ftrace_filter_write() for the write * routine if @flag has FTRACE_ITER_FILTER set, or * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set. * ftrace_regex_lseek() should be used as the lseek routine, and * release must call ftrace_regex_release(). */ int |
f45948e89 ftrace: Create a ... |
2324 |
ftrace_regex_open(struct ftrace_ops *ops, int flag, |
1cf41dd79 ftrace: Use hash ... |
2325 |
struct inode *inode, struct file *file) |
5072c59fd ftrace: add filte... |
2326 2327 |
{ struct ftrace_iterator *iter; |
f45948e89 ftrace: Create a ... |
2328 |
struct ftrace_hash *hash; |
5072c59fd ftrace: add filte... |
2329 |
int ret = 0; |
4eebcc81a ftrace: disable t... |
2330 2331 |
if (unlikely(ftrace_disabled)) return -ENODEV; |
5072c59fd ftrace: add filte... |
2332 2333 2334 |
iter = kzalloc(sizeof(*iter), GFP_KERNEL); if (!iter) return -ENOMEM; |
689fd8b65 tracing: trace pa... |
2335 2336 2337 2338 |
if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) { kfree(iter); return -ENOMEM; } |
f45948e89 ftrace: Create a ... |
2339 2340 2341 2342 |
if (flag & FTRACE_ITER_NOTRACE) hash = ops->notrace_hash; else hash = ops->filter_hash; |
33dc9b126 ftrace: Separate ... |
2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 |
iter->ops = ops; iter->flags = flag; if (file->f_mode & FMODE_WRITE) { mutex_lock(&ftrace_lock); iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash); mutex_unlock(&ftrace_lock); if (!iter->hash) { trace_parser_put(&iter->parser); kfree(iter); return -ENOMEM; } } |
1cf41dd79 ftrace: Use hash ... |
2357 |
|
41c52c0db ftrace: set_ftrac... |
2358 |
mutex_lock(&ftrace_regex_lock); |
33dc9b126 ftrace: Separate ... |
2359 |
|
5072c59fd ftrace: add filte... |
2360 |
if ((file->f_mode & FMODE_WRITE) && |
8650ae32e tracing: only tru... |
2361 |
(file->f_flags & O_TRUNC)) |
33dc9b126 ftrace: Separate ... |
2362 |
ftrace_filter_reset(iter->hash); |
5072c59fd ftrace: add filte... |
2363 2364 2365 |
if (file->f_mode & FMODE_READ) { iter->pg = ftrace_pages_start; |
5072c59fd ftrace: add filte... |
2366 2367 2368 2369 2370 |
ret = seq_open(file, &show_ftrace_seq_ops); if (!ret) { struct seq_file *m = file->private_data; m->private = iter; |
79fe249c8 tracing: Fix fail... |
2371 |
} else { |
33dc9b126 ftrace: Separate ... |
2372 2373 |
/* Failed */ free_ftrace_hash(iter->hash); |
79fe249c8 tracing: Fix fail... |
2374 |
trace_parser_put(&iter->parser); |
5072c59fd ftrace: add filte... |
2375 |
kfree(iter); |
79fe249c8 tracing: Fix fail... |
2376 |
} |
5072c59fd ftrace: add filte... |
2377 2378 |
} else file->private_data = iter; |
41c52c0db ftrace: set_ftrac... |
2379 |
mutex_unlock(&ftrace_regex_lock); |
5072c59fd ftrace: add filte... |
2380 2381 2382 |
return ret; } |
41c52c0db ftrace: set_ftrac... |
2383 2384 2385 |
static int ftrace_filter_open(struct inode *inode, struct file *file) { |
69a3083c4 ftrace: Decouple ... |
2386 2387 2388 |
return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH, inode, file); |
41c52c0db ftrace: set_ftrac... |
2389 2390 2391 2392 2393 |
} static int ftrace_notrace_open(struct inode *inode, struct file *file) { |
f45948e89 ftrace: Create a ... |
2394 |
return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE, |
1cf41dd79 ftrace: Use hash ... |
2395 |
inode, file); |
41c52c0db ftrace: set_ftrac... |
2396 |
} |
fc13cb0ce ftrace: Allow oth... |
2397 |
loff_t |
41c52c0db ftrace: set_ftrac... |
2398 |
ftrace_regex_lseek(struct file *file, loff_t offset, int origin) |
5072c59fd ftrace: add filte... |
2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 |
{ loff_t ret; if (file->f_mode & FMODE_READ) ret = seq_lseek(file, offset, origin); else file->f_pos = ret = 1; return ret; } |
64e7c4406 ftrace: add modul... |
2409 |
static int ftrace_match(char *str, char *regex, int len, int type) |
9f4801e30 ftrace: break up ... |
2410 |
{ |
9f4801e30 ftrace: break up ... |
2411 |
int matched = 0; |
751e9983e ftrace: Fix MATCH... |
2412 |
int slen; |
9f4801e30 ftrace: break up ... |
2413 |
|
9f4801e30 ftrace: break up ... |
2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 |
switch (type) { case MATCH_FULL: if (strcmp(str, regex) == 0) matched = 1; break; case MATCH_FRONT_ONLY: if (strncmp(str, regex, len) == 0) matched = 1; break; case MATCH_MIDDLE_ONLY: if (strstr(str, regex)) matched = 1; break; case MATCH_END_ONLY: |
751e9983e ftrace: Fix MATCH... |
2428 2429 |
slen = strlen(str); if (slen >= len && memcmp(str + slen - len, regex, len) == 0) |
9f4801e30 ftrace: break up ... |
2430 2431 2432 2433 2434 2435 |
matched = 1; break; } return matched; } |
b448c4e3a ftrace: Replace F... |
2436 |
static int |
1cf41dd79 ftrace: Use hash ... |
2437 |
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not) |
996e87be7 ftrace: Move reco... |
2438 |
{ |
b448c4e3a ftrace: Replace F... |
2439 |
struct ftrace_func_entry *entry; |
b448c4e3a ftrace: Replace F... |
2440 |
int ret = 0; |
1cf41dd79 ftrace: Use hash ... |
2441 2442 2443 2444 2445 |
entry = ftrace_lookup_ip(hash, rec->ip); if (not) { /* Do nothing if it doesn't exist */ if (!entry) return 0; |
b448c4e3a ftrace: Replace F... |
2446 |
|
33dc9b126 ftrace: Separate ... |
2447 |
free_hash_entry(hash, entry); |
1cf41dd79 ftrace: Use hash ... |
2448 2449 2450 2451 |
} else { /* Do nothing if it exists */ if (entry) return 0; |
b448c4e3a ftrace: Replace F... |
2452 |
|
1cf41dd79 ftrace: Use hash ... |
2453 |
ret = add_hash_entry(hash, rec->ip); |
b448c4e3a ftrace: Replace F... |
2454 2455 |
} return ret; |
996e87be7 ftrace: Move reco... |
2456 |
} |
64e7c4406 ftrace: add modul... |
2457 |
static int |
b9df92d2a ftrace: Consolida... |
2458 2459 |
ftrace_match_record(struct dyn_ftrace *rec, char *mod, char *regex, int len, int type) |
64e7c4406 ftrace: add modul... |
2460 2461 |
{ char str[KSYM_SYMBOL_LEN]; |
b9df92d2a ftrace: Consolida... |
2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 |
char *modname; kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); if (mod) { /* module lookup requires matching the module */ if (!modname || strcmp(modname, mod)) return 0; /* blank search means to match all funcs in the mod */ if (!len) return 1; } |
64e7c4406 ftrace: add modul... |
2475 |
|
64e7c4406 ftrace: add modul... |
2476 2477 |
return ftrace_match(str, regex, len, type); } |
1cf41dd79 ftrace: Use hash ... |
2478 2479 2480 |
static int match_records(struct ftrace_hash *hash, char *buff, int len, char *mod, int not) |
9f4801e30 ftrace: break up ... |
2481 |
{ |
b9df92d2a ftrace: Consolida... |
2482 |
unsigned search_len = 0; |
9f4801e30 ftrace: break up ... |
2483 2484 |
struct ftrace_page *pg; struct dyn_ftrace *rec; |
b9df92d2a ftrace: Consolida... |
2485 2486 |
int type = MATCH_FULL; char *search = buff; |
311d16da5 ftrace: Return EI... |
2487 |
int found = 0; |
b448c4e3a ftrace: Replace F... |
2488 |
int ret; |
9f4801e30 ftrace: break up ... |
2489 |
|
b9df92d2a ftrace: Consolida... |
2490 2491 2492 2493 |
if (len) { type = filter_parse_regex(buff, len, &search, ¬); search_len = strlen(search); } |
9f4801e30 ftrace: break up ... |
2494 |
|
52baf1192 ftrace: convert f... |
2495 |
mutex_lock(&ftrace_lock); |
265c831cb ftrace: add do_fo... |
2496 |
|
b9df92d2a ftrace: Consolida... |
2497 2498 |
if (unlikely(ftrace_disabled)) goto out_unlock; |
9f4801e30 ftrace: break up ... |
2499 |
|
265c831cb ftrace: add do_fo... |
2500 |
do_for_each_ftrace_rec(pg, rec) { |
b9df92d2a ftrace: Consolida... |
2501 |
if (ftrace_match_record(rec, mod, search, search_len, type)) { |
1cf41dd79 ftrace: Use hash ... |
2502 |
ret = enter_record(hash, rec, not); |
b448c4e3a ftrace: Replace F... |
2503 2504 2505 2506 |
if (ret < 0) { found = ret; goto out_unlock; } |
311d16da5 ftrace: Return EI... |
2507 |
found = 1; |
265c831cb ftrace: add do_fo... |
2508 2509 |
} } while_for_each_ftrace_rec(); |
b9df92d2a ftrace: Consolida... |
2510 |
out_unlock: |
52baf1192 ftrace: convert f... |
2511 |
mutex_unlock(&ftrace_lock); |
311d16da5 ftrace: Return EI... |
2512 2513 |
return found; |
5072c59fd ftrace: add filte... |
2514 |
} |
64e7c4406 ftrace: add modul... |
2515 |
static int |
1cf41dd79 ftrace: Use hash ... |
2516 |
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) |
64e7c4406 ftrace: add modul... |
2517 |
{ |
1cf41dd79 ftrace: Use hash ... |
2518 |
return match_records(hash, buff, len, NULL, 0); |
64e7c4406 ftrace: add modul... |
2519 |
} |
1cf41dd79 ftrace: Use hash ... |
2520 2521 |
static int ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod) |
64e7c4406 ftrace: add modul... |
2522 |
{ |
64e7c4406 ftrace: add modul... |
2523 |
int not = 0; |
6a24a244c ftrace: clean up ... |
2524 |
|
64e7c4406 ftrace: add modul... |
2525 2526 2527 2528 2529 2530 2531 2532 2533 |
/* blank or '*' mean the same */ if (strcmp(buff, "*") == 0) buff[0] = 0; /* handle the case of 'dont filter this module' */ if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) { buff[0] = 0; not = 1; } |
1cf41dd79 ftrace: Use hash ... |
2534 |
return match_records(hash, buff, strlen(buff), mod, not); |
64e7c4406 ftrace: add modul... |
2535 |
} |
f6180773d ftrace: add comma... |
2536 2537 2538 2539 2540 2541 |
/* * We register the module command as a template to show others how * to register the a command as well. */ static int |
43dd61c9a ftrace: Fix regre... |
2542 2543 |
ftrace_mod_callback(struct ftrace_hash *hash, char *func, char *cmd, char *param, int enable) |
f6180773d ftrace: add comma... |
2544 2545 |
{ char *mod; |
b448c4e3a ftrace: Replace F... |
2546 |
int ret = -EINVAL; |
f6180773d ftrace: add comma... |
2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 |
/* * cmd == 'mod' because we only registered this func * for the 'mod' ftrace_func_command. * But if you register one func with multiple commands, * you can tell which command was used by the cmd * parameter. */ /* we must have a module name */ if (!param) |
b448c4e3a ftrace: Replace F... |
2558 |
return ret; |
f6180773d ftrace: add comma... |
2559 2560 2561 |
mod = strsep(¶m, ":"); if (!strlen(mod)) |
b448c4e3a ftrace: Replace F... |
2562 |
return ret; |
f6180773d ftrace: add comma... |
2563 |
|
1cf41dd79 ftrace: Use hash ... |
2564 |
ret = ftrace_match_module_records(hash, func, mod); |
b448c4e3a ftrace: Replace F... |
2565 2566 2567 2568 2569 2570 |
if (!ret) ret = -EINVAL; if (ret < 0) return ret; return 0; |
f6180773d ftrace: add comma... |
2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 |
} static struct ftrace_func_command ftrace_mod_cmd = { .name = "mod", .func = ftrace_mod_callback, }; static int __init ftrace_mod_cmd_init(void) { return register_ftrace_command(&ftrace_mod_cmd); } device_initcall(ftrace_mod_cmd_init); |
59df055f1 ftrace: trace dif... |
2583 |
static void |
b6887d791 ftrace: rename _h... |
2584 |
function_trace_probe_call(unsigned long ip, unsigned long parent_ip) |
59df055f1 ftrace: trace dif... |
2585 |
{ |
b6887d791 ftrace: rename _h... |
2586 |
struct ftrace_func_probe *entry; |
59df055f1 ftrace: trace dif... |
2587 2588 2589 |
struct hlist_head *hhd; struct hlist_node *n; unsigned long key; |
59df055f1 ftrace: trace dif... |
2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 |
key = hash_long(ip, FTRACE_HASH_BITS); hhd = &ftrace_func_hash[key]; if (hlist_empty(hhd)) return; /* * Disable preemption for these calls to prevent a RCU grace * period. This syncs the hash iteration and freeing of items * on the hash. rcu_read_lock is too dangerous here. */ |
5168ae50a tracing: Remove f... |
2603 |
preempt_disable_notrace(); |
59df055f1 ftrace: trace dif... |
2604 2605 2606 2607 |
hlist_for_each_entry_rcu(entry, n, hhd, node) { if (entry->ip == ip) entry->ops->func(ip, parent_ip, &entry->data); } |
5168ae50a tracing: Remove f... |
2608 |
preempt_enable_notrace(); |
59df055f1 ftrace: trace dif... |
2609 |
} |
b6887d791 ftrace: rename _h... |
2610 |
static struct ftrace_ops trace_probe_ops __read_mostly = |
59df055f1 ftrace: trace dif... |
2611 |
{ |
fb9fb015e tracing: clean up... |
2612 |
.func = function_trace_probe_call, |
59df055f1 ftrace: trace dif... |
2613 |
}; |
b6887d791 ftrace: rename _h... |
2614 |
static int ftrace_probe_registered; |
59df055f1 ftrace: trace dif... |
2615 |
|
b6887d791 ftrace: rename _h... |
2616 |
static void __enable_ftrace_function_probe(void) |
59df055f1 ftrace: trace dif... |
2617 |
{ |
b848914ce ftrace: Implement... |
2618 |
int ret; |
59df055f1 ftrace: trace dif... |
2619 |
int i; |
b6887d791 ftrace: rename _h... |
2620 |
if (ftrace_probe_registered) |
59df055f1 ftrace: trace dif... |
2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 |
return; for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { struct hlist_head *hhd = &ftrace_func_hash[i]; if (hhd->first) break; } /* Nothing registered? */ if (i == FTRACE_FUNC_HASHSIZE) return; |
b848914ce ftrace: Implement... |
2631 2632 |
ret = __register_ftrace_function(&trace_probe_ops); if (!ret) |
a1cd61735 ftrace: Have ftra... |
2633 |
ret = ftrace_startup(&trace_probe_ops, 0); |
b848914ce ftrace: Implement... |
2634 |
|
b6887d791 ftrace: rename _h... |
2635 |
ftrace_probe_registered = 1; |
59df055f1 ftrace: trace dif... |
2636 |
} |
b6887d791 ftrace: rename _h... |
2637 |
static void __disable_ftrace_function_probe(void) |
59df055f1 ftrace: trace dif... |
2638 |
{ |
b848914ce ftrace: Implement... |
2639 |
int ret; |
59df055f1 ftrace: trace dif... |
2640 |
int i; |
b6887d791 ftrace: rename _h... |
2641 |
if (!ftrace_probe_registered) |
59df055f1 ftrace: trace dif... |
2642 2643 2644 2645 2646 2647 2648 2649 2650 |
return; for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { struct hlist_head *hhd = &ftrace_func_hash[i]; if (hhd->first) return; } /* no more funcs left */ |
b848914ce ftrace: Implement... |
2651 2652 2653 |
ret = __unregister_ftrace_function(&trace_probe_ops); if (!ret) ftrace_shutdown(&trace_probe_ops, 0); |
b6887d791 ftrace: rename _h... |
2654 |
ftrace_probe_registered = 0; |
59df055f1 ftrace: trace dif... |
2655 2656 2657 2658 2659 |
} static void ftrace_free_entry_rcu(struct rcu_head *rhp) { |
b6887d791 ftrace: rename _h... |
2660 2661 |
struct ftrace_func_probe *entry = container_of(rhp, struct ftrace_func_probe, rcu); |
59df055f1 ftrace: trace dif... |
2662 2663 2664 2665 2666 2667 2668 2669 |
if (entry->ops->free) entry->ops->free(&entry->data); kfree(entry); } int |
b6887d791 ftrace: rename _h... |
2670 |
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, |
59df055f1 ftrace: trace dif... |
2671 2672 |
void *data) { |
b6887d791 ftrace: rename _h... |
2673 |
struct ftrace_func_probe *entry; |
59df055f1 ftrace: trace dif... |
2674 2675 |
struct ftrace_page *pg; struct dyn_ftrace *rec; |
59df055f1 ftrace: trace dif... |
2676 |
int type, len, not; |
6a24a244c ftrace: clean up ... |
2677 |
unsigned long key; |
59df055f1 ftrace: trace dif... |
2678 2679 |
int count = 0; char *search; |
3f6fe06db tracing/filters: ... |
2680 |
type = filter_parse_regex(glob, strlen(glob), &search, ¬); |
59df055f1 ftrace: trace dif... |
2681 |
len = strlen(search); |
b6887d791 ftrace: rename _h... |
2682 |
/* we do not support '!' for function probes */ |
59df055f1 ftrace: trace dif... |
2683 2684 2685 2686 |
if (WARN_ON(not)) return -EINVAL; mutex_lock(&ftrace_lock); |
59df055f1 ftrace: trace dif... |
2687 |
|
45a4a2372 ftrace: Remove FT... |
2688 2689 |
if (unlikely(ftrace_disabled)) goto out_unlock; |
59df055f1 ftrace: trace dif... |
2690 |
|
45a4a2372 ftrace: Remove FT... |
2691 |
do_for_each_ftrace_rec(pg, rec) { |
59df055f1 ftrace: trace dif... |
2692 |
|
b9df92d2a ftrace: Consolida... |
2693 |
if (!ftrace_match_record(rec, NULL, search, len, type)) |
59df055f1 ftrace: trace dif... |
2694 2695 2696 2697 |
continue; entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) { |
b6887d791 ftrace: rename _h... |
2698 |
/* If we did not process any, then return error */ |
59df055f1 ftrace: trace dif... |
2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 |
if (!count) count = -ENOMEM; goto out_unlock; } count++; entry->data = data; /* * The caller might want to do something special * for each function we find. We call the callback * to give the caller an opportunity to do so. */ if (ops->callback) { if (ops->callback(rec->ip, &entry->data) < 0) { /* caller does not like this func */ kfree(entry); continue; } } entry->ops = ops; entry->ip = rec->ip; key = hash_long(entry->ip, FTRACE_HASH_BITS); hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]); } while_for_each_ftrace_rec(); |
b6887d791 ftrace: rename _h... |
2728 |
__enable_ftrace_function_probe(); |
59df055f1 ftrace: trace dif... |
2729 2730 2731 2732 2733 2734 2735 2736 |
out_unlock: mutex_unlock(&ftrace_lock); return count; } enum { |
b6887d791 ftrace: rename _h... |
2737 2738 |
PROBE_TEST_FUNC = 1, PROBE_TEST_DATA = 2 |
59df055f1 ftrace: trace dif... |
2739 2740 2741 |
}; static void |
b6887d791 ftrace: rename _h... |
2742 |
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, |
59df055f1 ftrace: trace dif... |
2743 2744 |
void *data, int flags) { |
b6887d791 ftrace: rename _h... |
2745 |
struct ftrace_func_probe *entry; |
59df055f1 ftrace: trace dif... |
2746 2747 2748 2749 2750 |
struct hlist_node *n, *tmp; char str[KSYM_SYMBOL_LEN]; int type = MATCH_FULL; int i, len = 0; char *search; |
b36461da2 tracing: Fix mino... |
2751 |
if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) |
59df055f1 ftrace: trace dif... |
2752 |
glob = NULL; |
b36461da2 tracing: Fix mino... |
2753 |
else if (glob) { |
59df055f1 ftrace: trace dif... |
2754 |
int not; |
3f6fe06db tracing/filters: ... |
2755 |
type = filter_parse_regex(glob, strlen(glob), &search, ¬); |
59df055f1 ftrace: trace dif... |
2756 |
len = strlen(search); |
b6887d791 ftrace: rename _h... |
2757 |
/* we do not support '!' for function probes */ |
59df055f1 ftrace: trace dif... |
2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 |
if (WARN_ON(not)) return; } mutex_lock(&ftrace_lock); for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { struct hlist_head *hhd = &ftrace_func_hash[i]; hlist_for_each_entry_safe(entry, n, tmp, hhd, node) { /* break up if statements for readability */ |
b6887d791 ftrace: rename _h... |
2769 |
if ((flags & PROBE_TEST_FUNC) && entry->ops != ops) |
59df055f1 ftrace: trace dif... |
2770 |
continue; |
b6887d791 ftrace: rename _h... |
2771 |
if ((flags & PROBE_TEST_DATA) && entry->data != data) |
59df055f1 ftrace: trace dif... |
2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 |
continue; /* do this last, since it is the most expensive */ if (glob) { kallsyms_lookup(entry->ip, NULL, NULL, NULL, str); if (!ftrace_match(str, glob, len, type)) continue; } hlist_del(&entry->node); call_rcu(&entry->rcu, ftrace_free_entry_rcu); } } |
b6887d791 ftrace: rename _h... |
2786 |
__disable_ftrace_function_probe(); |
59df055f1 ftrace: trace dif... |
2787 2788 2789 2790 |
mutex_unlock(&ftrace_lock); } void |
b6887d791 ftrace: rename _h... |
2791 |
unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, |
59df055f1 ftrace: trace dif... |
2792 2793 |
void *data) { |
b6887d791 ftrace: rename _h... |
2794 2795 |
__unregister_ftrace_function_probe(glob, ops, data, PROBE_TEST_FUNC | PROBE_TEST_DATA); |
59df055f1 ftrace: trace dif... |
2796 2797 2798 |
} void |
b6887d791 ftrace: rename _h... |
2799 |
unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) |
59df055f1 ftrace: trace dif... |
2800 |
{ |
b6887d791 ftrace: rename _h... |
2801 |
__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC); |
59df055f1 ftrace: trace dif... |
2802 |
} |
b6887d791 ftrace: rename _h... |
2803 |
void unregister_ftrace_function_probe_all(char *glob) |
59df055f1 ftrace: trace dif... |
2804 |
{ |
b6887d791 ftrace: rename _h... |
2805 |
__unregister_ftrace_function_probe(glob, NULL, NULL, 0); |
59df055f1 ftrace: trace dif... |
2806 |
} |
f6180773d ftrace: add comma... |
2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 |
static LIST_HEAD(ftrace_commands); static DEFINE_MUTEX(ftrace_cmd_mutex); int register_ftrace_command(struct ftrace_func_command *cmd) { struct ftrace_func_command *p; int ret = 0; mutex_lock(&ftrace_cmd_mutex); list_for_each_entry(p, &ftrace_commands, list) { if (strcmp(cmd->name, p->name) == 0) { ret = -EBUSY; goto out_unlock; } } list_add(&cmd->list, &ftrace_commands); out_unlock: mutex_unlock(&ftrace_cmd_mutex); return ret; } int unregister_ftrace_command(struct ftrace_func_command *cmd) { struct ftrace_func_command *p, *n; int ret = -ENODEV; mutex_lock(&ftrace_cmd_mutex); list_for_each_entry_safe(p, n, &ftrace_commands, list) { if (strcmp(cmd->name, p->name) == 0) { ret = 0; list_del_init(&p->list); goto out_unlock; } } out_unlock: mutex_unlock(&ftrace_cmd_mutex); return ret; } |
33dc9b126 ftrace: Separate ... |
2847 2848 |
static int ftrace_process_regex(struct ftrace_hash *hash, char *buff, int len, int enable) |
64e7c4406 ftrace: add modul... |
2849 |
{ |
f6180773d ftrace: add comma... |
2850 |
char *func, *command, *next = buff; |
6a24a244c ftrace: clean up ... |
2851 |
struct ftrace_func_command *p; |
0aff1c0ce ftrace: Fix possi... |
2852 |
int ret = -EINVAL; |
64e7c4406 ftrace: add modul... |
2853 2854 2855 2856 |
func = strsep(&next, ":"); if (!next) { |
1cf41dd79 ftrace: Use hash ... |
2857 |
ret = ftrace_match_records(hash, func, len); |
b448c4e3a ftrace: Replace F... |
2858 2859 2860 2861 2862 |
if (!ret) ret = -EINVAL; if (ret < 0) return ret; return 0; |
64e7c4406 ftrace: add modul... |
2863 |
} |
f6180773d ftrace: add comma... |
2864 |
/* command found */ |
64e7c4406 ftrace: add modul... |
2865 2866 |
command = strsep(&next, ":"); |
f6180773d ftrace: add comma... |
2867 2868 2869 |
mutex_lock(&ftrace_cmd_mutex); list_for_each_entry(p, &ftrace_commands, list) { if (strcmp(p->name, command) == 0) { |
43dd61c9a ftrace: Fix regre... |
2870 |
ret = p->func(hash, func, command, next, enable); |
f6180773d ftrace: add comma... |
2871 2872 |
goto out_unlock; } |
64e7c4406 ftrace: add modul... |
2873 |
} |
f6180773d ftrace: add comma... |
2874 2875 |
out_unlock: mutex_unlock(&ftrace_cmd_mutex); |
64e7c4406 ftrace: add modul... |
2876 |
|
f6180773d ftrace: add comma... |
2877 |
return ret; |
64e7c4406 ftrace: add modul... |
2878 |
} |
e309b41dd ftrace: remove no... |
2879 |
static ssize_t |
41c52c0db ftrace: set_ftrac... |
2880 2881 |
ftrace_regex_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos, int enable) |
5072c59fd ftrace: add filte... |
2882 2883 |
{ struct ftrace_iterator *iter; |
689fd8b65 tracing: trace pa... |
2884 2885 |
struct trace_parser *parser; ssize_t ret, read; |
5072c59fd ftrace: add filte... |
2886 |
|
4ba7978e9 tracing: Check th... |
2887 |
if (!cnt) |
5072c59fd ftrace: add filte... |
2888 |
return 0; |
41c52c0db ftrace: set_ftrac... |
2889 |
mutex_lock(&ftrace_regex_lock); |
5072c59fd ftrace: add filte... |
2890 |
|
45a4a2372 ftrace: Remove FT... |
2891 2892 2893 |
ret = -ENODEV; if (unlikely(ftrace_disabled)) goto out_unlock; |
5072c59fd ftrace: add filte... |
2894 2895 2896 2897 2898 |
if (file->f_mode & FMODE_READ) { struct seq_file *m = file->private_data; iter = m->private; } else iter = file->private_data; |
689fd8b65 tracing: trace pa... |
2899 2900 |
parser = &iter->parser; read = trace_get_user(parser, ubuf, cnt, ppos); |
5072c59fd ftrace: add filte... |
2901 |
|
4ba7978e9 tracing: Check th... |
2902 |
if (read >= 0 && trace_parser_loaded(parser) && |
689fd8b65 tracing: trace pa... |
2903 |
!trace_parser_cont(parser)) { |
33dc9b126 ftrace: Separate ... |
2904 |
ret = ftrace_process_regex(iter->hash, parser->buffer, |
689fd8b65 tracing: trace pa... |
2905 |
parser->idx, enable); |
313254a94 ftrace: Call trac... |
2906 |
trace_parser_clear(parser); |
5072c59fd ftrace: add filte... |
2907 |
if (ret) |
ed146b259 ftrace: Fix unmat... |
2908 |
goto out_unlock; |
eda1e3285 tracing: handle b... |
2909 |
} |
5072c59fd ftrace: add filte... |
2910 |
|
5072c59fd ftrace: add filte... |
2911 |
ret = read; |
ed146b259 ftrace: Fix unmat... |
2912 |
out_unlock: |
689fd8b65 tracing: trace pa... |
2913 |
mutex_unlock(&ftrace_regex_lock); |
ed146b259 ftrace: Fix unmat... |
2914 |
|
5072c59fd ftrace: add filte... |
2915 2916 |
return ret; } |
fc13cb0ce ftrace: Allow oth... |
2917 |
ssize_t |
41c52c0db ftrace: set_ftrac... |
2918 2919 2920 2921 2922 |
ftrace_filter_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { return ftrace_regex_write(file, ubuf, cnt, ppos, 1); } |
fc13cb0ce ftrace: Allow oth... |
2923 |
ssize_t |
41c52c0db ftrace: set_ftrac... |
2924 2925 2926 2927 2928 |
ftrace_notrace_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { return ftrace_regex_write(file, ubuf, cnt, ppos, 0); } |
33dc9b126 ftrace: Separate ... |
2929 |
static int |
f45948e89 ftrace: Create a ... |
2930 2931 |
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, int reset, int enable) |
41c52c0db ftrace: set_ftrac... |
2932 |
{ |
33dc9b126 ftrace: Separate ... |
2933 |
struct ftrace_hash **orig_hash; |
f45948e89 ftrace: Create a ... |
2934 |
struct ftrace_hash *hash; |
33dc9b126 ftrace: Separate ... |
2935 |
int ret; |
f45948e89 ftrace: Create a ... |
2936 |
|
936e074b2 ftrace: Modify ft... |
2937 2938 2939 |
/* All global ops uses the global ops filters */ if (ops->flags & FTRACE_OPS_FL_GLOBAL) ops = &global_ops; |
41c52c0db ftrace: set_ftrac... |
2940 |
if (unlikely(ftrace_disabled)) |
33dc9b126 ftrace: Separate ... |
2941 |
return -ENODEV; |
41c52c0db ftrace: set_ftrac... |
2942 |
|
f45948e89 ftrace: Create a ... |
2943 |
if (enable) |
33dc9b126 ftrace: Separate ... |
2944 |
orig_hash = &ops->filter_hash; |
f45948e89 ftrace: Create a ... |
2945 |
else |
33dc9b126 ftrace: Separate ... |
2946 2947 2948 2949 2950 |
orig_hash = &ops->notrace_hash; hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); if (!hash) return -ENOMEM; |
f45948e89 ftrace: Create a ... |
2951 |
|
41c52c0db ftrace: set_ftrac... |
2952 2953 |
mutex_lock(&ftrace_regex_lock); if (reset) |
1cf41dd79 ftrace: Use hash ... |
2954 |
ftrace_filter_reset(hash); |
41c52c0db ftrace: set_ftrac... |
2955 |
if (buf) |
1cf41dd79 ftrace: Use hash ... |
2956 |
ftrace_match_records(hash, buf, len); |
33dc9b126 ftrace: Separate ... |
2957 2958 |
mutex_lock(&ftrace_lock); |
41fb61c2d ftrace: Balance r... |
2959 |
ret = ftrace_hash_move(ops, enable, orig_hash, hash); |
072126f45 ftrace: Update fi... |
2960 2961 |
if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED && ftrace_enabled) |
30fb6aa74 ftrace: Fix unreg... |
2962 |
ftrace_run_update_code(FTRACE_UPDATE_CALLS); |
072126f45 ftrace: Update fi... |
2963 |
|
33dc9b126 ftrace: Separate ... |
2964 |
mutex_unlock(&ftrace_lock); |
41c52c0db ftrace: set_ftrac... |
2965 |
mutex_unlock(&ftrace_regex_lock); |
33dc9b126 ftrace: Separate ... |
2966 2967 2968 |
free_ftrace_hash(hash); return ret; |
41c52c0db ftrace: set_ftrac... |
2969 |
} |
77a2b37d2 ftrace: startup t... |
2970 2971 |
/** * ftrace_set_filter - set a function to filter on in ftrace |
936e074b2 ftrace: Modify ft... |
2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 |
* @ops - the ops to set the filter with * @buf - the string that holds the function filter text. * @len - the length of the string. * @reset - non zero to reset all filters before applying this filter. * * Filters denote which functions should be enabled when tracing is enabled. * If @buf is NULL and reset is set, all functions will be enabled for tracing. */ void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, int len, int reset) { ftrace_set_regex(ops, buf, len, reset, 1); } EXPORT_SYMBOL_GPL(ftrace_set_filter); /** * ftrace_set_notrace - set a function to not trace in ftrace * @ops - the ops to set the notrace filter with * @buf - the string that holds the function notrace text. * @len - the length of the string. * @reset - non zero to reset all filters before applying this filter. * * Notrace Filters denote which functions should not be enabled when tracing * is enabled. If @buf is NULL and reset is set, all functions will be enabled * for tracing. */ void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, int len, int reset) { ftrace_set_regex(ops, buf, len, reset, 0); } EXPORT_SYMBOL_GPL(ftrace_set_notrace); /** * ftrace_set_filter - set a function to filter on in ftrace * @ops - the ops to set the filter with |
77a2b37d2 ftrace: startup t... |
3007 3008 3009 3010 3011 3012 3013 |
* @buf - the string that holds the function filter text. * @len - the length of the string. * @reset - non zero to reset all filters before applying this filter. * * Filters denote which functions should be enabled when tracing is enabled. * If @buf is NULL and reset is set, all functions will be enabled for tracing. */ |
936e074b2 ftrace: Modify ft... |
3014 |
void ftrace_set_global_filter(unsigned char *buf, int len, int reset) |
77a2b37d2 ftrace: startup t... |
3015 |
{ |
f45948e89 ftrace: Create a ... |
3016 |
ftrace_set_regex(&global_ops, buf, len, reset, 1); |
41c52c0db ftrace: set_ftrac... |
3017 |
} |
936e074b2 ftrace: Modify ft... |
3018 |
EXPORT_SYMBOL_GPL(ftrace_set_global_filter); |
4eebcc81a ftrace: disable t... |
3019 |
|
41c52c0db ftrace: set_ftrac... |
3020 3021 |
/** * ftrace_set_notrace - set a function to not trace in ftrace |
936e074b2 ftrace: Modify ft... |
3022 |
* @ops - the ops to set the notrace filter with |
41c52c0db ftrace: set_ftrac... |
3023 3024 3025 3026 3027 3028 3029 3030 |
* @buf - the string that holds the function notrace text. * @len - the length of the string. * @reset - non zero to reset all filters before applying this filter. * * Notrace Filters denote which functions should not be enabled when tracing * is enabled. If @buf is NULL and reset is set, all functions will be enabled * for tracing. */ |
936e074b2 ftrace: Modify ft... |
3031 |
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) |
41c52c0db ftrace: set_ftrac... |
3032 |
{ |
f45948e89 ftrace: Create a ... |
3033 |
ftrace_set_regex(&global_ops, buf, len, reset, 0); |
77a2b37d2 ftrace: startup t... |
3034 |
} |
936e074b2 ftrace: Modify ft... |
3035 |
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace); |
77a2b37d2 ftrace: startup t... |
3036 |
|
2af15d6a4 ftrace: add kerne... |
3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 |
/* * command line interface to allow users to set filters on boot up. */ #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; static int __init set_ftrace_notrace(char *str) { strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); return 1; } __setup("ftrace_notrace=", set_ftrace_notrace); static int __init set_ftrace_filter(char *str) { strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); return 1; } __setup("ftrace_filter=", set_ftrace_filter); |
369bc18f9 ftrace: add kerne... |
3057 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER |
f6060f468 tracing: Prevent ... |
3058 |
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; |
801c29fd1 function-graph: F... |
3059 |
static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); |
369bc18f9 ftrace: add kerne... |
3060 3061 |
static int __init set_graph_function(char *str) { |
06f43d66e ftrace: Copy ftra... |
3062 |
strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); |
369bc18f9 ftrace: add kerne... |
3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 |
return 1; } __setup("ftrace_graph_filter=", set_graph_function); static void __init set_ftrace_early_graph(char *buf) { int ret; char *func; while (buf) { func = strsep(&buf, ","); /* we allow only one expression at a time */ ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, func); if (ret) printk(KERN_DEBUG "ftrace: function %s not " "traceable ", func); } } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
2a85a37f1 ftrace: Allow acc... |
3084 3085 |
void __init ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) |
2af15d6a4 ftrace: add kerne... |
3086 3087 3088 3089 3090 |
{ char *func; while (buf) { func = strsep(&buf, ","); |
f45948e89 ftrace: Create a ... |
3091 |
ftrace_set_regex(ops, func, strlen(func), 0, enable); |
2af15d6a4 ftrace: add kerne... |
3092 3093 3094 3095 3096 3097 |
} } static void __init set_ftrace_early_filters(void) { if (ftrace_filter_buf[0]) |
2a85a37f1 ftrace: Allow acc... |
3098 |
ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1); |
2af15d6a4 ftrace: add kerne... |
3099 |
if (ftrace_notrace_buf[0]) |
2a85a37f1 ftrace: Allow acc... |
3100 |
ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0); |
369bc18f9 ftrace: add kerne... |
3101 3102 3103 3104 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER if (ftrace_graph_buf[0]) set_ftrace_early_graph(ftrace_graph_buf); #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
2af15d6a4 ftrace: add kerne... |
3105 |
} |
fc13cb0ce ftrace: Allow oth... |
3106 |
int ftrace_regex_release(struct inode *inode, struct file *file) |
5072c59fd ftrace: add filte... |
3107 3108 3109 |
{ struct seq_file *m = (struct seq_file *)file->private_data; struct ftrace_iterator *iter; |
33dc9b126 ftrace: Separate ... |
3110 |
struct ftrace_hash **orig_hash; |
689fd8b65 tracing: trace pa... |
3111 |
struct trace_parser *parser; |
ed926f9b3 ftrace: Use count... |
3112 |
int filter_hash; |
33dc9b126 ftrace: Separate ... |
3113 |
int ret; |
5072c59fd ftrace: add filte... |
3114 |
|
41c52c0db ftrace: set_ftrac... |
3115 |
mutex_lock(&ftrace_regex_lock); |
5072c59fd ftrace: add filte... |
3116 3117 3118 3119 3120 3121 |
if (file->f_mode & FMODE_READ) { iter = m->private; seq_release(inode, file); } else iter = file->private_data; |
689fd8b65 tracing: trace pa... |
3122 3123 3124 |
parser = &iter->parser; if (trace_parser_loaded(parser)) { parser->buffer[parser->idx] = 0; |
1cf41dd79 ftrace: Use hash ... |
3125 |
ftrace_match_records(iter->hash, parser->buffer, parser->idx); |
5072c59fd ftrace: add filte... |
3126 |
} |
689fd8b65 tracing: trace pa... |
3127 |
trace_parser_put(parser); |
689fd8b65 tracing: trace pa... |
3128 |
|
058e297d3 ftrace: Only upda... |
3129 |
if (file->f_mode & FMODE_WRITE) { |
ed926f9b3 ftrace: Use count... |
3130 3131 3132 |
filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); if (filter_hash) |
33dc9b126 ftrace: Separate ... |
3133 |
orig_hash = &iter->ops->filter_hash; |
ed926f9b3 ftrace: Use count... |
3134 3135 |
else orig_hash = &iter->ops->notrace_hash; |
33dc9b126 ftrace: Separate ... |
3136 |
|
058e297d3 ftrace: Only upda... |
3137 |
mutex_lock(&ftrace_lock); |
41fb61c2d ftrace: Balance r... |
3138 3139 3140 3141 |
ret = ftrace_hash_move(iter->ops, filter_hash, orig_hash, iter->hash); if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED) && ftrace_enabled) |
30fb6aa74 ftrace: Fix unreg... |
3142 |
ftrace_run_update_code(FTRACE_UPDATE_CALLS); |
41fb61c2d ftrace: Balance r... |
3143 |
|
058e297d3 ftrace: Only upda... |
3144 3145 |
mutex_unlock(&ftrace_lock); } |
33dc9b126 ftrace: Separate ... |
3146 3147 |
free_ftrace_hash(iter->hash); kfree(iter); |
058e297d3 ftrace: Only upda... |
3148 |
|
41c52c0db ftrace: set_ftrac... |
3149 |
mutex_unlock(&ftrace_regex_lock); |
5072c59fd ftrace: add filte... |
3150 3151 |
return 0; } |
5e2336a0d tracing: make all... |
3152 |
static const struct file_operations ftrace_avail_fops = { |
5072c59fd ftrace: add filte... |
3153 3154 3155 |
.open = ftrace_avail_open, .read = seq_read, .llseek = seq_lseek, |
3be04b471 ftrace: Simplify ... |
3156 |
.release = seq_release_private, |
5072c59fd ftrace: add filte... |
3157 |
}; |
647bcd03d ftrace: Add enabl... |
3158 3159 3160 3161 3162 3163 |
static const struct file_operations ftrace_enabled_fops = { .open = ftrace_enabled_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release_private, }; |
5e2336a0d tracing: make all... |
3164 |
static const struct file_operations ftrace_filter_fops = { |
5072c59fd ftrace: add filte... |
3165 |
.open = ftrace_filter_open, |
850a80cfa ftrace: use seq_read |
3166 |
.read = seq_read, |
5072c59fd ftrace: add filte... |
3167 |
.write = ftrace_filter_write, |
98c4fd046 tracing: Keep tra... |
3168 |
.llseek = ftrace_regex_lseek, |
1cf41dd79 ftrace: Use hash ... |
3169 |
.release = ftrace_regex_release, |
5072c59fd ftrace: add filte... |
3170 |
}; |
5e2336a0d tracing: make all... |
3171 |
static const struct file_operations ftrace_notrace_fops = { |
41c52c0db ftrace: set_ftrac... |
3172 |
.open = ftrace_notrace_open, |
850a80cfa ftrace: use seq_read |
3173 |
.read = seq_read, |
41c52c0db ftrace: set_ftrac... |
3174 3175 |
.write = ftrace_notrace_write, .llseek = ftrace_regex_lseek, |
1cf41dd79 ftrace: Use hash ... |
3176 |
.release = ftrace_regex_release, |
41c52c0db ftrace: set_ftrac... |
3177 |
}; |
ea4e2bc4d ftrace: graph of ... |
3178 3179 3180 3181 3182 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER static DEFINE_MUTEX(graph_lock); int ftrace_graph_count; |
c7c6b1fe9 ftrace: Allow to ... |
3183 |
int ftrace_graph_filter_enabled; |
ea4e2bc4d ftrace: graph of ... |
3184 3185 3186 |
unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; static void * |
85951842a ftrace: Don't inc... |
3187 |
__g_next(struct seq_file *m, loff_t *pos) |
ea4e2bc4d ftrace: graph of ... |
3188 |
{ |
85951842a ftrace: Don't inc... |
3189 |
if (*pos >= ftrace_graph_count) |
ea4e2bc4d ftrace: graph of ... |
3190 |
return NULL; |
a4ec5e0c2 function-graph: u... |
3191 |
return &ftrace_graph_funcs[*pos]; |
85951842a ftrace: Don't inc... |
3192 |
} |
ea4e2bc4d ftrace: graph of ... |
3193 |
|
85951842a ftrace: Don't inc... |
3194 3195 3196 3197 3198 |
static void * g_next(struct seq_file *m, void *v, loff_t *pos) { (*pos)++; return __g_next(m, pos); |
ea4e2bc4d ftrace: graph of ... |
3199 3200 3201 3202 |
} static void *g_start(struct seq_file *m, loff_t *pos) { |
ea4e2bc4d ftrace: graph of ... |
3203 |
mutex_lock(&graph_lock); |
f9349a8f9 tracing/function-... |
3204 |
/* Nothing, tell g_show to print all functions are enabled */ |
c7c6b1fe9 ftrace: Allow to ... |
3205 |
if (!ftrace_graph_filter_enabled && !*pos) |
f9349a8f9 tracing/function-... |
3206 |
return (void *)1; |
85951842a ftrace: Don't inc... |
3207 |
return __g_next(m, pos); |
ea4e2bc4d ftrace: graph of ... |
3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 |
} static void g_stop(struct seq_file *m, void *p) { mutex_unlock(&graph_lock); } static int g_show(struct seq_file *m, void *v) { unsigned long *ptr = v; |
ea4e2bc4d ftrace: graph of ... |
3218 3219 3220 |
if (!ptr) return 0; |
f9349a8f9 tracing/function-... |
3221 3222 3223 3224 3225 |
if (ptr == (unsigned long *)1) { seq_printf(m, "#### all functions enabled #### "); return 0; } |
b375a11a2 tracing: switch f... |
3226 3227 |
seq_printf(m, "%ps ", (void *)*ptr); |
ea4e2bc4d ftrace: graph of ... |
3228 3229 3230 |
return 0; } |
88e9d34c7 seq_file: constif... |
3231 |
static const struct seq_operations ftrace_graph_seq_ops = { |
ea4e2bc4d ftrace: graph of ... |
3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 |
.start = g_start, .next = g_next, .stop = g_stop, .show = g_show, }; static int ftrace_graph_open(struct inode *inode, struct file *file) { int ret = 0; if (unlikely(ftrace_disabled)) return -ENODEV; mutex_lock(&graph_lock); if ((file->f_mode & FMODE_WRITE) && |
8650ae32e tracing: only tru... |
3248 |
(file->f_flags & O_TRUNC)) { |
c7c6b1fe9 ftrace: Allow to ... |
3249 |
ftrace_graph_filter_enabled = 0; |
ea4e2bc4d ftrace: graph of ... |
3250 3251 3252 |
ftrace_graph_count = 0; memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs)); } |
a4ec5e0c2 function-graph: u... |
3253 |
mutex_unlock(&graph_lock); |
ea4e2bc4d ftrace: graph of ... |
3254 |
|
a4ec5e0c2 function-graph: u... |
3255 |
if (file->f_mode & FMODE_READ) |
ea4e2bc4d ftrace: graph of ... |
3256 |
ret = seq_open(file, &ftrace_graph_seq_ops); |
ea4e2bc4d ftrace: graph of ... |
3257 3258 3259 |
return ret; } |
ea4e2bc4d ftrace: graph of ... |
3260 |
static int |
87827111a function-graph: F... |
3261 3262 3263 3264 3265 3266 3267 3268 |
ftrace_graph_release(struct inode *inode, struct file *file) { if (file->f_mode & FMODE_READ) seq_release(inode, file); return 0; } static int |
f9349a8f9 tracing/function-... |
3269 |
ftrace_set_func(unsigned long *array, int *idx, char *buffer) |
ea4e2bc4d ftrace: graph of ... |
3270 |
{ |
ea4e2bc4d ftrace: graph of ... |
3271 3272 |
struct dyn_ftrace *rec; struct ftrace_page *pg; |
f9349a8f9 tracing/function-... |
3273 |
int search_len; |
c7c6b1fe9 ftrace: Allow to ... |
3274 |
int fail = 1; |
f9349a8f9 tracing/function-... |
3275 3276 3277 3278 |
int type, not; char *search; bool exists; int i; |
ea4e2bc4d ftrace: graph of ... |
3279 |
|
f9349a8f9 tracing/function-... |
3280 |
/* decode regex */ |
3f6fe06db tracing/filters: ... |
3281 |
type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); |
c7c6b1fe9 ftrace: Allow to ... |
3282 3283 |
if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS) return -EBUSY; |
f9349a8f9 tracing/function-... |
3284 3285 |
search_len = strlen(search); |
52baf1192 ftrace: convert f... |
3286 |
mutex_lock(&ftrace_lock); |
45a4a2372 ftrace: Remove FT... |
3287 3288 3289 3290 3291 |
if (unlikely(ftrace_disabled)) { mutex_unlock(&ftrace_lock); return -ENODEV; } |
265c831cb ftrace: add do_fo... |
3292 |
do_for_each_ftrace_rec(pg, rec) { |
b9df92d2a ftrace: Consolida... |
3293 |
if (ftrace_match_record(rec, NULL, search, search_len, type)) { |
c7c6b1fe9 ftrace: Allow to ... |
3294 |
/* if it is in the array */ |
f9349a8f9 tracing/function-... |
3295 |
exists = false; |
c7c6b1fe9 ftrace: Allow to ... |
3296 |
for (i = 0; i < *idx; i++) { |
f9349a8f9 tracing/function-... |
3297 3298 |
if (array[i] == rec->ip) { exists = true; |
265c831cb ftrace: add do_fo... |
3299 3300 |
break; } |
c7c6b1fe9 ftrace: Allow to ... |
3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 |
} if (!not) { fail = 0; if (!exists) { array[(*idx)++] = rec->ip; if (*idx >= FTRACE_GRAPH_MAX_FUNCS) goto out; } } else { if (exists) { array[i] = array[--(*idx)]; array[*idx] = 0; fail = 0; } } |
ea4e2bc4d ftrace: graph of ... |
3317 |
} |
265c831cb ftrace: add do_fo... |
3318 |
} while_for_each_ftrace_rec(); |
c7c6b1fe9 ftrace: Allow to ... |
3319 |
out: |
52baf1192 ftrace: convert f... |
3320 |
mutex_unlock(&ftrace_lock); |
ea4e2bc4d ftrace: graph of ... |
3321 |
|
c7c6b1fe9 ftrace: Allow to ... |
3322 3323 3324 3325 3326 |
if (fail) return -EINVAL; ftrace_graph_filter_enabled = 1; return 0; |
ea4e2bc4d ftrace: graph of ... |
3327 3328 3329 3330 3331 3332 |
} static ssize_t ftrace_graph_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { |
689fd8b65 tracing: trace pa... |
3333 |
struct trace_parser parser; |
4ba7978e9 tracing: Check th... |
3334 |
ssize_t read, ret; |
ea4e2bc4d ftrace: graph of ... |
3335 |
|
c7c6b1fe9 ftrace: Allow to ... |
3336 |
if (!cnt) |
ea4e2bc4d ftrace: graph of ... |
3337 3338 3339 |
return 0; mutex_lock(&graph_lock); |
689fd8b65 tracing: trace pa... |
3340 3341 |
if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) { ret = -ENOMEM; |
1eb90f138 tracing: Fix fail... |
3342 |
goto out_unlock; |
ea4e2bc4d ftrace: graph of ... |
3343 |
} |
689fd8b65 tracing: trace pa... |
3344 |
read = trace_get_user(&parser, ubuf, cnt, ppos); |
ea4e2bc4d ftrace: graph of ... |
3345 |
|
4ba7978e9 tracing: Check th... |
3346 |
if (read >= 0 && trace_parser_loaded((&parser))) { |
689fd8b65 tracing: trace pa... |
3347 3348 3349 |
parser.buffer[parser.idx] = 0; /* we allow only one expression at a time */ |
a4ec5e0c2 function-graph: u... |
3350 |
ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, |
689fd8b65 tracing: trace pa... |
3351 |
parser.buffer); |
ea4e2bc4d ftrace: graph of ... |
3352 |
if (ret) |
1eb90f138 tracing: Fix fail... |
3353 |
goto out_free; |
ea4e2bc4d ftrace: graph of ... |
3354 |
} |
ea4e2bc4d ftrace: graph of ... |
3355 3356 |
ret = read; |
1eb90f138 tracing: Fix fail... |
3357 3358 |
out_free: |
689fd8b65 tracing: trace pa... |
3359 |
trace_parser_put(&parser); |
1eb90f138 tracing: Fix fail... |
3360 |
out_unlock: |
ea4e2bc4d ftrace: graph of ... |
3361 3362 3363 3364 3365 3366 |
mutex_unlock(&graph_lock); return ret; } static const struct file_operations ftrace_graph_fops = { |
87827111a function-graph: F... |
3367 3368 3369 3370 |
.open = ftrace_graph_open, .read = seq_read, .write = ftrace_graph_write, .release = ftrace_graph_release, |
6038f373a llseek: automatic... |
3371 |
.llseek = seq_lseek, |
ea4e2bc4d ftrace: graph of ... |
3372 3373 |
}; #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
df4fc3155 ftrace: add funct... |
3374 |
static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) |
5072c59fd ftrace: add filte... |
3375 |
{ |
5072c59fd ftrace: add filte... |
3376 |
|
5452af664 tracing/ftrace: f... |
3377 3378 |
trace_create_file("available_filter_functions", 0444, d_tracer, NULL, &ftrace_avail_fops); |
5072c59fd ftrace: add filte... |
3379 |
|
647bcd03d ftrace: Add enabl... |
3380 3381 |
trace_create_file("enabled_functions", 0444, d_tracer, NULL, &ftrace_enabled_fops); |
5452af664 tracing/ftrace: f... |
3382 3383 |
trace_create_file("set_ftrace_filter", 0644, d_tracer, NULL, &ftrace_filter_fops); |
41c52c0db ftrace: set_ftrac... |
3384 |
|
5452af664 tracing/ftrace: f... |
3385 |
trace_create_file("set_ftrace_notrace", 0644, d_tracer, |
41c52c0db ftrace: set_ftrac... |
3386 |
NULL, &ftrace_notrace_fops); |
ad90c0e3c ftrace: user upda... |
3387 |
|
ea4e2bc4d ftrace: graph of ... |
3388 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER |
5452af664 tracing/ftrace: f... |
3389 |
trace_create_file("set_graph_function", 0444, d_tracer, |
ea4e2bc4d ftrace: graph of ... |
3390 3391 |
NULL, &ftrace_graph_fops); |
ea4e2bc4d ftrace: graph of ... |
3392 |
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
5072c59fd ftrace: add filte... |
3393 3394 |
return 0; } |
68950619f ftrace: Sort the ... |
3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 |
static void ftrace_swap_recs(void *a, void *b, int size) { struct dyn_ftrace *reca = a; struct dyn_ftrace *recb = b; struct dyn_ftrace t; t = *reca; *reca = *recb; *recb = t; } |
5cb084bb1 tracing: Enable r... |
3405 |
static int ftrace_process_locs(struct module *mod, |
31e889098 ftrace: pass modu... |
3406 |
unsigned long *start, |
68bf21aa1 ftrace: mcount ca... |
3407 3408 |
unsigned long *end) { |
a79008755 ftrace: Allocate ... |
3409 3410 |
struct ftrace_page *pg; unsigned long count; |
68bf21aa1 ftrace: mcount ca... |
3411 3412 |
unsigned long *p; unsigned long addr; |
4376cac66 ftrace: Do not di... |
3413 |
unsigned long flags = 0; /* Shut up gcc */ |
a79008755 ftrace: Allocate ... |
3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 |
int ret = -ENOMEM; count = end - start; if (!count) return 0; pg = ftrace_allocate_pages(count); if (!pg) return -ENOMEM; |
68bf21aa1 ftrace: mcount ca... |
3424 |
|
e6ea44e9b ftrace: consolida... |
3425 |
mutex_lock(&ftrace_lock); |
a79008755 ftrace: Allocate ... |
3426 |
|
320823098 ftrace: Remove us... |
3427 3428 3429 3430 3431 |
/* * Core and each module needs their own pages, as * modules will free them when they are removed. * Force a new page to be allocated for modules. */ |
a79008755 ftrace: Allocate ... |
3432 3433 3434 3435 3436 |
if (!mod) { WARN_ON(ftrace_pages || ftrace_pages_start); /* First initialization */ ftrace_pages = ftrace_pages_start = pg; } else { |
320823098 ftrace: Remove us... |
3437 |
if (!ftrace_pages) |
a79008755 ftrace: Allocate ... |
3438 |
goto out; |
320823098 ftrace: Remove us... |
3439 |
|
a79008755 ftrace: Allocate ... |
3440 3441 3442 3443 |
if (WARN_ON(ftrace_pages->next)) { /* Hmm, we have free pages? */ while (ftrace_pages->next) ftrace_pages = ftrace_pages->next; |
320823098 ftrace: Remove us... |
3444 |
} |
a79008755 ftrace: Allocate ... |
3445 3446 3447 |
ftrace_pages->next = pg; ftrace_pages = pg; |
320823098 ftrace: Remove us... |
3448 |
} |
68bf21aa1 ftrace: mcount ca... |
3449 3450 3451 |
p = start; while (p < end) { addr = ftrace_call_adjust(*p++); |
20e5227e9 ftrace: allow NUL... |
3452 3453 3454 3455 3456 3457 3458 3459 |
/* * Some architecture linkers will pad between * the different mcount_loc sections of different * object files to satisfy alignments. * Skip any NULL pointers. */ if (!addr) continue; |
a79008755 ftrace: Allocate ... |
3460 3461 |
if (!ftrace_record_ip(addr)) break; |
68bf21aa1 ftrace: mcount ca... |
3462 |
} |
85ae32ae0 ftrace: Replace r... |
3463 3464 |
/* These new locations need to be initialized */ ftrace_new_pgs = pg; |
68950619f ftrace: Sort the ... |
3465 3466 3467 3468 |
/* Make each individual set of pages sorted by ips */ for (; pg; pg = pg->next) sort(pg->records, pg->index, sizeof(struct dyn_ftrace), ftrace_cmp_recs, ftrace_swap_recs); |
a4f18ed11 ftrace: Revert 8a... |
3469 |
/* |
4376cac66 ftrace: Do not di... |
3470 3471 3472 3473 3474 3475 |
* We only need to disable interrupts on start up * because we are modifying code that an interrupt * may execute, and the modification is not atomic. * But for modules, nothing runs the code we modify * until we are finished with it, and there's no * reason to cause large interrupt latencies while we do it. |
a4f18ed11 ftrace: Revert 8a... |
3476 |
*/ |
4376cac66 ftrace: Do not di... |
3477 3478 |
if (!mod) local_irq_save(flags); |
31e889098 ftrace: pass modu... |
3479 |
ftrace_update_code(mod); |
4376cac66 ftrace: Do not di... |
3480 3481 |
if (!mod) local_irq_restore(flags); |
a79008755 ftrace: Allocate ... |
3482 3483 |
ret = 0; out: |
e6ea44e9b ftrace: consolida... |
3484 |
mutex_unlock(&ftrace_lock); |
68bf21aa1 ftrace: mcount ca... |
3485 |
|
a79008755 ftrace: Allocate ... |
3486 |
return ret; |
68bf21aa1 ftrace: mcount ca... |
3487 |
} |
93eb677d7 ftrace: use modul... |
3488 |
#ifdef CONFIG_MODULES |
320823098 ftrace: Remove us... |
3489 3490 |
#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next) |
e7247a15f tracing: correct ... |
3491 |
void ftrace_release_mod(struct module *mod) |
93eb677d7 ftrace: use modul... |
3492 3493 |
{ struct dyn_ftrace *rec; |
320823098 ftrace: Remove us... |
3494 |
struct ftrace_page **last_pg; |
93eb677d7 ftrace: use modul... |
3495 |
struct ftrace_page *pg; |
a79008755 ftrace: Allocate ... |
3496 |
int order; |
93eb677d7 ftrace: use modul... |
3497 |
|
45a4a2372 ftrace: Remove FT... |
3498 |
mutex_lock(&ftrace_lock); |
e7247a15f tracing: correct ... |
3499 |
if (ftrace_disabled) |
45a4a2372 ftrace: Remove FT... |
3500 |
goto out_unlock; |
93eb677d7 ftrace: use modul... |
3501 |
|
320823098 ftrace: Remove us... |
3502 3503 3504 3505 3506 3507 3508 |
/* * Each module has its own ftrace_pages, remove * them from the list. */ last_pg = &ftrace_pages_start; for (pg = ftrace_pages_start; pg; pg = *last_pg) { rec = &pg->records[0]; |
e7247a15f tracing: correct ... |
3509 |
if (within_module_core(rec->ip, mod)) { |
93eb677d7 ftrace: use modul... |
3510 |
/* |
320823098 ftrace: Remove us... |
3511 3512 |
* As core pages are first, the first * page should never be a module page. |
93eb677d7 ftrace: use modul... |
3513 |
*/ |
320823098 ftrace: Remove us... |
3514 3515 3516 3517 3518 3519 3520 3521 |
if (WARN_ON(pg == ftrace_pages_start)) goto out_unlock; /* Check if we are deleting the last page */ if (pg == ftrace_pages) ftrace_pages = next_to_ftrace_page(last_pg); *last_pg = pg->next; |
a79008755 ftrace: Allocate ... |
3522 3523 3524 |
order = get_count_order(pg->size / ENTRIES_PER_PAGE); free_pages((unsigned long)pg->records, order); kfree(pg); |
320823098 ftrace: Remove us... |
3525 3526 3527 |
} else last_pg = &pg->next; } |
45a4a2372 ftrace: Remove FT... |
3528 |
out_unlock: |
93eb677d7 ftrace: use modul... |
3529 3530 3531 3532 3533 |
mutex_unlock(&ftrace_lock); } static void ftrace_init_module(struct module *mod, unsigned long *start, unsigned long *end) |
90d595fe5 ftrace: enable mc... |
3534 |
{ |
00fd61aee ftrace: do not in... |
3535 |
if (ftrace_disabled || start == end) |
fed1939c6 ftrace: remove ol... |
3536 |
return; |
5cb084bb1 tracing: Enable r... |
3537 |
ftrace_process_locs(mod, start, end); |
90d595fe5 ftrace: enable mc... |
3538 |
} |
93eb677d7 ftrace: use modul... |
3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 |
static int ftrace_module_notify(struct notifier_block *self, unsigned long val, void *data) { struct module *mod = data; switch (val) { case MODULE_STATE_COMING: ftrace_init_module(mod, mod->ftrace_callsites, mod->ftrace_callsites + mod->num_ftrace_callsites); break; case MODULE_STATE_GOING: |
e7247a15f tracing: correct ... |
3551 |
ftrace_release_mod(mod); |
93eb677d7 ftrace: use modul... |
3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 |
break; } return 0; } #else static int ftrace_module_notify(struct notifier_block *self, unsigned long val, void *data) { return 0; } #endif /* CONFIG_MODULES */ struct notifier_block ftrace_module_nb = { .notifier_call = ftrace_module_notify, .priority = 0, }; |
68bf21aa1 ftrace: mcount ca... |
3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 |
extern unsigned long __start_mcount_loc[]; extern unsigned long __stop_mcount_loc[]; void __init ftrace_init(void) { unsigned long count, addr, flags; int ret; /* Keep the ftrace pointer to the stub */ addr = (unsigned long)ftrace_stub; local_irq_save(flags); ftrace_dyn_arch_init(&addr); local_irq_restore(flags); /* ftrace_dyn_arch_init places the return code in addr */ if (addr) goto failed; count = __stop_mcount_loc - __start_mcount_loc; ret = ftrace_dyn_table_alloc(count); if (ret) goto failed; last_ftrace_enabled = ftrace_enabled = 1; |
5cb084bb1 tracing: Enable r... |
3595 |
ret = ftrace_process_locs(NULL, |
31e889098 ftrace: pass modu... |
3596 |
__start_mcount_loc, |
68bf21aa1 ftrace: mcount ca... |
3597 |
__stop_mcount_loc); |
93eb677d7 ftrace: use modul... |
3598 |
ret = register_module_notifier(&ftrace_module_nb); |
24ed0c4bf tracing: fix chec... |
3599 |
if (ret) |
93eb677d7 ftrace: use modul... |
3600 3601 |
pr_warning("Failed to register trace ftrace module notifier "); |
2af15d6a4 ftrace: add kerne... |
3602 |
set_ftrace_early_filters(); |
68bf21aa1 ftrace: mcount ca... |
3603 3604 3605 3606 |
return; failed: ftrace_disabled = 1; } |
68bf21aa1 ftrace: mcount ca... |
3607 |
|
3d0833953 ftrace: dynamic e... |
3608 |
#else |
0b6e4d56b ftrace: perform a... |
3609 |
|
2b499381b ftrace: Have glob... |
3610 |
static struct ftrace_ops global_ops = { |
bd69c30b1 ftrace: Add ops p... |
3611 3612 |
.func = ftrace_stub, }; |
0b6e4d56b ftrace: perform a... |
3613 3614 3615 3616 3617 3618 |
static int __init ftrace_nodyn_init(void) { ftrace_enabled = 1; return 0; } device_initcall(ftrace_nodyn_init); |
df4fc3155 ftrace: add funct... |
3619 3620 |
static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; } static inline void ftrace_startup_enable(int command) { } |
5a45cfe1c ftrace: use code ... |
3621 |
/* Keep as macros so we do not need to define the commands */ |
3b6cfdb17 ftrace: Set ops->... |
3622 3623 3624 3625 3626 |
# define ftrace_startup(ops, command) \ ({ \ (ops)->flags |= FTRACE_OPS_FL_ENABLED; \ 0; \ }) |
bd69c30b1 ftrace: Add ops p... |
3627 |
# define ftrace_shutdown(ops, command) do { } while (0) |
c7aafc549 ftrace: cleanups |
3628 3629 |
# define ftrace_startup_sysctl() do { } while (0) # define ftrace_shutdown_sysctl() do { } while (0) |
b848914ce ftrace: Implement... |
3630 3631 3632 3633 3634 3635 |
static inline int ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip) { return 1; } |
3d0833953 ftrace: dynamic e... |
3636 |
#endif /* CONFIG_DYNAMIC_FTRACE */ |
b848914ce ftrace: Implement... |
3637 3638 3639 |
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip) { |
cdbe61bfe ftrace: Allow dyn... |
3640 |
struct ftrace_ops *op; |
b848914ce ftrace: Implement... |
3641 |
|
b1cff0ad1 ftrace: Add inter... |
3642 3643 3644 3645 |
if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT))) return; trace_recursion_set(TRACE_INTERNAL_BIT); |
cdbe61bfe ftrace: Allow dyn... |
3646 3647 3648 3649 3650 3651 |
/* * Some of the ops may be dynamically allocated, * they must be freed after a synchronize_sched(). */ preempt_disable_notrace(); op = rcu_dereference_raw(ftrace_ops_list); |
b848914ce ftrace: Implement... |
3652 3653 3654 3655 3656 |
while (op != &ftrace_list_end) { if (ftrace_ops_test(op, ip)) op->func(ip, parent_ip); op = rcu_dereference_raw(op->next); }; |
cdbe61bfe ftrace: Allow dyn... |
3657 |
preempt_enable_notrace(); |
b1cff0ad1 ftrace: Add inter... |
3658 |
trace_recursion_clear(TRACE_INTERNAL_BIT); |
b848914ce ftrace: Implement... |
3659 |
} |
e32d89569 ftrace: add abili... |
3660 |
static void clear_ftrace_swapper(void) |
978f3a45d ftrace: use struc... |
3661 3662 |
{ struct task_struct *p; |
e32d89569 ftrace: add abili... |
3663 |
int cpu; |
978f3a45d ftrace: use struc... |
3664 |
|
e32d89569 ftrace: add abili... |
3665 3666 3667 |
get_online_cpus(); for_each_online_cpu(cpu) { p = idle_task(cpu); |
978f3a45d ftrace: use struc... |
3668 |
clear_tsk_trace_trace(p); |
e32d89569 ftrace: add abili... |
3669 3670 3671 |
} put_online_cpus(); } |
978f3a45d ftrace: use struc... |
3672 |
|
e32d89569 ftrace: add abili... |
3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 |
static void set_ftrace_swapper(void) { struct task_struct *p; int cpu; get_online_cpus(); for_each_online_cpu(cpu) { p = idle_task(cpu); set_tsk_trace_trace(p); } put_online_cpus(); |
978f3a45d ftrace: use struc... |
3684 |
} |
e32d89569 ftrace: add abili... |
3685 3686 3687 |
static void clear_ftrace_pid(struct pid *pid) { struct task_struct *p; |
229c4ef8a ftrace: do_each_p... |
3688 |
rcu_read_lock(); |
e32d89569 ftrace: add abili... |
3689 3690 3691 |
do_each_pid_task(pid, PIDTYPE_PID, p) { clear_tsk_trace_trace(p); } while_each_pid_task(pid, PIDTYPE_PID, p); |
229c4ef8a ftrace: do_each_p... |
3692 |
rcu_read_unlock(); |
e32d89569 ftrace: add abili... |
3693 3694 3695 3696 |
put_pid(pid); } static void set_ftrace_pid(struct pid *pid) |
978f3a45d ftrace: use struc... |
3697 3698 |
{ struct task_struct *p; |
229c4ef8a ftrace: do_each_p... |
3699 |
rcu_read_lock(); |
978f3a45d ftrace: use struc... |
3700 3701 3702 |
do_each_pid_task(pid, PIDTYPE_PID, p) { set_tsk_trace_trace(p); } while_each_pid_task(pid, PIDTYPE_PID, p); |
229c4ef8a ftrace: do_each_p... |
3703 |
rcu_read_unlock(); |
978f3a45d ftrace: use struc... |
3704 |
} |
756d17ee7 tracing: Support ... |
3705 |
static void clear_ftrace_pid_task(struct pid *pid) |
e32d89569 ftrace: add abili... |
3706 |
{ |
756d17ee7 tracing: Support ... |
3707 |
if (pid == ftrace_swapper_pid) |
e32d89569 ftrace: add abili... |
3708 3709 |
clear_ftrace_swapper(); else |
756d17ee7 tracing: Support ... |
3710 |
clear_ftrace_pid(pid); |
e32d89569 ftrace: add abili... |
3711 3712 3713 3714 3715 3716 3717 3718 3719 |
} static void set_ftrace_pid_task(struct pid *pid) { if (pid == ftrace_swapper_pid) set_ftrace_swapper(); else set_ftrace_pid(pid); } |
756d17ee7 tracing: Support ... |
3720 |
static int ftrace_pid_add(int p) |
df4fc3155 ftrace: add funct... |
3721 |
{ |
978f3a45d ftrace: use struc... |
3722 |
struct pid *pid; |
756d17ee7 tracing: Support ... |
3723 3724 |
struct ftrace_pid *fpid; int ret = -EINVAL; |
df4fc3155 ftrace: add funct... |
3725 |
|
756d17ee7 tracing: Support ... |
3726 |
mutex_lock(&ftrace_lock); |
df4fc3155 ftrace: add funct... |
3727 |
|
756d17ee7 tracing: Support ... |
3728 3729 3730 3731 |
if (!p) pid = ftrace_swapper_pid; else pid = find_get_pid(p); |
df4fc3155 ftrace: add funct... |
3732 |
|
756d17ee7 tracing: Support ... |
3733 3734 |
if (!pid) goto out; |
df4fc3155 ftrace: add funct... |
3735 |
|
756d17ee7 tracing: Support ... |
3736 |
ret = 0; |
df4fc3155 ftrace: add funct... |
3737 |
|
756d17ee7 tracing: Support ... |
3738 3739 3740 |
list_for_each_entry(fpid, &ftrace_pids, list) if (fpid->pid == pid) goto out_put; |
978f3a45d ftrace: use struc... |
3741 |
|
756d17ee7 tracing: Support ... |
3742 |
ret = -ENOMEM; |
df4fc3155 ftrace: add funct... |
3743 |
|
756d17ee7 tracing: Support ... |
3744 3745 3746 |
fpid = kmalloc(sizeof(*fpid), GFP_KERNEL); if (!fpid) goto out_put; |
df4fc3155 ftrace: add funct... |
3747 |
|
756d17ee7 tracing: Support ... |
3748 3749 |
list_add(&fpid->list, &ftrace_pids); fpid->pid = pid; |
0ef8cde56 ftrace: use task ... |
3750 |
|
756d17ee7 tracing: Support ... |
3751 |
set_ftrace_pid_task(pid); |
978f3a45d ftrace: use struc... |
3752 |
|
756d17ee7 tracing: Support ... |
3753 3754 3755 3756 3757 3758 3759 3760 3761 |
ftrace_update_pid_func(); ftrace_startup_enable(0); mutex_unlock(&ftrace_lock); return 0; out_put: if (pid != ftrace_swapper_pid) put_pid(pid); |
978f3a45d ftrace: use struc... |
3762 |
|
756d17ee7 tracing: Support ... |
3763 3764 3765 3766 3767 3768 3769 3770 |
out: mutex_unlock(&ftrace_lock); return ret; } static void ftrace_pid_reset(void) { struct ftrace_pid *fpid, *safe; |
978f3a45d ftrace: use struc... |
3771 |
|
756d17ee7 tracing: Support ... |
3772 3773 3774 3775 3776 3777 3778 3779 |
mutex_lock(&ftrace_lock); list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) { struct pid *pid = fpid->pid; clear_ftrace_pid_task(pid); list_del(&fpid->list); kfree(fpid); |
df4fc3155 ftrace: add funct... |
3780 |
} |
df4fc3155 ftrace: add funct... |
3781 3782 |
ftrace_update_pid_func(); ftrace_startup_enable(0); |
e6ea44e9b ftrace: consolida... |
3783 |
mutex_unlock(&ftrace_lock); |
756d17ee7 tracing: Support ... |
3784 |
} |
df4fc3155 ftrace: add funct... |
3785 |
|
756d17ee7 tracing: Support ... |
3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 |
static void *fpid_start(struct seq_file *m, loff_t *pos) { mutex_lock(&ftrace_lock); if (list_empty(&ftrace_pids) && (!*pos)) return (void *) 1; return seq_list_start(&ftrace_pids, *pos); } static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) { if (v == (void *)1) return NULL; return seq_list_next(v, &ftrace_pids, pos); } static void fpid_stop(struct seq_file *m, void *p) { mutex_unlock(&ftrace_lock); } static int fpid_show(struct seq_file *m, void *v) { const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list); if (v == (void *)1) { seq_printf(m, "no pid "); return 0; } if (fpid->pid == ftrace_swapper_pid) seq_printf(m, "swapper tasks "); else seq_printf(m, "%u ", pid_vnr(fpid->pid)); return 0; } static const struct seq_operations ftrace_pid_sops = { .start = fpid_start, .next = fpid_next, .stop = fpid_stop, .show = fpid_show, }; static int ftrace_pid_open(struct inode *inode, struct file *file) { int ret = 0; if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) ftrace_pid_reset(); if (file->f_mode & FMODE_READ) ret = seq_open(file, &ftrace_pid_sops); return ret; } |
df4fc3155 ftrace: add funct... |
3850 3851 3852 3853 |
static ssize_t ftrace_pid_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { |
457dc928f tracing, function... |
3854 |
char buf[64], *tmp; |
df4fc3155 ftrace: add funct... |
3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 |
long val; int ret; if (cnt >= sizeof(buf)) return -EINVAL; if (copy_from_user(&buf, ubuf, cnt)) return -EFAULT; buf[cnt] = 0; |
756d17ee7 tracing: Support ... |
3865 3866 3867 3868 |
/* * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid" * to clean the filter quietly. */ |
457dc928f tracing, function... |
3869 3870 |
tmp = strstrip(buf); if (strlen(tmp) == 0) |
756d17ee7 tracing: Support ... |
3871 |
return 1; |
457dc928f tracing, function... |
3872 |
ret = strict_strtol(tmp, 10, &val); |
df4fc3155 ftrace: add funct... |
3873 3874 |
if (ret < 0) return ret; |
756d17ee7 tracing: Support ... |
3875 |
ret = ftrace_pid_add(val); |
df4fc3155 ftrace: add funct... |
3876 |
|
756d17ee7 tracing: Support ... |
3877 3878 |
return ret ? ret : cnt; } |
df4fc3155 ftrace: add funct... |
3879 |
|
756d17ee7 tracing: Support ... |
3880 3881 3882 3883 3884 |
static int ftrace_pid_release(struct inode *inode, struct file *file) { if (file->f_mode & FMODE_READ) seq_release(inode, file); |
df4fc3155 ftrace: add funct... |
3885 |
|
756d17ee7 tracing: Support ... |
3886 |
return 0; |
df4fc3155 ftrace: add funct... |
3887 |
} |
5e2336a0d tracing: make all... |
3888 |
static const struct file_operations ftrace_pid_fops = { |
756d17ee7 tracing: Support ... |
3889 3890 3891 3892 3893 |
.open = ftrace_pid_open, .write = ftrace_pid_write, .read = seq_read, .llseek = seq_lseek, .release = ftrace_pid_release, |
df4fc3155 ftrace: add funct... |
3894 3895 3896 3897 3898 |
}; static __init int ftrace_init_debugfs(void) { struct dentry *d_tracer; |
df4fc3155 ftrace: add funct... |
3899 3900 3901 3902 3903 3904 |
d_tracer = tracing_init_dentry(); if (!d_tracer) return 0; ftrace_init_dyn_debugfs(d_tracer); |
5452af664 tracing/ftrace: f... |
3905 3906 |
trace_create_file("set_ftrace_pid", 0644, d_tracer, NULL, &ftrace_pid_fops); |
493762fc5 tracing: move fun... |
3907 3908 |
ftrace_profile_debugfs(d_tracer); |
df4fc3155 ftrace: add funct... |
3909 3910 |
return 0; } |
df4fc3155 ftrace: add funct... |
3911 |
fs_initcall(ftrace_init_debugfs); |
16444a8a4 ftrace: add basic... |
3912 |
/** |
81adbdc02 ftrace: only have... |
3913 |
* ftrace_kill - kill ftrace |
a2bb6a3d8 ftrace: add ftrac... |
3914 3915 3916 3917 3918 |
* * This function should be used by panic code. It stops ftrace * but in a not so nice way. If you need to simply kill ftrace * from a non-atomic section, use ftrace_kill. */ |
81adbdc02 ftrace: only have... |
3919 |
void ftrace_kill(void) |
a2bb6a3d8 ftrace: add ftrac... |
3920 3921 3922 |
{ ftrace_disabled = 1; ftrace_enabled = 0; |
a2bb6a3d8 ftrace: add ftrac... |
3923 3924 3925 3926 |
clear_ftrace_function(); } /** |
e0a413f61 tracing: Warn on ... |
3927 3928 3929 3930 3931 3932 3933 3934 |
* Test if ftrace is dead or not. */ int ftrace_is_dead(void) { return ftrace_disabled; } /** |
3d0833953 ftrace: dynamic e... |
3935 3936 |
* register_ftrace_function - register a function for profiling * @ops - ops structure that holds the function for profiling. |
16444a8a4 ftrace: add basic... |
3937 |
* |
3d0833953 ftrace: dynamic e... |
3938 3939 3940 3941 3942 3943 |
* Register a function to be called by all functions in the * kernel. * * Note: @ops->func and all the functions it calls must be labeled * with "notrace", otherwise it will go into a * recursive loop. |
16444a8a4 ftrace: add basic... |
3944 |
*/ |
3d0833953 ftrace: dynamic e... |
3945 |
int register_ftrace_function(struct ftrace_ops *ops) |
16444a8a4 ftrace: add basic... |
3946 |
{ |
45a4a2372 ftrace: Remove FT... |
3947 |
int ret = -1; |
4eebcc81a ftrace: disable t... |
3948 |
|
e6ea44e9b ftrace: consolida... |
3949 |
mutex_lock(&ftrace_lock); |
e7d3737ea tracing/function-... |
3950 |
|
45a4a2372 ftrace: Remove FT... |
3951 3952 |
if (unlikely(ftrace_disabled)) goto out_unlock; |
b0fc494fa ftrace: add ftrac... |
3953 |
ret = __register_ftrace_function(ops); |
b848914ce ftrace: Implement... |
3954 |
if (!ret) |
a1cd61735 ftrace: Have ftra... |
3955 |
ret = ftrace_startup(ops, 0); |
b848914ce ftrace: Implement... |
3956 |
|
b0fc494fa ftrace: add ftrac... |
3957 |
|
45a4a2372 ftrace: Remove FT... |
3958 |
out_unlock: |
e6ea44e9b ftrace: consolida... |
3959 |
mutex_unlock(&ftrace_lock); |
b0fc494fa ftrace: add ftrac... |
3960 |
return ret; |
3d0833953 ftrace: dynamic e... |
3961 |
} |
cdbe61bfe ftrace: Allow dyn... |
3962 |
EXPORT_SYMBOL_GPL(register_ftrace_function); |
3d0833953 ftrace: dynamic e... |
3963 3964 |
/** |
32632920a ftrace, trivial: ... |
3965 |
* unregister_ftrace_function - unregister a function for profiling. |
3d0833953 ftrace: dynamic e... |
3966 3967 3968 3969 3970 3971 3972 |
* @ops - ops structure that holds the function to unregister * * Unregister a function that was added to be called by ftrace profiling. */ int unregister_ftrace_function(struct ftrace_ops *ops) { int ret; |
e6ea44e9b ftrace: consolida... |
3973 |
mutex_lock(&ftrace_lock); |
3d0833953 ftrace: dynamic e... |
3974 |
ret = __unregister_ftrace_function(ops); |
b848914ce ftrace: Implement... |
3975 3976 |
if (!ret) ftrace_shutdown(ops, 0); |
e6ea44e9b ftrace: consolida... |
3977 |
mutex_unlock(&ftrace_lock); |
b0fc494fa ftrace: add ftrac... |
3978 3979 3980 |
return ret; } |
cdbe61bfe ftrace: Allow dyn... |
3981 |
EXPORT_SYMBOL_GPL(unregister_ftrace_function); |
b0fc494fa ftrace: add ftrac... |
3982 |
|
e309b41dd ftrace: remove no... |
3983 |
int |
b0fc494fa ftrace: add ftrac... |
3984 |
ftrace_enable_sysctl(struct ctl_table *table, int write, |
8d65af789 sysctl: remove "s... |
3985 |
void __user *buffer, size_t *lenp, |
b0fc494fa ftrace: add ftrac... |
3986 3987 |
loff_t *ppos) { |
45a4a2372 ftrace: Remove FT... |
3988 |
int ret = -ENODEV; |
4eebcc81a ftrace: disable t... |
3989 |
|
e6ea44e9b ftrace: consolida... |
3990 |
mutex_lock(&ftrace_lock); |
b0fc494fa ftrace: add ftrac... |
3991 |
|
45a4a2372 ftrace: Remove FT... |
3992 3993 3994 3995 |
if (unlikely(ftrace_disabled)) goto out; ret = proc_dointvec(table, write, buffer, lenp, ppos); |
b0fc494fa ftrace: add ftrac... |
3996 |
|
a32c7765e tracing: Fix stac... |
3997 |
if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) |
b0fc494fa ftrace: add ftrac... |
3998 |
goto out; |
a32c7765e tracing: Fix stac... |
3999 |
last_ftrace_enabled = !!ftrace_enabled; |
b0fc494fa ftrace: add ftrac... |
4000 4001 4002 4003 4004 4005 |
if (ftrace_enabled) { ftrace_startup_sysctl(); /* we are starting ftrace again */ |
b848914ce ftrace: Implement... |
4006 4007 4008 |
if (ftrace_ops_list != &ftrace_list_end) { if (ftrace_ops_list->next == &ftrace_list_end) ftrace_trace_function = ftrace_ops_list->func; |
b0fc494fa ftrace: add ftrac... |
4009 |
else |
b848914ce ftrace: Implement... |
4010 |
ftrace_trace_function = ftrace_ops_list_func; |
b0fc494fa ftrace: add ftrac... |
4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 |
} } else { /* stopping ftrace calls (just send to ftrace_stub) */ ftrace_trace_function = ftrace_stub; ftrace_shutdown_sysctl(); } out: |
e6ea44e9b ftrace: consolida... |
4021 |
mutex_unlock(&ftrace_lock); |
3d0833953 ftrace: dynamic e... |
4022 |
return ret; |
16444a8a4 ftrace: add basic... |
4023 |
} |
f17845e5d ftrace: warning i... |
4024 |
|
fb52607af tracing/function-... |
4025 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER |
e7d3737ea tracing/function-... |
4026 |
|
597af8153 function-graph: u... |
4027 |
static int ftrace_graph_active; |
4a2b8dda3 tracing/function-... |
4028 |
static struct notifier_block ftrace_suspend_notifier; |
e7d3737ea tracing/function-... |
4029 |
|
e49dc19c6 ftrace: function ... |
4030 4031 4032 4033 |
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) { return 0; } |
287b6e68c tracing/function-... |
4034 4035 4036 |
/* The callbacks that hook a function */ trace_func_graph_ret_t ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; |
e49dc19c6 ftrace: function ... |
4037 |
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; |
f201ae235 tracing/function-... |
4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 |
/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) { int i; int ret = 0; unsigned long flags; int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; struct task_struct *g, *t; for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH * sizeof(struct ftrace_ret_stack), GFP_KERNEL); if (!ret_stack_list[i]) { start = 0; end = i; ret = -ENOMEM; goto free; } } read_lock_irqsave(&tasklist_lock, flags); do_each_thread(g, t) { if (start == end) { ret = -EAGAIN; goto unlock; } if (t->ret_stack == NULL) { |
380c4b141 tracing/function-... |
4068 |
atomic_set(&t->tracing_graph_pause, 0); |
f201ae235 tracing/function-... |
4069 |
atomic_set(&t->trace_overrun, 0); |
26c01624a function-graph: a... |
4070 4071 4072 4073 |
t->curr_ret_stack = -1; /* Make sure the tasks see the -1 first: */ smp_wmb(); t->ret_stack = ret_stack_list[start++]; |
f201ae235 tracing/function-... |
4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 |
} } while_each_thread(g, t); unlock: read_unlock_irqrestore(&tasklist_lock, flags); free: for (i = start; i < end; i++) kfree(ret_stack_list[i]); return ret; } |
8aef2d285 function-graph: i... |
4084 |
static void |
38516ab59 tracing: Let trac... |
4085 4086 |
ftrace_graph_probe_sched_switch(void *ignore, struct task_struct *prev, struct task_struct *next) |
8aef2d285 function-graph: i... |
4087 4088 4089 |
{ unsigned long long timestamp; int index; |
be6f164a0 function-graph: a... |
4090 4091 4092 4093 4094 4095 |
/* * Does the user want to count the time a function was asleep. * If so, do not update the time stamps. */ if (trace_flags & TRACE_ITER_SLEEP_TIME) return; |
8aef2d285 function-graph: i... |
4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 |
timestamp = trace_clock_local(); prev->ftrace_timestamp = timestamp; /* only process tasks that we timestamped */ if (!next->ftrace_timestamp) return; /* * Update all the counters in next to make up for the * time next was sleeping. */ timestamp -= next->ftrace_timestamp; for (index = next->curr_ret_stack; index >= 0; index--) next->ret_stack[index].calltime += timestamp; } |
f201ae235 tracing/function-... |
4113 |
/* Allocate a return stack for each task */ |
fb52607af tracing/function-... |
4114 |
static int start_graph_tracing(void) |
f201ae235 tracing/function-... |
4115 4116 |
{ struct ftrace_ret_stack **ret_stack_list; |
5b058bcde tracing/function-... |
4117 |
int ret, cpu; |
f201ae235 tracing/function-... |
4118 4119 4120 4121 4122 4123 4124 |
ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE * sizeof(struct ftrace_ret_stack *), GFP_KERNEL); if (!ret_stack_list) return -ENOMEM; |
5b058bcde tracing/function-... |
4125 |
/* The cpu_boot init_task->ret_stack will never be freed */ |
179c498ae function-graph: o... |
4126 4127 |
for_each_online_cpu(cpu) { if (!idle_task(cpu)->ret_stack) |
868baf07b ftrace: Fix memor... |
4128 |
ftrace_graph_init_idle_task(idle_task(cpu), cpu); |
179c498ae function-graph: o... |
4129 |
} |
5b058bcde tracing/function-... |
4130 |
|
f201ae235 tracing/function-... |
4131 4132 4133 |
do { ret = alloc_retstack_tasklist(ret_stack_list); } while (ret == -EAGAIN); |
8aef2d285 function-graph: i... |
4134 |
if (!ret) { |
38516ab59 tracing: Let trac... |
4135 |
ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); |
8aef2d285 function-graph: i... |
4136 4137 4138 4139 4140 |
if (ret) pr_info("ftrace_graph: Couldn't activate tracepoint" " probe to kernel_sched_switch "); } |
f201ae235 tracing/function-... |
4141 4142 4143 |
kfree(ret_stack_list); return ret; } |
4a2b8dda3 tracing/function-... |
4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 |
/* * Hibernation protection. * The state of the current task is too much unstable during * suspend/restore to disk. We want to protect against that. */ static int ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, void *unused) { switch (state) { case PM_HIBERNATION_PREPARE: pause_graph_tracing(); break; case PM_POST_HIBERNATION: unpause_graph_tracing(); break; } return NOTIFY_DONE; } |
287b6e68c tracing/function-... |
4164 4165 |
int register_ftrace_graph(trace_func_graph_ret_t retfunc, trace_func_graph_ent_t entryfunc) |
15e6cb367 tracing: add a tr... |
4166 |
{ |
e7d3737ea tracing/function-... |
4167 |
int ret = 0; |
e6ea44e9b ftrace: consolida... |
4168 |
mutex_lock(&ftrace_lock); |
e7d3737ea tracing/function-... |
4169 |
|
05ce5818a function-graph: p... |
4170 |
/* we currently allow only one tracer registered at a time */ |
597af8153 function-graph: u... |
4171 |
if (ftrace_graph_active) { |
05ce5818a function-graph: p... |
4172 4173 4174 |
ret = -EBUSY; goto out; } |
4a2b8dda3 tracing/function-... |
4175 4176 |
ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call; register_pm_notifier(&ftrace_suspend_notifier); |
597af8153 function-graph: u... |
4177 |
ftrace_graph_active++; |
fb52607af tracing/function-... |
4178 |
ret = start_graph_tracing(); |
f201ae235 tracing/function-... |
4179 |
if (ret) { |
597af8153 function-graph: u... |
4180 |
ftrace_graph_active--; |
f201ae235 tracing/function-... |
4181 4182 |
goto out; } |
e53a6319c ftrace: let funct... |
4183 |
|
287b6e68c tracing/function-... |
4184 4185 |
ftrace_graph_return = retfunc; ftrace_graph_entry = entryfunc; |
e53a6319c ftrace: let funct... |
4186 |
|
a1cd61735 ftrace: Have ftra... |
4187 |
ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET); |
e7d3737ea tracing/function-... |
4188 4189 |
out: |
e6ea44e9b ftrace: consolida... |
4190 |
mutex_unlock(&ftrace_lock); |
e7d3737ea tracing/function-... |
4191 |
return ret; |
15e6cb367 tracing: add a tr... |
4192 |
} |
fb52607af tracing/function-... |
4193 |
void unregister_ftrace_graph(void) |
15e6cb367 tracing: add a tr... |
4194 |
{ |
e6ea44e9b ftrace: consolida... |
4195 |
mutex_lock(&ftrace_lock); |
e7d3737ea tracing/function-... |
4196 |
|
597af8153 function-graph: u... |
4197 |
if (unlikely(!ftrace_graph_active)) |
2aad1b76e function-graph: a... |
4198 |
goto out; |
597af8153 function-graph: u... |
4199 |
ftrace_graph_active--; |
287b6e68c tracing/function-... |
4200 |
ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; |
e49dc19c6 ftrace: function ... |
4201 |
ftrace_graph_entry = ftrace_graph_entry_stub; |
bd69c30b1 ftrace: Add ops p... |
4202 |
ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET); |
4a2b8dda3 tracing/function-... |
4203 |
unregister_pm_notifier(&ftrace_suspend_notifier); |
38516ab59 tracing: Let trac... |
4204 |
unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); |
e7d3737ea tracing/function-... |
4205 |
|
2aad1b76e function-graph: a... |
4206 |
out: |
e6ea44e9b ftrace: consolida... |
4207 |
mutex_unlock(&ftrace_lock); |
15e6cb367 tracing: add a tr... |
4208 |
} |
f201ae235 tracing/function-... |
4209 |
|
868baf07b ftrace: Fix memor... |
4210 4211 4212 4213 4214 4215 4216 4217 |
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack); static void graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) { atomic_set(&t->tracing_graph_pause, 0); atomic_set(&t->trace_overrun, 0); t->ftrace_timestamp = 0; |
25985edce Fix common misspe... |
4218 |
/* make curr_ret_stack visible before we add the ret_stack */ |
868baf07b ftrace: Fix memor... |
4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 |
smp_wmb(); t->ret_stack = ret_stack; } /* * Allocate a return stack for the idle task. May be the first * time through, or it may be done by CPU hotplug online. */ void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { t->curr_ret_stack = -1; /* * The idle task has no parent, it either has its own * stack or no stack at all. */ if (t->ret_stack) WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu)); if (ftrace_graph_active) { struct ftrace_ret_stack *ret_stack; ret_stack = per_cpu(idle_ret_stack, cpu); if (!ret_stack) { ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH * sizeof(struct ftrace_ret_stack), GFP_KERNEL); if (!ret_stack) return; per_cpu(idle_ret_stack, cpu) = ret_stack; } graph_init_task(t, ret_stack); } } |
f201ae235 tracing/function-... |
4252 |
/* Allocate a return stack for newly created task */ |
fb52607af tracing/function-... |
4253 |
void ftrace_graph_init_task(struct task_struct *t) |
f201ae235 tracing/function-... |
4254 |
{ |
84047e360 function-graph: a... |
4255 4256 |
/* Make sure we do not use the parent ret_stack */ t->ret_stack = NULL; |
ea14eb714 function-graph: I... |
4257 |
t->curr_ret_stack = -1; |
84047e360 function-graph: a... |
4258 |
|
597af8153 function-graph: u... |
4259 |
if (ftrace_graph_active) { |
82310a327 function-graph: e... |
4260 4261 4262 |
struct ftrace_ret_stack *ret_stack; ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH |
f201ae235 tracing/function-... |
4263 4264 |
* sizeof(struct ftrace_ret_stack), GFP_KERNEL); |
82310a327 function-graph: e... |
4265 |
if (!ret_stack) |
f201ae235 tracing/function-... |
4266 |
return; |
868baf07b ftrace: Fix memor... |
4267 |
graph_init_task(t, ret_stack); |
84047e360 function-graph: a... |
4268 |
} |
f201ae235 tracing/function-... |
4269 |
} |
fb52607af tracing/function-... |
4270 |
void ftrace_graph_exit_task(struct task_struct *t) |
f201ae235 tracing/function-... |
4271 |
{ |
eae849ca0 tracing/function-... |
4272 |
struct ftrace_ret_stack *ret_stack = t->ret_stack; |
f201ae235 tracing/function-... |
4273 |
t->ret_stack = NULL; |
eae849ca0 tracing/function-... |
4274 4275 4276 4277 |
/* NULL must become visible to IRQs before we free it: */ barrier(); kfree(ret_stack); |
f201ae235 tracing/function-... |
4278 |
} |
14a866c56 ftrace: add ftrac... |
4279 4280 4281 4282 4283 |
void ftrace_graph_stop(void) { ftrace_stop(); } |
15e6cb367 tracing: add a tr... |
4284 |
#endif |