Blame view
kernel/trace/trace_branch.c
10.1 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
1f0d69a9f tracing: profile ... |
2 3 4 5 6 7 8 9 |
/* * unlikely profiler * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */ #include <linux/kallsyms.h> #include <linux/seq_file.h> #include <linux/spinlock.h> |
65c6dc6ad tracing/branch-tr... |
10 |
#include <linux/irqflags.h> |
1f0d69a9f tracing: profile ... |
11 12 13 14 15 16 |
#include <linux/uaccess.h> #include <linux/module.h> #include <linux/ftrace.h> #include <linux/hash.h> #include <linux/fs.h> #include <asm/local.h> |
f633cef02 ftrace: change tr... |
17 |
|
1f0d69a9f tracing: profile ... |
18 |
#include "trace.h" |
002bb86d8 tracing/ftrace: s... |
19 |
#include "trace_stat.h" |
f633cef02 ftrace: change tr... |
20 |
#include "trace_output.h" |
1f0d69a9f tracing: profile ... |
21 |
|
2ed84eeb8 trace: rename unl... |
22 |
#ifdef CONFIG_BRANCH_TRACER |
52f232cb7 tracing: likely/u... |
23 |
|
002bb86d8 tracing/ftrace: s... |
24 |
static struct tracer branch_trace; |
9f029e83e ftrace: rename un... |
25 26 |
static int branch_tracing_enabled __read_mostly; static DEFINE_MUTEX(branch_tracing_mutex); |
e302cf3f9 tracing/branch-tr... |
27 |
|
9f029e83e ftrace: rename un... |
28 |
static struct trace_array *branch_tracer; |
52f232cb7 tracing: likely/u... |
29 30 |
static void |
068f530b3 tracing: Add the ... |
31 |
probe_likely_condition(struct ftrace_likely_data *f, int val, int expect) |
52f232cb7 tracing: likely/u... |
32 |
{ |
2425bcb92 tracing: Rename f... |
33 |
struct trace_event_call *call = &event_branch; |
9f029e83e ftrace: rename un... |
34 |
struct trace_array *tr = branch_tracer; |
132924943 tracing: Make str... |
35 |
struct trace_buffer *buffer; |
a7603ff4b tracing: Replace ... |
36 |
struct trace_array_cpu *data; |
52f232cb7 tracing: likely/u... |
37 |
struct ring_buffer_event *event; |
9f029e83e ftrace: rename un... |
38 |
struct trace_branch *entry; |
0a9877514 ring_buffer: remo... |
39 |
unsigned long flags; |
6224beb12 tracing: Have bra... |
40 |
int pc; |
52f232cb7 tracing: likely/u... |
41 |
const char *p; |
6224beb12 tracing: Have bra... |
42 43 |
if (current->trace_recursion & TRACE_BRANCH_BIT) return; |
52f232cb7 tracing: likely/u... |
44 45 46 47 48 49 50 51 52 |
/* * I would love to save just the ftrace_likely_data pointer, but * this code can also be used by modules. Ugly things can happen * if the module is unloaded, and then we go and read the * pointer. This is slower, but much safer. */ if (unlikely(!tr)) return; |
6224beb12 tracing: Have bra... |
53 54 |
raw_local_irq_save(flags); current->trace_recursion |= TRACE_BRANCH_BIT; |
1c5eb4481 tracing: Rename t... |
55 |
data = this_cpu_ptr(tr->array_buffer.data); |
6224beb12 tracing: Have bra... |
56 |
if (atomic_read(&data->disabled)) |
52f232cb7 tracing: likely/u... |
57 |
goto out; |
51a763dd8 tracing: Introduc... |
58 |
pc = preempt_count(); |
1c5eb4481 tracing: Rename t... |
59 |
buffer = tr->array_buffer.buffer; |
8f6e8a314 tracing: user loc... |
60 |
event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH, |
51a763dd8 tracing: Introduc... |
61 |
sizeof(*entry), flags, pc); |
52f232cb7 tracing: likely/u... |
62 63 |
if (!event) goto out; |
52f232cb7 tracing: likely/u... |
64 |
entry = ring_buffer_event_data(event); |
52f232cb7 tracing: likely/u... |
65 66 |
/* Strip off the path, only save the file */ |
068f530b3 tracing: Add the ... |
67 68 |
p = f->data.file + strlen(f->data.file); while (p >= f->data.file && *p != '/') |
52f232cb7 tracing: likely/u... |
69 70 |
p--; p++; |
068f530b3 tracing: Add the ... |
71 |
strncpy(entry->func, f->data.func, TRACE_FUNC_SIZE); |
52f232cb7 tracing: likely/u... |
72 73 74 |
strncpy(entry->file, p, TRACE_FILE_SIZE); entry->func[TRACE_FUNC_SIZE] = 0; entry->file[TRACE_FILE_SIZE] = 0; |
068f530b3 tracing: Add the ... |
75 76 |
entry->constant = f->constant; entry->line = f->data.line; |
52f232cb7 tracing: likely/u... |
77 |
entry->correct = val == expect; |
f306cc82a tracing: Update e... |
78 |
if (!call_filter_check_discard(call, entry, buffer, event)) |
52ffabe38 tracing: Make __b... |
79 |
trace_buffer_unlock_commit_nostack(buffer, event); |
52f232cb7 tracing: likely/u... |
80 81 |
out: |
6224beb12 tracing: Have bra... |
82 83 |
current->trace_recursion &= ~TRACE_BRANCH_BIT; raw_local_irq_restore(flags); |
52f232cb7 tracing: likely/u... |
84 85 86 |
} static inline |
068f530b3 tracing: Add the ... |
87 |
void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect) |
52f232cb7 tracing: likely/u... |
88 |
{ |
9f029e83e ftrace: rename un... |
89 |
if (!branch_tracing_enabled) |
52f232cb7 tracing: likely/u... |
90 91 92 93 |
return; probe_likely_condition(f, val, expect); } |
9f029e83e ftrace: rename un... |
94 |
int enable_branch_tracing(struct trace_array *tr) |
52f232cb7 tracing: likely/u... |
95 |
{ |
9f029e83e ftrace: rename un... |
96 97 |
mutex_lock(&branch_tracing_mutex); branch_tracer = tr; |
52f232cb7 tracing: likely/u... |
98 99 100 101 102 |
/* * Must be seen before enabling. The reader is a condition * where we do not need a matching rmb() */ smp_wmb(); |
9f029e83e ftrace: rename un... |
103 104 |
branch_tracing_enabled++; mutex_unlock(&branch_tracing_mutex); |
52f232cb7 tracing: likely/u... |
105 |
|
f54fc98aa tracing: remove u... |
106 |
return 0; |
52f232cb7 tracing: likely/u... |
107 |
} |
9f029e83e ftrace: rename un... |
108 |
void disable_branch_tracing(void) |
52f232cb7 tracing: likely/u... |
109 |
{ |
9f029e83e ftrace: rename un... |
110 |
mutex_lock(&branch_tracing_mutex); |
52f232cb7 tracing: likely/u... |
111 |
|
9f029e83e ftrace: rename un... |
112 |
if (!branch_tracing_enabled) |
52f232cb7 tracing: likely/u... |
113 |
goto out_unlock; |
9f029e83e ftrace: rename un... |
114 |
branch_tracing_enabled--; |
52f232cb7 tracing: likely/u... |
115 116 |
out_unlock: |
9f029e83e ftrace: rename un... |
117 |
mutex_unlock(&branch_tracing_mutex); |
52f232cb7 tracing: likely/u... |
118 |
} |
80e5ea450 ftrace: add trace... |
119 |
|
1c80025a4 tracing/ftrace: c... |
120 |
static int branch_trace_init(struct trace_array *tr) |
80e5ea450 ftrace: add trace... |
121 |
{ |
306169292 tracing: Remove {... |
122 |
return enable_branch_tracing(tr); |
80e5ea450 ftrace: add trace... |
123 124 125 126 |
} static void branch_trace_reset(struct trace_array *tr) { |
306169292 tracing: Remove {... |
127 |
disable_branch_tracing(); |
80e5ea450 ftrace: add trace... |
128 |
} |
ae7462b4f trace: make the t... |
129 |
static enum print_line_t trace_branch_print(struct trace_iterator *iter, |
a9a577638 tracing: Allow ev... |
130 |
int flags, struct trace_event *event) |
f633cef02 ftrace: change tr... |
131 132 |
{ struct trace_branch *field; |
2c9b238eb trace: Change str... |
133 |
trace_assign_type(field, iter->ent); |
f633cef02 ftrace: change tr... |
134 |
|
7d40f6716 tracing: Have bra... |
135 136 137 138 139 140 141 142 |
trace_seq_printf(&iter->seq, "[%s] %s:%s:%d ", field->correct ? " ok " : " MISS ", field->func, field->file, field->line); return trace_handle_return(&iter->seq); |
f633cef02 ftrace: change tr... |
143 |
} |
557055beb tracing: Fix bran... |
144 145 146 |
static void branch_print_header(struct seq_file *s) { seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT" |
d79ac28fd tracing: Merge co... |
147 148 149 150 151 |
" FUNC:FILE:LINE " "# | | | | | " " | "); |
557055beb tracing: Fix bran... |
152 |
} |
e302cf3f9 tracing/branch-tr... |
153 |
|
a9a577638 tracing: Allow ev... |
154 155 156 |
static struct trace_event_functions trace_branch_funcs = { .trace = trace_branch_print, }; |
f633cef02 ftrace: change tr... |
157 |
static struct trace_event trace_branch_event = { |
ef18012b2 tracing: remove f... |
158 |
.type = TRACE_BRANCH, |
a9a577638 tracing: Allow ev... |
159 |
.funcs = &trace_branch_funcs, |
f633cef02 ftrace: change tr... |
160 |
}; |
002bb86d8 tracing/ftrace: s... |
161 162 163 164 165 166 167 168 |
static struct tracer branch_trace __read_mostly = { .name = "branch", .init = branch_trace_init, .reset = branch_trace_reset, #ifdef CONFIG_FTRACE_SELFTEST .selftest = trace_selftest_startup_branch, #endif /* CONFIG_FTRACE_SELFTEST */ |
557055beb tracing: Fix bran... |
169 |
.print_header = branch_print_header, |
002bb86d8 tracing/ftrace: s... |
170 171 172 173 174 |
}; __init static int init_branch_tracer(void) { int ret; |
9023c9309 tracing: Rename (... |
175 |
ret = register_trace_event(&trace_branch_event); |
002bb86d8 tracing/ftrace: s... |
176 177 178 179 180 181 182 183 |
if (!ret) { printk(KERN_WARNING "Warning: could not register " "branch events "); return 1; } return register_tracer(&branch_trace); } |
6f4156723 tracing: Allow tr... |
184 |
core_initcall(init_branch_tracer); |
002bb86d8 tracing/ftrace: s... |
185 |
|
52f232cb7 tracing: likely/u... |
186 187 |
#else static inline |
068f530b3 tracing: Add the ... |
188 |
void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect) |
52f232cb7 tracing: likely/u... |
189 190 |
{ } |
2ed84eeb8 trace: rename unl... |
191 |
#endif /* CONFIG_BRANCH_TRACER */ |
52f232cb7 tracing: likely/u... |
192 |
|
134e6a034 tracing: Show num... |
193 |
void ftrace_likely_update(struct ftrace_likely_data *f, int val, |
d45ae1f70 tracing: Process ... |
194 |
int expect, int is_constant) |
1f0d69a9f tracing: profile ... |
195 |
{ |
4a6c91fbd x86/uaccess, ftra... |
196 |
unsigned long flags = user_access_save(); |
d45ae1f70 tracing: Process ... |
197 |
/* A constant is always correct */ |
134e6a034 tracing: Show num... |
198 199 |
if (is_constant) { f->constant++; |
d45ae1f70 tracing: Process ... |
200 |
val = expect; |
134e6a034 tracing: Show num... |
201 |
} |
52f232cb7 tracing: likely/u... |
202 203 204 205 206 207 |
/* * I would love to have a trace point here instead, but the * trace point code is so inundated with unlikely and likely * conditions that the recursive nightmare that exists is too * much to try to get working. At least for now. */ |
068f530b3 tracing: Add the ... |
208 |
trace_likely_condition(f, val, expect); |
52f232cb7 tracing: likely/u... |
209 |
|
1f0d69a9f tracing: profile ... |
210 211 |
/* FIXME: Make this atomic! */ if (val == expect) |
134e6a034 tracing: Show num... |
212 |
f->data.correct++; |
1f0d69a9f tracing: profile ... |
213 |
else |
134e6a034 tracing: Show num... |
214 |
f->data.incorrect++; |
4a6c91fbd x86/uaccess, ftra... |
215 216 |
user_access_restore(flags); |
1f0d69a9f tracing: profile ... |
217 218 |
} EXPORT_SYMBOL(ftrace_likely_update); |
e302cf3f9 tracing/branch-tr... |
219 220 |
extern unsigned long __start_annotated_branch_profile[]; extern unsigned long __stop_annotated_branch_profile[]; |
1f0d69a9f tracing: profile ... |
221 |
|
e302cf3f9 tracing/branch-tr... |
222 |
static int annotated_branch_stat_headers(struct seq_file *m) |
1f0d69a9f tracing: profile ... |
223 |
{ |
d79ac28fd tracing: Merge co... |
224 225 |
seq_puts(m, " correct incorrect % " " Function " |
fa6f0cc75 tracing: Replace ... |
226 227 228 229 230 231 |
" File Line " " ------- --------- - " " -------- " " ---- ---- "); |
e302cf3f9 tracing/branch-tr... |
232 |
return 0; |
1f0d69a9f tracing: profile ... |
233 |
} |
80042c8f0 tracing: Use gene... |
234 |
static inline long get_incorrect_percent(const struct ftrace_branch_data *p) |
1f0d69a9f tracing: profile ... |
235 |
{ |
e302cf3f9 tracing/branch-tr... |
236 |
long percent; |
1f0d69a9f tracing: profile ... |
237 |
|
e302cf3f9 tracing/branch-tr... |
238 239 240 241 242 |
if (p->correct) { percent = p->incorrect * 100; percent /= p->correct + p->incorrect; } else percent = p->incorrect ? 100 : -1; |
1f0d69a9f tracing: profile ... |
243 |
|
e302cf3f9 tracing/branch-tr... |
244 |
return percent; |
1f0d69a9f tracing: profile ... |
245 |
} |
134e6a034 tracing: Show num... |
246 |
static const char *branch_stat_process_file(struct ftrace_branch_data *p) |
1f0d69a9f tracing: profile ... |
247 |
{ |
1f0d69a9f tracing: profile ... |
248 |
const char *f; |
1f0d69a9f tracing: profile ... |
249 |
|
1f0d69a9f tracing: profile ... |
250 251 252 253 |
/* Only print the file, not the path */ f = p->file + strlen(p->file); while (f >= p->file && *f != '/') f--; |
134e6a034 tracing: Show num... |
254 255 256 257 258 259 260 |
return ++f; } static void branch_stat_show(struct seq_file *m, struct ftrace_branch_data *p, const char *f) { long percent; |
1f0d69a9f tracing: profile ... |
261 |
|
2bcd521a6 trace: profile al... |
262 263 264 |
/* * The miss is overlayed on correct, and hit on incorrect. */ |
e302cf3f9 tracing/branch-tr... |
265 |
percent = get_incorrect_percent(p); |
1f0d69a9f tracing: profile ... |
266 |
|
bac28bfe4 trace: branch pro... |
267 |
if (percent < 0) |
fa6f0cc75 tracing: Replace ... |
268 |
seq_puts(m, " X "); |
bac28bfe4 trace: branch pro... |
269 270 |
else seq_printf(m, "%3ld ", percent); |
134e6a034 tracing: Show num... |
271 |
|
1f0d69a9f tracing: profile ... |
272 273 |
seq_printf(m, "%-30.30s %-20.20s %d ", p->func, f, p->line); |
134e6a034 tracing: Show num... |
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 |
} static int branch_stat_show_normal(struct seq_file *m, struct ftrace_branch_data *p, const char *f) { seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); branch_stat_show(m, p, f); return 0; } static int annotate_branch_stat_show(struct seq_file *m, void *v) { struct ftrace_likely_data *p = v; const char *f; int l; f = branch_stat_process_file(&p->data); if (!p->constant) return branch_stat_show_normal(m, &p->data, f); l = snprintf(NULL, 0, "/%lu", p->constant); l = l > 8 ? 0 : 8 - l; seq_printf(m, "%8lu/%lu %*lu ", p->data.correct, p->constant, l, p->data.incorrect); branch_stat_show(m, &p->data, f); |
1f0d69a9f tracing: profile ... |
301 302 |
return 0; } |
425480081 tracing: add hand... |
303 |
static void *annotated_branch_stat_start(struct tracer_stat *trace) |
e302cf3f9 tracing/branch-tr... |
304 305 306 |
{ return __start_annotated_branch_profile; } |
1f0d69a9f tracing: profile ... |
307 |
|
e302cf3f9 tracing/branch-tr... |
308 309 |
static void * annotated_branch_stat_next(void *v, int idx) |
1f0d69a9f tracing: profile ... |
310 |
{ |
134e6a034 tracing: Show num... |
311 |
struct ftrace_likely_data *p = v; |
1f0d69a9f tracing: profile ... |
312 |
|
e302cf3f9 tracing/branch-tr... |
313 |
++p; |
1f0d69a9f tracing: profile ... |
314 |
|
e302cf3f9 tracing/branch-tr... |
315 316 317 318 |
if ((void *)p >= (void *)__stop_annotated_branch_profile) return NULL; return p; |
1f0d69a9f tracing: profile ... |
319 |
} |
80042c8f0 tracing: Use gene... |
320 |
static int annotated_branch_stat_cmp(const void *p1, const void *p2) |
e302cf3f9 tracing/branch-tr... |
321 |
{ |
80042c8f0 tracing: Use gene... |
322 323 |
const struct ftrace_branch_data *a = p1; const struct ftrace_branch_data *b = p2; |
e302cf3f9 tracing/branch-tr... |
324 325 326 327 328 329 330 331 332 333 |
long percent_a, percent_b; percent_a = get_incorrect_percent(a); percent_b = get_incorrect_percent(b); if (percent_a < percent_b) return -1; if (percent_a > percent_b) return 1; |
ede55c9d7 tracing: Add corr... |
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 |
if (a->incorrect < b->incorrect) return -1; if (a->incorrect > b->incorrect) return 1; /* * Since the above shows worse (incorrect) cases * first, we continue that by showing best (correct) * cases last. */ if (a->correct > b->correct) return -1; if (a->correct < b->correct) return 1; return 0; |
e302cf3f9 tracing/branch-tr... |
351 |
} |
1f0d69a9f tracing: profile ... |
352 |
|
002bb86d8 tracing/ftrace: s... |
353 354 355 356 357 358 |
static struct tracer_stat annotated_branch_stats = { .name = "branch_annotated", .stat_start = annotated_branch_stat_start, .stat_next = annotated_branch_stat_next, .stat_cmp = annotated_branch_stat_cmp, .stat_headers = annotated_branch_stat_headers, |
134e6a034 tracing: Show num... |
359 |
.stat_show = annotate_branch_stat_show |
002bb86d8 tracing/ftrace: s... |
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 |
}; __init static int init_annotated_branch_stats(void) { int ret; ret = register_stat_tracer(&annotated_branch_stats); if (!ret) { printk(KERN_WARNING "Warning: could not register " "annotated branches stats "); return 1; } return 0; } fs_initcall(init_annotated_branch_stats); |
2bcd521a6 trace: profile al... |
376 |
#ifdef CONFIG_PROFILE_ALL_BRANCHES |
2bcd521a6 trace: profile al... |
377 |
|
e302cf3f9 tracing/branch-tr... |
378 379 |
extern unsigned long __start_branch_profile[]; extern unsigned long __stop_branch_profile[]; |
1f0d69a9f tracing: profile ... |
380 |
|
e302cf3f9 tracing/branch-tr... |
381 382 |
static int all_branch_stat_headers(struct seq_file *m) { |
d79ac28fd tracing: Merge co... |
383 384 |
seq_puts(m, " miss hit % " " Function " |
fa6f0cc75 tracing: Replace ... |
385 386 387 388 389 390 |
" File Line " " ------- --------- - " " -------- " " ---- ---- "); |
e302cf3f9 tracing/branch-tr... |
391 392 |
return 0; } |
1f0d69a9f tracing: profile ... |
393 |
|
425480081 tracing: add hand... |
394 |
static void *all_branch_stat_start(struct tracer_stat *trace) |
1f0d69a9f tracing: profile ... |
395 |
{ |
e302cf3f9 tracing/branch-tr... |
396 397 398 399 400 401 402 |
return __start_branch_profile; } static void * all_branch_stat_next(void *v, int idx) { struct ftrace_branch_data *p = v; |
1f0d69a9f tracing: profile ... |
403 |
|
e302cf3f9 tracing/branch-tr... |
404 |
++p; |
1f0d69a9f tracing: profile ... |
405 |
|
e302cf3f9 tracing/branch-tr... |
406 407 |
if ((void *)p >= (void *)__stop_branch_profile) return NULL; |
1f0d69a9f tracing: profile ... |
408 |
|
e302cf3f9 tracing/branch-tr... |
409 410 |
return p; } |
2bcd521a6 trace: profile al... |
411 |
|
134e6a034 tracing: Show num... |
412 413 414 415 416 417 418 419 |
static int all_branch_stat_show(struct seq_file *m, void *v) { struct ftrace_branch_data *p = v; const char *f; f = branch_stat_process_file(p); return branch_stat_show_normal(m, p, f); } |
002bb86d8 tracing/ftrace: s... |
420 421 |
static struct tracer_stat all_branch_stats = { .name = "branch_all", |
034939b65 tracing/ftrace: h... |
422 423 424 |
.stat_start = all_branch_stat_start, .stat_next = all_branch_stat_next, .stat_headers = all_branch_stat_headers, |
134e6a034 tracing: Show num... |
425 |
.stat_show = all_branch_stat_show |
034939b65 tracing/ftrace: h... |
426 |
}; |
e302cf3f9 tracing/branch-tr... |
427 |
|
002bb86d8 tracing/ftrace: s... |
428 |
__init static int all_annotated_branch_stats(void) |
e302cf3f9 tracing/branch-tr... |
429 |
{ |
e302cf3f9 tracing/branch-tr... |
430 |
int ret; |
002bb86d8 tracing/ftrace: s... |
431 432 |
ret = register_stat_tracer(&all_branch_stats); |
e302cf3f9 tracing/branch-tr... |
433 |
if (!ret) { |
002bb86d8 tracing/ftrace: s... |
434 435 436 |
printk(KERN_WARNING "Warning: could not register " "all branches stats "); |
e302cf3f9 tracing/branch-tr... |
437 438 |
return 1; } |
002bb86d8 tracing/ftrace: s... |
439 |
return 0; |
e302cf3f9 tracing/branch-tr... |
440 |
} |
002bb86d8 tracing/ftrace: s... |
441 442 |
fs_initcall(all_annotated_branch_stats); #endif /* CONFIG_PROFILE_ALL_BRANCHES */ |