Blame view
kernel/trace/trace_branch.c
9.06 KB
1f0d69a9f tracing: profile ... |
1 2 3 4 5 6 7 8 |
/* * unlikely profiler * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */ #include <linux/kallsyms.h> #include <linux/seq_file.h> #include <linux/spinlock.h> |
65c6dc6ad tracing/branch-tr... |
9 |
#include <linux/irqflags.h> |
1f0d69a9f tracing: profile ... |
10 11 12 13 14 15 16 |
#include <linux/debugfs.h> #include <linux/uaccess.h> #include <linux/module.h> #include <linux/ftrace.h> #include <linux/hash.h> #include <linux/fs.h> #include <asm/local.h> |
f633cef02 ftrace: change tr... |
17 |
|
1f0d69a9f tracing: profile ... |
18 |
#include "trace.h" |
002bb86d8 tracing/ftrace: s... |
19 |
#include "trace_stat.h" |
f633cef02 ftrace: change tr... |
20 |
#include "trace_output.h" |
1f0d69a9f tracing: profile ... |
21 |
|
2ed84eeb8 trace: rename unl... |
22 |
#ifdef CONFIG_BRANCH_TRACER |
52f232cb7 tracing: likely/u... |
23 |
|
002bb86d8 tracing/ftrace: s... |
24 |
static struct tracer branch_trace; |
9f029e83e ftrace: rename un... |
25 26 |
static int branch_tracing_enabled __read_mostly; static DEFINE_MUTEX(branch_tracing_mutex); |
e302cf3f9 tracing/branch-tr... |
27 |
|
9f029e83e ftrace: rename un... |
28 |
static struct trace_array *branch_tracer; |
52f232cb7 tracing: likely/u... |
29 30 |
static void |
9f029e83e ftrace: rename un... |
31 |
probe_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb7 tracing: likely/u... |
32 |
{ |
e1112b4d9 tracing/filters: ... |
33 |
struct ftrace_event_call *call = &event_branch; |
9f029e83e ftrace: rename un... |
34 |
struct trace_array *tr = branch_tracer; |
52f232cb7 tracing: likely/u... |
35 |
struct ring_buffer_event *event; |
9f029e83e ftrace: rename un... |
36 |
struct trace_branch *entry; |
8f6e8a314 tracing: user loc... |
37 |
struct ring_buffer *buffer; |
0a9877514 ring_buffer: remo... |
38 |
unsigned long flags; |
52f232cb7 tracing: likely/u... |
39 40 41 42 43 44 45 46 47 48 49 50 |
int cpu, pc; const char *p; /* * I would love to save just the ftrace_likely_data pointer, but * this code can also be used by modules. Ugly things can happen * if the module is unloaded, and then we go and read the * pointer. This is slower, but much safer. */ if (unlikely(!tr)) return; |
a5e25883a ftrace: replace r... |
51 |
local_irq_save(flags); |
52f232cb7 tracing: likely/u... |
52 53 54 |
cpu = raw_smp_processor_id(); if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) goto out; |
51a763dd8 tracing: Introduc... |
55 |
pc = preempt_count(); |
8f6e8a314 tracing: user loc... |
56 57 |
buffer = tr->buffer; event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH, |
51a763dd8 tracing: Introduc... |
58 |
sizeof(*entry), flags, pc); |
52f232cb7 tracing: likely/u... |
59 60 |
if (!event) goto out; |
52f232cb7 tracing: likely/u... |
61 |
entry = ring_buffer_event_data(event); |
52f232cb7 tracing: likely/u... |
62 63 64 65 66 67 68 69 70 71 72 73 74 |
/* Strip off the path, only save the file */ p = f->file + strlen(f->file); while (p >= f->file && *p != '/') p--; p++; strncpy(entry->func, f->func, TRACE_FUNC_SIZE); strncpy(entry->file, p, TRACE_FILE_SIZE); entry->func[TRACE_FUNC_SIZE] = 0; entry->file[TRACE_FILE_SIZE] = 0; entry->line = f->line; entry->correct = val == expect; |
8f6e8a314 tracing: user loc... |
75 76 |
if (!filter_check_discard(call, entry, buffer, event)) ring_buffer_unlock_commit(buffer, event); |
52f232cb7 tracing: likely/u... |
77 78 79 |
out: atomic_dec(&tr->data[cpu]->disabled); |
a5e25883a ftrace: replace r... |
80 |
local_irq_restore(flags); |
52f232cb7 tracing: likely/u... |
81 82 83 |
} static inline |
9f029e83e ftrace: rename un... |
84 |
void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb7 tracing: likely/u... |
85 |
{ |
9f029e83e ftrace: rename un... |
86 |
if (!branch_tracing_enabled) |
52f232cb7 tracing: likely/u... |
87 88 89 90 |
return; probe_likely_condition(f, val, expect); } |
9f029e83e ftrace: rename un... |
91 |
int enable_branch_tracing(struct trace_array *tr) |
52f232cb7 tracing: likely/u... |
92 |
{ |
9f029e83e ftrace: rename un... |
93 94 |
mutex_lock(&branch_tracing_mutex); branch_tracer = tr; |
52f232cb7 tracing: likely/u... |
95 96 97 98 99 |
/* * Must be seen before enabling. The reader is a condition * where we do not need a matching rmb() */ smp_wmb(); |
9f029e83e ftrace: rename un... |
100 101 |
branch_tracing_enabled++; mutex_unlock(&branch_tracing_mutex); |
52f232cb7 tracing: likely/u... |
102 |
|
f54fc98aa tracing: remove u... |
103 |
return 0; |
52f232cb7 tracing: likely/u... |
104 |
} |
9f029e83e ftrace: rename un... |
105 |
void disable_branch_tracing(void) |
52f232cb7 tracing: likely/u... |
106 |
{ |
9f029e83e ftrace: rename un... |
107 |
mutex_lock(&branch_tracing_mutex); |
52f232cb7 tracing: likely/u... |
108 |
|
9f029e83e ftrace: rename un... |
109 |
if (!branch_tracing_enabled) |
52f232cb7 tracing: likely/u... |
110 |
goto out_unlock; |
9f029e83e ftrace: rename un... |
111 |
branch_tracing_enabled--; |
52f232cb7 tracing: likely/u... |
112 113 |
out_unlock: |
9f029e83e ftrace: rename un... |
114 |
mutex_unlock(&branch_tracing_mutex); |
52f232cb7 tracing: likely/u... |
115 |
} |
80e5ea450 ftrace: add trace... |
116 117 118 119 120 121 122 123 124 125 |
static void start_branch_trace(struct trace_array *tr) { enable_branch_tracing(tr); } static void stop_branch_trace(struct trace_array *tr) { disable_branch_tracing(); } |
1c80025a4 tracing/ftrace: c... |
126 |
static int branch_trace_init(struct trace_array *tr) |
80e5ea450 ftrace: add trace... |
127 |
{ |
80e5ea450 ftrace: add trace... |
128 |
start_branch_trace(tr); |
1c80025a4 tracing/ftrace: c... |
129 |
return 0; |
80e5ea450 ftrace: add trace... |
130 131 132 133 134 135 |
} static void branch_trace_reset(struct trace_array *tr) { stop_branch_trace(tr); } |
ae7462b4f trace: make the t... |
136 |
static enum print_line_t trace_branch_print(struct trace_iterator *iter, |
a9a577638 tracing: Allow ev... |
137 |
int flags, struct trace_event *event) |
f633cef02 ftrace: change tr... |
138 139 |
{ struct trace_branch *field; |
2c9b238eb trace: Change str... |
140 |
trace_assign_type(field, iter->ent); |
f633cef02 ftrace: change tr... |
141 |
|
2c9b238eb trace: Change str... |
142 143 |
if (trace_seq_printf(&iter->seq, "[%s] %s:%s:%d ", |
f633cef02 ftrace: change tr... |
144 145 146 147 148 |
field->correct ? " ok " : " MISS ", field->func, field->file, field->line)) return TRACE_TYPE_PARTIAL_LINE; |
d9793bd80 trace: judicious ... |
149 |
return TRACE_TYPE_HANDLED; |
f633cef02 ftrace: change tr... |
150 |
} |
557055beb tracing: Fix bran... |
151 152 153 154 155 156 157 158 159 |
static void branch_print_header(struct seq_file *s) { seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT" " FUNC:FILE:LINE "); seq_puts(s, "# | | | | | " " | "); } |
e302cf3f9 tracing/branch-tr... |
160 |
|
a9a577638 tracing: Allow ev... |
161 162 163 |
static struct trace_event_functions trace_branch_funcs = { .trace = trace_branch_print, }; |
f633cef02 ftrace: change tr... |
164 |
static struct trace_event trace_branch_event = { |
ef18012b2 tracing: remove f... |
165 |
.type = TRACE_BRANCH, |
a9a577638 tracing: Allow ev... |
166 |
.funcs = &trace_branch_funcs, |
f633cef02 ftrace: change tr... |
167 |
}; |
002bb86d8 tracing/ftrace: s... |
168 169 170 171 172 173 174 175 |
static struct tracer branch_trace __read_mostly = { .name = "branch", .init = branch_trace_init, .reset = branch_trace_reset, #ifdef CONFIG_FTRACE_SELFTEST .selftest = trace_selftest_startup_branch, #endif /* CONFIG_FTRACE_SELFTEST */ |
557055beb tracing: Fix bran... |
176 |
.print_header = branch_print_header, |
002bb86d8 tracing/ftrace: s... |
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 |
}; __init static int init_branch_tracer(void) { int ret; ret = register_ftrace_event(&trace_branch_event); if (!ret) { printk(KERN_WARNING "Warning: could not register " "branch events "); return 1; } return register_tracer(&branch_trace); } device_initcall(init_branch_tracer); |
52f232cb7 tracing: likely/u... |
193 194 |
#else static inline |
9f029e83e ftrace: rename un... |
195 |
void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb7 tracing: likely/u... |
196 197 |
{ } |
2ed84eeb8 trace: rename unl... |
198 |
#endif /* CONFIG_BRANCH_TRACER */ |
52f232cb7 tracing: likely/u... |
199 |
|
9f029e83e ftrace: rename un... |
200 |
void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect) |
1f0d69a9f tracing: profile ... |
201 |
{ |
52f232cb7 tracing: likely/u... |
202 203 204 205 206 207 208 |
/* * I would love to have a trace point here instead, but the * trace point code is so inundated with unlikely and likely * conditions that the recursive nightmare that exists is too * much to try to get working. At least for now. */ trace_likely_condition(f, val, expect); |
1f0d69a9f tracing: profile ... |
209 210 211 212 213 214 215 |
/* FIXME: Make this atomic! */ if (val == expect) f->correct++; else f->incorrect++; } EXPORT_SYMBOL(ftrace_likely_update); |
e302cf3f9 tracing/branch-tr... |
216 217 |
extern unsigned long __start_annotated_branch_profile[]; extern unsigned long __stop_annotated_branch_profile[]; |
1f0d69a9f tracing: profile ... |
218 |
|
e302cf3f9 tracing/branch-tr... |
219 |
static int annotated_branch_stat_headers(struct seq_file *m) |
1f0d69a9f tracing: profile ... |
220 |
{ |
e302cf3f9 tracing/branch-tr... |
221 222 223 224 225 226 227 228 229 |
seq_printf(m, " correct incorrect %% "); seq_printf(m, " Function " " File Line " " ------- --------- - " " -------- " " ---- ---- "); return 0; |
1f0d69a9f tracing: profile ... |
230 |
} |
e302cf3f9 tracing/branch-tr... |
231 |
static inline long get_incorrect_percent(struct ftrace_branch_data *p) |
1f0d69a9f tracing: profile ... |
232 |
{ |
e302cf3f9 tracing/branch-tr... |
233 |
long percent; |
1f0d69a9f tracing: profile ... |
234 |
|
e302cf3f9 tracing/branch-tr... |
235 236 237 238 239 |
if (p->correct) { percent = p->incorrect * 100; percent /= p->correct + p->incorrect; } else percent = p->incorrect ? 100 : -1; |
1f0d69a9f tracing: profile ... |
240 |
|
e302cf3f9 tracing/branch-tr... |
241 |
return percent; |
1f0d69a9f tracing: profile ... |
242 |
} |
e302cf3f9 tracing/branch-tr... |
243 |
static int branch_stat_show(struct seq_file *m, void *v) |
1f0d69a9f tracing: profile ... |
244 |
{ |
9f029e83e ftrace: rename un... |
245 |
struct ftrace_branch_data *p = v; |
1f0d69a9f tracing: profile ... |
246 |
const char *f; |
bac28bfe4 trace: branch pro... |
247 |
long percent; |
1f0d69a9f tracing: profile ... |
248 |
|
1f0d69a9f tracing: profile ... |
249 250 251 252 253 |
/* Only print the file, not the path */ f = p->file + strlen(p->file); while (f >= p->file && *f != '/') f--; f++; |
2bcd521a6 trace: profile al... |
254 255 256 |
/* * The miss is overlayed on correct, and hit on incorrect. */ |
e302cf3f9 tracing/branch-tr... |
257 |
percent = get_incorrect_percent(p); |
1f0d69a9f tracing: profile ... |
258 |
|
bac28bfe4 trace: branch pro... |
259 260 261 262 263 |
seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); if (percent < 0) seq_printf(m, " X "); else seq_printf(m, "%3ld ", percent); |
1f0d69a9f tracing: profile ... |
264 265 266 267 |
seq_printf(m, "%-30.30s %-20.20s %d ", p->func, f, p->line); return 0; } |
425480081 tracing: add hand... |
268 |
static void *annotated_branch_stat_start(struct tracer_stat *trace) |
e302cf3f9 tracing/branch-tr... |
269 270 271 |
{ return __start_annotated_branch_profile; } |
1f0d69a9f tracing: profile ... |
272 |
|
e302cf3f9 tracing/branch-tr... |
273 274 |
static void * annotated_branch_stat_next(void *v, int idx) |
1f0d69a9f tracing: profile ... |
275 |
{ |
e302cf3f9 tracing/branch-tr... |
276 |
struct ftrace_branch_data *p = v; |
1f0d69a9f tracing: profile ... |
277 |
|
e302cf3f9 tracing/branch-tr... |
278 |
++p; |
1f0d69a9f tracing: profile ... |
279 |
|
e302cf3f9 tracing/branch-tr... |
280 281 282 283 |
if ((void *)p >= (void *)__stop_annotated_branch_profile) return NULL; return p; |
1f0d69a9f tracing: profile ... |
284 |
} |
e302cf3f9 tracing/branch-tr... |
285 286 287 288 289 290 291 292 293 294 295 296 297 298 |
static int annotated_branch_stat_cmp(void *p1, void *p2) { struct ftrace_branch_data *a = p1; struct ftrace_branch_data *b = p2; long percent_a, percent_b; percent_a = get_incorrect_percent(a); percent_b = get_incorrect_percent(b); if (percent_a < percent_b) return -1; if (percent_a > percent_b) return 1; |
ede55c9d7 tracing: Add corr... |
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 |
if (a->incorrect < b->incorrect) return -1; if (a->incorrect > b->incorrect) return 1; /* * Since the above shows worse (incorrect) cases * first, we continue that by showing best (correct) * cases last. */ if (a->correct > b->correct) return -1; if (a->correct < b->correct) return 1; return 0; |
e302cf3f9 tracing/branch-tr... |
316 |
} |
1f0d69a9f tracing: profile ... |
317 |
|
002bb86d8 tracing/ftrace: s... |
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 |
static struct tracer_stat annotated_branch_stats = { .name = "branch_annotated", .stat_start = annotated_branch_stat_start, .stat_next = annotated_branch_stat_next, .stat_cmp = annotated_branch_stat_cmp, .stat_headers = annotated_branch_stat_headers, .stat_show = branch_stat_show }; __init static int init_annotated_branch_stats(void) { int ret; ret = register_stat_tracer(&annotated_branch_stats); if (!ret) { printk(KERN_WARNING "Warning: could not register " "annotated branches stats "); return 1; } return 0; } fs_initcall(init_annotated_branch_stats); |
2bcd521a6 trace: profile al... |
341 |
#ifdef CONFIG_PROFILE_ALL_BRANCHES |
2bcd521a6 trace: profile al... |
342 |
|
e302cf3f9 tracing/branch-tr... |
343 344 |
extern unsigned long __start_branch_profile[]; extern unsigned long __stop_branch_profile[]; |
1f0d69a9f tracing: profile ... |
345 |
|
e302cf3f9 tracing/branch-tr... |
346 347 348 349 350 351 352 353 354 355 356 357 |
static int all_branch_stat_headers(struct seq_file *m) { seq_printf(m, " miss hit %% "); seq_printf(m, " Function " " File Line " " ------- --------- - " " -------- " " ---- ---- "); return 0; } |
1f0d69a9f tracing: profile ... |
358 |
|
425480081 tracing: add hand... |
359 |
static void *all_branch_stat_start(struct tracer_stat *trace) |
1f0d69a9f tracing: profile ... |
360 |
{ |
e302cf3f9 tracing/branch-tr... |
361 362 363 364 365 366 367 |
return __start_branch_profile; } static void * all_branch_stat_next(void *v, int idx) { struct ftrace_branch_data *p = v; |
1f0d69a9f tracing: profile ... |
368 |
|
e302cf3f9 tracing/branch-tr... |
369 |
++p; |
1f0d69a9f tracing: profile ... |
370 |
|
e302cf3f9 tracing/branch-tr... |
371 372 |
if ((void *)p >= (void *)__stop_branch_profile) return NULL; |
1f0d69a9f tracing: profile ... |
373 |
|
e302cf3f9 tracing/branch-tr... |
374 375 |
return p; } |
2bcd521a6 trace: profile al... |
376 |
|
002bb86d8 tracing/ftrace: s... |
377 378 |
static struct tracer_stat all_branch_stats = { .name = "branch_all", |
034939b65 tracing/ftrace: h... |
379 380 381 |
.stat_start = all_branch_stat_start, .stat_next = all_branch_stat_next, .stat_headers = all_branch_stat_headers, |
002bb86d8 tracing/ftrace: s... |
382 |
.stat_show = branch_stat_show |
034939b65 tracing/ftrace: h... |
383 |
}; |
e302cf3f9 tracing/branch-tr... |
384 |
|
002bb86d8 tracing/ftrace: s... |
385 |
__init static int all_annotated_branch_stats(void) |
e302cf3f9 tracing/branch-tr... |
386 |
{ |
e302cf3f9 tracing/branch-tr... |
387 |
int ret; |
002bb86d8 tracing/ftrace: s... |
388 389 |
ret = register_stat_tracer(&all_branch_stats); |
e302cf3f9 tracing/branch-tr... |
390 |
if (!ret) { |
002bb86d8 tracing/ftrace: s... |
391 392 393 |
printk(KERN_WARNING "Warning: could not register " "all branches stats "); |
e302cf3f9 tracing/branch-tr... |
394 395 |
return 1; } |
002bb86d8 tracing/ftrace: s... |
396 |
return 0; |
e302cf3f9 tracing/branch-tr... |
397 |
} |
002bb86d8 tracing/ftrace: s... |
398 399 |
fs_initcall(all_annotated_branch_stats); #endif /* CONFIG_PROFILE_ALL_BRANCHES */ |