Blame view
arch/mips/kernel/ftrace.c
8.47 KB
538f19526 MIPS: Tracing: Ad... |
1 2 3 4 |
/* * Code for replacing ftrace calls with jumps. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> |
e42405400 MIPS: Tracing: Re... |
5 |
* Copyright (C) 2009, 2010 DSLab, Lanzhou University, China |
f7a904dff MIPS: Loongson: C... |
6 |
* Author: Wu Zhangjin <wuzhangjin@gmail.com> |
538f19526 MIPS: Tracing: Ad... |
7 8 9 10 11 12 13 |
* * Thanks goes to Steven Rostedt for writing the original x86 version. */ #include <linux/uaccess.h> #include <linux/init.h> #include <linux/ftrace.h> |
29c5d3462 MIPS: Tracing: Ad... |
14 15 |
#include <asm/asm.h> #include <asm/asm-offsets.h> |
e42405400 MIPS: Tracing: Re... |
16 17 |
#include <asm/cacheflush.h> #include <asm/uasm.h> |
538f19526 MIPS: Tracing: Ad... |
18 |
|
d9cdb2f10 MIPS, Tracing: Su... |
19 |
#include <asm-generic/sections.h> |
c9f84873c MIPS: Tracing: Cl... |
20 |
|
49de830ad MIPS: Ftrace: Fix... |
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) #define MCOUNT_OFFSET_INSNS 5 #else #define MCOUNT_OFFSET_INSNS 4 #endif /* * Check if the address is in kernel space * * Clone core_kernel_text() from kernel/extable.c, but doesn't call * init_kernel_text() for Ftrace doesn't trace functions in init sections. */ static inline int in_kernel_space(unsigned long ip) { if (ip >= (unsigned long)_stext && ip <= (unsigned long)_etext) return 1; return 0; } |
538f19526 MIPS: Tracing: Ad... |
40 41 42 43 |
#ifdef CONFIG_DYNAMIC_FTRACE #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ |
c54794d19 MIPS: Mask jump t... |
44 |
#define JUMP_RANGE_MASK ((1UL << 28) - 1) |
538f19526 MIPS: Tracing: Ad... |
45 |
|
4d6829f92 MIPS: Tracing: Cl... |
46 |
#define INSN_NOP 0x00000000 /* nop */ |
e42405400 MIPS: Tracing: Re... |
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
#define INSN_JAL(addr) \ ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) static unsigned int insn_jal_ftrace_caller __read_mostly; static unsigned int insn_lui_v1_hi16_mcount __read_mostly; static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly; static inline void ftrace_dyn_arch_init_insns(void) { u32 *buf; unsigned int v1; /* lui v1, hi16_mcount */ v1 = 3; buf = (u32 *)&insn_lui_v1_hi16_mcount; UASM_i_LA_mostly(&buf, v1, MCOUNT_ADDR); /* jal (ftrace_caller + 8), jump over the first two instruction */ buf = (u32 *)&insn_jal_ftrace_caller; |
c54794d19 MIPS: Mask jump t... |
66 |
uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK); |
e42405400 MIPS: Tracing: Re... |
67 68 69 70 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER /* j ftrace_graph_caller */ buf = (u32 *)&insn_j_ftrace_graph_caller; |
c54794d19 MIPS: Mask jump t... |
71 |
uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK); |
e42405400 MIPS: Tracing: Re... |
72 73 |
#endif } |
538f19526 MIPS: Tracing: Ad... |
74 75 76 |
static int ftrace_modify_code(unsigned long ip, unsigned int new_code) { |
046199cae MIPS: Tracing: Ma... |
77 78 79 80 81 82 83 |
int faulted; /* *(unsigned int *)ip = new_code; */ safe_store_code(new_code, ip, faulted); if (unlikely(faulted)) return -EFAULT; |
538f19526 MIPS: Tracing: Ad... |
84 85 86 87 88 |
flush_icache_range(ip, ip + 8); return 0; } |
7f21a6096 MIPS, Tracing: Cl... |
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
/* * The details about the calling site of mcount on MIPS * * 1. For kernel: * * move at, ra * jal _mcount --> nop * * 2. For modules: * * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT * * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) * addiu v1, v1, low_16bit_of_mcount * move at, ra * move $12, ra_address * jalr v1 * sub sp, sp, 8 * 1: offset = 5 instructions * 2.2 For the Other situations * * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) * addiu v1, v1, low_16bit_of_mcount * move at, ra * jalr v1 * nop | move $12, ra_address | sub sp, sp, 8 * 1: offset = 4 instructions */ |
7f21a6096 MIPS, Tracing: Cl... |
117 |
#define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS) |
538f19526 MIPS: Tracing: Ad... |
118 119 120 121 122 |
int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { unsigned int new; unsigned long ip = rec->ip; |
e42405400 MIPS: Tracing: Re... |
123 |
/* |
d9cdb2f10 MIPS, Tracing: Su... |
124 125 |
* If ip is in kernel space, no long call, otherwise, long call is * needed. |
e42405400 MIPS: Tracing: Re... |
126 |
*/ |
7f21a6096 MIPS, Tracing: Cl... |
127 |
new = in_kernel_space(ip) ? INSN_NOP : INSN_B_1F; |
538f19526 MIPS: Tracing: Ad... |
128 129 |
return ftrace_modify_code(ip, new); } |
538f19526 MIPS: Tracing: Ad... |
130 131 132 133 |
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { unsigned int new; unsigned long ip = rec->ip; |
d9cdb2f10 MIPS, Tracing: Su... |
134 135 |
new = in_kernel_space(ip) ? insn_jal_ftrace_caller : insn_lui_v1_hi16_mcount; |
538f19526 MIPS: Tracing: Ad... |
136 137 138 139 140 141 142 143 144 |
return ftrace_modify_code(ip, new); } #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) int ftrace_update_ftrace_func(ftrace_func_t func) { unsigned int new; |
4d6829f92 MIPS: Tracing: Cl... |
145 |
new = INSN_JAL((unsigned long)func); |
538f19526 MIPS: Tracing: Ad... |
146 147 148 149 150 151 |
return ftrace_modify_code(FTRACE_CALL_IP, new); } int __init ftrace_dyn_arch_init(void *data) { |
e42405400 MIPS: Tracing: Re... |
152 153 154 155 156 |
/* Encode the instructions when booting */ ftrace_dyn_arch_init_insns(); /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); |
538f19526 MIPS: Tracing: Ad... |
157 158 159 160 161 |
/* The return code is retured via data */ *(unsigned long *)data = 0; return 0; } |
68ccf7521 MIPS: Tracing: Cl... |
162 |
#endif /* CONFIG_DYNAMIC_FTRACE */ |
29c5d3462 MIPS: Tracing: Ad... |
163 164 |
#ifdef CONFIG_FUNCTION_GRAPH_TRACER |
e17ff5fec MIPS: Tracing: Ad... |
165 166 167 |
#ifdef CONFIG_DYNAMIC_FTRACE extern void ftrace_graph_call(void); |
e17ff5fec MIPS: Tracing: Ad... |
168 169 170 171 172 |
#define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) int ftrace_enable_ftrace_graph_caller(void) { return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, |
e42405400 MIPS: Tracing: Re... |
173 |
insn_j_ftrace_graph_caller); |
e17ff5fec MIPS: Tracing: Ad... |
174 175 176 177 |
} int ftrace_disable_ftrace_graph_caller(void) { |
4d6829f92 MIPS: Tracing: Cl... |
178 |
return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP); |
e17ff5fec MIPS: Tracing: Ad... |
179 |
} |
68ccf7521 MIPS: Tracing: Cl... |
180 |
#endif /* CONFIG_DYNAMIC_FTRACE */ |
e17ff5fec MIPS: Tracing: Ad... |
181 |
|
7326c4e56 MIPS: Tracing: Ma... |
182 |
#ifndef KBUILD_MCOUNT_RA_ADDRESS |
68ccf7521 MIPS: Tracing: Cl... |
183 |
|
29c5d3462 MIPS: Tracing: Ad... |
184 185 186 |
#define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ |
2816e3259 MIPS, Tracing: Cl... |
187 188 |
unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long old_parent_ra, unsigned long parent_ra_addr, unsigned long fp) |
29c5d3462 MIPS: Tracing: Ad... |
189 |
{ |
2816e3259 MIPS, Tracing: Cl... |
190 |
unsigned long sp, ip, tmp; |
29c5d3462 MIPS: Tracing: Ad... |
191 |
unsigned int code; |
046199cae MIPS: Tracing: Ma... |
192 |
int faulted; |
29c5d3462 MIPS: Tracing: Ad... |
193 |
|
68ccf7521 MIPS: Tracing: Cl... |
194 |
/* |
2816e3259 MIPS, Tracing: Cl... |
195 |
* For module, move the ip from the return address after the |
9a620a559 MIPS, Tracing: Sp... |
196 197 |
* instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for * kernel, move after the instruction "move ra, at"(offset is 16) |
68ccf7521 MIPS: Tracing: Cl... |
198 |
*/ |
2816e3259 MIPS, Tracing: Cl... |
199 |
ip = self_ra - (in_kernel_space(self_ra) ? 16 : 24); |
29c5d3462 MIPS: Tracing: Ad... |
200 |
|
68ccf7521 MIPS: Tracing: Cl... |
201 202 203 204 |
/* * search the text until finding the non-store instruction or "s{d,w} * ra, offset(sp)" instruction */ |
29c5d3462 MIPS: Tracing: Ad... |
205 |
do { |
046199cae MIPS: Tracing: Ma... |
206 207 208 209 210 |
/* get the code at "ip": code = *(unsigned int *)ip; */ safe_load_code(code, ip, faulted); if (unlikely(faulted)) return 0; |
68ccf7521 MIPS: Tracing: Cl... |
211 212 |
/* * If we hit the non-store instruction before finding where the |
29c5d3462 MIPS: Tracing: Ad... |
213 |
* ra is stored, then this is a leaf function and it does not |
68ccf7521 MIPS: Tracing: Cl... |
214 215 |
* store the ra on the stack */ |
29c5d3462 MIPS: Tracing: Ad... |
216 |
if ((code & S_R_SP) != S_R_SP) |
2816e3259 MIPS, Tracing: Cl... |
217 |
return parent_ra_addr; |
29c5d3462 MIPS: Tracing: Ad... |
218 |
|
9a620a559 MIPS, Tracing: Sp... |
219 220 221 |
/* Move to the next instruction */ ip -= 4; } while ((code & S_RA_SP) != S_RA_SP); |
29c5d3462 MIPS: Tracing: Ad... |
222 223 |
sp = fp + (code & OFFSET_MASK); |
046199cae MIPS: Tracing: Ma... |
224 |
|
2816e3259 MIPS, Tracing: Cl... |
225 226 |
/* tmp = *(unsigned long *)sp; */ safe_load_stack(tmp, sp, faulted); |
046199cae MIPS: Tracing: Ma... |
227 228 |
if (unlikely(faulted)) return 0; |
29c5d3462 MIPS: Tracing: Ad... |
229 |
|
2816e3259 MIPS, Tracing: Cl... |
230 |
if (tmp == old_parent_ra) |
29c5d3462 MIPS: Tracing: Ad... |
231 |
return sp; |
29c5d3462 MIPS: Tracing: Ad... |
232 233 |
return 0; } |
68ccf7521 MIPS: Tracing: Cl... |
234 |
#endif /* !KBUILD_MCOUNT_RA_ADDRESS */ |
7326c4e56 MIPS: Tracing: Ma... |
235 |
|
29c5d3462 MIPS: Tracing: Ad... |
236 237 238 239 |
/* * Hook the return address and push it in the stack of return addrs * in current thread info. */ |
2816e3259 MIPS, Tracing: Cl... |
240 |
void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, |
29c5d3462 MIPS: Tracing: Ad... |
241 242 |
unsigned long fp) { |
2816e3259 MIPS, Tracing: Cl... |
243 |
unsigned long old_parent_ra; |
29c5d3462 MIPS: Tracing: Ad... |
244 245 246 |
struct ftrace_graph_ent trace; unsigned long return_hooker = (unsigned long) &return_to_handler; |
b9f07eb2f MIPS, Tracing: Fi... |
247 |
int faulted, insns; |
29c5d3462 MIPS: Tracing: Ad... |
248 249 250 |
if (unlikely(atomic_read(¤t->tracing_graph_pause))) return; |
68ccf7521 MIPS: Tracing: Cl... |
251 |
/* |
2816e3259 MIPS, Tracing: Cl... |
252 253 |
* "parent_ra_addr" is the stack address saved the return address of * the caller of _mcount. |
7326c4e56 MIPS: Tracing: Ma... |
254 255 256 257 258 259 |
* * if the gcc < 4.5, a leaf function does not save the return address * in the stack address, so, we "emulate" one in _mcount's stack space, * and hijack it directly, but for a non-leaf function, it save the * return address to the its own stack space, we can not hijack it * directly, but need to find the real stack address, |
29c5d3462 MIPS: Tracing: Ad... |
260 |
* ftrace_get_parent_addr() does it! |
7326c4e56 MIPS: Tracing: Ma... |
261 262 263 264 265 |
* * if gcc>= 4.5, with the new -mmcount-ra-address option, for a * non-leaf function, the location of the return address will be saved * to $12 for us, and for a leaf function, only put a zero into $12. we * do it in ftrace_graph_caller of mcount.S. |
29c5d3462 MIPS: Tracing: Ad... |
266 |
*/ |
2816e3259 MIPS, Tracing: Cl... |
267 268 |
/* old_parent_ra = *parent_ra_addr; */ safe_load_stack(old_parent_ra, parent_ra_addr, faulted); |
046199cae MIPS: Tracing: Ma... |
269 270 |
if (unlikely(faulted)) goto out; |
7326c4e56 MIPS: Tracing: Ma... |
271 |
#ifndef KBUILD_MCOUNT_RA_ADDRESS |
2816e3259 MIPS, Tracing: Cl... |
272 273 |
parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra, old_parent_ra, (unsigned long)parent_ra_addr, fp); |
68ccf7521 MIPS: Tracing: Cl... |
274 275 276 277 |
/* * If fails when getting the stack address of the non-leaf function's * ra, stop function graph tracer and return */ |
2816e3259 MIPS, Tracing: Cl... |
278 |
if (parent_ra_addr == 0) |
046199cae MIPS: Tracing: Ma... |
279 |
goto out; |
7326c4e56 MIPS: Tracing: Ma... |
280 |
#endif |
2816e3259 MIPS, Tracing: Cl... |
281 282 |
/* *parent_ra_addr = return_hooker; */ safe_store_stack(return_hooker, parent_ra_addr, faulted); |
046199cae MIPS: Tracing: Ma... |
283 284 |
if (unlikely(faulted)) goto out; |
29c5d3462 MIPS: Tracing: Ad... |
285 |
|
2816e3259 MIPS, Tracing: Cl... |
286 287 288 |
if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp) == -EBUSY) { *parent_ra_addr = old_parent_ra; |
29c5d3462 MIPS: Tracing: Ad... |
289 290 |
return; } |
b9f07eb2f MIPS, Tracing: Fi... |
291 292 293 294 295 296 297 298 |
/* * Get the recorded ip of the current mcount calling site in the * __mcount_loc section, which will be used to filter the function * entries configured through the tracing/set_graph_function interface. */ insns = in_kernel_space(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; trace.func = self_ra - (MCOUNT_INSN_SIZE * insns); |
29c5d3462 MIPS: Tracing: Ad... |
299 300 301 302 |
/* Only trace if the calling function expects to */ if (!ftrace_graph_entry(&trace)) { current->curr_ret_stack--; |
2816e3259 MIPS, Tracing: Cl... |
303 |
*parent_ra_addr = old_parent_ra; |
29c5d3462 MIPS: Tracing: Ad... |
304 |
} |
046199cae MIPS: Tracing: Ma... |
305 306 307 308 |
return; out: ftrace_graph_stop(); WARN_ON(1); |
29c5d3462 MIPS: Tracing: Ad... |
309 |
} |
68ccf7521 MIPS: Tracing: Cl... |
310 |
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |